curvefs/client: fix s3 object will not be removed

signed-off-by: hzwuhongsong hzwuhongsong@corp.netease.com

.bazelignore

@@ -1 +0,0 @@
-thirdparties/aws/aws-sdk-cpp

.gitignore

@@ -119,6 +119,7 @@ thirdparties/rocksdb/lib/
 thirdparties/rocksdb/include/
 thirdparties/rocksdb/rocksdb/
 thirdparties/rocksdb/*.tar.gz
+thirdparties/aws/*.tar.gz
 
 /external
 /bazel-*

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "nbd"]
 	path = nbd
 	url = https://github.com/opencurve/curve-nbd
-[submodule "thirdparties/aws/aws-sdk-cpp"]
-	path = thirdparties/aws/aws-sdk-cpp
-	url = https://github.com/opencurve/aws-sdk-cpp

Makefile

@@ -1,22 +1,27 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build install image
+.PHONY: list build dep install image
 
+stor?=""
 prefix?= "$(PWD)/projects"
 release?= 0
+dep?= 0
 only?= "*"
 tag?= "curvebs:unknown"
 case?= "*"
 os?= "debian9"
 
 list:
-	@bash util/build.sh --list
+	@bash util/build.sh --stor=${stor} --list
 
 build:
-	@bash util/build.sh --only=$(only) --release=$(release) --os=$(os)
+	@bash util/build.sh --stor=${stor} --only=$(only) --dep=$(dep) --release=$(release) --os=$(os)
+
+dep:
+	@bash util/build.sh --stor=${stor} --only="" --dep=1
 
 install:
-	@bash util/install.sh --prefix=$(prefix) --only=$(only)
+	@bash util/install.sh --stor=${stor} --prefix=$(prefix) --only=$(only)
 
 image:
-	@bash util/image.sh $(tag) $(os)
+	@bash util/image.sh ${stor} $(tag) $(os)

WORKSPACE

@@ -173,12 +173,37 @@ new_local_repository(
     path = "thirdparties/etcdclient",
 )
 
-new_local_repository(
+http_archive(
     name = "aws",
+    urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"],
+    sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf",
+    strip_prefix = "aws-sdk-cpp-1.7.340",
     build_file = "//:thirdparties/aws/aws.BUILD",
-    path = "thirdparties/aws/aws-sdk-cpp",
 )
 
+http_archive(
+    name = "aws_c_common",
+    urls = ["https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz"],
+    sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
+    strip_prefix = "aws-c-common-0.4.29",
+    build_file = "//:thirdparties/aws/aws-c-common.BUILD",
+)
+
+http_archive(
+    name = "aws_c_event_stream",
+    urls = ["https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz"],
+    sha256 = "31d880d1c868d3f3df1e1f4b45e56ac73724a4dc3449d04d47fc0746f6f077b6",
+    strip_prefix = "aws-c-event-stream-0.1.4",
+    build_file = "//:thirdparties/aws/aws-c-event-stream.BUILD",
+)
+
+http_archive(
+    name = "aws_checksums",
+    urls = ["https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz"],
+    sha256 = "6e6bed6f75cf54006b6bafb01b3b96df19605572131a2260fddaf0e87949ced0",
+    strip_prefix = "aws-checksums-0.1.5",
+    build_file = "//:thirdparties/aws/aws-checksums.BUILD",
+)
 
 # C++ rules for Bazel.
 http_archive(

conf/s3.conf

@@ -7,8 +7,8 @@ s3.endpoint=
 # reserved for backward compatible
 s3.snapshot_bucket_name=
 s3.bucket_name=
-s3.ak=
-s3.sk=
+s3.ak=fake
+s3.sk=fake
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=false

curvefs/Makefile

@@ -1,23 +0,0 @@
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-.PHONY: build dep install image
-
-prefix?= "$(PWD)/devops/projects"
-only?= "*"
-dep?= 0
-release?= 0
-os?= "debian9"
-build_rocksdb?= 0
-tag?= "curvefs:unknown"
-
-build:
-	@bash util/build.sh --only=$(only) --dep=$(dep) --release=$(release) --os=$(os) --build_rocksdb=$(build_rocksdb)
-
-dep:
-	@bash util/build.sh --only="" --dep=1 --build_rocksdb=$(build_rocksdb)
-
-install:
-	@bash util/install.sh --prefix=$(prefix) --only=$(only)
-
-image:
-	@bash util/image.sh $(tag) $(os)

curvefs/conf/client.conf

@@ -82,10 +82,17 @@ fuseClient.listDentryLimit=65536
 fuseClient.flushPeriodSec=5
 fuseClient.maxNameLength=255
 fuseClient.iCacheLruSize=65536
-fuseClient.dCacheLruSize=65536
+fuseClient.dCacheLruSize=1000000
 fuseClient.enableICacheMetrics=true
 fuseClient.enableDCacheMetrics=true
-fuseClient.cto=false
+fuseClient.cto=true
+fuseClient.downloadMaxRetryTimes=3
+
+### kvcache opt
+fuseClient.supportKVcache=false
+fuseClient.setThreadPool=4
+fuseClient.getThreadPool=4
+
 # you shoudle enable it when mount one filesystem to multi mountpoints,
 # it gurantee the consistent of file after rename, otherwise you should
 # disable it for performance.
@@ -103,6 +110,7 @@ fuseClient.disableXattr=false
 fuseClient.maxDataSize=1024
 # default refresh data interval 30s
 fuseClient.refreshDataIntervalSec=30
+fuseClient.warmupThreadsNum=10
 
 #### volume
 volume.bigFileSize=1048576
@@ -147,13 +155,14 @@ s3.readCacheMaxByte=209715200
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=False
-s3.max_connections=32
+s3.region=us-east-1
+s3.max_connections=500
 s3.connect_timeout=60000
 s3.request_timeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
 s3.loglevel=4
 s3.logPrefix=/data/logs/curvefs/aws_ # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__
-s3.async_thread_num=30
+s3.async_thread_num=500
 # limit all inflight async requests' bytes, |0| means not limited
 s3.max_async_request_inflight_bytes=104857600
 s3.chunkFlushThreads=5

curvefs/docker/debian10/Dockerfile

@@ -4,7 +4,5 @@ RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
 COPY entrypoint.sh /
 COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libaws-cpp-sdk-core.so /usr/lib/
-COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian11/Dockerfile

@@ -4,7 +4,5 @@ RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
 COPY entrypoint.sh /
 COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libaws-cpp-sdk-core.so /usr/lib/
-COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian9/Dockerfile

@@ -1,10 +1,9 @@
 FROM opencurvedocker/curve-base:debian9
 ENV TZ=Asia/Shanghai
+COPY entrypoint.sh /
 RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
-COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libaws-cpp-sdk-core.so /usr/lib/
-COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
-RUN chmod a+x /entrypoint.sh
+RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \
+    && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \
+    && cp /curvefs/tools-v2/sbin/curve /usr/bin/
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian9/compile/Dockerfile

@@ -1,5 +1,4 @@
 FROM opencurvedocker/curve-base:debian9
-
 RUN apt-get clean \
     && apt-get -y update \
     && apt-get -y install \

curvefs/docker/debian9/entrypoint.sh

@@ -8,6 +8,7 @@ g_args=""
 g_prefix=""
 g_binary=""
 g_start_args=""
+g_preexec="/curvefs/tools-v2/sbin/daemon"
 
 ############################  BASIC FUNCTIONS
 function msg() {
@@ -119,6 +120,7 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
+    [[ ! -z $g_preexec ]] && $g_preexec &
     if [ $g_role == "etcd" ]; then
         exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
     elif [ $g_role == "monitor" ]; then

curvefs/monitor/grafana/provisioning/dashboards/client.json

@@ -3474,7 +3474,7 @@
                 "type": "prometheus"
               },
               "exemplar": true,
-              "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_io_size_count\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_read_io_size\", instance=~\"$instance\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -3484,7 +3484,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "read_io_size_count",
+          "title": "read_io_size",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -3570,7 +3570,7 @@
                 "type": "prometheus"
               },
               "exemplar": true,
-              "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_io_size_count\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\"curvefs.*[[fs:regex]]_user_write_io_size\", instance=~\"$instance\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -3580,7 +3580,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "write_io_size_count",
+          "title": "write_io_size",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -4631,7 +4631,7 @@
                 "type": "prometheus"
               },
               "exemplar": true,
-              "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_size_count\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_read_size\", instance=~\"$instance\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -4641,7 +4641,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "read_size",
+          "title": "read_io_size",
           "tooltip": {
             "shared": true,
             "sort": 0,
@@ -4727,7 +4727,7 @@
                 "type": "prometheus"
               },
               "exemplar": true,
-              "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_size_count\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\"curvefs_s3.*[[fs:regex]]_adaptor_write_size\", instance=~\"$instance\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -4737,7 +4737,7 @@
           "timeFrom": null,
           "timeRegions": [],
           "timeShift": null,
-          "title": "write_size",
+          "title": "write_io_size",
           "tooltip": {
             "shared": true,
             "sort": 0,

curvefs/proto/metaserver.proto

@@ -218,7 +218,7 @@ message Inode {
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index
     optional uint32 dtime = 19;
     optional uint32 openmpcount = 20; // openmpcount mount points had the file open
-    map<string, string> xattr = 21;
+    map<string, bytes> xattr = 21;
     repeated uint64 parent = 22;
 }
 
@@ -265,12 +265,6 @@ message CreateRootInodeResponse {
     optional uint64 appliedIndex = 2;
 }
 
-enum InodeOpenStatusChange {
-    OPEN = 1;
-    CLOSE = 2;
-    NOCHANGE = 3;
-}
-
 message UpdateInodeRequest {
     required uint32 poolId = 1;
     required uint32 copysetId = 2;
@@ -290,8 +284,8 @@ message UpdateInodeRequest {
     // field 16 is left for compatibility
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 17;
     optional uint32 nlink = 18;
-    optional InodeOpenStatusChange inodeOpenstatusChange = 19;
-    map<string, string> xattr = 20;
+    // field 19 is left for compatibility
+    map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
     optional VolumeExtentList volumeExtents = 23;
@@ -384,7 +378,7 @@ message InodeAttr {
     optional uint64 rdev = 16;
     optional uint32 dtime = 17;
     optional uint32 openmpcount = 18;
-    map<string, string> xattr = 19;
+    map<string, bytes> xattr = 19;
     repeated uint64 parent = 20;
 }
 
@@ -415,7 +409,7 @@ message BatchGetXAttrRequest {
 message XAttr {
     required uint64 inodeId = 1;
     required uint32 fsId = 2;
-    map<string, string> xAttrInfos = 3;
+    map<string, bytes> xAttrInfos = 3;
 }
 
 message BatchGetXAttrResponse {

curvefs/src/client/BUILD

@@ -72,5 +72,6 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",
+        "@com_google_googletest//:gtest_prod",
     ],
 )

curvefs/src/client/client_operator.cpp

@@ -183,11 +183,11 @@ CURVEFS_ERROR RenameOperator::Precheck() {
 // record old inode info if overwrite
 CURVEFS_ERROR RenameOperator::RecordOldInodeInfo() {
     if (oldInodeId_ != 0) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        auto rc = inodeManager_->GetInode(oldInodeId_, inodeWrapper);
+        InodeAttr attr;
+        CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(oldInodeId_, &attr);
         if (rc == CURVEFS_ERROR::OK) {
-            oldInodeSize_ = inodeWrapper->GetLength();
-            oldInodeType_ = inodeWrapper->GetType();
+            oldInodeSize_ = attr.length();
+            oldInodeType_ = attr.type();
         } else {
             LOG_ERROR("GetInode", rc);
             return CURVEFS_ERROR::NOTEXIST;
@@ -279,7 +279,7 @@ CURVEFS_ERROR RenameOperator::LinkInode(uint64_t inodeId, uint64_t parent) {
         return rc;
     }
 
-    rc = inodeWrapper->LinkLocked(parent);
+    rc = inodeWrapper->Link(parent);
     if (rc != CURVEFS_ERROR::OK) {
         LOG_ERROR("Link", rc);
         return rc;
@@ -297,7 +297,7 @@ CURVEFS_ERROR RenameOperator::UnLinkInode(uint64_t inodeId, uint64_t parent) {
         return rc;
     }
 
-    rc = inodeWrapper->UnLinkLocked(parent);
+    rc = inodeWrapper->UnLink(parent);
     if (rc != CURVEFS_ERROR::OK) {
         LOG_ERROR("UnLink", rc);
         return rc;
@@ -316,10 +316,7 @@ CURVEFS_ERROR RenameOperator::UpdateMCTime(uint64_t inodeId) {
     }
 
     curve::common::UniqueLock lk = inodeWrapper->GetUniqueLock();
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    inodeWrapper->SetMTime(now.tv_sec, now.tv_nsec);
-    inodeWrapper->SetCTime(now.tv_sec, now.tv_nsec);
+    inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
 
     rc = inodeWrapper->SyncAttr();
     if (rc != CURVEFS_ERROR::OK) {
@@ -388,7 +385,7 @@ CURVEFS_ERROR RenameOperator::UpdateInodeParent() {
         return rc;
     }
 
-    rc = inodeWrapper->UpdateParentLocked(parentId_, newParentId_);
+    rc = inodeWrapper->UpdateParent(parentId_, newParentId_);
     if (rc != CURVEFS_ERROR::OK) {
         LOG_ERROR("UpdateInodeParent", rc);
         return rc;

curvefs/src/client/client_operator.h

@@ -68,9 +68,9 @@ class RenameOperator {
         *oldInodeType = oldInodeType_;
     }
 
- private:
     std::string DebugString();
 
+ private:
     CURVEFS_ERROR CheckOverwrite();
 
     CURVEFS_ERROR GetLatestTxIdWithLock();

curvefs/src/client/common/common.cpp

@@ -76,6 +76,29 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) {
     return os;
 }
 
+const char kCurveFsWarmupOpAdd[] = "add";
+const char kCurveFsWarmupTypeList[] = "list";
+const char kCurveFsWarmupTypeSingle[] = "single";
+
+WarmupOpType GetWarmupOpType(const std::string& op) {
+    auto ret = WarmupOpType::kWarmupOpUnknown;
+    if (op == kCurveFsWarmupOpAdd) {
+        ret = WarmupOpType::kWarmupOpAdd;
+    }
+    return ret;
+}
+
+WarmupType GetWarmupType(const std::string& type) {
+    auto ret =
+        WarmupType::kWarmupTypeUnknown;
+    if (type == kCurveFsWarmupTypeList) {
+        ret = WarmupType::kWarmupTypeList;
+    } else if (type == kCurveFsWarmupTypeSingle) {
+        ret = WarmupType::kWarmupTypeSingle;
+    }
+    return ret;
+}
+
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/common/common.h

@@ -27,6 +27,7 @@
 
 #include <bthread/mutex.h>
 #include <ostream>
+#include <string>
 
 namespace curvefs {
 namespace client {
@@ -62,7 +63,35 @@ enum class MetaServerOpType {
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);
 
-const uint32_t MAXXATTRLENGTH = 256;
+const uint32_t MAX_XATTR_NAME_LENGTH = 255;
+const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024;
+
+const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op";
+
+enum class WarmupOpType {
+    kWarmupOpUnknown = 0,
+    kWarmupOpAdd = 1,
+};
+
+WarmupOpType GetWarmupOpType(const std::string& op);
+
+enum class WarmupType {
+    kWarmupTypeUnknown = 0,
+    kWarmupTypeList = 1,
+    kWarmupTypeSingle = 2,
+};
+
+WarmupType GetWarmupType(const std::string& type);
+
+enum class FileHandle : uint64_t {
+    kDefaultValue = 0,
+    kKeepCache = 1,
+};
+
+enum class NlinkChange : int32_t {
+    kAddOne = 1,
+    kSubOne = -1,
+};
 
 }  // namespace common
 }  // namespace client

curvefs/src/client/common/config.cpp

@@ -279,6 +279,10 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
     conf->GetValueFatalIfFail("fuseClient.disableXattr",
                               &clientOption->disableXattr);
     conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
+    conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
+                              &clientOption->warmupThreadsNum);
+    conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
+                              &clientOption->downloadMaxRetryTimes);
 
     LOG_IF(WARNING, conf->GetBoolValue("fuseClient.enableSplice",
                                        &clientOption->enableFuseSplice))

curvefs/src/client/common/config.h

@@ -197,6 +197,8 @@ struct FuseClientOption {
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;
     bool disableXattr = false;
+    uint32_t warmupThreadsNum;
+    uint32_t downloadMaxRetryTimes;
 };
 
 void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption);

curvefs/src/client/curve_fuse_op.cpp

@@ -23,6 +23,9 @@
 
 #include <string>
 #include <memory>
+#include <cstring>
+#include <utility>
+#include <vector>
 
 #include "curvefs/src/client/curve_fuse_op.h"
 #include "curvefs/src/client/fuse_client.h"
@@ -46,12 +49,12 @@ using ::curvefs::client::FuseClient;
 using ::curvefs::client::FuseS3Client;
 using ::curvefs::client::FuseVolumeClient;
 using ::curvefs::client::common::FuseClientOption;
-using ::curvefs::client::common::MAXXATTRLENGTH;
 using ::curvefs::client::rpcclient::MdsClientImpl;
 using ::curvefs::client::rpcclient::MDSBaseClient;
 using ::curvefs::client::metric::ClientOpMetric;
 using ::curvefs::common::LatencyUpdater;
 using ::curvefs::client::metric::InflightGuard;
+using ::curvefs::client::common::FileHandle;
 
 using ::curvefs::common::FLAGS_vlog_level;
 
@@ -130,18 +133,17 @@ int InitGlog(const char *confPath, const char *argv0) {
     return 0;
 }
 
-int InitFuseClient(const char *confPath, const char* fsName,
-    const char *fsType, const char *mdsAddr) {
+int InitFuseClient(const struct MountOption *mountOption) {
     g_clientOpMetric = new ClientOpMetric();
 
     Configuration conf;
-    conf.SetConfigPath(confPath);
+    conf.SetConfigPath(mountOption->conf);
     if (!conf.LoadConfig()) {
-        LOG(ERROR) << "LoadConfig failed, confPath = " << confPath;
+        LOG(ERROR) << "LoadConfig failed, confPath = " << mountOption->conf;
         return -1;
     }
-    if (mdsAddr)
-        conf.SetStringValue("mdsOpt.rpcRetryOpt.addrs", mdsAddr);
+    if (mountOption->mdsAddr)
+        conf.SetStringValue("mdsOpt.rpcRetryOpt.addrs", mountOption->mdsAddr);
 
     conf.PrintConfig();
 
@@ -149,11 +151,12 @@ int InitFuseClient(const char *confPath, const char* fsName,
     curvefs::client::common::InitFuseClientOption(&conf, g_fuseClientOption);
 
     std::shared_ptr<FsInfo> fsInfo = std::make_shared<FsInfo>();
-    if (GetFsInfo(fsName, fsInfo.get()) != 0) {
+    if (GetFsInfo(mountOption->fsName, fsInfo.get()) != 0) {
         return -1;
     }
 
-    std::string fsTypeStr = (fsType == nullptr) ? "" : fsType;
+    std::string fsTypeStr =
+        (mountOption->fsType == nullptr) ? "" : mountOption->fsType;
     std::string fsTypeMds;
     if (fsInfo->fstype() == FSType::TYPE_S3) {
        fsTypeMds = "s3";
@@ -183,6 +186,11 @@ int InitFuseClient(const char *confPath, const char* fsName,
         return -1;
     }
 
+    ret = g_ClientInstance->SetMountStatus(mountOption);
+    if (ret != CURVEFS_ERROR::OK) {
+        return -1;
+    }
+
     return 0;
 }
 
@@ -273,13 +281,73 @@ void FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) {
     fuse_reply_attr(req, &attr, g_fuseClientOption->attrTimeOut);
 }
 
+int AddWarmupTask(const std::string& type, const std::string& path) {
+    int ret = 0;
+    switch (curvefs::client::common::GetWarmupType(type)) {
+        case curvefs::client::common::WarmupType::kWarmupTypeList:
+            g_ClientInstance->PutWarmTask(path);
+            break;
+        case curvefs::client::common::WarmupType::kWarmupTypeSingle:
+            g_ClientInstance->FetchDentryEnqueue(path);
+            break;
+        default:
+            // not support add warmup type (warmup single file/dir or filelist)
+            LOG(ERROR) << "not support warmup type, only support single/list";
+            ret = ERANGE;
+    }
+    return ret;
+}
+
+int Warmup(const std::string& name, const std::string& value) {
+    // warmup
+    std::vector<std::string> opTypePath;
+    curve::common::SplitString(value, "\n", &opTypePath);
+    if (opTypePath.size() != 3) {
+        LOG(ERROR) << name << " has invalid xattr value " << value;
+        return ERANGE;
+    }
+    int ret = 0;
+    switch (curvefs::client::common::GetWarmupOpType(opTypePath[0])) {
+        case curvefs::client::common::WarmupOpType::kWarmupOpAdd:
+            ret = AddWarmupTask(opTypePath[1], opTypePath[2]);
+            if (ret != 0) {
+                LOG(ERROR) << name << " has invalid xattr value " << value;
+            }
+            break;
+        default:
+            LOG(ERROR) << name << " has invalid xattr value " << value;
+            ret = ERANGE;
+    }
+    return ret;
+}
+
+void FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino, const char* name,
+                    const char* value, size_t size, int flags) {
+    std::string xattrValue(value, size);
+    VLOG(9) << "FuseOpSetXattr"
+            << " ino " << ino << " name " << name << " value " << xattrValue
+            << " flags " << flags;
+    if (strcmp(name, curvefs::client::common::kCurveFsWarmupXAttr) == 0) {
+        // warmup
+        int code = Warmup(name, xattrValue);
+        fuse_reply_err(req, code);
+    } else {
+        // set xattr
+        CURVEFS_ERROR ret = g_ClientInstance->FuseOpSetXattr(req, ino, name,
+            value, size, flags);
+        FuseReplyErrByErrCode(req, ret);
+    }
+
+    VLOG(9) << "FuseOpSetXattr done";
+}
+
 void FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino, const char *name,
     size_t size) {
     InflightGuard guard(&g_clientOpMetric->opGetXattr.inflightOpNum);
     LatencyUpdater updater(&g_clientOpMetric->opGetXattr.latency);
-    char buf[MAXXATTRLENGTH] = {0};
+    std::string buf;
     CURVEFS_ERROR ret = g_ClientInstance->FuseOpGetXattr(req, ino, name,
-                                                         buf, size);
+                                                         &buf, size);
     if (ret != CURVEFS_ERROR::OK) {
         g_clientOpMetric->opGetXattr.ecount << 1;
         FuseReplyErrByErrCode(req, ret);
@@ -287,9 +355,9 @@ void FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino, const char *name,
     }
 
     if (size == 0) {
-        fuse_reply_xattr(req, strlen(buf));
+        fuse_reply_xattr(req, buf.length());
     } else {
-        fuse_reply_buf(req, buf, strlen(buf));
+        fuse_reply_buf(req, buf.data(), buf.length());
     }
 }
 

curvefs/src/client/curve_fuse_op.h

@@ -39,8 +39,7 @@ extern "C" {
 
 int InitGlog(const char *confPath, const char *argv0);
 
-int InitFuseClient(const char *confPath, const char *fsName,
-        const char* fsType, const char* mdsAddr);
+int InitFuseClient(const struct MountOption *mountOption);
 
 void UnInitFuseClient();
 

curvefs/src/client/dentry_cache_manager.cpp

@@ -78,7 +78,6 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
             << ", parent = " << parent << ", name = " << name;
         return MetaStatusCodeToCurvefsErrCode(ret);
     }
-
     if (!curvefs::client::common::FLAGS_enableCto) {
         dCache_->Put(key, *out);
     }

curvefs/src/client/fuse_client.cpp

@@ -33,9 +33,13 @@
 #include <utility>
 
 #include "curvefs/proto/mds.pb.h"
+#include "curvefs/src/client/common/common.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/fuse_common.h"
 #include "curvefs/src/client/client_operator.h"
+#include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/xattr_manager.h"
+#include "curvefs/src/common/define.h"
 #include "src/common/net_common.h"
 #include "src/common/dummyserver.h"
 #include "src/client/client_common.h"
@@ -47,7 +51,9 @@ using ::curvefs::common::S3Info;
 using ::curvefs::common::Volume;
 using ::curvefs::mds::topology::PartitionTxId;
 using ::curvefs::mds::FSStatusCode_Name;
-using ::curvefs::client::common::MAXXATTRLENGTH;
+using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH;
+using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
+using ::curvefs::client::common::FileHandle;
 
 #define RETURN_IF_UNSUCCESS(action)                                            \
     do {                                                                       \
@@ -122,17 +128,215 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     if (ret3 != CURVEFS_ERROR::OK) {
         return ret3;
     }
-
     ret3 =
         dentryManager_->Init(option.dCacheLruSize, option.enableDCacheMetrics);
     if (ret3 != CURVEFS_ERROR::OK) {
         return ret3;
     }
-
+    warmUpFile_.exist = false;
+    bgCmdStop_.store(false, std::memory_order_release);
+    bgCmdTaskThread_ = Thread(&FuseClient::WarmUpTask, this);
+    taskFetchMetaPool_.Start(option_.warmupThreadsNum);
     return ret3;
 }
 
+void FuseClient::WarmUpTask() {
+    // TODO(hzwuhongsong): Maybe we can start the warmup thread after mount
+    while (!mounted_.load(std::memory_order_acquire)) {
+        usleep(WARMUP_CHECKINTERVAL_US);
+        VLOG(6) << "wait mount success.";
+        continue;
+    }
+    while (!bgCmdStop_.load(std::memory_order_acquire)) {
+        std::list<std::string> readAheadPaths;
+        WaitWarmUp();
+        while (hasWarmTask()) {
+            std::string warmUpTask;
+            GetwarmTask(&warmUpTask);
+            VLOG(9) << "warmup task is: " << warmUpTask;
+            std::string pDelimiter = "/";
+            char* pToken = nullptr;
+            char* pSave = nullptr;
+            pToken = strtok_r(const_cast<char*>(warmUpTask.c_str()),
+              const_cast<char*>(pDelimiter.c_str()), &pSave);
+            if (nullptr == pToken) {
+                VLOG(6) << "warmUpTask nullptr";
+                continue;
+            }
+            Dentry dentry;
+            CURVEFS_ERROR ret = dentryManager_->GetDentry(
+              fsInfo_->rootinodeid(), pToken, &dentry);
+            if (ret != CURVEFS_ERROR::OK) {
+                LOG(WARNING) << "FetchDentry error: " << ret
+                             << ", name: " << warmUpTask;
+                return;
+            }
+            if (FsFileType::TYPE_S3 != dentry.type()) {
+                LOG(WARNING) << "not a file: " << warmUpTask
+                        << "type is: " << dentry.type();
+                return;
+            }
+            fuse_ino_t  ino = dentry.inodeid();
+            std::shared_ptr<InodeWrapper> inodeWrapper;
+            ret = inodeManager_->GetInode(ino, inodeWrapper);
+            if (ret != CURVEFS_ERROR::OK) {
+                LOG(ERROR) << "inodeManager get inode fail, ret = "
+                           << ret << ", inodeid = " << ino;
+                return;
+            }
+            uint64_t len = inodeWrapper->GetLength();
+            VLOG(9) << "ino is: " << ino << ", len is: " << len;
+            WarmUpFileContext_t warmUpFile{ino, len, true};
+            SetWarmUpFile(warmUpFile);
+        }
+    }
+}
+
+void FuseClient::FetchDentryEnqueue(std::string file) {
+    VLOG(9) << "FetchDentryEnqueue start: " << file;
+    auto task = [this, file]() {
+        LookPath(file);
+    };
+    taskFetchMetaPool_.Enqueue(task);
+}
+
+void FuseClient::LookPath(std::string file) {
+    VLOG(9) << "LookPath start: " << file;
+    std::vector<std::string> splitPath;
+    // remove enter, newline, blank
+    std::string blanks("\r\n ");
+    file.erase(0, file.find_first_not_of(blanks));
+    file.erase(file.find_last_not_of(blanks) + 1);
+    if (file.empty()) {
+        VLOG(9) << "empty path";
+        return;
+    }
+    bool isRoot = false;
+    if (file == "/") {
+        splitPath.push_back(file);
+        isRoot = true;
+    } else {
+        splitStr(file, "/", &splitPath);
+    }
+    VLOG(6) << "splitPath size is: " << splitPath.size();
+    if (splitPath.size() == 1 && isRoot) {
+        VLOG(9) << "i am root";
+        FetchChildDentryEnqueue(fsInfo_->rootinodeid());
+        return;
+    } else if (splitPath.size() == 1) {
+        VLOG(9) << "parent is root: " << fsInfo_->rootinodeid()
+                << ", path is: " << splitPath[0];
+        this->FetchDentry(fsInfo_->rootinodeid(), splitPath[0]);
+        return;
+    } else if (splitPath.size() > 1) {  // travel path
+        VLOG(9) << "traverse path start: " << splitPath.size();
+        std::string lastName = splitPath.back();
+        splitPath.pop_back();
+        fuse_ino_t ino = fsInfo_->rootinodeid();
+        for (auto iter : splitPath) {
+            VLOG(9) << "traverse path: " << iter
+                    << "ino is: " << ino;
+            Dentry dentry;
+            std::string pathName = iter;
+            CURVEFS_ERROR ret = dentryManager_->GetDentry(
+              ino, pathName, &dentry);
+            if (ret != CURVEFS_ERROR::OK) {
+                if (ret != CURVEFS_ERROR::NOTEXIST) {
+                  LOG(WARNING) << "dentryManager_ get dentry fail, ret = "
+                    << ret << ", parent inodeid = " << ino
+                    << ", name = " << file;
+                }
+                VLOG(9) << "FetchDentry error: " << ret;
+                return;
+            }
+            ino = dentry.inodeid();
+        }
+        this->FetchDentry(ino, lastName);
+        VLOG(9) << "ino is: " << ino
+                << "lastname is: " << lastName;
+        return;
+    } else {
+        VLOG(3) << "unknown path";
+    }
+    return;
+}
+
+void FuseClient::FetchChildDentryEnqueue(fuse_ino_t  ino) {
+    auto task = [this, ino]() {
+       // reverse from root
+       this->FetchChildDentry(ino);
+    };
+    taskFetchMetaPool_.Enqueue(task);
+}
+
+void FuseClient::FetchChildDentry(fuse_ino_t ino) {
+    VLOG(9) << "FetchChildDentry start: " <<  ino;
+    std::list<Dentry> dentryList;
+    auto limit = option_.listDentryLimit;
+    CURVEFS_ERROR ret = dentryManager_->ListDentry(
+      ino, &dentryList, limit);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
+                    << ", parent = " << ino;
+        return;
+    }
+    for (auto iter : dentryList) {
+        VLOG(9) << "FetchChildDentry: " <<  iter.name();
+        if (FsFileType::TYPE_S3 == iter.type()) {
+            std::unique_lock<std::mutex> lck(fetchMtx_);
+            readAheadFiles_.push_front(iter.inodeid());
+            VLOG(9) << "FetchChildDentry: " << iter.inodeid();;
+        } else if (FsFileType::TYPE_DIRECTORY == iter.type()) {
+            FetchChildDentryEnqueue(iter.inodeid());
+            VLOG(9) << "FetchChildDentry: " << iter.inodeid();
+        } else if (FsFileType::TYPE_SYM_LINK == iter.type()) {  // need todo
+        } else {
+            VLOG(9) << "unknown type";
+        }
+    }
+    return;
+}
+
+void FuseClient::FetchDentry(fuse_ino_t ino, std::string file) {
+    VLOG(9) << "FetchDentry start: " << file
+            << ", ino: " << ino;
+    Dentry dentry;
+    CURVEFS_ERROR ret = dentryManager_->GetDentry(ino, file, &dentry);
+    if (ret != CURVEFS_ERROR::OK) {
+        if (ret != CURVEFS_ERROR::NOTEXIST) {
+            LOG(WARNING) << "dentryManager_ get dentry fail, ret = " << ret
+                         << ", parent inodeid = " << ino
+                         << ", name = " << file;
+        }
+        VLOG(1) << "FetchDentry error: " << ret;
+        return;
+    }
+    if (FsFileType::TYPE_S3 == dentry.type()) {
+        std::unique_lock<std::mutex> lck(fetchMtx_);
+        readAheadFiles_.push_front(dentry.inodeid());
+        return;
+    } else if (FsFileType::TYPE_DIRECTORY == dentry.type()) {
+        FetchChildDentryEnqueue(dentry.inodeid());
+        VLOG(9) << "FetchDentry: " << dentry.inodeid();
+        return;
+
+    } else if (FsFileType::TYPE_SYM_LINK == dentry.type()) {
+    } else {
+        VLOG(3) << "unkown, file: " << file
+                << ", ino: " << ino;
+    }
+    VLOG(9) << "FetchDentry end: " << file
+            << ", ino: " << ino;
+    return;
+}
+
 void FuseClient::UnInit() {
+    bgCmdStop_.store(true, std::memory_order_release);
+    WarmUpRun();
+    if (bgCmdTaskThread_.joinable()) {
+        bgCmdTaskThread_.join();
+    }
+    taskFetchMetaPool_.Stop();
     delete mdsBase_;
     mdsBase_ = nullptr;
 }
@@ -154,49 +358,6 @@ void FuseClient::Fini() {
 
 CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                      struct fuse_conn_info *conn) {
-    struct MountOption* mOpts = (struct MountOption*)userdata;
-    // set path
-    mountpoint_.set_path((mOpts->mountPoint == nullptr) ? ""
-                                                       : mOpts->mountPoint);
-    std::string fsName = (mOpts->fsName == nullptr) ? "" : mOpts->fsName;
-
-    mountpoint_.set_cto(FLAGS_enableCto);
-
-    int retVal = SetHostPortInMountPoint(&mountpoint_);
-    if (retVal < 0) {
-        LOG(ERROR) << "Set Host and Port in MountPoint failed, ret = "
-                   << retVal;
-        return CURVEFS_ERROR::INTERNAL;
-    }
-
-    auto ret = mdsClient_->MountFs(fsName, mountpoint_, fsInfo_.get());
-    if (ret != FSStatusCode::OK && ret != FSStatusCode::MOUNT_POINT_EXIST) {
-        LOG(ERROR) << "MountFs failed, FSStatusCode = " << ret
-                   << ", FSStatusCode_Name = "
-                   << FSStatusCode_Name(ret)
-                   << ", fsName = " << fsName
-                   << ", mountPoint = " << mountpoint_.ShortDebugString();
-        return CURVEFS_ERROR::MOUNT_FAILED;
-    }
-
-    inodeManager_->SetFsId(fsInfo_->fsid());
-    dentryManager_->SetFsId(fsInfo_->fsid());
-    enableSumInDir_ = fsInfo_->enablesumindir() && !FLAGS_enableCto;
-    LOG(INFO) << "Mount " << fsName << " on " << mountpoint_.ShortDebugString()
-              << " success!"
-              << " enableSumInDir = " << enableSumInDir_;
-
-    fsMetric_ = std::make_shared<FSMetric>(fsName);
-
-    // init fsname and mountpoint
-    leaseExecutor_->SetFsName(fsName);
-    leaseExecutor_->SetMountPoint(mountpoint_);
-    if (!leaseExecutor_->Start()) {
-        return CURVEFS_ERROR::INTERNAL;
-    }
-
-    init_ = true;
-
     return CURVEFS_ERROR::OK;
 }
 
@@ -318,25 +479,18 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
                    << ", inodeid = " << ino;
         return ret;
     }
-
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
     if (fi->flags & O_TRUNC) {
         if (fi->flags & O_WRONLY || fi->flags & O_RDWR) {
-            Inode *inode = inodeWrapper->GetMutableInodeUnlocked();
-            uint64_t length = inode->length();
-            CURVEFS_ERROR tRet = Truncate(inode, 0);
+            uint64_t length = inodeWrapper->GetLengthLocked();
+            CURVEFS_ERROR tRet = Truncate(inodeWrapper.get(), 0);
             if (tRet != CURVEFS_ERROR::OK) {
                 LOG(ERROR) << "truncate file fail, ret = " << ret
                            << ", inodeid = " << ino;
                 return CURVEFS_ERROR::INTERNAL;
             }
-            inode->set_length(0);
-            struct timespec now;
-            clock_gettime(CLOCK_REALTIME, &now);
-            inode->set_ctime(now.tv_sec);
-            inode->set_ctime_ns(now.tv_nsec);
-            inode->set_mtime(now.tv_sec);
-            inode->set_mtime_ns(now.tv_nsec);
+            inodeWrapper->SetLengthLocked(0);
+            inodeWrapper->UpdateTimestampLocked(kChangeTime | kModifyTime);
             if (length != 0) {
                 ret = inodeWrapper->Sync();
                 if (ret != CURVEFS_ERROR::OK) {
@@ -348,6 +502,7 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
 
             if (enableSumInDir_ && length != 0) {
                 // update parent summary info
+                const Inode *inode = inodeWrapper->GetInodeLocked();
                 XAttr xattr;
                 xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
                     std::to_string(length)});
@@ -365,12 +520,14 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
             return CURVEFS_ERROR::NOPERMISSION;
         }
     }
-
+    if (FLAGS_enableCto) {
+        inodeManager_->AddOpenedInode(ino);
+    }
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::UpdateParentInodeMCTimeAndInvalidNlink(
-    fuse_ino_t parent, FsFileType type) {
+CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
+    fuse_ino_t parent, FsFileType type, NlinkChange nlink) {
     std::shared_ptr<InodeWrapper> parentInodeWrapper;
     auto ret = inodeManager_->GetInode(parent, parentInodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
@@ -381,15 +538,13 @@ CURVEFS_ERROR FuseClient::UpdateParentInodeMCTimeAndInvalidNlink(
 
     {
         curve::common::UniqueLock lk = parentInodeWrapper->GetUniqueLock();
-        struct timespec now;
-        clock_gettime(CLOCK_REALTIME, &now);
-        parentInodeWrapper->SetMTime(now.tv_sec, now.tv_nsec);
-        parentInodeWrapper->SetCTime(now.tv_sec, now.tv_nsec);
+        parentInodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
 
         if (FsFileType::TYPE_DIRECTORY == type) {
-            parentInodeWrapper->InvalidateNlink();
+            parentInodeWrapper->UpdateNlinkLocked(nlink);
         }
     }
+    inodeManager_->ShipToFlush(parentInodeWrapper);
     return CURVEFS_ERROR::OK;
 }
 
@@ -454,9 +609,9 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
         return ret;
     }
 
-    ret = UpdateParentInodeMCTimeAndInvalidNlink(parent, type);
+    ret = UpdateParentMCTimeAndNlink(parent, type, NlinkChange::kAddOne);
     if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "UpdateParentInodeMCTimeAndInvalidNlink failed"
+        LOG(ERROR) << "UpdateParentMCTimeAndNlink failed"
                    << ", parent: " << parent
                    << ", name: " << name
                    << ", type: " << type;
@@ -487,7 +642,7 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
     }
 
     InodeAttr attr;
-    inodeWrapper->GetInodeAttrLocked(&attr);
+    inodeWrapper->GetInodeAttr(&attr);
     GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
@@ -522,6 +677,7 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
 
     uint64_t ino = dentry.inodeid();
 
+    // judge dir empty
     if (FsFileType::TYPE_DIRECTORY == type) {
         std::list<Dentry> dentryList;
         auto limit = option_.listDentryLimit;
@@ -544,9 +700,9 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
         return ret;
     }
 
-    ret = UpdateParentInodeMCTimeAndInvalidNlink(parent, type);
+    ret = UpdateParentMCTimeAndNlink(parent, type, NlinkChange::kSubOne);
     if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "UpdateParentInodeMCTimeAndInvalidNlink failed"
+        LOG(ERROR) << "UpdateParentMCTimeAndNlink failed"
                    << ", parent: " << parent
                    << ", name: " << name
                    << ", type: " << type;
@@ -561,7 +717,7 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
         return ret;
     }
 
-    ret = inodeWrapper->UnLinkLocked(parent);
+    ret = inodeWrapper->UnLink(parent);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "UnLink failed, ret = " << ret << ", inodeid = " << ino
                    << ", parent = " << parent << ", name = " << name;
@@ -661,8 +817,6 @@ CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
         return ret;
     }
 
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-
     uint64_t dindex = fi->fh;
     DirBufferHead *bufHead = dirBuf_->DirBufferGet(dindex);
     if (!bufHead->wasRead) {
@@ -686,7 +840,7 @@ CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
                 inodeIds.emplace(dentry.inodeid());
             }
             VLOG(3) << "batch get inode size = " << inodeIds.size();
-            ret = inodeManager_->BatchGetInodeAttrAsync(ino, inodeIds,
+            ret = inodeManager_->BatchGetInodeAttrAsync(ino, &inodeIds,
                                                         &inodeAttrMap);
             if (ret != CURVEFS_ERROR::OK) {
                 LOG(ERROR) << "BatchGetInodeAttr failed when FuseOpReadDir"
@@ -736,6 +890,7 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
 
     curve::common::LockGuard lg(renameMutex_);
     CURVEFS_ERROR rc = CURVEFS_ERROR::OK;
+    VLOG(3) << "FuseOpRename [start]: " << renameOp.DebugString();
     RETURN_IF_UNSUCCESS(GetTxId);
     RETURN_IF_UNSUCCESS(Precheck);
     RETURN_IF_UNSUCCESS(RecordOldInodeInfo);
@@ -744,6 +899,7 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
     RETURN_IF_UNSUCCESS(LinkDestParentInode);
     RETURN_IF_UNSUCCESS(PrepareTx);
     RETURN_IF_UNSUCCESS(CommitTx);
+    VLOG(3) << "FuseOpRename [success]: " << renameOp.DebugString();
     // Do not check UnlinkSrcParentInode, beause rename is already success
     renameOp.UnlinkSrcParentInode();
     renameOp.UnlinkOldInode();
@@ -762,15 +918,6 @@ CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi,
                                         struct stat *attr) {
     VLOG(1) << "FuseOpGetAttr ino = " << ino;
-    if (FLAGS_enableCto) {
-        CURVEFS_ERROR ret = inodeManager_->RefreshInode(ino);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                       << ", inodeid = " << ino;
-            return ret;
-        }
-    }
-
     InodeAttr inodeAttr;
     CURVEFS_ERROR ret =
         inodeManager_->GetInodeAttr(ino, &inodeAttr);
@@ -796,64 +943,60 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                    << ", inodeid = " << ino;
         return ret;
     }
-
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-    Inode *inode = inodeWrapper->GetMutableInodeUnlocked();
+    if (to_set & FUSE_SET_ATTR_MODE) {
+        inodeWrapper->SetMode(attr->st_mode);
+    }
+    if (to_set & FUSE_SET_ATTR_UID) {
+        inodeWrapper->SetUid(attr->st_uid);
+    }
+    if (to_set & FUSE_SET_ATTR_GID) {
+        inodeWrapper->SetGid(attr->st_gid);
+    }
 
     struct timespec now;
     clock_gettime(CLOCK_REALTIME, &now);
 
-    if (to_set & FUSE_SET_ATTR_MODE) {
-        inode->set_mode(attr->st_mode);
-    }
-    if (to_set & FUSE_SET_ATTR_UID) {
-        inode->set_uid(attr->st_uid);
-    }
-    if (to_set & FUSE_SET_ATTR_GID) {
-        inode->set_gid(attr->st_gid);
-    }
     if (to_set & FUSE_SET_ATTR_ATIME) {
-        inode->set_atime(attr->st_atim.tv_sec);
-        inode->set_atime_ns(attr->st_atim.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(attr->st_atim, kAccessTime);
     }
     if (to_set & FUSE_SET_ATTR_ATIME_NOW) {
-        inode->set_atime(now.tv_sec);
-        inode->set_atime_ns(now.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(now, kAccessTime);
     }
     if (to_set & FUSE_SET_ATTR_MTIME) {
-        inode->set_mtime(attr->st_mtim.tv_sec);
-        inode->set_mtime_ns(attr->st_mtim.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(attr->st_mtim, kModifyTime);
     }
     if (to_set & FUSE_SET_ATTR_MTIME_NOW) {
-        inode->set_mtime(now.tv_sec);
-        inode->set_mtime_ns(now.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(now, kModifyTime);
     }
     if (to_set & FUSE_SET_ATTR_CTIME) {
-        inode->set_ctime(attr->st_ctim.tv_sec);
-        inode->set_ctime_ns(attr->st_ctim.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(attr->st_ctim, kChangeTime);
     } else {
-        inode->set_ctime(now.tv_sec);
-        inode->set_ctime_ns(now.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(now, kChangeTime);
     }
+
     if (to_set & FUSE_SET_ATTR_SIZE) {
-        int64_t changeSize =  attr->st_size - inode->length();
-        CURVEFS_ERROR tRet = Truncate(inode, attr->st_size);
+        int64_t changeSize =
+            attr->st_size -
+            static_cast<int64_t>(inodeWrapper->GetLengthLocked());
+        CURVEFS_ERROR tRet = Truncate(inodeWrapper.get(), attr->st_size);
         if (tRet != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "truncate file fail, ret = " << ret
                        << ", inodeid = " << ino;
             return tRet;
         }
-        inode->set_length(attr->st_size);
+        inodeWrapper->SetLengthLocked(attr->st_size);
         ret = inodeWrapper->Sync();
         if (ret != CURVEFS_ERROR::OK) {
             return ret;
         }
         InodeAttr inodeAttr;
-        inodeWrapper->GetInodeAttrUnlocked(&inodeAttr);
+        inodeWrapper->GetInodeAttrLocked(&inodeAttr);
         InodeAttr2ParamAttr(inodeAttr, attrOut);
 
         if (enableSumInDir_ && changeSize != 0) {
             // update parent summary info
+            const Inode* inode = inodeWrapper->GetInodeLocked();
             XAttr xattr;
             xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
                 std::to_string(std::abs(changeSize))});
@@ -875,27 +1018,13 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
         return ret;
     }
     InodeAttr inodeAttr;
-    inodeWrapper->GetInodeAttrUnlocked(&inodeAttr);
+    inodeWrapper->GetInodeAttrLocked(&inodeAttr);
     InodeAttr2ParamAttr(inodeAttr, attrOut);
     return ret;
 }
 
-bool IsSummaryInfo(const char *name) {
-    return std::strstr(name, SUMMARYPREFIX);
-}
-
-bool IsOneLayer(const char *name) {
-    if (std::strcmp(name, XATTRFILES) == 0 ||
-        std::strcmp(name, XATTRSUBDIRS) == 0 ||
-        std::strcmp(name, XATTRENTRIES) == 0 ||
-        std::strcmp(name, XATTRFBYTES) == 0) {
-        return true;
-    }
-    return false;
-}
-
 CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                          size_t size) {
     VLOG(9) << "FuseOpGetXattr, ino: " << ino
             << ", name: " << name << ", size = " << size;
@@ -903,51 +1032,28 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
         return CURVEFS_ERROR::NOTSUPPORT;
     }
 
-    std::string xValue;
-    // get summary info
-    if (IsSummaryInfo(name)) {
-        InodeAttr inodeAttr;
-        CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                       << ", inodeid = " << ino;
-            return ret;
-        }
-
-        if (inodeAttr.type() == FsFileType::TYPE_DIRECTORY) {
-            // not enable record summary info in dir xattr,
-            // need recursive computation all files
-            if (!enableSumInDir_) {
-                if (IsOneLayer(name)) {
-                    ret = xattrManager_->CalOneLayerSumInfo(&inodeAttr);
-                } else {
-                    ret = xattrManager_->CalAllLayerSumInfo(&inodeAttr);
-                }
-            } else {
-                if (IsOneLayer(name)) {
-                    ret = xattrManager_->FastCalOneLayerSumInfo(&inodeAttr);
-                } else {
-                    ret = xattrManager_->FastCalAllLayerSumInfo(&inodeAttr);
-                }
-            }
-
-            if (CURVEFS_ERROR::OK != ret) {
-                return ret;
-            }
-            LOG(INFO) << "After calculate summary info:\n"
-                      << inodeAttr.DebugString();
-            auto it = inodeAttr.xattr().find(name);
-            if (it != inodeAttr.xattr().end()) {
-                xValue = it->second;
-            }
-        }
+    InodeAttr inodeAttr;
+    CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
+                    << ", inodeid = " << ino;
+        return ret;
     }
 
-    CURVEFS_ERROR ret = CURVEFS_ERROR::NODATA;
-    if (xValue.length() > 0) {
-        if ((size == 0 && xValue.length() <= MAXXATTRLENGTH) ||
-            (size >= xValue.length() && size <= MAXXATTRLENGTH)) {
-            memcpy(value, xValue.c_str(), xValue.length());
+    ret = xattrManager_->GetXattr(name, value, &inodeAttr, enableSumInDir_);
+    if (CURVEFS_ERROR::OK != ret) {
+        LOG(ERROR) << "xattrManager get xattr failed, name = " << name;
+        return ret;
+    }
+
+    ret = CURVEFS_ERROR::NODATA;
+    if (value->length() > 0) {
+        if ((size == 0 && value->length() <= MAX_XATTR_VALUE_LENGTH) ||
+            (size >= value->length() &&
+                value->length() <= MAX_XATTR_VALUE_LENGTH)) {
+            VLOG(1) << "FuseOpGetXattr name = " << name
+                    << ", length = " << value->length()
+                    << ", value = " << *value;
             ret = CURVEFS_ERROR::OK;
         } else {
             ret = CURVEFS_ERROR::OUT_OF_RANGE;
@@ -956,6 +1062,46 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
     return ret;
 }
 
+CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
+                                         const char* name, const char* value,
+                                         size_t size, int flags) {
+    if (option_.disableXattr) {
+        return CURVEFS_ERROR::NOTSUPPORT;
+    }
+
+    std::string strname(name);
+    std::string strvalue(value, size);
+    VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
+            << ", size = " << size
+            << ", strvalue: " << strvalue;
+    if (strname.length() > MAX_XATTR_NAME_LENGTH  ||
+        size > MAX_XATTR_VALUE_LENGTH) {
+        LOG(ERROR) << "xattr length is too long, name = " << name
+                   << ", name length = " << strname.length()
+                   << ", value length = " << size;
+        return CURVEFS_ERROR::OUT_OF_RANGE;
+    }
+
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
+    }
+
+    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+    inodeWrapper->SetXattrLocked(strname, strvalue);
+    ret = inodeWrapper->SyncAttr();
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "set xattr fail, ret = " << ret << ", inodeid = " << ino
+                   << ", name = " << strname << ", value = " << strvalue;
+        return ret;
+    }
+    VLOG(1) << "FuseOpSetXattr end";
+    return CURVEFS_ERROR::OK;
+}
+
 CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize) {
     VLOG(1) << "FuseOpListXattr, ino: " << ino << ", size = " << size;
@@ -1050,10 +1196,10 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
         return ret;
     }
 
-    ret = UpdateParentInodeMCTimeAndInvalidNlink(
-        parent, FsFileType::TYPE_SYM_LINK);
+    ret = UpdateParentMCTimeAndNlink(parent, FsFileType::TYPE_SYM_LINK,
+        NlinkChange::kAddOne);
     if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "UpdateParentInodeMCTimeAndInvalidNlink failed"
+        LOG(ERROR) << "UpdateParentMCTimeAndNlink failed"
                    << ", link:" << link
                    << ", parent: " << parent
                    << ", name: " << name
@@ -1077,7 +1223,7 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
     }
 
     InodeAttr attr;
-    inodeWrapper->GetInodeAttrLocked(&attr);
+    inodeWrapper->GetInodeAttr(&attr);
     GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
@@ -1096,7 +1242,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
                    << ", inodeid = " << ino;
         return ret;
     }
-    ret = inodeWrapper->LinkLocked(newparent);
+    ret = inodeWrapper->Link(newparent);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "Link Inode fail, ret = " << ret << ", inodeid = " << ino
                    << ", newparent = " << newparent
@@ -1114,7 +1260,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
         LOG(ERROR) << "dentryManager_ CreateDentry fail, ret = " << ret
                    << ", parent = " << newparent << ", name = " << newname;
 
-        CURVEFS_ERROR ret2 = inodeWrapper->UnLinkLocked(newparent);
+        CURVEFS_ERROR ret2 = inodeWrapper->UnLink(newparent);
         if (ret2 != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "Also unlink inode failed, ret = " << ret2
                        << ", inodeid = " << inodeWrapper->GetInodeId();
@@ -1122,9 +1268,9 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
         return ret;
     }
 
-    ret = UpdateParentInodeMCTimeAndInvalidNlink(newparent, type);
+    ret = UpdateParentMCTimeAndNlink(newparent, type, NlinkChange::kAddOne);
     if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "UpdateParentInodeMCTimeAndInvalidNlink failed"
+        LOG(ERROR) << "UpdateParentMCTimeAndNlink failed"
                    << ", parent: " << newparent
                    << ", name: " << newname
                    << ", type: " << type;
@@ -1148,7 +1294,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
     }
 
     InodeAttr attr;
-    inodeWrapper->GetInodeAttrLocked(&attr);
+    inodeWrapper->GetInodeAttr(&attr);
     GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
@@ -1156,20 +1302,23 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string *linkStr) {
     VLOG(1) << "FuseOpReadLink, ino: " << ino << ", linkStr: " << linkStr;
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    InodeAttr attr;
+    CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &attr);
     if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
                    << ", inodeid = " << ino;
         return ret;
     }
-    *linkStr = inodeWrapper->GetSymlinkStr();
+    *linkStr = attr.symlink();
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
     VLOG(1) << "FuseOpRelease, ino: " << ino;
+    if (FLAGS_enableCto) {
+        inodeManager_->RemoveOpenedInode(ino);
+    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -1182,5 +1331,51 @@ void FuseClient::FlushAll() {
     FlushInodeAll();
 }
 
+CURVEFS_ERROR
+FuseClient::SetMountStatus(const struct MountOption *mountOption) {
+    mountpoint_.set_path(
+        (mountOption->mountPoint == nullptr) ? "" : mountOption->mountPoint);
+    std::string fsName =
+        (mountOption->fsName == nullptr) ? "" : mountOption->fsName;
+
+    mountpoint_.set_cto(FLAGS_enableCto);
+
+    int retVal = SetHostPortInMountPoint(&mountpoint_);
+    if (retVal < 0) {
+        LOG(ERROR) << "Set Host and Port in MountPoint failed, ret = "
+                   << retVal;
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    auto ret = mdsClient_->MountFs(fsName, mountpoint_, fsInfo_.get());
+    if (ret != FSStatusCode::OK && ret != FSStatusCode::MOUNT_POINT_EXIST) {
+        LOG(ERROR) << "MountFs failed, FSStatusCode = " << ret
+                   << ", FSStatusCode_Name = " << FSStatusCode_Name(ret)
+                   << ", fsName = " << fsName
+                   << ", mountPoint = " << mountpoint_.ShortDebugString();
+        return CURVEFS_ERROR::MOUNT_FAILED;
+    }
+    inodeManager_->SetFsId(fsInfo_->fsid());
+    dentryManager_->SetFsId(fsInfo_->fsid());
+    enableSumInDir_ = fsInfo_->enablesumindir() && !FLAGS_enableCto;
+
+    LOG(INFO) << "Mount " << fsName << " on " << mountpoint_.ShortDebugString()
+              << " success!"
+              << " enableSumInDir = " << enableSumInDir_;
+
+    fsMetric_ = std::make_shared<FSMetric>(fsName);
+
+    // init fsname and mountpoint
+    leaseExecutor_->SetFsName(fsName);
+    leaseExecutor_->SetMountPoint(mountpoint_);
+    if (!leaseExecutor_->Start()) {
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    init_ = true;
+    mounted_.store(true, std::memory_order_release);
+    return CURVEFS_ERROR::OK;
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/fuse_client.h

@@ -30,6 +30,8 @@
 #include <memory>
 #include <string>
 #include <list>
+#include <utility>
+#include <vector>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/mds.pb.h"
@@ -45,12 +47,14 @@
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/common/define.h"
+#include "curvefs/src/common/s3util.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/lease/lease_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
 
 #define DirectIOAlignment 512
+#define WARMUP_CHECKINTERVAL_US 1000*1000
 
 using ::curve::common::Atomic;
 using ::curve::common::InterruptibleSleeper;
@@ -74,7 +78,11 @@ using curvefs::common::is_aligned;
 const uint32_t kMaxHostNameLength = 255u;
 
 using mds::Mountpoint;
-
+typedef struct WarmUpFileContext {
+    fuse_ino_t inode;
+    uint64_t fileLen;
+    bool exist;
+} WarmUpFileContext_t;
 class FuseClient {
  public:
     FuseClient()
@@ -87,6 +95,7 @@ class FuseClient {
         mdsBase_(nullptr),
         isStop_(true),
         init_(false),
+        mounted_(false),
         enableSumInDir_(false) {}
 
     virtual ~FuseClient() {}
@@ -104,6 +113,7 @@ class FuseClient {
             mdsBase_(nullptr),
             isStop_(true),
             init_(false),
+            mounted_(false),
             enableSumInDir_(false) {}
 
     virtual CURVEFS_ERROR Init(const FuseClientOption &option);
@@ -180,9 +190,13 @@ class FuseClient {
                                         struct stat* attrOut);
 
     virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                          size_t size);
 
+    virtual CURVEFS_ERROR FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
+                                         const char* name, const char* value,
+                                         size_t size, int flags);
+
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 
@@ -230,6 +244,15 @@ class FuseClient {
         init_ = true;
     }
 
+    void SetMounted(bool mounted) {
+        mounted_ = mounted;
+    }
+
+    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable> &
+      GetTaskFetchPool() {
+        return taskFetchMetaPool_;
+    }
+
     std::shared_ptr<FsInfo> GetFsInfo() {
         return fsInfo_;
     }
@@ -244,6 +267,36 @@ class FuseClient {
     void SetEnableSumInDir(bool enable) {
         enableSumInDir_ = enable;
     }
+    void GetReadAheadFiles(std::list<fuse_ino_t>* readAheadFiles) {
+        std::unique_lock<std::mutex> lck(fetchMtx_);
+        *readAheadFiles = std::move(readAheadFiles_);
+    }
+
+    void GetWarmUpFile(WarmUpFileContext_t* warmUpFile) {
+        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
+        *warmUpFile = std::move(warmUpFile_);
+        warmUpFile_.exist = false;
+        return;
+    }
+    void SetWarmUpFile(WarmUpFileContext_t warmUpFile) {
+        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
+        warmUpFile_ = warmUpFile;
+        warmUpFile_.exist = true;
+    }
+    bool hasWarmUpTask() {
+        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
+        return warmUpFile_.exist;
+    }
+
+    void FetchDentryEnqueue(std::string file);
+
+    void PutWarmTask(const std::string& warmUpTask) {
+        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
+        warmUpTasks_.push_back(warmUpTask);
+        WarmUpRun();
+    }
+
+    CURVEFS_ERROR SetMountStatus(const struct MountOption *mountOption);
 
  protected:
     CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
@@ -271,13 +324,54 @@ class FuseClient {
         return 0;
     }
 
+    void splitStr(const std::string& srcStr, const std::string& delimiter,
+      std::vector<std::string>* splitPath) {
+        char* pToken = nullptr;
+        char* pSave = nullptr;
+        pToken = strtok_r(const_cast<char*>(srcStr.c_str()),
+          const_cast<char*>(delimiter.c_str()), &pSave);
+        if (nullptr == pToken) {
+            VLOG(6) << "del lookpath end";
+            return;
+        }
+        splitPath->push_back(pToken);
+        while (true) {
+            pToken = strtok_r(NULL, const_cast<char*>(
+              delimiter.c_str()), &pSave);
+            if (nullptr == pToken) {
+                VLOG(6) << "del lookpath end";
+                break;
+            }
+            VLOG(9) << "del pToken is:" << pToken
+                    << "pSave:" << pSave;
+            splitPath->push_back(pToken);
+        }
+    }
+
  private:
-    virtual CURVEFS_ERROR Truncate(Inode* inode, uint64_t length) = 0;
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
 
     virtual void FlushData() = 0;
 
-    CURVEFS_ERROR UpdateParentInodeMCTimeAndInvalidNlink(
-        fuse_ino_t parent, FsFileType type);
+    CURVEFS_ERROR UpdateParentMCTimeAndNlink(
+        fuse_ino_t parent, FsFileType type,  NlinkChange nlink);
+
+    void WarmUpTask();
+
+    void WarmUpRun() {
+        std::lock_guard<std::mutex> lk(mtx_);
+        runned_ = true;
+        cond_.notify_one();
+    }
+    void WaitWarmUp() {
+        std::unique_lock<std::mutex> lk(mtx_);
+        cond_.wait(lk, [this]() { return runned_; });
+        runned_ = false;
+    }
+    std::mutex mtx_;
+    std::condition_variable cond_;
+    bool runned_ = false;
+
 
  protected:
     // mds client
@@ -308,6 +402,8 @@ class FuseClient {
     // init flags
     bool init_;
 
+    std::atomic<bool>  mounted_;
+
     // enable record summary info in dir inode xattr
     bool enableSumInDir_;
 
@@ -321,6 +417,40 @@ class FuseClient {
     Atomic<bool> isStop_;
 
     curve::common::Mutex renameMutex_;
+
+    Thread bgCmdTaskThread_;
+    std::atomic<bool> bgCmdStop_;
+    std::mutex cmdMtx_;
+
+    void FetchChildDentryEnqueue(fuse_ino_t  ino);
+    void FetchChildDentry(fuse_ino_t ino);
+    void FetchDentry(fuse_ino_t ino, std::string file);
+    void LookPath(std::string file);
+    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
+        taskFetchMetaPool_;
+
+    // need warmup files
+    std::list<fuse_ino_t> readAheadFiles_;
+    std::mutex fetchMtx_;
+
+    //  one warmup file provided by the user
+    WarmUpFileContext_t warmUpFile_;
+    std::mutex warmUpFileMtx_;
+
+    std::list<std::string> warmUpTasks_;  // todo: need size control ?
+    std::mutex warmUpTaskMtx_;
+
+    void GetwarmTask(std::string *warmUpTask) {
+        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
+        if (warmUpTasks_.empty())
+            return;
+        *warmUpTask = std::move(warmUpTasks_.front());
+        warmUpTasks_.pop_front();
+    }
+    bool hasWarmTask() {
+        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
+        return !warmUpTasks_.empty();
+    }
 };
 
 }  // namespace client

curvefs/src/client/fuse_s3_client.cpp

@@ -23,9 +23,7 @@
 #include "curvefs/src/client/fuse_s3_client.h"
 
 #include <memory>
-#include <string>
-#include <list>
-
+#include <vector>
 namespace curvefs {
 namespace client {
 namespace common {
@@ -43,6 +41,7 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
+    downloadMaxRetryTimes_ = option.downloadMaxRetryTimes;
 
     // set fsS3Option
     const auto& s3Info = fsInfo_->detail().s3info();
@@ -75,13 +74,302 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
                                inodeManager_, mdsClient_, fsCacheManager,
                                nullptr, true);
     }
-
+    isWarmUping_.store(false);
+    bgFetchStop_.store(false, std::memory_order_release);
+    bgFetchThread_ = Thread(&FuseS3Client::BackGroundFetch, this);
+    GetTaskFetchPool();
     return ret;
 }
 
+void FuseS3Client::GetWarmUpFileList(const WarmUpFileContext_t&warmUpFile,
+  std::vector<std::string>& warmUpFilelist) {
+        struct fuse_file_info fi{};
+        fi.flags &= ~O_DIRECT;
+        size_t rSize = 0;
+        std::unique_ptr<char[]> data(new char[warmUpFile.fileLen+1]);
+        std::memset(data.get(), 0, warmUpFile.fileLen);
+        data[warmUpFile.fileLen] = '\n';
+        FuseOpRead(nullptr, warmUpFile.inode,
+            warmUpFile.fileLen, 0, &fi, data.get(), &rSize);
+        std::string file = data.get();
+        VLOG(9) << "file is: " << file;
+        // remove enter, newline, blank
+        std::string blanks("\r\n ");
+        file.erase(0, file.find_first_not_of(blanks));
+        file.erase(file.find_last_not_of(blanks) + 1);
+        VLOG(9) << "after del file is: " << file;
+        splitStr(file, "\n", &warmUpFilelist);
+}
+
+void FuseS3Client::BackGroundFetch() {
+    while (!bgFetchStop_.load(std::memory_order_acquire)) {
+        usleep(WARMUP_CHECKINTERVAL_US);
+        if (hasWarmUpTask()) {  // new warmup task
+            WarmUpFileContext_t warmUpFile;
+            GetWarmUpFile(&warmUpFile);
+            VLOG(9) << " len is: " << warmUpFile.fileLen
+                << "ino is: " << warmUpFile.inode;
+
+            std::vector<std::string> warmUpFilelist;
+            GetWarmUpFileList(warmUpFile, warmUpFilelist);
+            for (auto filePath : warmUpFilelist) {
+                FetchDentryEnqueue(filePath);
+            }
+        }
+        {   // file need warmup
+            std::list<fuse_ino_t> readAheadFiles;
+            GetReadAheadFiles(&readAheadFiles);
+            if (!readAheadFiles.empty()) {
+                LOG(INFO) << "num of files is need loaded is: "
+                        << readAheadFiles.size();
+                for (auto iter : readAheadFiles) {
+                    VLOG(9) << "BackGroundFetch: " << iter;
+                    fetchDataEnqueue(iter);
+                }
+            }
+        }
+        {  // objs will be downloaded
+            {
+                std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                if (needWarmupObjs_.empty()) {
+                    continue;
+                }
+            }
+            if (isWarmUping_.exchange(true)) {
+                continue;
+            }
+            std::thread downloadThread =
+              std::thread(&FuseS3Client::WarmUpAllObjs, this);
+            downloadThread.detach();
+        }
+    }
+    return;
+}
+
+void FuseS3Client::fetchDataEnqueue(fuse_ino_t ino) {
+    VLOG(9) << "fetchDataEnqueue start: " << ino;
+    auto task = [this, ino]() {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                    << ", inodeid = " << ino;
+            return;
+        }
+        google::protobuf::Map<uint64_t, S3ChunkInfoList> s3ChunkInfoMap;
+        {
+            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+            s3ChunkInfoMap = *inodeWrapper->GetChunkInfoMap();
+        }
+        if (s3ChunkInfoMap.empty()) {
+            return;
+        }
+        travelChunks(ino, s3ChunkInfoMap);
+    };
+    GetTaskFetchPool().Enqueue(task);
+}
+
+// travel and download all objs belong to the chunk
+void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo) {
+    uint64_t blockSize = s3Adaptor_->GetBlockSize();
+    uint64_t chunkSize = s3Adaptor_->GetChunkSize();
+    uint64_t offset, len, chunkid, compaction;
+    for (size_t i = 0; i < chunkInfo.s3chunks_size(); i++) {
+        auto chunkinfo = chunkInfo.mutable_s3chunks(i);
+        auto fsId = fsInfo_->fsid();
+        chunkid = chunkinfo->chunkid();
+        compaction = chunkinfo->compaction();
+        offset = chunkinfo->offset();
+        len = chunkinfo->len();
+        // the offset in the chunk
+        uint64_t chunkPos = offset % chunkSize;
+        // the offset in the block
+        uint64_t blockPos = chunkPos % blockSize;
+        // the first blockIndex
+        uint64_t blockIndexBegin = chunkPos / blockSize;
+
+        if (len < blockSize) {  // just one block
+            auto objectName = curvefs::common::s3util::GenObjName(
+                chunkid, blockIndexBegin, compaction, fsId, ino);
+            std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+            needWarmupObjs_.push_back(std::make_pair(objectName, len));
+        } else {
+            // the offset in the block
+            uint64_t blockPos = chunkPos % blockSize;
+
+            // firstly, let's get the size in the first block
+            // then, subtract the length in the first block
+            // to obtain the remaining length
+            // lastly, We need to judge the last block is full or not
+            uint64_t firstBlockSize = (blockPos != 0) ?
+              blockSize - blockPos : blockSize;
+            uint64_t leftSize = len - firstBlockSize;
+            uint32_t blockCounts = (leftSize % blockSize == 0) ?
+              (leftSize / blockSize + 1) : (leftSize / blockSize + 1 + 1);
+            // so we can get the last blockIndex
+            // because the bolck Index is cumulative
+            uint64_t blockIndexEnd = blockIndexBegin + blockCounts - 1;
+
+            // the size of the last block
+            uint64_t lastBlockSize = leftSize % blockSize;
+            // whether the first block or the last block is full or not
+            bool firstBlockFull = (blockPos == 0) ? true : false;
+            bool lastBlockFull = (lastBlockSize == 0) ? true : false;
+            // the start and end block Index that need travel
+            uint64_t travelStartIndex, travelEndIndex;
+            // if the block is full, the size is needed download
+            // of the obj is blockSize. Otherwise, the value is special.
+            if (!firstBlockFull) {
+                travelStartIndex = blockIndexBegin + 1;
+                auto objectName = curvefs::common::s3util::GenObjName(
+                  chunkid, blockIndexBegin, compaction, fsId, ino);
+                std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                needWarmupObjs_.push_back(std::make_pair(
+                  objectName, firstBlockSize));
+            } else {
+                travelStartIndex = blockIndexBegin;
+            }
+            if (!lastBlockFull) {
+                // block index is greater than or equal to 0
+                travelEndIndex = (blockIndexEnd == blockIndexBegin) ?
+                  blockIndexEnd : blockIndexEnd - 1;
+                auto objectName = curvefs::common::s3util::GenObjName(
+                  chunkid, blockIndexEnd, compaction, fsId, ino);
+                // there is no need to care about the order
+                // in which objects are downloaded
+                std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                needWarmupObjs_.push_back(
+                  std::make_pair(objectName, lastBlockSize));
+            } else {
+                travelEndIndex = blockIndexEnd;
+            }
+            VLOG(9) << "travel obj, ino: " << ino
+                << ", chunkid: " << chunkid
+                << ", blockCounts: " << blockCounts
+                << ", compaction: " << compaction
+                << ", blockSize: " << blockSize
+                << ", chunkSize: " << chunkSize
+                << ", offset: "  << offset
+                << ", blockIndexBegin: " << blockIndexBegin
+                << ", blockIndexEnd: " << blockIndexEnd
+                << ", len: " << len
+                << ", firstBlockSize: " << firstBlockSize
+                << ", lastBlockSize: " << lastBlockSize
+                << ", blockPos: " << blockPos
+                << ", chunkPos: " << chunkPos;
+            for (auto blockIndex = travelStartIndex;
+              blockIndex <= travelEndIndex ; blockIndex++) {
+                auto objectName = curvefs::common::s3util::GenObjName(
+                    chunkid, blockIndex, compaction, fsId, ino);
+                {
+                    std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                    needWarmupObjs_.push_back(
+                      std::make_pair(objectName, blockSize));
+                }
+            }
+        }
+    }
+}
+
+// TODO(hzwuhongsong): These logics are very similar to other place,
+// try to merge it
+void FuseS3Client::WarmUpAllObjs() {
+    std::list<std::pair<std::string, uint64_t>> needWarmupObjs;
+    {
+        std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+        LOG(INFO) << "num of objs need loaded is: " << needWarmupObjs_.size();
+        needWarmupObjs = std::move(needWarmupObjs_);
+    }
+    std::atomic<uint64_t> pendingReq(0);
+    curve::common::CountDownEvent cond(1);
+    // callback function
+    GetObjectAsyncCallBack cb =
+        [&](const S3Adapter *adapter,
+            const std::shared_ptr<GetObjectAsyncContext> &context) {
+            if (bgFetchStop_.load()) {
+                LOG(INFO) << "need stop warmup";
+                cond.Signal();
+                return;
+            }
+            if (context->retCode == 0) {
+                VLOG(9) << "Get Object success: " << context->key;
+                int ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(
+                  context->key, context->buf, context->len);
+                if (ret < 0) {
+                    LOG_EVERY_SECOND(INFO) <<
+                    "write read directly failed, key: " << context->key;
+                }
+                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
+                    VLOG(6) << "pendingReq is over";
+                    cond.Signal();
+                }
+                delete []context->buf;
+                return;
+            }
+            if (++context->retry >= downloadMaxRetryTimes_) {
+                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
+                    VLOG(6) << "pendingReq is over";
+                    cond.Signal();
+                }
+                LOG(WARNING) << "Up to max retry times, "
+                             << "download object failed, key: "
+                             << context->key;
+                delete []context->buf;
+                return;
+            }
+
+            LOG(WARNING) << "Get Object failed, key: " << context->key
+                         << ", offset: " << context->offset;
+            s3Adaptor_->GetS3Client()->DownloadAsync(context);
+    };
+
+    pendingReq.fetch_add(needWarmupObjs.size(), std::memory_order_seq_cst);
+    if (pendingReq.load()) {
+        VLOG(9) << "wait for pendingReq" << pendingReq.load();
+        for (auto iter : needWarmupObjs) {
+            VLOG(9) << "download start: " << iter.first;
+            std::string name = iter.first;
+            uint64_t readLen = iter.second;
+            if (s3Adaptor_->GetDiskCacheManager()->IsCached(name)) {
+                pendingReq.fetch_sub(1);
+                continue;
+            }
+            char *cacheS3 = new char[readLen];
+            memset(cacheS3, 0, readLen);
+            auto context = std::make_shared<GetObjectAsyncContext>();
+            context->key = name;
+            context->buf = cacheS3;
+            context->offset = 0;
+            context->len = readLen;
+            context->cb = cb;
+            context->retry = 0;
+            s3Adaptor_->GetS3Client()->DownloadAsync(context);
+        }
+        if (pendingReq.load())
+            cond.Wait();
+    }
+    isWarmUping_.exchange(false);
+    LOG(INFO) << "num of objs is loaded over ";
+}
+
+void FuseS3Client::travelChunks(
+    fuse_ino_t ino,
+    const google::protobuf::Map<uint64_t, S3ChunkInfoList>& s3ChunkInfoMap) {
+    VLOG(9) << "travel chunk start: " << ino
+            << ", size: " << s3ChunkInfoMap.size();
+    std::list<std::pair<std::string, uint64_t>> prefetchObjs;
+    for (auto const& iter : s3ChunkInfoMap) {
+        VLOG(9) << "travel chunk: " << iter.first;
+        travelChunk(ino, iter.second);
+    }
+    VLOG(9) << "travel chunks end";
+}
+
 void FuseS3Client::UnInit() {
-    s3Adaptor_->Stop();
+    bgFetchStop_.store(true, std::memory_order_release);
+    bgFetchThread_.join();
     FuseClient::UnInit();
+    s3Adaptor_->Stop();
     curve::common::S3Adapter::Shutdown();
 }
 
@@ -117,7 +405,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         fsMetric_->userWrite.qps.count << 1;
         uint64_t duration = butil::cpuwide_time_us() - start;
         fsMetric_->userWrite.latency << duration;
-        fsMetric_->userWriteIoSize << wRet;
+        fsMetric_->userWriteIoSize.set_value(wRet);
     }
 
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -129,21 +417,16 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
     }
 
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-    Inode *inode = inodeWrapper->GetMutableInodeUnlocked();
 
     *wSize = wRet;
     size_t changeSize = 0;
     // update file len
-    if (inode->length() < off + *wSize) {
-        changeSize = off + *wSize - inode->length();
-        inode->set_length(off + *wSize);
+    if (inodeWrapper->GetLengthLocked() < off + *wSize) {
+        changeSize = off + *wSize - inodeWrapper->GetLengthLocked();
+        inodeWrapper->SetLengthLocked(off + *wSize);
     }
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    inode->set_mtime(now.tv_sec);
-    inode->set_mtime_ns(now.tv_nsec);
-    inode->set_ctime(now.tv_sec);
-    inode->set_ctime_ns(now.tv_nsec);
+
+    inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
 
     inodeManager_->ShipToFlush(inodeWrapper);
 
@@ -152,6 +435,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
     }
 
     if (enableSumInDir_ && changeSize != 0) {
+        const Inode* inode = inodeWrapper->GetInodeLocked();
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
             std::to_string(changeSize)});
@@ -211,20 +495,14 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
         fsMetric_->userRead.qps.count << 1;
         uint64_t duration = butil::cpuwide_time_us() - start;
         fsMetric_->userRead.latency << duration;
-        fsMetric_->userReadIoSize << rRet;
+        fsMetric_->userReadIoSize.set_value(rRet);
     }
 
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-    Inode *newInode = inodeWrapper->GetMutableInodeUnlocked();
-
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    newInode->set_atime(now.tv_sec);
-    newInode->set_atime_ns(now.tv_nsec);
-
+    inodeWrapper->UpdateTimestampLocked(kAccessTime);
     inodeManager_->ShipToFlush(inodeWrapper);
 
-    VLOG(6) << "read end, read size = " << *rSize;
+    VLOG(9) << "read end, read size = " << *rSize;
     return ret;
 }
 
@@ -290,7 +568,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
     return inodeWrapper->Sync();
 }
 
-CURVEFS_ERROR FuseS3Client::Truncate(Inode *inode, uint64_t length) {
+CURVEFS_ERROR FuseS3Client::Truncate(InodeWrapper *inode, uint64_t length) {
     return s3Adaptor_->Truncate(inode, length);
 }
 

curvefs/src/client/fuse_s3_client.h

@@ -25,12 +25,21 @@
 #define CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
 
 #include <memory>
+#include <string>
+#include <list>
+#include <vector>
+#include <utility>
 
 #include "curvefs/src/client/fuse_client.h"
 #include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "src/common/s3_adapter.h"
+
 namespace curvefs {
 namespace client {
 
+using curve::common::GetObjectAsyncContext;
+using curve::common::GetObjectAsyncCallBack;
+
 class FuseS3Client : public FuseClient {
  public:
     FuseS3Client()
@@ -85,13 +94,36 @@ class FuseS3Client : public FuseClient {
         struct fuse_file_info *fi) override;
 
  private:
-    CURVEFS_ERROR Truncate(Inode *inode, uint64_t length) override;
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
 
     void FlushData() override;
+    // get the warmUp filelist
+    void GetWarmUpFileList(const WarmUpFileContext_t&,
+      std::vector<std::string>&);
+    void BackGroundFetch();
+    // put the file needed warmup to queue,
+    // then can downlaod the objs belong to it
+    void fetchDataEnqueue(fuse_ino_t ino);
+    // travel all chunks
+    void travelChunks(
+        fuse_ino_t ino,
+        const google::protobuf::Map<uint64_t, S3ChunkInfoList>& s3ChunkInfoMap);
+    // travel and download all objs belong to the chunk
+    void travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo);
+    // warmup all the prefetchObjs
+    void WarmUpAllObjs();
 
  private:
     // s3 adaptor
     std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
+
+    Thread bgFetchThread_;
+    std::atomic<bool> bgFetchStop_;
+    std::mutex warmupObjsMtx_;
+    std::atomic<bool> isWarmUping_;
+    std::list<std::pair<std::string, uint64_t>> needWarmupObjs_;
+    std::mutex fetchMtx_;
+    uint32_t downloadMaxRetryTimes_;
 };
 
 

curvefs/src/client/fuse_volume_client.cpp

@@ -173,7 +173,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
         fsMetric_->userWrite.bps.count << size;
         fsMetric_->userWrite.qps.count << 1;
         fsMetric_->userWrite.latency << timer.u_elapsed();
-        fsMetric_->userWriteIoSize << size;
+        fsMetric_->userWriteIoSize.set_value(size);
     }
 
     VLOG(9) << "write end, ino: " << ino << ", offset: " << off
@@ -219,7 +219,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
         fsMetric_->userRead.bps.count << size;
         fsMetric_->userRead.qps.count << 1;
         fsMetric_->userRead.latency << timer.u_elapsed();
-        fsMetric_->userReadIoSize << size;
+        fsMetric_->userReadIoSize.set_value(size);
     }
 
     *rSize = size;
@@ -296,7 +296,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
     return inodeWrapper->Sync();
 }
 
-CURVEFS_ERROR FuseVolumeClient::Truncate(Inode *inode, uint64_t length) {
+CURVEFS_ERROR FuseVolumeClient::Truncate(InodeWrapper *inode, uint64_t length) {
     // Todo: call volume truncate
     return CURVEFS_ERROR::OK;
 }

curvefs/src/client/fuse_volume_client.h

@@ -97,7 +97,7 @@ class FuseVolumeClient : public FuseClient {
     void SetVolumeStorageForTesting(VolumeStorage *storage);
 
  private:
-    CURVEFS_ERROR Truncate(Inode *inode, uint64_t length) override;
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
 
     void FlushData() override;
 

curvefs/src/client/inode_cache_manager.cpp

@@ -47,6 +47,7 @@ namespace curvefs {
 namespace client {
 
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
+using curvefs::client::common::FLAGS_enableCto;
 
 class TrimICacheAsyncDone : public MetaServerClientDone {
  public:
@@ -107,7 +108,7 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
     // get inode from cache
     bool ok = iCache_->Get(inodeId, &out);
-    if (ok) {
+    if (ok && NeedUseCahce(inodeId, out->IsDirty())) {
         curve::common::UniqueLock lgGuard = out->GetUniqueLock();
         if (out->GetType() == FsFileType::TYPE_FILE) {
             return CURVEFS_ERROR::OK;
@@ -134,48 +135,14 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR
-InodeCacheManagerImpl::RefreshInode(uint64_t inodeId) {
-    NameLockGuard lock(nameLock_, std::to_string(inodeId));
-
-    // get inode from metaserver
-    Inode inode;
-    bool streaming = false;
-    GET_INODE_REMOTE(fsId_, inodeId, &inode, &streaming);
-
-    // get inode from cache
-    std::shared_ptr<InodeWrapper> out;
-    bool ok = iCache_->Get(inodeId, &out);
-    curve::common::UniqueLock lgGuard;
-    if (!ok) {
-        out = std::make_shared<InodeWrapper>(
-            std::move(inode), metaClient_, s3ChunkInfoMetric_,
-            option_.maxDataSize, option_.refreshDataIntervalSec);
-    } else {
-        lgGuard = out->GetUniqueLock();
-    }
-
-    // refresh data
-    REFRESH_DATA_REMOTE(out, streaming);
-
-    // put to cache or refresh length
-    if (!ok) {
-        PUT_INODE_CACHE(inodeId, out);
-    } else {
-        out->SetLength(inode.length());
-    }
-
-    return CURVEFS_ERROR::OK;
-}
-
 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
                                                   InodeAttr *out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
     // 1. find in icache
     std::shared_ptr<InodeWrapper> inodeWrapper;
     bool ok = iCache_->Get(inodeId, &inodeWrapper);
-    if (ok) {
-        inodeWrapper->GetInodeAttrLocked(out);
+    if (ok && NeedUseCahce(inodeId, inodeWrapper->IsDirty())) {
+        inodeWrapper->GetInodeAttr(out);
         return CURVEFS_ERROR::OK;
     }
 
@@ -205,27 +172,28 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
     std::set<uint64_t> *inodeIds,
-    std::list<InodeAttr> *attr) {
-    // get some inode in icache
+    std::list<InodeAttr> *attrs) {
+    // get some inode attr in icache
     for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok) {
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
             InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttrLocked(&tmpAttr);
-            attr->emplace_back(tmpAttr);
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace_back(std::move(tmpAttr));
             iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
+            continue;
         }
+        ++iter;
     }
 
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
 
-    MetaStatusCode ret = metaClient_->BatchGetInodeAttr(fsId_, *inodeIds, attr);
+    MetaStatusCode ret = metaClient_->BatchGetInodeAttr(fsId_, *inodeIds,
+        attrs);
     if (MetaStatusCode::OK != ret) {
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed, MetaStatusCode = "
                    << ret << ", MetaStatusCode_Name = "
@@ -236,21 +204,37 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     uint64_t parentId,
-    const std::set<uint64_t> &inodeIds,
+    std::set<uint64_t> *inodeIds,
     std::map<uint64_t, InodeAttr> *attrs) {
-    if (inodeIds.empty()) {
-        return CURVEFS_ERROR::OK;
+    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
+    std::map<uint64_t, InodeAttr> cachedAttr;
+    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
+
+    // get some inode attr in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+            InodeAttr tmpAttr;
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace(*iter, std::move(tmpAttr));
+            iter = inodeIds->erase(iter);
+        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
+            attrs->emplace(*iter, cachedAttr[*iter]);
+            iter = inodeIds->erase(iter);
+        } else {
+            ++iter;
+        }
     }
 
-    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    bool ok  = iAttrCache_->Get(parentId, attrs);
-    if (ok) {
+    if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
 
     // split inodeIds by partitionId and batch limit
     std::vector<std::vector<uint64_t>> inodeGroups;
-    if (!metaClient_->SplitRequestInodes(fsId_, inodeIds, &inodeGroups)) {
+    if (!metaClient_->SplitRequestInodes(fsId_, *inodeIds, &inodeGroups)) {
         return CURVEFS_ERROR::NOTEXIST;
     }
 
@@ -273,7 +257,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     // wait for all sudrequest finished
     cond->Wait();
 
-    ok  = iAttrCache_->Get(parentId, attrs);
+    bool ok  = iAttrCache_->Get(parentId, attrs);
     if (!ok) {
         LOG(WARNING) << "get attrs form iAttrCache_ failed.";
     }
@@ -288,10 +272,8 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok) {
-            XAttr tmpXattr;
-            inodeWrapper->GetXattrLocked(&tmpXattr);
-            xattr->emplace_back(tmpXattr);
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+            xattr->emplace_back(inodeWrapper->GetXattr());
             iter = inodeIds->erase(iter);
         } else {
             ++iter;
@@ -462,6 +444,11 @@ void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
                 iCache_->Remove(inodeId);
             }
             trimSize--;
+            // remove the attr of the inode in iattrcache
+            auto parents = inodeWrapper->GetParentLocked();
+            for (uint64_t parent : parents) {
+                iAttrCache_->Remove(parent, inodeId);
+            }
         } else {
             LOG(ERROR) << "icache size " << iCache_->Size();
             assert(0);
@@ -503,5 +490,33 @@ InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
     return rc;
 }
 
+void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    openedInodes_.emplace(inodeId);
+}
+
+void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    if (iter != openedInodes_.end()) {
+        openedInodes_.erase(iter);
+    }
+}
+
+bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    return iter != openedInodes_.end();
+}
+
+bool InodeCacheManagerImpl::NeedUseCahce(uint64_t inodeId, bool IsDirty) {
+    if (!FLAGS_enableCto || OpenInodeCached(inodeId) || IsDirty) {
+        return true;
+    }
+    return false;
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_cache_manager.h

@@ -74,7 +74,7 @@ class InodeAttrCache {
         curve::common::LockGuard lg(iAttrCacheMutex_);
         auto iter = iAttrCache_.find(parentId);
         if (iter != iAttrCache_.end()) {
-            *imap = iter->second;
+            imap->insert(iter->second.begin(), iter->second.end());
             return true;
         }
         return false;
@@ -102,6 +102,14 @@ class InodeAttrCache {
                 << size << ", after = " << iAttrCache_.size();
     }
 
+    void Remove(uint64_t parentId, uint64_t inodeId) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            iter->second.erase(inodeId);
+        }
+    }
+
  private:
     // inodeAttr cache; <parentId, <inodeId, inodeAttr>>
     std::map<uint64_t, std::map<uint64_t, InodeAttr>> iAttrCache_;
@@ -130,8 +138,6 @@ class InodeCacheManager {
     GetInode(uint64_t inodeId,
              std::shared_ptr<InodeWrapper> &out) = 0;  // NOLINT
 
-    virtual CURVEFS_ERROR RefreshInode(uint64_t inodeId) = 0;
-
     virtual CURVEFS_ERROR GetInodeAttr(uint64_t inodeId, InodeAttr *out) = 0;
 
     virtual CURVEFS_ERROR BatchGetInodeAttr(
@@ -140,7 +146,7 @@ class InodeCacheManager {
 
     virtual CURVEFS_ERROR BatchGetInodeAttrAsync(
         uint64_t parentId,
-        const std::set<uint64_t> &inodeIds,
+        std::set<uint64_t> *inodeIds,
         std::map<uint64_t, InodeAttr> *attrs) = 0;
 
     virtual CURVEFS_ERROR BatchGetXAttr(std::set<uint64_t> *inodeIds,
@@ -165,6 +171,10 @@ class InodeCacheManager {
 
     virtual void ReleaseCache(uint64_t parentId) = 0;
 
+    virtual void AddOpenedInode(uint64_t inodeId) = 0;
+
+    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
+
  protected:
     uint32_t fsId_;
 };
@@ -225,15 +235,13 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     CURVEFS_ERROR GetInode(uint64_t inodeId,
                            std::shared_ptr<InodeWrapper> &out) override;
 
-    CURVEFS_ERROR RefreshInode(uint64_t inodeId) override;
-
     CURVEFS_ERROR GetInodeAttr(uint64_t inodeId, InodeAttr *out) override;
 
     CURVEFS_ERROR BatchGetInodeAttr(std::set<uint64_t> *inodeIds,
         std::list<InodeAttr> *attrs) override;
 
     CURVEFS_ERROR BatchGetInodeAttrAsync(uint64_t parentId,
-        const std::set<uint64_t> &inodeIds,
+        std::set<uint64_t> *inodeIds,
         std::map<uint64_t, InodeAttr> *attrs = nullptr) override;
 
     CURVEFS_ERROR BatchGetXAttr(std::set<uint64_t> *inodeIds,
@@ -260,11 +268,18 @@ class InodeCacheManagerImpl : public InodeCacheManager,
 
     void RemoveICache(const std::shared_ptr<InodeWrapper> &inode);
 
+    void AddOpenedInode(uint64_t inodeId) override;
+
+    void RemoveOpenedInode(uint64_t inodeId) override;
+
+    bool NeedUseCahce(uint64_t inodeId, bool IsDirty);
+
  private:
     virtual void FlushInodeBackground();
     void TrimIcache(uint64_t trimSize);
     CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
                               bool streaming = true);
+    bool OpenInodeCached(uint64_t inodeId);
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
@@ -277,6 +292,10 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
     curve::common::Mutex dirtyMapMutex_;
 
+    // record opened inode
+    std::multiset<uint64_t> openedInodes_;
+    curve::common::Mutex openInodesMutex_;
+
     curve::common::GenericNameLock<Mutex> nameLock_;
 
     curve::common::GenericNameLock<Mutex> asyncNameLock_;

curvefs/src/client/inode_wrapper.cpp

@@ -25,6 +25,7 @@
 #include <glog/logging.h>
 
 #include <cstddef>
+#include <ctime>
 #include <memory>
 #include <mutex>
 #include <sstream>
@@ -49,6 +50,14 @@ using rpcclient::DataIndices;
 
 bvar::Adder<int64_t> g_alive_inode_count{"alive_inode_count"};
 
+#define REFRESH_NLINK                       \
+do {                                        \
+    CURVEFS_ERROR ret = RefreshNlink();     \
+    if (ret != CURVEFS_ERROR::OK) {         \
+        return ret;                         \
+    }                                       \
+} while (0)
+
 std::ostream &operator<<(std::ostream &os, const struct stat &attr) {
     os << "{ st_ino = " << attr.st_ino << ", st_mode = " << attr.st_mode
        << ", st_nlink = " << attr.st_nlink << ", st_uid = " << attr.st_uid
@@ -135,21 +144,59 @@ class GetOrModifyS3ChunkInfoAsyncDone : public MetaServerClientDone {
     std::shared_ptr<InodeWrapper> inodeWrapper_;
 };
 
+CURVEFS_ERROR InodeWrapper::GetInodeAttrLocked(InodeAttr *attr) {
+    attr->set_inodeid(inode_.inodeid());
+    attr->set_fsid(inode_.fsid());
+    attr->set_length(inode_.length());
+    attr->set_ctime(inode_.ctime());
+    attr->set_ctime_ns(inode_.ctime_ns());
+    attr->set_mtime(inode_.mtime());
+    attr->set_mtime_ns(inode_.mtime_ns());
+    attr->set_atime(inode_.atime());
+    attr->set_atime_ns(inode_.atime_ns());
+    attr->set_uid(inode_.uid());
+    attr->set_gid(inode_.gid());
+    attr->set_mode(inode_.mode());
+    attr->set_nlink(inode_.nlink());
+    attr->set_type(inode_.type());
+    *(attr->mutable_parent()) = inode_.parent();
+    if (inode_.has_symlink()) {
+        attr->set_symlink(inode_.symlink());
+    }
+    if (inode_.has_rdev()) {
+        attr->set_rdev(inode_.rdev());
+    }
+    if (inode_.has_dtime()) {
+        attr->set_dtime(inode_.dtime());
+    }
+    if (inode_.xattr_size() > 0) {
+        *(attr->mutable_xattr()) = inode_.xattr();
+    }
+    return CURVEFS_ERROR::OK;
+}
+
+void InodeWrapper::GetInodeAttr(InodeAttr *attr) {
+    curve::common::UniqueLock lg(mtx_);
+    GetInodeAttrLocked(attr);
+}
+
 CURVEFS_ERROR InodeWrapper::SyncAttr(bool internal) {
     curve::common::UniqueLock lock = GetSyncingInodeUniqueLock();
     if (dirty_) {
-        dirty_ = false;
         MetaStatusCode ret = metaClient_->UpdateInodeAttrWithOutNlink(
-            inode_, InodeOpenStatusChange::NOCHANGE, nullptr, internal);
+            inode_.fsid(), inode_.inodeid(), dirtyAttr_,
+            nullptr, internal);
 
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "metaClient_ UpdateInodeAttrWithOutNlink failed, "
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            dirty_ = true;
             return MetaStatusCodeToCurvefsErrCode(ret);
         }
+
+        dirty_ = false;
+        dirtyAttr_.Clear();
     }
     return CURVEFS_ERROR::OK;
 }
@@ -177,8 +224,10 @@ void InodeWrapper::AsyncFlushAttr(MetaServerClientDone* done,
     if (dirty_) {
         LockSyncingInode();
         metaClient_->UpdateInodeWithOutNlinkAsync(
-            inode_, new UpdateInodeAsyncDone(shared_from_this(), done));
+            inode_.fsid(), inode_.inodeid(), dirtyAttr_,
+            new UpdateInodeAsyncDone(shared_from_this(), done));
         dirty_ = false;
+        dirtyAttr_.Clear();
         return;
     }
 
@@ -256,24 +305,23 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR InodeWrapper::LinkLocked(uint64_t parent) {
+CURVEFS_ERROR InodeWrapper::Link(uint64_t parent) {
     curve::common::UniqueLock lg(mtx_);
-    REFRESH_NLINK_IF_NEED;
+    REFRESH_NLINK;
     uint32_t old = inode_.nlink();
     inode_.set_nlink(old + 1);
+    dirtyAttr_.set_nlink(inode_.nlink());
     VLOG(3) << "LinkLocked, inodeid = " << inode_.inodeid()
             << ", newnlink = " << inode_.nlink();
 
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    inode_.set_ctime(now.tv_sec);
-    inode_.set_ctime_ns(now.tv_nsec);
-    inode_.set_mtime(now.tv_sec);
-    inode_.set_mtime_ns(now.tv_nsec);
+    UpdateTimestampLocked(kChangeTime | kModifyTime);
     if (inode_.type() != FsFileType::TYPE_DIRECTORY && parent != 0) {
         inode_.add_parent(parent);
+        dirtyAttr_.add_parent(parent);
     }
-    MetaStatusCode ret = metaClient_->UpdateInodeAttr(inode_);
+
+    MetaStatusCode ret = metaClient_->UpdateInodeAttr(
+        inode_.fsid(), inode_.inodeid(), dirtyAttr_);
     if (ret != MetaStatusCode::OK) {
         inode_.set_nlink(old);
         LOG(ERROR) << "metaClient_ UpdateInodeAttr failed"
@@ -282,13 +330,15 @@ CURVEFS_ERROR InodeWrapper::LinkLocked(uint64_t parent) {
                    << ", inodeid = " << inode_.inodeid();
         return MetaStatusCodeToCurvefsErrCode(ret);
     }
+
     dirty_ = false;
+    dirtyAttr_.Clear();
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
+CURVEFS_ERROR InodeWrapper::UnLink(uint64_t parent) {
     curve::common::UniqueLock lg(mtx_);
-    REFRESH_NLINK_IF_NEED;
+    REFRESH_NLINK;
     uint32_t old = inode_.nlink();
     VLOG(1) << "Unlink inode = " << inode_.DebugString();
     if (old > 0) {
@@ -300,17 +350,12 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
         VLOG(3) << "UnLinkLocked, inodeid = " << inode_.inodeid()
                 << ", newnlink = " << inode_.nlink()
                 << ", type = " << inode_.type();
-        struct timespec now;
-        clock_gettime(CLOCK_REALTIME, &now);
-        inode_.set_ctime(now.tv_sec);
-        inode_.set_ctime_ns(now.tv_nsec);
-        inode_.set_mtime(now.tv_sec);
-        inode_.set_mtime_ns(now.tv_nsec);
-        // newlink == 0 will be deleted at metasever
+        UpdateTimestampLocked(kChangeTime | kModifyTime);
+        // newnlink == 0 will be deleted at metaserver
         // dir will not update parent
         // parent = 0; is useless
-        if (newnlink != 0 && inode_.type() != FsFileType::TYPE_DIRECTORY
-            && parent != 0) {
+        if (newnlink != 0 && inode_.type() != FsFileType::TYPE_DIRECTORY &&
+            parent != 0) {
             auto parents = inode_.mutable_parent();
             for (auto iter = parents->begin(); iter != parents->end(); iter++) {
                 if (*iter == parent) {
@@ -340,7 +385,12 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
                 return err;
             }
         }
-        MetaStatusCode ret = metaClient_->UpdateInodeAttr(inode_);
+
+        dirtyAttr_.set_nlink(inode_.nlink());
+        *dirtyAttr_.mutable_parent() = inode_.parent();
+
+        MetaStatusCode ret = metaClient_->UpdateInodeAttr(
+            inode_.fsid(), inode_.inodeid(), dirtyAttr_);
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "metaClient_ UpdateInodeAttr failed"
                        << ", MetaStatusCode = " << ret
@@ -349,6 +399,7 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
             return MetaStatusCodeToCurvefsErrCode(ret);
         }
         dirty_ = false;
+        dirtyAttr_.Clear();
         return CURVEFS_ERROR::OK;
     }
     LOG(ERROR) << "Unlink find nlink <= 0, nlink = " << old
@@ -356,22 +407,7 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
     return CURVEFS_ERROR::INTERNAL;
 }
 
-CURVEFS_ERROR
-InodeWrapper::UpdateInodeStatus(InodeOpenStatusChange statusChange) {
-    MetaStatusCode ret =
-        metaClient_->UpdateInodeAttrWithOutNlink(inode_, statusChange);
-    if (ret != MetaStatusCode::OK) {
-        LOG(ERROR) << "metaClient_ UpdateInodeAttrWithOutNlink failed"
-                   << ", MetaStatusCode = " << ret
-                   << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
-                   << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
-    }
-    dirty_ = false;
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR InodeWrapper::UpdateParentLocked(
+CURVEFS_ERROR InodeWrapper::UpdateParent(
     uint64_t oldParent, uint64_t newParent) {
     curve::common::UniqueLock lg(mtx_);
     auto parents = inode_.mutable_parent();
@@ -382,8 +418,10 @@ CURVEFS_ERROR InodeWrapper::UpdateParentLocked(
         }
     }
     inode_.add_parent(newParent);
+    *dirtyAttr_.mutable_parent() = inode_.parent();
 
-    MetaStatusCode ret = metaClient_->UpdateInodeAttrWithOutNlink(inode_);
+    MetaStatusCode ret = metaClient_->UpdateInodeAttrWithOutNlink(
+        inode_.fsid(), inode_.inodeid(), dirtyAttr_);
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ UpdateInodeAttrWithOutNlink failed"
                    << ", MetaStatusCode = " << ret
@@ -392,6 +430,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParentLocked(
         return MetaStatusCodeToCurvefsErrCode(ret);
     }
     dirty_ = false;
+    dirtyAttr_.Clear();
     return CURVEFS_ERROR::OK;
 }
 
@@ -418,7 +457,7 @@ CURVEFS_ERROR InodeWrapper::Sync(bool internal) {
 }
 
 void InodeWrapper::Async(MetaServerClientDone *done, bool internal) {
-    VLOG(3) << "async inode: " << inode_.ShortDebugString();
+    VLOG(9) << "async inode: " << inode_.ShortDebugString();
 
     switch (inode_.type()) {
         case FsFileType::TYPE_S3:
@@ -446,11 +485,12 @@ void InodeWrapper::AsyncFlushAttrAndExtents(MetaServerClientDone *done,
         }
 
         metaClient_->UpdateInodeWithOutNlinkAsync(
-            inode_,
+            inode_.fsid(), inode_.inodeid(), dirtyAttr_,
             new UpdateInodeAttrAndExtentClosure{shared_from_this(), done},
-            InodeOpenStatusChange::NOCHANGE, std::move(indices));
+            std::move(indices));
 
         dirty_ = false;
+        dirtyAttr_.Clear();
         return;
     }
 
@@ -466,20 +506,20 @@ CURVEFS_ERROR InodeWrapper::SyncS3(bool internal) {
     curve::common::UniqueLock lockS3chunkInfo =
         GetSyncingS3ChunkInfoUniqueLock();
     if (dirty_ || !s3ChunkInfoAdd_.empty()) {
-        dirty_ = false;
         MetaStatusCode ret = metaClient_->UpdateInodeAttrWithOutNlink(
-            inode_, InodeOpenStatusChange::NOCHANGE, &s3ChunkInfoAdd_,
-            internal);
+            inode_.fsid(), inode_.inodeid(), dirtyAttr_,
+            &s3ChunkInfoAdd_, internal);
 
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "metaClient_ UpdateInodeAttrWithOutNlink failed, "
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            dirty_ = true;
             return MetaStatusCodeToCurvefsErrCode(ret);
         }
         ClearS3ChunkInfoAdd();
+        dirty_ = false;
+        dirtyAttr_.Clear();
     }
     return CURVEFS_ERROR::OK;
 }
@@ -527,9 +567,11 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
             indices.s3ChunkInfoMap = std::move(s3ChunkInfoAdd_);
         }
         metaClient_->UpdateInodeWithOutNlinkAsync(
-            inode_, new UpdateInodeAsyncS3Done{shared_from_this(), done},
-            InodeOpenStatusChange::NOCHANGE, std::move(indices));
+            inode_.fsid(), inode_.inodeid(), dirtyAttr_,
+            new UpdateInodeAsyncS3Done{shared_from_this(), done},
+            std::move(indices));
         dirty_ = false;
+        dirtyAttr_.Clear();
         ClearS3ChunkInfoAdd();
         return;
     }
@@ -569,9 +611,55 @@ CURVEFS_ERROR InodeWrapper::RefreshNlink() {
     }
     inode_.set_nlink(attr.nlink());
     VLOG(3) << "RefreshNlink from metaserver, newnlink: " << attr.nlink();
-    ResetNlinkValid();
     return CURVEFS_ERROR::OK;
 }
 
+void InodeWrapper::MergeXAttrLocked(
+    const google::protobuf::Map<std::string, std::string>& xattrs) {
+    auto helper =
+        [](const google::protobuf::Map<std::string, std::string>& incoming,
+           google::protobuf::Map<std::string, std::string>* xattrs) {
+            for (const auto& attr : incoming) {
+                (*xattrs)[attr.first] = attr.second;
+            }
+        };
+
+    helper(xattrs, inode_.mutable_xattr());
+    helper(xattrs, dirtyAttr_.mutable_xattr());
+    dirty_ = true;
+}
+
+void InodeWrapper::UpdateTimestampLocked(int flags) {
+    struct timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+    return UpdateTimestampLocked(now, flags);
+}
+
+void InodeWrapper::UpdateTimestampLocked(const timespec& now, int flags) {
+    if (flags & kAccessTime) {
+        inode_.set_atime(now.tv_sec);
+        inode_.set_atime_ns(now.tv_nsec);
+        dirtyAttr_.set_atime(now.tv_sec);
+        dirtyAttr_.set_atime_ns(now.tv_nsec);
+        dirty_ = true;
+    }
+
+    if (flags & kChangeTime) {
+        inode_.set_ctime(now.tv_sec);
+        inode_.set_ctime_ns(now.tv_nsec);
+        dirtyAttr_.set_ctime(now.tv_sec);
+        dirtyAttr_.set_ctime_ns(now.tv_nsec);
+        dirty_ = true;
+    }
+
+    if (flags & kModifyTime) {
+        inode_.set_mtime(now.tv_sec);
+        inode_.set_mtime_ns(now.tv_nsec);
+        dirtyAttr_.set_mtime(now.tv_sec);
+        dirtyAttr_.set_mtime_ns(now.tv_nsec);
+        dirty_ = true;
+    }
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_wrapper.h

@@ -25,6 +25,7 @@
 #define CURVEFS_SRC_CLIENT_INODE_WRAPPER_H_
 
 #include <sys/stat.h>
+#include <gtest/gtest_prod.h>
 #include <climits>
 #include <cstdint>
 #include <utility>
@@ -41,28 +42,21 @@
 #include "src/common/timeutility.h"
 
 using ::curvefs::metaserver::Inode;
-using ::curvefs::metaserver::InodeOpenStatusChange;
 using ::curvefs::metaserver::S3ChunkInfoList;
 using ::curvefs::metaserver::S3ChunkInfo;
 
 namespace curvefs {
 namespace client {
 
-#define REFRESH_NLINK_IF_NEED               \
-do {                                        \
-    if (!isNlinkValid_) {                   \
-        CURVEFS_ERROR ret = RefreshNlink(); \
-        if (ret != CURVEFS_ERROR::OK) {     \
-            return ret;                     \
-        }                                   \
-    }                                       \
-} while (0)
+constexpr int kAccessTime = 1 << 0;
+constexpr int kChangeTime = 1 << 1;
+constexpr int kModifyTime = 1 << 2;
 
 using ::curvefs::metaserver::VolumeExtentList;
 
-enum InodeStatus {
-    Normal = 0,
-    Error = -1,
+enum class InodeStatus {
+    kNormal = 0,
+    kError = -1,
 };
 
 // TODO(xuchaojie) : get from conf maybe?
@@ -72,6 +66,7 @@ using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;
 using metric::S3ChunkInfoMetric;
+using common::NlinkChange;
 
 std::ostream &operator<<(std::ostream &os, const struct stat &attr);
 void AppendS3ChunkInfoToMap(uint64_t chunkIndex, const S3ChunkInfo &info,
@@ -81,47 +76,24 @@ extern bvar::Adder<int64_t> g_alive_inode_count;
 
 class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
  public:
-    InodeWrapper(const Inode &inode,
-                 const std::shared_ptr<MetaServerClient> &metaClient,
-                 const std::shared_ptr<S3ChunkInfoMetric>
-                    &s3ChunkInfoMetric = nullptr,
-                 uint64_t maxDataSize = ULONG_MAX,
-                 uint32_t refreshDataInterval = UINT_MAX)
-        : inode_(inode),
-          status_(InodeStatus::Normal),
-          isNlinkValid_(true),
-          metaClient_(metaClient),
-          s3ChunkInfoMetric_(s3ChunkInfoMetric),
-          dirty_(false),
-          baseMaxDataSize_(maxDataSize),
-          maxDataSize_(maxDataSize),
-          refreshDataInterval_(refreshDataInterval),
-          lastRefreshTime_(::curve::common::TimeUtility::GetTimeofDaySec()),
-          s3ChunkInfoAddSize_(0) {
-              UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize());
-              g_alive_inode_count << 1;
-          }
-
-    InodeWrapper(Inode &&inode,
-                 const std::shared_ptr<MetaServerClient> &metaClient,
-                 const std::shared_ptr<S3ChunkInfoMetric>
-                    &s3ChunkInfoMetric = nullptr,
+    InodeWrapper(Inode inode,
+                 std::shared_ptr<MetaServerClient> metaClient,
+                 std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric = nullptr,
                  uint64_t maxDataSize = ULONG_MAX,
                  uint32_t refreshDataInterval = UINT_MAX)
         : inode_(std::move(inode)),
-          status_(InodeStatus::Normal),
-          isNlinkValid_(true),
-          metaClient_(metaClient),
-          s3ChunkInfoMetric_(s3ChunkInfoMetric),
-          dirty_(false),
+          status_(InodeStatus::kNormal),
           baseMaxDataSize_(maxDataSize),
           maxDataSize_(maxDataSize),
           refreshDataInterval_(refreshDataInterval),
           lastRefreshTime_(::curve::common::TimeUtility::GetTimeofDaySec()),
-          s3ChunkInfoAddSize_(0) {
-              UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize());
-              g_alive_inode_count << 1;
-          }
+          s3ChunkInfoAddSize_(0),
+          metaClient_(std::move(metaClient)),
+          s3ChunkInfoMetric_(std::move(s3ChunkInfoMetric)),
+          dirty_(false) {
+        UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize());
+        g_alive_inode_count << 1;
+    }
 
     ~InodeWrapper() {
         UpdateS3ChunkInfoMetric(-s3ChunkInfoSize_ - s3ChunkInfoAddSize_);
@@ -146,19 +118,26 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     }
 
     void SetLength(uint64_t len) {
-        if (inode_.length() > len) {
-            return;
-        }
+        curve::common::UniqueLock lg(mtx_);
+        SetLengthLocked(len);
+    }
 
+    void SetLengthLocked(uint64_t len) {
         inode_.set_length(len);
+        dirtyAttr_.set_length(len);
         dirty_ = true;
     }
 
     void SetType(FsFileType type) {
         inode_.set_type(type);
+        dirtyAttr_.set_type(type);
         dirty_ = true;
     }
 
+    uint64_t GetLengthLocked() const {
+        return inode_.length();
+    }
+
     uint64_t GetLength() const {
         curve::common::UniqueLock lg(mtx_);
         return inode_.length();
@@ -166,103 +145,63 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
 
     void SetUid(uint32_t uid) {
         inode_.set_uid(uid);
+        dirtyAttr_.set_uid(uid);
         dirty_ = true;
     }
 
     void SetGid(uint32_t gid) {
         inode_.set_gid(gid);
+        dirtyAttr_.set_gid(gid);
         dirty_ = true;
     }
 
     void SetMode(uint32_t mode) {
         inode_.set_mode(mode);
+        dirtyAttr_.set_mode(mode);
         dirty_ = true;
     }
 
-    void SetMTime(uint64_t mtime, uint32_t mtime_ns) {
-        inode_.set_mtime(mtime);
-        inode_.set_mtime_ns(mtime_ns);
-        dirty_ = true;
-    }
-
-    void SetCTime(uint64_t ctime, uint32_t ctime_ns) {
-        inode_.set_ctime(ctime);
-        inode_.set_ctime_ns(ctime_ns);
-        dirty_ = true;
-    }
-
-    void SetATime(uint64_t atime, uint32_t atime_ns) {
-        inode_.set_atime(atime);
-        inode_.set_atime_ns(atime_ns);
-        dirty_ = true;
-    }
-
-    Inode GetInodeUnlocked() const {
-        return inode_;
-    }
-
-    Inode GetInodeLocked() const {
+    Inode GetInode() const {
         curve::common::UniqueLock lg(mtx_);
         return inode_;
     }
 
-    Inode* GetMutableInodeUnlocked() {
-        dirty_ = true;
+    // Get an immutable inode.
+    //
+    // The const return value is used to forbid modify inode through this
+    // interface, all modification operations should using `SetXXX()`.
+    const Inode* GetInodeLocked() const {
         return &inode_;
     }
 
-    CURVEFS_ERROR GetInodeAttrUnlocked(InodeAttr *attr) {
-        REFRESH_NLINK_IF_NEED;
+    // Update timestamp of inode.
+    //
+    // flags can be any combination of kAccessTime/kModifyTime/kChangeTime
+    void UpdateTimestampLocked(int flags);
+    void UpdateTimestampLocked(const timespec& now, int flags);
 
-        attr->set_inodeid(inode_.inodeid());
-        attr->set_fsid(inode_.fsid());
-        attr->set_length(inode_.length());
-        attr->set_ctime(inode_.ctime());
-        attr->set_ctime_ns(inode_.ctime_ns());
-        attr->set_mtime(inode_.mtime());
-        attr->set_mtime_ns(inode_.mtime_ns());
-        attr->set_atime(inode_.atime());
-        attr->set_atime_ns(inode_.atime_ns());
-        attr->set_uid(inode_.uid());
-        attr->set_gid(inode_.gid());
-        attr->set_mode(inode_.mode());
-        attr->set_nlink(inode_.nlink());
-        attr->set_type(inode_.type());
-        *(attr->mutable_parent()) = inode_.parent();
-        if (inode_.has_symlink()) {
-            attr->set_symlink(inode_.symlink());
-        }
-        if (inode_.has_rdev()) {
-            attr->set_rdev(inode_.rdev());
-        }
-        if (inode_.has_dtime()) {
-            attr->set_dtime(inode_.dtime());
-        }
-        if (inode_.xattr_size() > 0) {
-            *(attr->mutable_xattr()) = inode_.xattr();
-        }
-        return CURVEFS_ERROR::OK;
-    }
+    // Merge incoming extended attributes.
+    //
+    // Existing attributes will be overwritten, new attributes well be inserted.
+    void MergeXAttrLocked(
+        const google::protobuf::Map<std::string, std::string>& xattrs);
 
-    void GetInodeAttrLocked(InodeAttr *attr) {
+    CURVEFS_ERROR GetInodeAttrLocked(InodeAttr *attr);
+
+    void GetInodeAttr(InodeAttr *attr);
+
+    XAttr GetXattr() const {
+        XAttr ret;
         curve::common::UniqueLock lg(mtx_);
-        GetInodeAttrUnlocked(attr);
+        ret.set_fsid(inode_.fsid());
+        ret.set_inodeid(inode_.inodeid());
+        *(ret.mutable_xattrinfos()) = inode_.xattr();
+        return ret;
     }
 
-    void GetXattrLocked(XAttr *xattr) {
-        curve::common::UniqueLock lg(mtx_);
-        xattr->set_fsid(inode_.fsid());
-        xattr->set_inodeid(inode_.inodeid());
-        *(xattr->mutable_xattrinfos()) = inode_.xattr();
-    }
-
-    void UpdateInode(const Inode &inode) {
-        inode_ = inode;
-        dirty_ = true;
-    }
-
-    void SwapInode(Inode *other) {
-        inode_.Swap(other);
+    void SetXattrLocked(const std::string &key, const std::string &value) {
+        (*inode_.mutable_xattr())[key] = value;
+        (*dirtyAttr_.mutable_xattr()) = inode_.xattr();
         dirty_ = true;
     }
 
@@ -270,25 +209,16 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         return curve::common::UniqueLock(mtx_);
     }
 
-    CURVEFS_ERROR UpdateParentLocked(uint64_t oldParent, uint64_t newParent);
+    const google::protobuf::RepeatedField<uint64_t>& GetParentLocked() {
+        return inode_.parent();
+    }
+
+    CURVEFS_ERROR UpdateParent(uint64_t oldParent, uint64_t newParent);
 
     // dir will not update parent
-    CURVEFS_ERROR LinkLocked(uint64_t parent = 0);
+    CURVEFS_ERROR Link(uint64_t parent = 0);
 
-    CURVEFS_ERROR UnLinkLocked(uint64_t parent = 0);
-
-    // mark nlink invalid, need to refresh from metaserver
-    void InvalidateNlink() {
-        isNlinkValid_ = false;
-    }
-
-    void ResetNlinkValid() {
-        isNlinkValid_ = true;
-    }
-
-    bool IsNlinkValid() {
-        return isNlinkValid_;
-    }
+    CURVEFS_ERROR UnLink(uint64_t parent = 0);
 
     CURVEFS_ERROR RefreshNlink();
 
@@ -335,6 +265,14 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         return s3ChunkInfoAdd_.empty();
     }
 
+    uint32_t GetNlinkLocked() {
+        return inode_.nlink();
+    }
+
+    void UpdateNlinkLocked(NlinkChange nlink) {
+        inode_.set_nlink(inode_.nlink() + static_cast<int32_t>(nlink));
+    }
+
     void AppendS3ChunkInfo(uint64_t chunkIndex, const S3ChunkInfo &info) {
         curve::common::UniqueLock lg(mtx_);
         AppendS3ChunkInfoToMap(chunkIndex, info, &s3ChunkInfoAdd_);
@@ -345,9 +283,13 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         UpdateS3ChunkInfoMetric(2);
     }
 
+    google::protobuf::Map<uint64_t, S3ChunkInfoList>* GetChunkInfoMap() {
+        return inode_.mutable_s3chunkinfomap();
+    }
+
     void MarkInodeError() {
         // TODO(xuchaojie) : when inode is marked error, prevent futher write.
-        status_ = InodeStatus::Error;
+        status_ = InodeStatus::kError;
     }
 
     void LockSyncingInode() const {
@@ -398,8 +340,6 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     }
 
  private:
-    CURVEFS_ERROR UpdateInodeStatus(InodeOpenStatusChange statusChange);
-
     CURVEFS_ERROR SyncS3ChunkInfo(bool internal = false);
 
     int64_t CalS3ChunkInfoSize() {
@@ -446,15 +386,19 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     void FlushVolumeExtentAsync();
 
  private:
+    FRIEND_TEST(TestInodeWrapper, TestUpdateInodeAttrIncrementally);
+
     Inode inode_;
+
+    // dirty attributes, and needs update to metaserver
+    InodeAttr dirtyAttr_;
+
     InodeStatus status_;
     uint64_t baseMaxDataSize_;
     uint64_t maxDataSize_;
     uint32_t refreshDataInterval_;
     uint64_t lastRefreshTime_;
 
-    bool isNlinkValid_;
-
     google::protobuf::Map<uint64_t, S3ChunkInfoList> s3ChunkInfoAdd_;
     int64_t s3ChunkInfoAddSize_;
     int64_t s3ChunkInfoSize_;

curvefs/src/client/main.cpp

@@ -21,6 +21,9 @@
  * Author: xuchaojie
  */
 
+#include <glog/logging.h>
+
+#include <string>
 #include <unordered_map>
 #include "curvefs/src/client/curve_fuse_op.h"
 #include "curvefs/src/client/fuse_common.h"
@@ -47,15 +50,17 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     release : FuseOpRelease,
     fsync : FuseOpFsync,
     opendir : FuseOpOpenDir,
-    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    readdir : 0,
-    #else
+    // TODO(wuhongsong): readdirplus is problematic,
+    // resulting in inconsistent metadata
+    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    // readdir : 0,
+    // #else
     readdir : FuseOpReadDir,
-    #endif
+    // #endif
     releasedir : FuseOpReleaseDir,
     fsyncdir : 0,
     statfs : FuseOpStatFs,
-    setxattr : 0,
+    setxattr : FuseOpSetXattr,
     getxattr : FuseOpGetXattr,
     listxattr : FuseOpListXattr,
     removexattr : 0,
@@ -75,9 +80,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     flock : 0,
     fallocate : 0,
     #endif
-    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    readdirplus : FuseOpReadDirPlus,
-    #endif
+    // TODO(wuhongsong): The current implementation is problematic,
+    // resulting in inconsistent metadata
+    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    readdirplus : 0,
+    // #endif
     #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 4)
     copy_file_range : 0,
     #endif
@@ -218,14 +225,14 @@ int main(int argc, char *argv[]) {
         printf("Init glog failed, confpath = %s\n", mOpts.conf);
     }
 
-    ret = InitFuseClient(mOpts.conf, mOpts.fsName, mOpts.fsType, mOpts.mdsAddr);
+    ret = InitFuseClient(&mOpts);
     if (ret < 0) {
-        printf("init fuse client fail, conf =%s\n", mOpts.conf);
+        LOG(ERROR) << "init fuse client fail, conf = " << mOpts.conf;
         goto err_out4;
     }
 
-    printf("fuse start loop, singlethread = %d, max_idle_threads = %d\n",
-        opts.singlethread, opts.max_idle_threads);
+    LOG(INFO) << "fuse start loop, singlethread = " << opts.singlethread
+              << ", max_idle_threads = " << opts.max_idle_threads;
 
     /* Block until ctrl+c or fusermount -u */
     if (opts.singlethread) {

curvefs/src/client/metric/client_metric.h

@@ -25,6 +25,7 @@
 #define CURVEFS_SRC_CLIENT_METRIC_CLIENT_METRIC_H_
 
 #include <bvar/bvar.h>
+#include <cstdint>
 #include <string>
 #include "src/client/client_metric.h"
 
@@ -222,16 +223,16 @@ struct FSMetric {
 
     InterfaceMetric userWrite;
     InterfaceMetric userRead;
-    bvar::LatencyRecorder userWriteIoSize;
-    bvar::LatencyRecorder userReadIoSize;
+    bvar::Status<uint32_t> userWriteIoSize;
+    bvar::Status<uint32_t> userReadIoSize;
 
     explicit FSMetric(const std::string &name = "")
         : fsName(!name.empty() ? name
                                : prefix + curve::common::ToHexString(this)),
           userWrite(prefix, fsName + "_userWrite"),
           userRead(prefix, fsName + "_userRead"),
-          userWriteIoSize(prefix, fsName + "_userWriteIoSize"),
-          userReadIoSize(prefix, fsName + "_userReadIoSize") {}
+          userWriteIoSize(prefix, fsName + "_userWriteIoSize", 0),
+          userReadIoSize(prefix, fsName + "_userReadIoSize", 0) {}
 };
 
 struct S3Metric {
@@ -244,8 +245,8 @@ struct S3Metric {
     InterfaceMetric adaptorWriteDiskCache;
     InterfaceMetric adaptorReadS3;
     InterfaceMetric adaptorReadDiskCache;
-    bvar::LatencyRecorder readSize;
-    bvar::LatencyRecorder writeSize;
+    bvar::Status<uint32_t> readSize;
+    bvar::Status<uint32_t> writeSize;
 
     explicit S3Metric(const std::string &name = "")
         : fsName(!name.empty() ? name
@@ -256,8 +257,8 @@ struct S3Metric {
           adaptorWriteDiskCache(prefix, fsName + "_adaptor_write_disk_cache"),
           adaptorReadS3(prefix, fsName + "_adaptor_read_s3"),
           adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache"),
-          readSize(prefix, fsName + "_adaptor_read_size"),
-          writeSize(prefix, fsName + "_adaptor_write_size") {}
+          readSize(prefix, fsName + "_adaptor_read_size", 0),
+          writeSize(prefix, fsName + "_adaptor_write_size", 0) {}
 };
 
 struct DiskCacheMetric {

curvefs/src/client/rpcclient/mds_client.cpp

@@ -486,6 +486,7 @@ MdsClientImpl::RefreshSession(const std::vector<PartitionTxId> &txIds,
 FSStatusCode MdsClientImpl::GetLatestTxId(const GetLatestTxIdRequest& request,
                                           GetLatestTxIdResponse* response) {
     auto task = RPCTask {
+        VLOG(3) << "GetLatestTxId [request]: " << request.DebugString();
         mdsClientMetric_.getLatestTxId.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.getLatestTxId.latency);
         mdsbasecli_->GetLatestTxId(request, response, cntl, channel);
@@ -509,6 +510,8 @@ FSStatusCode MdsClientImpl::GetLatestTxId(const GetLatestTxIdRequest& request,
             LOG(WARNING) << "GetLatestTxId fail, errcode = " << rc
                          << ", errmsg = " << FSStatusCode_Name(rc);
         }
+
+        VLOG(3) << "GetLatestTxId [response]: " << response->DebugString();
         return rc;
     };
 
@@ -518,6 +521,7 @@ FSStatusCode MdsClientImpl::GetLatestTxId(const GetLatestTxIdRequest& request,
 
 FSStatusCode MdsClientImpl::CommitTx(const CommitTxRequest& request) {
     auto task = RPCTask {
+        VLOG(3) << "CommitTx [request]: " << request.DebugString();
         mdsClientMetric_.commitTx.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.commitTx.latency);
         CommitTxResponse response;
@@ -542,15 +546,18 @@ FSStatusCode MdsClientImpl::CommitTx(const CommitTxRequest& request) {
             LOG(WARNING) << "CommitTx: retCode = " << rc
                          << ", message = " << FSStatusCode_Name(rc);
         }
+        VLOG(3) << "CommitTx [response]: " << response.DebugString();
         return rc;
     };
     // for rpc error or get lock failed/timeout, we will retry until success
     return ReturnError(rpcexcutor_.DoRPCTask(task, 0));
 }
 
-FSStatusCode MdsClientImpl::GetLatestTxId(std::vector<PartitionTxId>* txIds) {
+FSStatusCode MdsClientImpl::GetLatestTxId(uint32_t fsId,
+                                          std::vector<PartitionTxId>* txIds) {
     GetLatestTxIdRequest request;
     GetLatestTxIdResponse response;
+    request.set_fsid(fsId);
     FSStatusCode rc = GetLatestTxId(request, &response);
     if (rc == FSStatusCode::OK) {
         *txIds = { response.txids().begin(), response.txids().end() };

curvefs/src/client/rpcclient/mds_client.h

@@ -112,7 +112,8 @@ class MdsClient {
                    const std::string& fsName,
                    const Mountpoint& mountpoint) = 0;
 
-    virtual FSStatusCode GetLatestTxId(std::vector<PartitionTxId>* txIds) = 0;
+    virtual FSStatusCode GetLatestTxId(uint32_t fsId,
+                                       std::vector<PartitionTxId>* txIds) = 0;
 
     virtual FSStatusCode
     GetLatestTxIdWithLock(uint32_t fsId,
@@ -196,7 +197,8 @@ class MdsClientImpl : public MdsClient {
                                 const std::string& fsName,
                                 const Mountpoint& mountpoint) override;
 
-    FSStatusCode GetLatestTxId(std::vector<PartitionTxId>* txIds) override;
+    FSStatusCode GetLatestTxId(uint32_t fsId,
+                               std::vector<PartitionTxId>* txIds) override;
 
     FSStatusCode
     GetLatestTxIdWithLock(uint32_t fsId,

curvefs/src/client/rpcclient/metacache.cpp

@@ -79,7 +79,7 @@ void MetaCache::GetAllTxIds(std::vector<PartitionTxId> *txIds) {
 
 bool MetaCache::RefreshTxId() {
     std::vector<PartitionTxId> txIds;
-    FSStatusCode rc = mdsClient_->GetLatestTxId(&txIds);
+    FSStatusCode rc = mdsClient_->GetLatestTxId(fsID_, &txIds);
     if (rc != FSStatusCode::OK) {
         LOG(ERROR) << "Get latest txid failed, retCode=" << rc;
         return false;

curvefs/src/client/rpcclient/metaserver_client.cpp

@@ -830,30 +830,44 @@ MetaServerClientImpl::UpdateInode(const UpdateInodeRequest &request,
 
 namespace {
 
-void FillInodeAttr(const Inode& inode,
-                  InodeOpenStatusChange statusChange,
-                  bool nlink,
-                  UpdateInodeRequest* request) {
-    request->set_inodeid(inode.inodeid());
-    request->set_fsid(inode.fsid());
-    request->set_length(inode.length());
-    request->set_ctime(inode.ctime());
-    request->set_mtime(inode.mtime());
-    request->set_atime(inode.atime());
-    request->set_uid(inode.uid());
-    request->set_gid(inode.gid());
-    request->set_mode(inode.mode());
-    request->set_inodeopenstatuschange(statusChange);
-    *request->mutable_parent() = inode.parent();
-    if (inode.xattr_size() > 0) {
-        *request->mutable_xattr() = inode.xattr();
+#define SET_REQUEST_FIELD_IF_HAS(request, attr, field) \
+    do {                                               \
+        if ((attr).has_##field()) {                    \
+            (request)->set_##field((attr).field());     \
+        }                                              \
+    } while (false)
+
+void FillInodeAttr(uint32_t fsId,
+                   uint64_t inodeId,
+                   const InodeAttr& attr,
+                   bool nlink,
+                   UpdateInodeRequest* request) {
+    request->set_fsid(fsId);
+    request->set_inodeid(inodeId);
+
+    SET_REQUEST_FIELD_IF_HAS(request, attr, length);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, atime);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, atime_ns);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, ctime);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, ctime_ns);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, mtime);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, mtime_ns);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, uid);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, gid);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, mode);
+
+    *request->mutable_parent() = attr.parent();
+    if (attr.xattr_size() > 0) {
+        *request->mutable_xattr() = attr.xattr();
     }
 
     if (nlink) {
-        request->set_nlink(inode.nlink());
+        request->set_nlink(attr.nlink());
     }
 }
 
+#undef SET_REQUEST_FIELD_IF_HAS
+
 void FillDataIndices(DataIndices&& indices, UpdateInodeRequest* request) {
     if (indices.s3ChunkInfoMap && !indices.s3ChunkInfoMap->empty()) {
         *request->mutable_s3chunkinfoadd() =
@@ -868,19 +882,24 @@ void FillDataIndices(DataIndices&& indices, UpdateInodeRequest* request) {
 
 }  // namespace
 
-MetaStatusCode
-MetaServerClientImpl::UpdateInodeAttr(const Inode &inode,
-                                      InodeOpenStatusChange statusChange) {
+MetaStatusCode MetaServerClientImpl::UpdateInodeAttr(
+    uint32_t fsId,
+    uint64_t inodeId,
+    const InodeAttr& attr) {
     UpdateInodeRequest request;
-    FillInodeAttr(inode, statusChange, /*nlink=*/true, &request);
+    FillInodeAttr(fsId, inodeId, attr, /*nlink=*/true, &request);
     return UpdateInode(request);
 }
 
 MetaStatusCode MetaServerClientImpl::UpdateInodeAttrWithOutNlink(
-    const Inode &inode, InodeOpenStatusChange statusChange,
-    S3ChunkInfoMap *s3ChunkInfoAdd, bool internal) {
+    uint32_t fsId,
+    uint64_t inodeId,
+    const InodeAttr& attr,
+    S3ChunkInfoMap* s3ChunkInfoAdd,
+    bool internal) {
     UpdateInodeRequest request;
-    FillInodeAttr(inode, statusChange, /*nlink=*/false, &request);
+    FillInodeAttr(fsId, inodeId, attr, /*nlink=*/false,
+                  &request);
     if (s3ChunkInfoAdd != nullptr) {
         DataIndices indices;
         indices.s3ChunkInfoMap = *s3ChunkInfoAdd;
@@ -934,7 +953,6 @@ void UpdateInodeRpcDone::Run() {
     VLOG(6) << "UpdateInode done, "
             << "response: " << response.DebugString();
     done_->SetRetCode(ret);
-    return;
 }
 
 void MetaServerClientImpl::UpdateInodeAsync(const UpdateInodeRequest &request,
@@ -962,21 +980,14 @@ void MetaServerClientImpl::UpdateInodeAsync(const UpdateInodeRequest &request,
     excutor->DoAsyncRPCTask(taskDone);
 }
 
-void MetaServerClientImpl::UpdateInodeAttrAsync(
-    const Inode &inode, MetaServerClientDone *done,
-    InodeOpenStatusChange statusChange) {
-    UpdateInodeRequest request;
-    FillInodeAttr(inode, statusChange, /*nlink=*/true, &request);
-    UpdateInodeAsync(request, done);
-}
-
 void MetaServerClientImpl::UpdateInodeWithOutNlinkAsync(
-    const Inode &inode,
-    MetaServerClientDone *done,
-    InodeOpenStatusChange statusChange,
-    DataIndices &&indices) {
+    uint32_t fsId,
+    uint64_t inodeId,
+    const InodeAttr& attr,
+    MetaServerClientDone* done,
+    DataIndices&& indices) {
     UpdateInodeRequest request;
-    FillInodeAttr(inode, statusChange, /*nlink=*/false, &request);
+    FillInodeAttr(fsId, inodeId, attr, /*nlink=*/false, &request);
     FillDataIndices(std::move(indices), &request);
     UpdateInodeAsync(request, done);
 }

curvefs/src/client/rpcclient/metaserver_client.h

@@ -45,7 +45,6 @@ using ::curvefs::client::metric::MetaServerClientMetric;
 using ::curvefs::metaserver::Dentry;
 using ::curvefs::metaserver::FsFileType;
 using ::curvefs::metaserver::Inode;
-using ::curvefs::metaserver::InodeOpenStatusChange;
 using ::curvefs::metaserver::InodeAttr;
 using ::curvefs::metaserver::XAttr;
 using ::curvefs::metaserver::MetaStatusCode;
@@ -114,25 +113,23 @@ class MetaServerClient {
         const std::set<uint64_t> &inodeIds,
         std::list<XAttr> *xattr) = 0;
 
-    virtual MetaStatusCode UpdateInodeAttr(const Inode &inode,
-        InodeOpenStatusChange statusChange =
-            InodeOpenStatusChange::NOCHANGE) = 0;
+    virtual MetaStatusCode UpdateInodeAttr(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr) = 0;
 
     virtual MetaStatusCode UpdateInodeAttrWithOutNlink(
-        const Inode &inode,
-        InodeOpenStatusChange statusChange = InodeOpenStatusChange::NOCHANGE,
-        S3ChunkInfoMap *s3ChunkInfoAdd = nullptr,
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
+        S3ChunkInfoMap* s3ChunkInfoAdd = nullptr,
         bool internal = false) = 0;
 
-    virtual void UpdateInodeAttrAsync(const Inode &inode,
-        MetaServerClientDone *done,
-        InodeOpenStatusChange statusChange =
-            InodeOpenStatusChange::NOCHANGE) = 0;
-
     virtual void UpdateInodeWithOutNlinkAsync(
-        const Inode& inode,
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
         MetaServerClientDone* done,
-        InodeOpenStatusChange statusChange = InodeOpenStatusChange::NOCHANGE,
         DataIndices&& indices = {}) = 0;
 
     virtual MetaStatusCode GetOrModifyS3ChunkInfo(
@@ -218,25 +215,24 @@ class MetaServerClientImpl : public MetaServerClient {
         const std::set<uint64_t> &inodeIds,
         std::list<XAttr> *xattr) override;
 
-    MetaStatusCode UpdateInodeAttr(const Inode &inode,
-        InodeOpenStatusChange statusChange =
-            InodeOpenStatusChange::NOCHANGE) override;
+    MetaStatusCode UpdateInodeAttr(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr) override;
 
     MetaStatusCode UpdateInodeAttrWithOutNlink(
-        const Inode &inode,
-        InodeOpenStatusChange statusChange = InodeOpenStatusChange::NOCHANGE,
-        S3ChunkInfoMap *s3ChunkInfoAdd = nullptr,
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
+        S3ChunkInfoMap* s3ChunkInfoAdd = nullptr,
         bool internal = false) override;
 
-    void UpdateInodeAttrAsync(const Inode &inode, MetaServerClientDone *done,
-                          InodeOpenStatusChange statusChange =
-                              InodeOpenStatusChange::NOCHANGE) override;
-
     void UpdateInodeWithOutNlinkAsync(
-        const Inode &inode,
-        MetaServerClientDone *done,
-        InodeOpenStatusChange statusChange = InodeOpenStatusChange::NOCHANGE,
-        DataIndices &&indices = {}) override;
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
+        MetaServerClientDone* done,
+        DataIndices&& indices = {}) override;
 
     MetaStatusCode GetOrModifyS3ChunkInfo(
         uint32_t fsId, uint64_t inodeId,

curvefs/src/client/s3/client_s3_adaptor.cpp

@@ -147,7 +147,7 @@ int S3ClientAdaptorImpl::Write(uint64_t inodeId, uint64_t offset,
     fsCacheManager_->DataCacheByteDec(length);
     if (s3Metric_.get() != nullptr) {
         CollectMetrics(&s3Metric_->adaptorWrite, ret, start);
-        s3Metric_->writeSize << length;
+        s3Metric_->writeSize.set_value(length);
     }
     VLOG(6) << "write end inodeId:" << inodeId << ",ret:" << ret
             << ", pendingReq_ is: " << pendingReq_;
@@ -169,14 +169,16 @@ int S3ClientAdaptorImpl::Read(uint64_t inodeId, uint64_t offset,
     }
     if (s3Metric_.get() != nullptr) {
         CollectMetrics(&s3Metric_->adaptorRead, ret, start);
-        s3Metric_->readSize << length;
+        s3Metric_->readSize.set_value(length);
     }
     VLOG(6) << "read end offset:" << offset << ", len:" << length
             << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
     return ret;
 }
 
-CURVEFS_ERROR S3ClientAdaptorImpl::Truncate(Inode *inode, uint64_t size) {
+CURVEFS_ERROR S3ClientAdaptorImpl::Truncate(InodeWrapper *inodeWrapper,
+                                            uint64_t size) {
+    const auto *inode = inodeWrapper->GetInodeLocked();
     uint64_t fileSize = inode->length();
 
     if (size < fileSize) {
@@ -215,7 +217,7 @@ CURVEFS_ERROR S3ClientAdaptorImpl::Truncate(Inode *inode, uint64_t size) {
             }
             assert(chunkId <= (beginChunkId + chunkIdNum - 1));
             S3ChunkInfo *tmp;
-            auto s3ChunkInfoMap = inode->mutable_s3chunkinfomap();
+            auto* s3ChunkInfoMap = inodeWrapper->GetChunkInfoMap();
             auto s3chunkInfoListIter = s3ChunkInfoMap->find(index);
             if (s3chunkInfoListIter == s3ChunkInfoMap->end()) {
                 S3ChunkInfoList s3chunkInfoList;

curvefs/src/client/s3/client_s3_adaptor.h

@@ -80,7 +80,8 @@ class S3ClientAdaptor {
                       const char *buf) = 0;
     virtual int Read(uint64_t inodeId, uint64_t offset, uint64_t length,
                      char *buf) = 0;
-    virtual CURVEFS_ERROR Truncate(Inode *inode, uint64_t size) = 0;
+    virtual CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper,
+                                   uint64_t size) = 0;
     virtual void ReleaseCache(uint64_t inodeId) = 0;
     virtual CURVEFS_ERROR Flush(uint64_t inodeId) = 0;
     virtual CURVEFS_ERROR FlushAllCache(uint64_t inodeId) = 0;
@@ -92,6 +93,11 @@ class S3ClientAdaptor {
     virtual void InitMetrics(const std::string &fsName) = 0;
     virtual void CollectMetrics(InterfaceMetric *interface, int count,
                                 uint64_t start) = 0;
+    virtual std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() = 0;
+    virtual std::shared_ptr<S3Client> GetS3Client() = 0;
+    virtual uint64_t GetBlockSize() = 0;
+    virtual uint64_t GetChunkSize() = 0;
+    virtual bool HasDiskCache() = 0;
 };
 
 using FlushChunkCacheCallBack = std::function<
@@ -130,7 +136,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
     int Write(uint64_t inodeId, uint64_t offset, uint64_t length,
               const char *buf);
     int Read(uint64_t inodeId, uint64_t offset, uint64_t length, char *buf);
-    CURVEFS_ERROR Truncate(Inode *inode, uint64_t size);
+    CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper, uint64_t size);
     void ReleaseCache(uint64_t inodeId);
     CURVEFS_ERROR Flush(uint64_t inodeId);
     CURVEFS_ERROR FlushAllCache(uint64_t inodeId);

curvefs/src/client/s3/client_s3_cache_manager.cpp

@@ -356,7 +356,8 @@ int FileCacheManager::Read(uint64_t inodeId, uint64_t offset, uint64_t length,
             {
                 ::curve::common::UniqueLock lgGuard =
                     inodeWrapper->GetUniqueLock();
-                Inode* inode = inodeWrapper->GetMutableInodeUnlocked();
+                const Inode* inode = inodeWrapper->GetInodeLocked();
+                const auto* s3chunkinfo = inodeWrapper->GetChunkInfoMap();
                 VLOG(9) << "FileCacheManager::Read Inode: "
                           << inode->DebugString();
                 fileLen = inode->length();
@@ -366,8 +367,8 @@ int FileCacheManager::Read(uint64_t inodeId, uint64_t offset, uint64_t length,
                             << ",len:" << iter->len
                             << ",bufOffset:" << iter->bufOffset;
                     auto s3InfoListIter =
-                        inode->s3chunkinfomap().find(iter->index);
-                    if (s3InfoListIter == inode->s3chunkinfomap().end()) {
+                        s3chunkinfo->find(iter->index);
+                    if (s3InfoListIter == s3chunkinfo->end()) {
                         VLOG(6)
                             << "s3infolist is not found.index:" << iter->index;
                         memset(dataBuf + iter->bufOffset, 0, iter->len);
@@ -2237,7 +2238,7 @@ CURVEFS_ERROR DataCache::Flush(uint64_t inodeId, bool toS3) {
                 // will be destructed before being accessed
                 if (pendingReq.fetch_sub(
                     1, std::memory_order_seq_cst) == 1) {
-                    VLOG(9) << "pendingReq is over";
+                    VLOG(9) << "pendingReq is over, " << context->key;
                     cond.Signal();
                 }
                 VLOG(9) << "PutObjectAsyncCallBack: " << context->key

curvefs/src/client/s3/disk_cache_manager.cpp

@@ -63,6 +63,7 @@ DiskCacheManager::DiskCacheManager(std::shared_ptr<PosixWrapper> posixWrapper,
     diskFsUsedRatio_ = 0;
     fullRatio_ = 0;
     safeRatio_ = 0;
+    diskUsedInit_ = false;
     maxUsableSpaceBytes_ = 0;
     // cannot limit the size,
     // because cache is been delete must after upload to s3
@@ -174,6 +175,9 @@ bool DiskCacheManager::IsCached(const std::string name) {
 int DiskCacheManager::UmountDiskCache() {
     LOG(INFO) << "umount disk cache.";
     int ret;
+    if (diskInitThread_.joinable()) {
+        diskInitThread_.join();
+    }
     ret = cacheWrite_->UploadAllCacheWriteFile();
     if (ret < 0) {
         LOG(ERROR) << "umount disk cache error.";
@@ -277,7 +281,7 @@ int64_t DiskCacheManager::SetDiskFsUsedRatio() {
         return -1;
     }
     int64_t usedPercent = 100 * usedBytes / (usedBytes + availableBytes) + 1;
-    diskFsUsedRatio_.store(usedPercent, std::memory_order_seq_cst);
+    diskFsUsedRatio_.store(usedPercent);
     return usedPercent;
 }
 
@@ -297,15 +301,16 @@ void DiskCacheManager::SetDiskInitUsedBytes() {
             << "get disk used size failed.";
         return;
     }
-    usedBytes_.fetch_add(usedBytes, std::memory_order_seq_cst);
+    usedBytes_.fetch_add(usedBytes);
     if (metric_.get() != nullptr)
         metric_->diskUsedBytes.set_value(usedBytes_);
+    diskUsedInit_.store(true);
     VLOG(9) << "cache disk used size is: " << result;
     return;
 }
 
 bool DiskCacheManager::IsDiskCacheFull() {
-    int64_t ratio = diskFsUsedRatio_.load(std::memory_order_seq_cst);
+    int64_t ratio = diskFsUsedRatio_.load();
     uint64_t usedBytes = GetDiskUsedbytes();
     if (ratio >= fullRatio_ || usedBytes >= maxUsableSpaceBytes_) {
         VLOG(6) << "disk cache is full"
@@ -325,7 +330,7 @@ bool DiskCacheManager::IsDiskCacheSafe() {
     if (IsExceedFileNums()) {
         return false;
     }
-    int64_t ratio = diskFsUsedRatio_.load(std::memory_order_seq_cst);
+    int64_t ratio = diskFsUsedRatio_.load();
     uint64_t usedBytes = GetDiskUsedbytes();
     if ((usedBytes < (safeRatio_ * maxUsableSpaceBytes_ / 100))
       && (ratio < safeRatio_)) {
@@ -354,6 +359,13 @@ void DiskCacheManager::TrimCache() {
     const std::chrono::seconds sleepSec(trimCheckIntervalSec_);
     LOG(INFO) << "trim function start.";
     waitIntervalSec_.Init(trimCheckIntervalSec_ * 1000);
+    // trim will start after get the disk size
+    while (!IsDiskUsedInited()) {
+        if (!isRunning_) {
+            return;
+        }
+        waitIntervalSec_.WaitForNextExcution();
+    }
     // 1. check cache disk usage every sleepSec seconds.
     // 2. if cache disk is full,
     //    then remove disk file until cache disk is lower than safeRatio_.
@@ -450,8 +462,11 @@ void DiskCacheManager::InitMetrics(const std::string &fsName) {
     cacheWrite_->InitMetrics(metric_);
     cacheRead_->InitMetrics(metric_);
     // this function move to here from init，
-    // Otherwise, you can't get the original metric
-    SetDiskInitUsedBytes();
+    // Otherwise, you can't get the original metric.
+    // SetDiskInitUsedBytes may takes a long time,
+    // so use a separate thread to do this.
+    diskInitThread_ = curve::common::Thread(
+      &DiskCacheManager::SetDiskInitUsedBytes, this);
 }
 
 }  // namespace client

curvefs/src/client/s3/disk_cache_manager.h

@@ -108,8 +108,16 @@ class DiskCacheManager {
      * @brief: stop trim thread.
      */
     int TrimStop();
+
     void InitMetrics(const std::string &fsName);
 
+    /**
+     * @brief: has geted the origin used size or not.
+     */
+    virtual bool IsDiskUsedInited() {
+        return diskUsedInit_.load();
+    }
+
  private:
     /**
      * @brief add the used bytes of disk cache.
@@ -179,6 +187,10 @@ class DiskCacheManager {
     Throttle diskCacheThrottle_;
 
     S3ClientAdaptorOption option_;
+
+    // has geted the origin used size or not
+    std::atomic<bool> diskUsedInit_;
+    curve::common::Thread diskInitThread_;
 };
 
 

curvefs/src/client/s3/disk_cache_manager_impl.cpp

@@ -80,22 +80,17 @@ int DiskCacheManagerImpl::Write(const std::string name, const char *buf,
     VLOG(9) << "write name = " << name << ", length = " << length;
     int ret = 0;
     ret = WriteDiskFile(name, buf, length);
-    if (ret < 0) {
-        ret = client_->Upload(name, buf, length);
-        if (ret < 0) {
-            LOG(ERROR) << "upload object fail. object: " << name;
-            return -1;
-        }
-    }
-    VLOG(9) << "write success, write name = " << name;
-    return 0;
+    VLOG(9) << "write end, write name: " << name
+            << "ret: " << ret;
+    return ret;
 }
 
 int DiskCacheManagerImpl::WriteDiskFile(const std::string name, const char *buf,
                                         uint64_t length) {
     VLOG(9) << "write name = " << name << ", length = " << length;
     // if cache disk is full
-    if (diskCacheManager_->IsDiskCacheFull()) {
+    if (!diskCacheManager_->IsDiskUsedInited() ||
+      diskCacheManager_->IsDiskCacheFull()) {
         VLOG(6) << "write disk file fail, disk full.";
         return -1;
     }
@@ -121,12 +116,13 @@ int DiskCacheManagerImpl::WriteDiskFile(const std::string name, const char *buf,
 
     // notify async load to s3
     diskCacheManager_->AsyncUploadEnqueue(name);
-    return writeRet;
+    return 0;
 }
 
 int DiskCacheManagerImpl::WriteReadDirect(const std::string fileName,
                                           const char *buf, uint64_t length) {
-    if (diskCacheManager_->IsDiskCacheFull()) {
+    if (!diskCacheManager_->IsDiskUsedInited() ||
+      diskCacheManager_->IsDiskCacheFull()) {
         VLOG(6) << "write disk file fail, disk full.";
         return -1;
     }

curvefs/src/client/volume/default_volume_storage.cpp

@@ -111,12 +111,11 @@ CURVEFS_ERROR DefaultVolumeStorage::Write(uint64_t ino,
 
     {
         auto lk = inodeWrapper->GetUniqueLock();
-        auto* inode = inodeWrapper->GetMutableInodeUnlocked();
-        UpdateInodeTimestamp(inode, kModifyTime | kChangeTime);
-
-        if (offset + len > inode->length()) {
-            inode->set_length(offset + len);
+        if (offset + len > inodeWrapper->GetLengthLocked()) {
+            inodeWrapper->SetLengthLocked(offset + len);
         }
+
+        inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
     }
 
     inodeCacheManager_->ShipToFlush(inodeWrapper);
@@ -170,8 +169,10 @@ CURVEFS_ERROR DefaultVolumeStorage::Read(uint64_t ino,
     }
 
     // TODO(all): check whether inode is opened with 'NO_ATIME'
-    auto* inode = inodeWrapper->GetMutableInodeUnlocked();
-    UpdateInodeTimestamp(inode, kAccessTime);
+    {
+        auto lock = inodeWrapper->GetUniqueLock();
+        inodeWrapper->UpdateTimestampLocked(kAccessTime);
+    }
     inodeCacheManager_->ShipToFlush(inodeWrapper);
 
     VLOG(9) << "read end, ino: " << ino << ", offset: " << offset

curvefs/src/client/volume/utils.cpp

@@ -36,26 +36,6 @@ namespace client {
 using ::curvefs::metaserver::Inode;
 using ::curvefs::volume::AllocateHint;
 
-void UpdateInodeTimestamp(Inode* inode, int flags) {
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-
-    if (flags & kAccessTime) {
-        inode->set_atime(now.tv_sec);
-        inode->set_atime_ns(now.tv_nsec);
-    }
-
-    if (flags & kChangeTime) {
-        inode->set_ctime(now.tv_sec);
-        inode->set_ctime_ns(now.tv_nsec);
-    }
-
-    if (flags & kModifyTime) {
-        inode->set_mtime(now.tv_sec);
-        inode->set_mtime_ns(now.tv_nsec);
-    }
-}
-
 bool AllocSpace(SpaceManager* space,
                 const AllocPart& part,
                 std::map<uint64_t, WritePart>* writes,

curvefs/src/client/volume/utils.h

@@ -43,14 +43,6 @@ using ::curvefs::volume::Extent;
 using ::curvefs::volume::SpaceManager;
 using ::curvefs::volume::WritePart;
 
-enum {
-    kAccessTime = 1 << 0,
-    kChangeTime = 1 << 1,
-    kModifyTime = 1 << 2,
-};
-
-void UpdateInodeTimestamp(Inode* inode, int flags);
-
 /**
  * @brief Allocate space for a write request
  * @param space space manager that used for allocating space

curvefs/src/client/xattr_manager.cpp

@@ -31,6 +31,21 @@ namespace client {
 using ::curve::common::StringToUll;
 using ::curve::common::Thread;
 
+
+bool IsSummaryInfo(const char *name) {
+    return std::strstr(name, SUMMARYPREFIX);
+}
+
+bool IsOneLayer(const char *name) {
+    if (std::strcmp(name, XATTRFILES) == 0 ||
+        std::strcmp(name, XATTRSUBDIRS) == 0 ||
+        std::strcmp(name, XATTRENTRIES) == 0 ||
+        std::strcmp(name, XATTRFBYTES) == 0) {
+        return true;
+    }
+    return false;
+}
+
 // if direction is true means '+', false means '-'
 bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction) {
     uint64_t firstNum = 0;
@@ -441,6 +456,43 @@ CURVEFS_ERROR XattrManager::FastCalAllLayerSumInfo(InodeAttr *attr) {
     return CURVEFS_ERROR::OK;
 }
 
+CURVEFS_ERROR XattrManager::GetXattr(const char* name, std::string *value,
+    InodeAttr *attr, bool enableSumInDir) {
+    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
+    // get summary info if the xattr name is summary type
+    if (IsSummaryInfo(name) && attr->type() == FsFileType::TYPE_DIRECTORY) {
+        // if not enable record summary info in dir xattr,
+        // need recursive computation all files;
+        // otherwise only recursive computation all dirs.
+        if (!enableSumInDir) {
+            if (IsOneLayer(name)) {
+                ret = CalOneLayerSumInfo(attr);
+            } else {
+                ret = CalAllLayerSumInfo(attr);
+            }
+        } else {
+            if (IsOneLayer(name)) {
+                ret = FastCalOneLayerSumInfo(attr);
+            } else {
+                ret = FastCalAllLayerSumInfo(attr);
+            }
+        }
+
+        if (CURVEFS_ERROR::OK != ret) {
+            return ret;
+        }
+        LOG(INFO) << "After calculate summary info:\n"
+                  << attr->DebugString();
+    }
+
+    auto it = attr->xattr().find(name);
+    if (it != attr->xattr().end()) {
+        *value = it->second;
+    }
+    return ret;
+}
+
+
 CURVEFS_ERROR XattrManager::UpdateParentInodeXattr(uint64_t parentId,
     const XAttr &xattr, bool direction) {
     VLOG(9) << "UpdateParentInodeXattr inodeId = " << parentId
@@ -455,10 +507,11 @@ CURVEFS_ERROR XattrManager::UpdateParentInodeXattr(uint64_t parentId,
     }
 
     ::curve::common::UniqueLock lgGuard = pInodeWrapper->GetUniqueLock();
-    auto inode = pInodeWrapper->GetMutableInodeUnlocked();
+    auto inodeXAttr = pInodeWrapper->GetInodeLocked()->xattr();
+    bool update = false;
     for (const auto &it : xattr.xattrinfos()) {
-        auto iter = inode->mutable_xattr()->find(it.first);
-        if (iter != inode->mutable_xattr()->end()) {
+        auto iter = inodeXAttr.find(it.first);
+        if (iter != inodeXAttr.end()) {
             uint64_t dat = 0;
             if (StringToUll(it.second, &dat)) {
                 if (!AddUllStringToFirst(&(iter->second), dat, direction)) {
@@ -468,9 +521,15 @@ CURVEFS_ERROR XattrManager::UpdateParentInodeXattr(uint64_t parentId,
                 LOG(ERROR) << "StringToUll failed, first = " << it.second;
                 return CURVEFS_ERROR::INTERNAL;
             }
+            update = true;
         }
     }
-    inodeManager_->ShipToFlush(pInodeWrapper);
+
+    if (update) {
+        pInodeWrapper->MergeXAttrLocked(inodeXAttr);
+        inodeManager_->ShipToFlush(pInodeWrapper);
+    }
+
     return CURVEFS_ERROR::OK;
 }
 

curvefs/src/client/xattr_manager.h

@@ -74,13 +74,8 @@ class XattrManager {
         isStop_.store(true);
     }
 
-    CURVEFS_ERROR CalOneLayerSumInfo(InodeAttr *attr);
-
-    CURVEFS_ERROR CalAllLayerSumInfo(InodeAttr *attr);
-
-    CURVEFS_ERROR FastCalOneLayerSumInfo(InodeAttr *attr);
-
-    CURVEFS_ERROR FastCalAllLayerSumInfo(InodeAttr *attr);
+    CURVEFS_ERROR GetXattr(const char* name, std::string *value,
+        InodeAttr *attr, bool enableSumInDir);
 
     CURVEFS_ERROR UpdateParentInodeXattr(uint64_t parentId,
         const XAttr &xattr, bool direction);
@@ -116,6 +111,14 @@ class XattrManager {
         Atomic<uint32_t> *inflightNum,
         Atomic<bool> *ret);
 
+    CURVEFS_ERROR CalOneLayerSumInfo(InodeAttr *attr);
+
+    CURVEFS_ERROR CalAllLayerSumInfo(InodeAttr *attr);
+
+    CURVEFS_ERROR FastCalOneLayerSumInfo(InodeAttr *attr);
+
+    CURVEFS_ERROR FastCalAllLayerSumInfo(InodeAttr *attr);
+
  private:
     // inode cache manager
     std::shared_ptr<InodeCacheManager> inodeManager_;

curvefs/src/mds/fs_manager.cpp

@@ -808,6 +808,13 @@ FSStatusCode FsManager::GetFsTxSequence(const std::string& fsName,
 void FsManager::GetLatestTxId(const GetLatestTxIdRequest* request,
                               GetLatestTxIdResponse* response) {
     std::vector<PartitionTxId> txIds;
+    if (!request->has_fsid()) {
+        response->set_statuscode(FSStatusCode::PARAM_ERROR);
+        LOG(ERROR) << "Bad GetLatestTxId request which missing fsid"
+                   << ", request=" << request->DebugString();
+        return;
+    }
+
     uint32_t fsId = request->fsid();
     if (!request->lock()) {
         GetLatestTxId(fsId, &txIds);

curvefs/src/mds/mds_service.cpp

@@ -325,7 +325,9 @@ void MdsServiceImpl::GetLatestTxId(
     GetLatestTxIdResponse* response,
     ::google::protobuf::Closure* done) {
     brpc::ClosureGuard guard(done);
+    VLOG(3) << "GetLatestTxId [request]: " << request->DebugString();
     fsManager_->GetLatestTxId(request, response);
+    VLOG(3) << "GetLatestTxId [response]: " << response->DebugString();
 }
 
 void MdsServiceImpl::CommitTx(::google::protobuf::RpcController* controller,
@@ -333,7 +335,9 @@ void MdsServiceImpl::CommitTx(::google::protobuf::RpcController* controller,
                               CommitTxResponse* response,
                               ::google::protobuf::Closure* done) {
     brpc::ClosureGuard guard(done);
+    VLOG(3) << "CommitTx [request]: " << request->DebugString();
     fsManager_->CommitTx(request, response);
+    VLOG(3) << "CommitTx [response]: " << request->DebugString();
 }
 
 }  // namespace mds

curvefs/src/mds/schedule/copysetSchedule.cpp

@@ -218,6 +218,12 @@ int CopySetScheduler::CheckAndBalanceZoneByMetaserverUsage(ZoneIdType zoneId) {
 
         std::vector<CopySetInfo> copysetVector =
             topo_->GetCopySetInfosInMetaServer(sourceId);
+
+        // TODO(chenwei): select copyset which uses less disk
+        thread_local static std::random_device rd;
+        thread_local static std::mt19937 randomGenerator(rd());
+        std::shuffle(copysetVector.begin(), copysetVector.end(),
+                     randomGenerator);
         for (const auto &copyset : copysetVector) {
             if (!IsCopysetCanTransfer(copyset)) {
                 continue;
@@ -276,6 +282,12 @@ int CopySetScheduler::CheckAndBalanceZoneByCopysetNum(ZoneIdType zoneId) {
 
         std::vector<CopySetInfo> copysetVector =
             topo_->GetCopySetInfosInMetaServer(sourceId);
+
+        // TODO(chenwei): select copyset which uses less disk
+        thread_local static std::random_device rd;
+        thread_local static std::mt19937 randomGenerator(rd());
+        std::shuffle(copysetVector.begin(), copysetVector.end(),
+                     randomGenerator);
         for (const auto &copyset : copysetVector) {
             if (!IsCopysetCanTransfer(copyset)) {
                 continue;

curvefs/src/mds/topology/topology_metric.cpp

@@ -161,6 +161,17 @@ void TopologyMetricService::UpdateTopologyMetrics() {
             totalDiskUsed += ix->second->diskUsed.get_value();
             totalMemoryThreshold += ix->second->memoryThreshold.get_value();
             totalMemoryUsed += ix->second->memoryUsed.get_value();
+
+            // process the metric of metaserver which has no copyset
+            if (metaServerMetricInfo.find(msId) == metaServerMetricInfo.end()) {
+                auto ix = gMetaServerMetrics.find(msId);
+                if (ix != gMetaServerMetrics.end()) {
+                    ix->second->scatterWidth.set_value(0);
+                    ix->second->copysetNum.set_value(0);
+                    ix->second->leaderNum.set_value(0);
+                    ix->second->partitionNum.set_value(0);
+                }
+            }
         }
 
         it->second->metaServerNum.set_value(msIdInPool.size());

curvefs/src/metaserver/inode_manager.cpp

@@ -295,7 +295,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request,
     const S3ChunkInfoList *list2add;
     VLOG(9) << "UpdateInode inode " << old->inodeid() << " map2add size "
             << map2add.size();
-    uint64_t size4add = 0;
     for (const auto &item : map2add) {
         uint64_t chunkIndex = item.first;
         list2add = &item.second;
@@ -438,13 +437,6 @@ MetaStatusCode InodeManager::UpdateInodeWhenCreateOrRemoveSubNode(
         }
     }
 
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    inode.set_ctime(now.tv_sec);
-    inode.set_ctime_ns(now.tv_nsec);
-    inode.set_mtime(now.tv_sec);
-    inode.set_mtime_ns(now.tv_nsec);
-
     ret = inodeStorage_->Update(inode);
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "UpdateInode fail, " << inode.ShortDebugString()

curvefs/src/metaserver/partition_cleaner.cpp

@@ -19,16 +19,15 @@
  * @Date: 2021-12-15 10:54:37
  * @Author: chenwei
  */
-
 #include "curvefs/src/metaserver/partition_cleaner.h"
+
 #include <list>
+
 #include "curvefs/src/metaserver/copyset/meta_operator.h"
-#include "curvefs/proto/mds.pb.h"
 
 namespace curvefs {
 namespace metaserver {
 
-using ::curvefs::mds::FsInfo;
 using ::curvefs::mds::FSStatusCode;
 using ::curvefs::mds::FSStatusCode_Name;
 
@@ -89,18 +88,24 @@ MetaStatusCode PartitionCleaner::CleanDataAndDeleteInode(const Inode& inode) {
     if (FsFileType::TYPE_S3 == inode.type()) {
          // get s3info from mds
         FsInfo fsInfo;
-        auto ret = mdsClient_->GetFsInfo(inode.fsid(), &fsInfo);
-        if (ret != FSStatusCode::OK) {
-            if (FSStatusCode::NOT_FOUND == ret) {
-                LOG(ERROR) << "The fsName not exist, fsId = " << inode.fsid();
-                return MetaStatusCode::S3_DELETE_ERR;
-            } else {
-                LOG(ERROR) << "GetFsInfo failed, FSStatusCode = " << ret
-                        << ", FSStatusCode_Name = "
-                        << FSStatusCode_Name(ret)
+        if (fsInfoMap_.find(inode.fsid()) == fsInfoMap_.end()) {
+            auto ret = mdsClient_->GetFsInfo(inode.fsid(), &fsInfo);
+            if (ret != FSStatusCode::OK) {
+                if (FSStatusCode::NOT_FOUND == ret) {
+                    LOG(ERROR)
+                        << "The fsName not exist, fsId = " << inode.fsid();
+                    return MetaStatusCode::S3_DELETE_ERR;
+                } else {
+                    LOG(ERROR)
+                        << "GetFsInfo failed, FSStatusCode = " << ret
+                        << ", FSStatusCode_Name = " << FSStatusCode_Name(ret)
                         << ", fsId = " << inode.fsid();
-                return MetaStatusCode::S3_DELETE_ERR;
+                    return MetaStatusCode::S3_DELETE_ERR;
+                }
             }
+            fsInfoMap_.insert({inode.fsid(), fsInfo});
+        } else {
+            fsInfo = fsInfoMap_.find(inode.fsid())->second;
         }
         const auto& s3Info = fsInfo.detail().s3info();
         // reinit s3 adaptor

curvefs/src/metaserver/partition_cleaner.h

@@ -24,15 +24,19 @@
 #define CURVEFS_SRC_METASERVER_PARTITION_CLEANER_H_
 
 #include <memory>
+#include <unordered_map>
+
+#include "curvefs/proto/mds.pb.h"
+#include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
 #include "curvefs/src/metaserver/partition.h"
 #include "curvefs/src/metaserver/s3/metaserver_s3_adaptor.h"
-#include "curvefs/src/client/rpcclient/mds_client.h"
 
 namespace curvefs {
 namespace metaserver {
 
 using ::curvefs::client::rpcclient::MdsClient;
+using ::curvefs::mds::FsInfo;
 
 class PartitionCleaner {
  public:
@@ -79,6 +83,7 @@ class PartitionCleaner {
     std::shared_ptr<MdsClient> mdsClient_;
     bool isStop_;
     uint32_t inodeDeletePeriodMs_;
+    std::unordered_map<uint32_t, FsInfo> fsInfoMap_;
 };
 
 class PartitionCleanerClosure : public google::protobuf::Closure {

curvefs/src/metaserver/s3compact_inode.cpp

@@ -563,6 +563,7 @@ void CompactInodeJob::CompactChunks(const S3CompactTask& task) {
         GetNeedCompact(inode.s3chunkinfomap(), inode.length(), chunkSize);
     if (needCompact.empty()) {
         VLOG(6) << "s3compact: no need to compact " << inode.inodeid();
+        opts_->s3adapterManager->ReleaseS3Adapter(s3adapterIndex);
         return;
     }
 

curvefs/src/metaserver/trash.cpp

@@ -142,18 +142,23 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
     } else if (FsFileType::TYPE_S3 == inode.type()) {
         // get s3info from mds
         FsInfo fsInfo;
-        auto ret = mdsClient_->GetFsInfo(item.fsId, &fsInfo);
-        if (ret != FSStatusCode::OK) {
-            if (FSStatusCode::NOT_FOUND == ret) {
-                LOG(ERROR) << "The fsName not exist, fsId = " << item.fsId;
-                return MetaStatusCode::S3_DELETE_ERR;
-            } else {
-                LOG(ERROR) << "GetFsInfo failed, FSStatusCode = " << ret
-                        << ", FSStatusCode_Name = "
-                        << FSStatusCode_Name(ret)
+        if (fsInfoMap_.find(item.fsId) == fsInfoMap_.end()) {
+            auto ret = mdsClient_->GetFsInfo(item.fsId, &fsInfo);
+            if (ret != FSStatusCode::OK) {
+                if (FSStatusCode::NOT_FOUND == ret) {
+                    LOG(ERROR) << "The fsName not exist, fsId = " << item.fsId;
+                    return MetaStatusCode::S3_DELETE_ERR;
+                } else {
+                    LOG(ERROR)
+                        << "GetFsInfo failed, FSStatusCode = " << ret
+                        << ", FSStatusCode_Name = " << FSStatusCode_Name(ret)
                         << ", fsId = " << item.fsId;
-                return MetaStatusCode::S3_DELETE_ERR;
+                    return MetaStatusCode::S3_DELETE_ERR;
+                }
             }
+            fsInfoMap_.insert({item.fsId, fsInfo});
+        } else {
+            fsInfo = fsInfoMap_.find(item.fsId)->second;
         }
         const auto& s3Info = fsInfo.detail().s3info();
         // reinit s3 adaptor
@@ -163,6 +168,21 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
         clientAdaptorOption.chunkSize = s3Info.chunksize();
         s3Adaptor_->Reinit(clientAdaptorOption, s3Info.ak(), s3Info.sk(),
             s3Info.endpoint(), s3Info.bucketname());
+        ret = inodeStorage_->PaddingInodeS3ChunkInfo(item.fsId,
+          item.inodeId, inode.mutable_s3chunkinfomap());
+        if (ret != MetaStatusCode::OK) {
+            LOG(ERROR) << "GetInode chunklist fail, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId
+                << ", retCode = " << MetaStatusCode_Name(ret);
+            return ret;
+        }
+        if (inode.s3chunkinfomap().empty()) {
+            LOG(WARNING) << "GetInode chunklist empty, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId;
+            return MetaStatusCode::NOT_FOUND;
+        }
+        VLOG(9) << "DeleteInodeAndData, inode: "
+            << inode.ShortDebugString();
         int retVal = s3Adaptor_->Delete(inode);
         if (retVal != 0) {
             LOG(ERROR) << "S3ClientAdaptor delete s3 data failed"
@@ -171,7 +191,6 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
             return MetaStatusCode::S3_DELETE_ERR;
         }
     }
-
     ret = inodeStorage_->Delete(Key4Inode(item.fsId, item.inodeId));
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "Delete Inode fail, fsId = " << item.fsId

curvefs/src/metaserver/trash.h

@@ -26,6 +26,7 @@
 #include <cstdint>
 #include <memory>
 #include <list>
+#include <unordered_map>
 
 #include "src/common/configuration.h"
 #include "src/common/concurrent/concurrent.h"
@@ -116,6 +117,7 @@ class TrashImpl : public Trash {
     std::shared_ptr<InodeStorage> inodeStorage_;
     std::shared_ptr<S3ClientAdaptor>  s3Adaptor_;
     std::shared_ptr<MdsClient> mdsClient_;
+    std::unordered_map<uint32_t, FsInfo> fsInfoMap_;
 
     std::list<TrashItem> trashItems_;
 

curvefs/src/volume/block_device_aio.h

@@ -23,6 +23,7 @@
 #ifndef CURVEFS_SRC_VOLUME_BLOCK_DEVICE_AIO_H_
 #define CURVEFS_SRC_VOLUME_BLOCK_DEVICE_AIO_H_
 
+#include <array>
 #include <atomic>
 #include <condition_variable>
 #include <memory>

curvefs/test/client/BUILD

@@ -36,6 +36,21 @@ cc_library(
     linkopts = ["-L/usr/local/lib/x86_64-linux-gnu"],
 )
 
+cc_library(
+    name = "client_test_utils",
+    srcs = [
+        "utils.cpp",
+    ],
+    hdrs = [
+        "utils.h",
+    ],
+    copts = CURVE_TEST_COPTS,
+    visibility = ["//visibility:public"],
+    deps = [
+        "//curvefs/proto:metaserver_cc_proto",
+    ],
+)
+
 cc_test(
     name = "client_s3_test",
     srcs = glob([

curvefs/test/client/client_s3_adaptor_test.cpp

@@ -26,6 +26,7 @@
 #include <google/protobuf/util/message_differencer.h>
 #include <gtest/gtest.h>
 
+#include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/test/client/mock_client_s3.h"
 #include "curvefs/test/client/mock_client_s3_cache_manager.h"
 #include "curvefs/test/client/mock_disk_cache_manager.h"
@@ -87,19 +88,23 @@ class ClientS3AdaptorTest : public testing::Test {
 };
 
 uint64_t gInodeId = 1;
-void InitInode(Inode *inode) {
-    inode->set_inodeid(gInodeId);
-    inode->set_fsid(2);
-    inode->set_length(0);
-    inode->set_ctime(1623835517);
-    inode->set_mtime(1623835517);
-    inode->set_atime(1623835517);
-    inode->set_uid(1);
-    inode->set_gid(1);
-    inode->set_mode(1);
-    inode->set_nlink(1);
-    inode->set_type(curvefs::metaserver::FsFileType::TYPE_S3);
+std::unique_ptr<InodeWrapper> InitInode() {
+    Inode inode;
+
+    inode.set_inodeid(gInodeId);
+    inode.set_fsid(2);
+    inode.set_length(0);
+    inode.set_ctime(1623835517);
+    inode.set_mtime(1623835517);
+    inode.set_atime(1623835517);
+    inode.set_uid(1);
+    inode.set_gid(1);
+    inode.set_mode(1);
+    inode.set_nlink(1);
+    inode.set_type(curvefs::metaserver::FsFileType::TYPE_S3);
     gInodeId++;
+
+    return absl::make_unique<InodeWrapper>(std::move(inode), nullptr);
 }
 
 TEST_F(ClientS3AdaptorTest, test_init) {
@@ -154,55 +159,50 @@ TEST_F(ClientS3AdaptorTest, read_fail) {
 }
 
 TEST_F(ClientS3AdaptorTest, truncate_small) {
-    curvefs::metaserver::Inode inode;
-    InitInode(&inode);
-    inode.set_length(1000);
+    auto inode = InitInode();
+    inode->SetLength(1000);
 
     auto fileCache = std::make_shared<MockFileCacheManager>();
     EXPECT_CALL(*mockFsCacheManager_, FindOrCreateFileCacheManager(_, _))
         .WillOnce(Return(fileCache));
     EXPECT_CALL(*fileCache, TruncateCache(_, _)).WillOnce(Return());
-    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(&inode, 100));
+    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(inode.get(), 100));
 }
 
 TEST_F(ClientS3AdaptorTest, truncate_unchange) {
-    curvefs::metaserver::Inode inode;
-    InitInode(&inode);
-    inode.set_length(1000);
+    auto inode = InitInode();
+    inode->SetLength(1000);
 
-    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(&inode, 1000));
+    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(inode.get(), 1000));
 }
 
 TEST_F(ClientS3AdaptorTest, truncate_big_alloc_chunkId_fail) {
-    curvefs::metaserver::Inode inode;
-    InitInode(&inode);
+    auto inode = InitInode();
 
     EXPECT_CALL(*mockMdsClient_, AllocS3ChunkId(_, _, _))
         .WillOnce(Return(FSStatusCode::UNKNOWN_ERROR));
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL,
-              s3ClientAdaptor_->Truncate(&inode, 1000));
+              s3ClientAdaptor_->Truncate(inode.get(), 1000));
 }
 
 TEST_F(ClientS3AdaptorTest, truncate_big_success) {
-    curvefs::metaserver::Inode inode;
-    InitInode(&inode);
+    auto inode = InitInode();
 
     uint64_t chunkId = 999;
     EXPECT_CALL(*mockMdsClient_, AllocS3ChunkId(_, _, _))
         .WillOnce(DoAll(SetArgPointee<2>(chunkId), Return(FSStatusCode::OK)));
-    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(&inode, 1000));
+    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(inode.get(), 1000));
 }
 
 TEST_F(ClientS3AdaptorTest, truncate_big_more_chunkId) {
-    curvefs::metaserver::Inode inode;
-    InitInode(&inode);
+    auto inode = InitInode();
 
     uint64_t chunkId = 999;
     EXPECT_CALL(*mockMdsClient_, AllocS3ChunkId(_, _, _))
         .WillOnce(DoAll(SetArgPointee<2>(chunkId), Return(FSStatusCode::OK)));
     ASSERT_EQ(CURVEFS_ERROR::OK,
-              s3ClientAdaptor_->Truncate(&inode, 8 * 1024 * 1024));
-    auto s3ChunkInfoMap = inode.mutable_s3chunkinfomap();
+              s3ClientAdaptor_->Truncate(inode.get(), 8 * 1024 * 1024));
+    auto s3ChunkInfoMap = inode->GetChunkInfoMap();
     auto s3chunkInfoListIter = s3ChunkInfoMap->find(0);
     auto s3ChunkInfo = s3chunkInfoListIter->second.s3chunks(0);
     ASSERT_EQ(999, s3ChunkInfo.chunkid());

curvefs/test/client/mock_client_s3_adaptor.h

@@ -26,6 +26,7 @@
 #include <memory>
 #include <string>
 
+#include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
 
 namespace curvefs {
@@ -33,21 +34,6 @@ namespace client {
 
 class MockS3ClientAdaptor : public S3ClientAdaptor {
  public:
-    MockS3ClientAdaptor() {}
-    ~MockS3ClientAdaptor() {}
-/*
-    MOCK_METHOD5(Init,
-                 CURVEFS_ERROR(const S3ClientAdaptorOption &option,
-                               std::shared_ptr<S3Client> client,
-                               std::shared_ptr<InodeCacheManager> inodeManager,
-                               std::shared_ptr<MdsClient> mdsClient,
-                               std::shared_ptr<DiskCacheManagerImpl>
-                                   diskcacheManagerImpl));
-    MOCK_METHOD4(Init,
-                 CURVEFS_ERROR(const S3ClientAdaptorOption& option,
-                               std::shared_ptr<S3Client> client,
-                               std::shared_ptr<InodeCacheManager> inodeManager,
-                               std::shared_ptr<MdsClient> mdsClient));*/
     MOCK_METHOD7(Init,
                  CURVEFS_ERROR(const S3ClientAdaptorOption &option,
                                std::shared_ptr<S3Client> client,
@@ -68,13 +54,18 @@ class MockS3ClientAdaptor : public S3ClientAdaptor {
     MOCK_METHOD1(FlushAllCache, CURVEFS_ERROR(uint64_t inodeId));
     MOCK_METHOD0(FsSync, CURVEFS_ERROR());
     MOCK_METHOD0(Stop, int());
-    MOCK_METHOD2(Truncate, CURVEFS_ERROR(Inode* inode, uint64_t size));
+    MOCK_METHOD2(Truncate, CURVEFS_ERROR(InodeWrapper* inode, uint64_t size));
     MOCK_METHOD3(AllocS3ChunkId, FSStatusCode(uint32_t fsId, uint32_t idNum,
                                               uint64_t *chunkId));
     MOCK_METHOD1(SetFsId, void(uint32_t fsId));
     MOCK_METHOD1(InitMetrics, void(const std::string &fsName));
     MOCK_METHOD3(CollectMetrics,
                  void(InterfaceMetric *interface, int count, uint64_t start));
+    MOCK_METHOD0(GetDiskCacheManager, std::shared_ptr<DiskCacheManagerImpl>());
+    MOCK_METHOD0(GetS3Client, std::shared_ptr<S3Client>());
+    MOCK_METHOD0(GetBlockSize, uint64_t());
+    MOCK_METHOD0(GetChunkSize, uint64_t());
+    MOCK_METHOD0(HasDiskCache, bool());
 };
 
 }  // namespace client

curvefs/test/client/mock_disk_cache_manager.h

@@ -54,6 +54,8 @@ class MockDiskCacheManager : public DiskCacheManager {
     MOCK_METHOD3(WriteReadDirect,
                   int(const std::string fileName,
                       const char* buf, uint64_t length));
+    MOCK_METHOD0(IsDiskUsedInited,
+                 bool());
 };
 
 class MockDiskCacheManager2 : public DiskCacheManager {

curvefs/test/client/mock_inode_cache_manager.h

@@ -56,13 +56,11 @@ class MockInodeCacheManager : public InodeCacheManager {
     MOCK_METHOD2(GetInodeAttr, CURVEFS_ERROR(
         uint64_t inodeId, InodeAttr *out));
 
-    MOCK_METHOD1(RefreshInode, CURVEFS_ERROR(uint64_t inodeId));
-
     MOCK_METHOD2(BatchGetInodeAttr, CURVEFS_ERROR(
         std::set<uint64_t> *inodeIds, std::list<InodeAttr> *attrs));
 
     MOCK_METHOD3(BatchGetInodeAttrAsync,
-        CURVEFS_ERROR(uint64_t parentId, const std::set<uint64_t> &inodeIds,
+        CURVEFS_ERROR(uint64_t parentId, std::set<uint64_t> *inodeIds,
         std::map<uint64_t, InodeAttr> *attrs));
 
     MOCK_METHOD2(BatchGetXAttr, CURVEFS_ERROR(
@@ -88,6 +86,10 @@ class MockInodeCacheManager : public InodeCacheManager {
     MOCK_METHOD0(FlushInodeOnce, void());
 
     MOCK_METHOD1(ReleaseCache, void(uint64_t parentId));
+
+    MOCK_METHOD1(AddOpenedInode, void(uint64_t inodeId));
+
+    MOCK_METHOD1(RemoveOpenedInode, void(uint64_t inodeId));
 };
 
 }  // namespace client

curvefs/test/client/mock_metaserver_client.h

@@ -35,18 +35,12 @@
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 
-using ::testing::Return;
-using ::testing::_;
-
 namespace curvefs {
 namespace client {
 namespace rpcclient {
 
 class MockMetaServerClient : public MetaServerClient {
  public:
-    MockMetaServerClient() {}
-    ~MockMetaServerClient() {}
-
     MOCK_METHOD4(Init,
                  MetaStatusCode(const ExcutorOpt &excutorOpt,
                                 const ExcutorOpt &excutorInternalOpt,
@@ -95,34 +89,34 @@ class MockMetaServerClient : public MetaServerClient {
         uint32_t fsId, const std::set<uint64_t> &inodeIds,
         std::list<XAttr> *xattr));
 
-    MOCK_METHOD2(UpdateInodeAttr,
-                 MetaStatusCode(const Inode &inode,
-                                InodeOpenStatusChange statusChange));
+    MOCK_METHOD3(UpdateInodeAttr,
+                 MetaStatusCode(uint32_t,
+                                uint64_t,
+                                const InodeAttr&));
 
-    MOCK_METHOD4(UpdateInodeAttrWithOutNlink,
-                 MetaStatusCode(const Inode &inode,
-                                InodeOpenStatusChange statusChange,
-                                S3ChunkInfoMap *s3ChunkInfoAdd,
+    MOCK_METHOD5(UpdateInodeAttrWithOutNlink,
+                 MetaStatusCode(uint32_t,
+                                uint64_t,
+                                const InodeAttr&,
+                                S3ChunkInfoMap* s3ChunkInfoAdd,
                                 bool internal));
 
-    MOCK_METHOD3(UpdateInodeAttrAsync,
-                 void(const Inode &inode, MetaServerClientDone *done,
-                      InodeOpenStatusChange statusChange));
-
     // Workaround for rvalue parameters
     // https://stackoverflow.com/questions/12088537/workaround-for-gmock-to-support-rvalue-reference
-    void UpdateInodeWithOutNlinkAsync(const Inode& inode,
+    void UpdateInodeWithOutNlinkAsync(uint32_t fsId,
+                                      uint64_t inodeId,
+                                      const InodeAttr& attr,
                                       MetaServerClientDone* done,
-                                      InodeOpenStatusChange change,
                                       DataIndices&& indices) override {
-        return UpdateInodeWithOutNlinkAsync_rvr(inode, done, change,
+        return UpdateInodeWithOutNlinkAsync_rvr(fsId, inodeId, attr, done,
                                                 std::move(indices));
     }
 
-    MOCK_METHOD4(UpdateInodeWithOutNlinkAsync_rvr,
-                 void(const Inode& inode,
+    MOCK_METHOD5(UpdateInodeWithOutNlinkAsync_rvr,
+                 void(uint32_t,
+                      uint64_t,
+                      const InodeAttr&,
                       MetaServerClientDone* done,
-                      InodeOpenStatusChange statusChange,
                       DataIndices));
 
     MOCK_METHOD2(UpdateXattrAsync, void(const Inode &inode,

curvefs/test/client/rpcclient/BUILD

@@ -30,6 +30,7 @@ cc_test(
        "//curvefs/proto:curvefs_topology_cc_proto",
        "//curvefs/src/common:curvefs_common",
        "//curvefs/test/utils:curvefs_test_utils",
+       "//curvefs/test/client:client_test_utils"
    ],
    visibility = ["//visibility:public"],
 )

curvefs/test/client/rpcclient/mds_client_test.cpp

@@ -440,22 +440,40 @@ TEST_F(MdsClientImplTest, CommitTxWithLock) {
 
 TEST_F(MdsClientImplTest, GetLatestTxId) {
     std::vector<PartitionTxId> txIds;
+    uint32_t fsId = 1;
 
     // CASE 1: GetLatestTxId success
-    GetLatestTxIdResponse response;
-    response.set_statuscode(FSStatusCode::OK);
     EXPECT_CALL(mockmdsbasecli_, GetLatestTxId(_, _, _, _))
-        .WillOnce(SetArgPointee<1>(response));
+        .WillOnce(
+            Invoke([&](const GetLatestTxIdRequest& request,
+                       GetLatestTxIdResponse* response,
+                       brpc::Controller *cntl,
+                       brpc::Channel *channel) {
+                if (request.fsid() != fsId) {
+                    response->set_statuscode(FSStatusCode::PARAM_ERROR);
+                } else {
+                    response->set_statuscode(FSStatusCode::OK);
+                }
+            }));
 
-    auto rc = mdsclient_.GetLatestTxId(&txIds);
+    auto rc = mdsclient_.GetLatestTxId(fsId, &txIds);
     ASSERT_EQ(rc, FSStatusCode::OK);
 
     // CASE 2: GetLatestTxId fail
-    response.set_statuscode(FSStatusCode::UNKNOWN_ERROR);
     EXPECT_CALL(mockmdsbasecli_, GetLatestTxId(_, _, _, _))
-        .WillOnce(SetArgPointee<1>(response));
+        .WillOnce(
+            Invoke([&](const GetLatestTxIdRequest& request,
+                       GetLatestTxIdResponse* response,
+                       brpc::Controller *cntl,
+                       brpc::Channel *channel) {
+                if (request.fsid() != fsId) {
+                    response->set_statuscode(FSStatusCode::PARAM_ERROR);
+                } else {
+                    response->set_statuscode(FSStatusCode::UNKNOWN_ERROR);
+                }
+            }));
 
-    rc = mdsclient_.GetLatestTxId(&txIds);
+    rc = mdsclient_.GetLatestTxId(fsId, &txIds);
     ASSERT_EQ(rc, FSStatusCode::UNKNOWN_ERROR);
 
     // CASE 3: RPC error, retry until success
@@ -467,14 +485,16 @@ TEST_F(MdsClientImplTest, GetLatestTxId) {
                        GetLatestTxIdResponse* response,
                        brpc::Controller *cntl,
                        brpc::Channel *channel) {
-                if (++count <= 5) {
+                if (request.fsid() != fsId) {
+                    response->set_statuscode(FSStatusCode::PARAM_ERROR);
+                } else if (++count <= 5) {
                     cntl->SetFailed(112, "Not connected to");
                 } else {
                     response->set_statuscode(FSStatusCode::OK);
                 }
             }));
 
-    rc = mdsclient_.GetLatestTxId(&txIds);
+    rc = mdsclient_.GetLatestTxId(fsId, &txIds);
     ASSERT_EQ(rc, FSStatusCode::OK);
 }
 

curvefs/test/client/rpcclient/metaserver_client_test.cpp

@@ -43,6 +43,7 @@
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/common/define.h"
 #include "curvefs/src/common/process.h"
+#include "curvefs/test/client/utils.h"
 
 namespace curvefs {
 namespace client {
@@ -743,7 +744,8 @@ TEST_F(MetaServerClientImplTest, test_UpdateInodeAttr) {
         .WillRepeatedly(DoAll(SetArgPointee<2>(target_),
                               SetArgPointee<3>(applyIndex), Return(true)));
 
-    MetaStatusCode status = metaserverCli_.UpdateInodeAttr(inode);
+    MetaStatusCode status = metaserverCli_.UpdateInodeAttr(
+        inode.fsid(), inode.inodeid(), ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 
     // test1: update inode ok
@@ -757,7 +759,8 @@ TEST_F(MetaServerClientImplTest, test_UpdateInodeAttr) {
         .WillRepeatedly(DoAll(SetArgPointee<2>(target_),
                               SetArgPointee<3>(applyIndex), Return(true)));
     EXPECT_CALL(*mockMetacache_.get(), UpdateApplyIndex(_, _));
-    status = metaserverCli_.UpdateInodeAttr(inode);
+    status = metaserverCli_.UpdateInodeAttr(
+        inode.fsid(), inode.inodeid(), ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::OK, status);
 
     // test2: update inode with overload
@@ -766,7 +769,8 @@ TEST_F(MetaServerClientImplTest, test_UpdateInodeAttr) {
         .WillRepeatedly(DoAll(
             SetArgPointee<2>(response),
             Invoke(SetRpcService<UpdateInodeRequest, UpdateInodeResponse>)));
-    status = metaserverCli_.UpdateInodeAttr(inode);
+    status = metaserverCli_.UpdateInodeAttr(inode.fsid(), inode.inodeid(),
+                                            ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::OVERLOAD, status);
 
     // test3: response has no applyindex
@@ -777,13 +781,15 @@ TEST_F(MetaServerClientImplTest, test_UpdateInodeAttr) {
             SetArgPointee<2>(response),
             Invoke(SetRpcService<UpdateInodeRequest, UpdateInodeResponse>)));
 
-    status = metaserverCli_.UpdateInodeAttr(inode);
+    status = metaserverCli_.UpdateInodeAttr(inode.fsid(), inode.inodeid(),
+                                            ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 
     // test4: get target always fail
     EXPECT_CALL(*mockMetacache_.get(), GetTarget(_, _, _, _, _))
         .WillRepeatedly(Return(false));
-    status = metaserverCli_.UpdateInodeAttr(inode);
+    status = metaserverCli_.UpdateInodeAttr(inode.fsid(), inode.inodeid(),
+                                            ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 }
 

curvefs/test/client/rpcclient/mock_mds_client.h

@@ -63,8 +63,9 @@ class MockMdsClient : public MdsClient {
     MOCK_METHOD3(AllocS3ChunkId, FSStatusCode(uint32_t fsId, uint32_t idNum,
                                               uint64_t *chunkId));
 
-    MOCK_METHOD1(GetLatestTxId,
-                 FSStatusCode(std::vector<PartitionTxId>* txIds));
+    MOCK_METHOD2(GetLatestTxId,
+                 FSStatusCode(uint32_t fsId,
+                              std::vector<PartitionTxId>* txIds));
 
     MOCK_METHOD5(GetLatestTxIdWithLock,
                  FSStatusCode(uint32_t fsId,

curvefs/test/client/test_disk_cache_manager.cpp

@@ -272,11 +272,20 @@ TEST_F(TestDiskCacheManager, TrimStop) {
 }
 
 TEST_F(TestDiskCacheManager, TrimRun_1) {
+    S3ClientAdaptorOption option;
+    option.diskCacheOpt.cacheDir = "/tmp";
+    option.diskCacheOpt.trimCheckIntervalSec = 1;
+    EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(-1));
+    EXPECT_CALL(*wrapper, mkdir(_, _)).WillOnce(Return(-1));
+    EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
+        .WillOnce(Return(0));
+    diskCacheManager_->Init(client_, option);
+    diskCacheManager_->InitMetrics("test");
     EXPECT_CALL(*wrapper, statfs(NotNull(), NotNull()))
         .WillRepeatedly(Return(-1));
     int ret = diskCacheManager_->TrimRun();
     sleep(6);
-    diskCacheManager_->TrimStop();
+    diskCacheManager_->UmountDiskCache();
 }
 
 TEST_F(TestDiskCacheManager, TrimCache_2) {
@@ -298,15 +307,19 @@ TEST_F(TestDiskCacheManager, TrimCache_2) {
         .WillRepeatedly(Return(buf));
     EXPECT_CALL(*diskCacheRead_, GetCacheIoFullDir())
         .WillRepeatedly(Return(buf));
-    EXPECT_CALL(*wrapper, stat(NotNull(), NotNull()))
-        .Times(3)
-        .WillOnce(Return(0))
-        .WillOnce(Return(-1))
+    S3ClientAdaptorOption option;
+    option.diskCacheOpt.cacheDir = "/tmp";
+    option.diskCacheOpt.trimCheckIntervalSec = 1;
+    EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(-1));
+    EXPECT_CALL(*wrapper, mkdir(_, _)).WillOnce(Return(-1));
+    EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
         .WillOnce(Return(0));
+    diskCacheManager_->Init(client_, option);
+    diskCacheManager_->InitMetrics("test");
     diskCacheManager_->AddCache("test");
     int ret = diskCacheManager_->TrimRun();
     sleep(6);
-    diskCacheManager_->TrimStop();
+    diskCacheManager_->UmountDiskCache();
 }
 
 TEST_F(TestDiskCacheManager, TrimCache_4) {
@@ -331,10 +344,19 @@ TEST_F(TestDiskCacheManager, TrimCache_4) {
     EXPECT_CALL(*wrapper, stat(NotNull(), NotNull()))
         .WillRepeatedly(Return(-1));
     EXPECT_CALL(*wrapper, remove(_)).WillRepeatedly(Return(-1));
+    S3ClientAdaptorOption option;
+    option.diskCacheOpt.cacheDir = "/tmp";
+    option.diskCacheOpt.trimCheckIntervalSec = 1;
+    EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(-1));
+    EXPECT_CALL(*wrapper, mkdir(_, _)).WillOnce(Return(-1));
+    EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
+        .WillOnce(Return(0));
+    diskCacheManager_->Init(client_, option);
+    diskCacheManager_->InitMetrics("test");
     diskCacheManager_->AddCache("test");
     int ret = diskCacheManager_->TrimRun();
     sleep(6);
-    diskCacheManager_->TrimStop();
+    diskCacheManager_->UmountDiskCache();
 }
 
 TEST_F(TestDiskCacheManager, TrimCache_5) {
@@ -360,17 +382,26 @@ TEST_F(TestDiskCacheManager, TrimCache_5) {
     EXPECT_CALL(*wrapper, stat(NotNull(), NotNull()))
         .WillRepeatedly(Return(-1));
     EXPECT_CALL(*wrapper, remove(_)).WillRepeatedly(Return(0));
+    S3ClientAdaptorOption option;
+    option.diskCacheOpt.cacheDir = "/tmp";
+    option.diskCacheOpt.trimCheckIntervalSec = 1;
+    EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(-1));
+    EXPECT_CALL(*wrapper, mkdir(_, _)).WillOnce(Return(-1));
+    EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
+        .WillOnce(Return(0));
+    diskCacheManager_->Init(client_, option);
+    diskCacheManager_->InitMetrics("test");
     diskCacheManager_->AddCache("test");
     int ret = diskCacheManager_->TrimRun();
     sleep(6);
-    diskCacheManager_->TrimStop();
+    diskCacheManager_->UmountDiskCache();
 }
 
 TEST_F(TestDiskCacheManager, TrimCache_noexceed) {
-    S3ClientAdaptorOption option;
+      S3ClientAdaptorOption option;
     option.diskCacheOpt.maxFileNums = 5;
     option.diskCacheOpt.diskCacheType = (DiskCacheType)2;
-    option.diskCacheOpt.cacheDir = "/mnt/test_unit";
+    option.diskCacheOpt.cacheDir = "/tmp";
     option.diskCacheOpt.trimCheckIntervalSec = 1;
     option.diskCacheOpt.fullRatio = 0;
     option.diskCacheOpt.safeRatio = 0;
@@ -397,25 +428,26 @@ TEST_F(TestDiskCacheManager, TrimCache_noexceed) {
         .WillOnce(Return(0))
         .WillOnce(Return(-1))
         .WillOnce(Return(0));
-
+    EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
+        .WillOnce(Return(0));
     int ret = diskCacheManager_->TrimRun();
+    diskCacheManager_->InitMetrics("test");
     sleep(6);
-    diskCacheManager_->TrimStop();
+    diskCacheManager_->UmountDiskCache();
 }
 
 TEST_F(TestDiskCacheManager, TrimCache_exceed) {
     S3ClientAdaptorOption option;
     option.diskCacheOpt.maxFileNums = 5;
     option.diskCacheOpt.diskCacheType = (DiskCacheType)2;
-    option.diskCacheOpt.cacheDir = "/mnt/test_unit";
+    option.diskCacheOpt.cacheDir = "/tmp";
     option.diskCacheOpt.trimCheckIntervalSec = 1;
-    option.diskCacheOpt.fullRatio = 0;
-    option.diskCacheOpt.safeRatio = 5;
-    option.diskCacheOpt.maxUsableSpaceBytes = 100;
+    option.diskCacheOpt.fullRatio = 90;
+    option.diskCacheOpt.safeRatio = 70;
+    option.diskCacheOpt.maxUsableSpaceBytes =
+      std::numeric_limits<uint64_t>::max();
     option.diskCacheOpt.cmdTimeoutSec = 5;
     option.diskCacheOpt.asyncLoadPeriodMs = 10;
-    option.diskCacheOpt.maxFileNums =
-      std::numeric_limits<uint64_t>::max();
     diskCacheManager_->Init(client_, option);
 
     std::string buf = "test";
@@ -430,16 +462,23 @@ TEST_F(TestDiskCacheManager, TrimCache_exceed) {
     stat.f_bavail = 0;
     EXPECT_CALL(*wrapper, statfs(NotNull(), _))
         .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(-1)));
+    EXPECT_CALL(*wrapper, remove(_)).WillRepeatedly(Return(0));
 
     diskCacheManager_->AddCache("test00");
     diskCacheManager_->AddCache("test01");
     diskCacheManager_->AddCache("test02");
     diskCacheManager_->AddCache("test03");
     diskCacheManager_->AddCache("test04");
-    diskCacheManager_->AddCache("test05");
-    int ret = diskCacheManager_->TrimRun();
+    EXPECT_CALL(*wrapper, stat(NotNull(), NotNull()))
+        .Times(2)
+        .WillOnce(Return(-1))
+        .WillOnce(Return(0));
+    EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
+        .WillOnce(Return(0));
+    diskCacheManager_->TrimRun();
+    diskCacheManager_->InitMetrics("test");
     sleep(6);
-    diskCacheManager_->TrimStop();
+    diskCacheManager_->UmountDiskCache();
 }
 
 TEST_F(TestDiskCacheManager, WriteReadDirect) {

curvefs/test/client/test_disk_cache_manager_impl.cpp

@@ -96,9 +96,7 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
 
 TEST_F(TestDiskCacheManagerImpl, Init) {
     S3ClientAdaptorOption s3AdaptorOption;
-    s3AdaptorOption.diskCacheOpt.forceFlush = false;
-    s3AdaptorOption.diskCacheOpt.threads = 0;
-
+    s3AdaptorOption.diskCacheOpt.threads = 10;
     EXPECT_CALL(*diskCacheManager_, Init(_, _)).WillOnce(Return(-1));
     int ret = diskCacheManagerImpl_->Init(s3AdaptorOption);
     ASSERT_EQ(-1, ret);
@@ -129,6 +127,7 @@ TEST_F(TestDiskCacheManagerImpl, WriteClosure) {
     EXPECT_CALL(*diskCacheManager_, IsDiskCacheFull()).WillOnce(Return(false));
     EXPECT_CALL(*diskCacheWrite_, WriteDiskFile(_, _, _, _))
         .WillOnce(Return(0));
+    EXPECT_CALL(*diskCacheManager_, IsDiskUsedInited()).WillOnce(Return(true));
     EXPECT_CALL(*diskCacheWrite_, GetCacheIoFullDir()).WillOnce(Return(buf));
     EXPECT_CALL(*diskCacheRead_, GetCacheIoFullDir()).WillOnce(Return(buf));
     EXPECT_CALL(*diskCacheRead_, LinkWriteToRead(_, _, _)).WillOnce(Return(0));
@@ -141,8 +140,8 @@ TEST_F(TestDiskCacheManagerImpl, Write) {
     std::string fileName = "test";
     std::string buf = "test";
     int ret;
-    EXPECT_CALL(*client_, Upload(_, _, _)).WillOnce(Return(-1));
     EXPECT_CALL(*diskCacheManager_, IsDiskCacheFull()).WillOnce(Return(true));
+    EXPECT_CALL(*diskCacheManager_, IsDiskUsedInited()).WillOnce(Return(true));
     ret = diskCacheManagerImpl_->Write(fileName,
                                        const_cast<char *>(buf.c_str()), 10);
     ASSERT_EQ(-1, ret);
@@ -150,7 +149,7 @@ TEST_F(TestDiskCacheManagerImpl, Write) {
     EXPECT_CALL(*diskCacheManager_, IsDiskCacheFull()).WillOnce(Return(false));
     EXPECT_CALL(*diskCacheWrite_, WriteDiskFile(_, _, _, _))
         .WillOnce(Return(-1));
-    EXPECT_CALL(*client_, Upload(_, _, _)).WillOnce(Return(-1));
+    EXPECT_CALL(*diskCacheManager_, IsDiskUsedInited()).WillOnce(Return(true));
     ret = diskCacheManagerImpl_->Write(fileName,
                                        const_cast<char *>(buf.c_str()), 10);
     ASSERT_EQ(-1, ret);
@@ -161,7 +160,7 @@ TEST_F(TestDiskCacheManagerImpl, Write) {
     EXPECT_CALL(*diskCacheWrite_, GetCacheIoFullDir()).WillOnce(Return(buf));
     EXPECT_CALL(*diskCacheRead_, GetCacheIoFullDir()).WillOnce(Return(buf));
     EXPECT_CALL(*diskCacheRead_, LinkWriteToRead(_, _, _)).WillOnce(Return(-1));
-    EXPECT_CALL(*client_, Upload(_, _, _)).WillOnce(Return(-1));
+    EXPECT_CALL(*diskCacheManager_, IsDiskUsedInited()).WillOnce(Return(true));
     ret = diskCacheManagerImpl_->Write(fileName,
                                        const_cast<char *>(buf.c_str()), 10);
     ASSERT_EQ(-1, ret);
@@ -173,12 +172,7 @@ TEST_F(TestDiskCacheManagerImpl, Write) {
     EXPECT_CALL(*diskCacheRead_, GetCacheIoFullDir()).WillOnce(Return(buf));
     EXPECT_CALL(*diskCacheRead_, LinkWriteToRead(_, _, _)).WillOnce(Return(0));
     EXPECT_CALL(*diskCacheWrite_, AsyncUploadEnqueue(_)).WillOnce(Return());
-    ret = diskCacheManagerImpl_->Write(fileName,
-                                       const_cast<char *>(buf.c_str()), 10);
-    ASSERT_EQ(0, ret);
-
-    EXPECT_CALL(*diskCacheManager_, IsDiskCacheFull()).WillOnce(Return(true));
-    EXPECT_CALL(*client_, Upload(_, _, _)).WillOnce(Return(0));
+    EXPECT_CALL(*diskCacheManager_, IsDiskUsedInited()).WillOnce(Return(true));
     ret = diskCacheManagerImpl_->Write(fileName,
                                        const_cast<char *>(buf.c_str()), 10);
     ASSERT_EQ(0, ret);
@@ -225,11 +219,13 @@ TEST_F(TestDiskCacheManagerImpl, IsCached) {
 TEST_F(TestDiskCacheManagerImpl, UmountDiskCache) {
     EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
         .WillOnce(Return(-1));
+    diskCacheManagerImpl_->InitMetrics("test");
     int ret = diskCacheManagerImpl_->UmountDiskCache();
     ASSERT_EQ(0, ret);
 
     EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
         .WillOnce(Return(0));
+    diskCacheManagerImpl_->InitMetrics("test");
     ret = diskCacheManagerImpl_->UmountDiskCache();
     ASSERT_EQ(0, ret);
 }
@@ -237,12 +233,13 @@ TEST_F(TestDiskCacheManagerImpl, UmountDiskCache) {
 TEST_F(TestDiskCacheManagerImpl, WriteReadDirect) {
     std::string fileName = "test";
     std::string buf = "test";
-    EXPECT_CALL(*diskCacheManager_, IsDiskCacheFull()).WillOnce(Return(true));
+    EXPECT_CALL(*diskCacheManager_, IsDiskUsedInited()).WillOnce(Return(false));
     int ret = diskCacheManagerImpl_->WriteReadDirect(
         fileName, const_cast<char *>(buf.c_str()), 10);
     ASSERT_EQ(-1, ret);
 
     EXPECT_CALL(*diskCacheManager_, IsDiskCacheFull()).WillOnce(Return(false));
+    EXPECT_CALL(*diskCacheManager_, IsDiskUsedInited()).WillOnce(Return(true));
     EXPECT_CALL(*diskCacheManager_, WriteReadDirect(_, _, _))
         .WillOnce(Return(-1));
     ret = diskCacheManagerImpl_->WriteReadDirect(
@@ -250,6 +247,7 @@ TEST_F(TestDiskCacheManagerImpl, WriteReadDirect) {
     ASSERT_EQ(-1, ret);
 
     EXPECT_CALL(*diskCacheManager_, IsDiskCacheFull()).WillOnce(Return(false));
+    EXPECT_CALL(*diskCacheManager_, IsDiskUsedInited()).WillOnce(Return(true));
     EXPECT_CALL(*diskCacheManager_, WriteReadDirect(_, _, _))
         .WillOnce(Return(0));
     ret = diskCacheManagerImpl_->WriteReadDirect(

curvefs/test/client/test_inodeWrapper.cpp

@@ -37,6 +37,7 @@
 #include "curvefs/src/client/volume/extent.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/test/client/mock_metaserver_client.h"
+#include "src/common/timeutility.h"
 
 using ::google::protobuf::util::MessageDifferencer;
 
@@ -57,15 +58,12 @@ using rpcclient::MockMetaServerClient;
 
 class TestInodeWrapper : public ::testing::Test {
  protected:
-    TestInodeWrapper() {}
-    ~TestInodeWrapper() {}
-
-    virtual void SetUp() {
+    void SetUp() override {
         metaClient_ = std::make_shared<MockMetaServerClient>();
         inodeWrapper_ = std::make_shared<InodeWrapper>(Inode(), metaClient_);
     }
 
-    virtual void TearDown() {
+    void TearDown() override {
         metaClient_ = nullptr;
         inodeWrapper_ = nullptr;
     }
@@ -144,7 +142,7 @@ TEST_F(TestInodeWrapper, testSyncSuccess) {
     uint64_t chunkIndex1 = 1;
     inodeWrapper_->AppendS3ChunkInfo(chunkIndex1, info1);
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .WillOnce(Return(MetaStatusCode::OK));
 
     CURVEFS_ERROR ret = inodeWrapper_->Sync();
@@ -166,7 +164,7 @@ TEST_F(TestInodeWrapper, testSyncFailed) {
     uint64_t chunkIndex1 = 1;
     inodeWrapper_->AppendS3ChunkInfo(chunkIndex1, info1);
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .WillOnce(Return(MetaStatusCode::NOT_FOUND));
 
     CURVEFS_ERROR ret = inodeWrapper_->Sync();
@@ -178,7 +176,7 @@ TEST_F(TestInodeWrapper, TestFlushVolumeExtent_NoNeedFlush) {
 
     inodeWrapper_->SetType(FsFileType::TYPE_FILE);
     inodeWrapper_->ClearDirty();
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .Times(0);
     EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(_, _, _, _))
         .Times(0);
@@ -197,7 +195,7 @@ TEST_F(TestInodeWrapper, TestFlushVolumeExtent) {
     pext.pOffset = 0;
     pext.UnWritten = true;
     extentCache->Merge(0, pext);
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .Times(0);
     EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(_, _, _, _))
         .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentList&,
@@ -216,7 +214,7 @@ TEST_F(TestInodeWrapper, TestRefreshNlink) {
     EXPECT_CALL(*metaClient_, GetInodeAttr(_, _, _))
         .WillOnce(DoAll(SetArgPointee<2>(attr), Return(MetaStatusCode::OK)));
     inodeWrapper_->RefreshNlink();
-    Inode inode = inodeWrapper_->GetInodeUnlocked();
+    Inode inode = inodeWrapper_->GetInode();
     ASSERT_EQ(nlink, inode.nlink());
 }
 
@@ -262,9 +260,10 @@ struct FakeCallback : public MetaServerClientDone {
 };
 
 struct FakeUpdateInodeWithOutNlinkAsync {
-    void operator()(const Inode& inode,
+    void operator()(uint32_t /*fsid*/,
+                    uint64_t /*inodeId*/,
+                    const InodeAttr& attr,
                     MetaServerClientDone* done,
-                    InodeOpenStatusChange statusChange,
                     DataIndices indices) const {
         std::thread th{[done]() {
             std::this_thread::sleep_for(std::chrono::seconds(1));
@@ -288,7 +287,7 @@ TEST_F(TestInodeWrapper, TestAsyncInode) {
             }
 
             EXPECT_CALL(*metaClient_,
-                        UpdateInodeWithOutNlinkAsync_rvr(_, _, _, _))
+                        UpdateInodeWithOutNlinkAsync_rvr(_, _, _, _, _))
                 .Times(dirty ? 1 : 0)
                 .WillRepeatedly(Invoke(FakeUpdateInodeWithOutNlinkAsync{}));
 
@@ -300,5 +299,47 @@ TEST_F(TestInodeWrapper, TestAsyncInode) {
     }
 }
 
+TEST_F(TestInodeWrapper, TestUpdateInodeAttrIncrementally) {
+    Inode inode;
+    inode.set_type(FsFileType::TYPE_S3);
+    inode.set_length(0);
+    inode.set_atime(0);
+    inode.set_atime_ns(0);
+    inode.set_ctime(0);
+    inode.set_ctime_ns(0);
+    inode.set_mtime(0);
+    inode.set_mtime_ns(0);
+
+    InodeWrapper wrapper(std::move(inode), metaClient_);
+
+    {
+        auto lock = wrapper.GetUniqueLock();
+        wrapper.UpdateTimestampLocked(kAccessTime);
+    }
+
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
+        .WillOnce(Invoke([](uint32_t /*fsId*/, uint64_t /*inodeId*/,
+                            const InodeAttr& attr,
+                            S3ChunkInfoMap* /*s3info*/, bool /*internal*/
+                         ) {
+            EXPECT_FALSE(attr.has_length());
+            return MetaStatusCode::OK;
+        }));
+
+    ASSERT_EQ(CURVEFS_ERROR::OK, wrapper.Sync());
+
+    ASSERT_FALSE(wrapper.dirty_);
+    ASSERT_FALSE(wrapper.dirtyAttr_.has_atime());
+    ASSERT_FALSE(wrapper.dirtyAttr_.has_atime_ns());
+}
+
+TEST_F(TestInodeWrapper, TestSetXattr) {
+    inodeWrapper_->SetXattrLocked("name", "value");
+    XAttr xattr = inodeWrapper_->GetXattr();
+    ASSERT_TRUE(xattr.xattrinfos().find("name") != xattr.xattrinfos().end());
+    ASSERT_EQ((*xattr.mutable_xattrinfos())["name"], "value");
+    ASSERT_TRUE(inodeWrapper_->IsDirty());
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/test/client/test_inode_cache_manager.cpp

@@ -20,12 +20,14 @@
  * Author: xuchaojie
  */
 
+#include <gmock/gmock-spec-builders.h>
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
 #include <chrono>
 #include <cstdint>
 #include <thread>
 
+#include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/test/client/mock_metaserver_client.h"
 #include "curvefs/src/client/inode_cache_manager.h"
@@ -136,7 +138,7 @@ TEST_F(TestInodeCacheManager, GetInode) {
     ret = iCacheManager_->GetInode(inodeId, inodeWrapper);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 
-    Inode out = inodeWrapper->GetInodeUnlocked();
+    Inode out = inodeWrapper->GetInode();
     ASSERT_EQ(inodeId, out.inodeid());
     ASSERT_EQ(fsId_, out.fsid());
     ASSERT_EQ(fileLength, out.length());
@@ -144,83 +146,32 @@ TEST_F(TestInodeCacheManager, GetInode) {
     ret = iCacheManager_->GetInode(inodeId, inodeWrapper);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 
-    out = inodeWrapper->GetInodeUnlocked();
+    out = inodeWrapper->GetInode();
     ASSERT_EQ(inodeId, out.inodeid());
     ASSERT_EQ(fsId_, out.fsid());
     ASSERT_EQ(fileLength, out.length());
-}
 
-TEST_F(TestInodeCacheManager, RefreshInode) {
-    uint64_t inodeId = 100;
-    uint64_t fileLength = 100;
-
-    Inode inode;
-    inode.set_inodeid(inodeId);
-    inode.set_fsid(fsId_);
-    inode.set_length(fileLength);
-    inode.set_type(FsFileType::TYPE_S3);
-    auto s3ChunkInfoMap = inode.mutable_s3chunkinfomap();
-    S3ChunkInfoList *s3ChunkInfoList = new S3ChunkInfoList();
-    S3ChunkInfo *s3ChunkInfo = s3ChunkInfoList->add_s3chunks();
-    s3ChunkInfo->set_chunkid(1);
-    s3ChunkInfo->set_compaction(1);
-    s3ChunkInfo->set_offset(0);
-    s3ChunkInfo->set_len(1024);
-    s3ChunkInfo->set_size(65536);
-    s3ChunkInfo->set_zero(true);
-    s3ChunkInfoMap->insert({1, *s3ChunkInfoList});
-
-    // cache miss, get s3-inode from metaserver failed
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(Return(MetaStatusCode::NOT_FOUND));
-    ASSERT_EQ(CURVEFS_ERROR::NOTEXIST, iCacheManager_->RefreshInode(inodeId));
-
-    // cache miss, get s3-inode from metaserver ok, do not need streaming
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(DoAll(SetArgPointee<2>(inode), SetArgPointee<3>(false),
-                        Return(MetaStatusCode::OK)));
-    ASSERT_EQ(CURVEFS_ERROR::OK, iCacheManager_->RefreshInode(inodeId));
-
-    // cache miss, get s3-inode from metaserver, need streaming
-    uint64_t inodeId2 = 200;
-    Inode inode2 = inode;
-    inode2.set_inodeid(inodeId2);
+    // enable cto and not opened and not dirty
+    curvefs::client::common::FLAGS_enableCto = true;
     EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId2, _, _))
         .WillOnce(DoAll(SetArgPointee<2>(inode2), SetArgPointee<3>(true),
                         Return(MetaStatusCode::OK)));
     EXPECT_CALL(*metaClient_,
                 GetOrModifyS3ChunkInfo(fsId_, inodeId2, _, true, _, _))
         .WillOnce(Return(MetaStatusCode::OK));
-    ASSERT_EQ(CURVEFS_ERROR::OK, iCacheManager_->RefreshInode(inodeId2));
-
-    // cache hit, refresh s3-inode from metaserver, do not need streaming
-    Inode inodenew = inode;
-    inodenew.set_length(fileLength * 3);
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(DoAll(SetArgPointee<2>(inodenew), SetArgPointee<3>(false),
-                        Return(MetaStatusCode::OK)));
-    EXPECT_CALL(*metaClient_,
-                GetOrModifyS3ChunkInfo(fsId_, inodeId, _, true, _, _))
-        .WillOnce(Return(MetaStatusCode::OK));
-    ASSERT_EQ(CURVEFS_ERROR::OK, iCacheManager_->RefreshInode(inodeId));
-    std::shared_ptr<InodeWrapper> inodeWrapper;
     ASSERT_EQ(CURVEFS_ERROR::OK,
-              iCacheManager_->GetInode(inodeId, inodeWrapper));
-    ASSERT_EQ(inodenew.length(), inodeWrapper->GetLength());
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
 
-    // cache miss, get file-inode from metaserver
-    Inode inodefile;
-    uint64_t inodefileid = 300;
-    inodefile.set_inodeid(inodefileid);
-    inodefile.set_fsid(fsId_);
-    inodefile.set_type(FsFileType::TYPE_FILE);
-    inodefile.set_length(1);
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodefileid, _, _))
-        .WillOnce(DoAll(SetArgPointee<2>(inodefile), SetArgPointee<3>(false),
-                        Return(MetaStatusCode::OK)));
-    EXPECT_CALL(*metaClient_, GetVolumeExtent(fsId_, inodefileid, _, _))
-        .WillOnce(Return(MetaStatusCode::OK));
-    ASSERT_EQ(CURVEFS_ERROR::OK, iCacheManager_->RefreshInode(inodefileid));
+    // enable cto and opened will hit cache
+    iCacheManager_->AddOpenedInode(inodeId2);
+    ASSERT_EQ(CURVEFS_ERROR::OK,
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
+
+    // enable cto and not opened and inode dirty will hit cache
+    iCacheManager_->RemoveOpenedInode(inodeId2);
+    inodeWrapper->MarkDirty();
+    ASSERT_EQ(CURVEFS_ERROR::OK,
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
 }
 
 TEST_F(TestInodeCacheManager, GetInodeAttr) {
@@ -271,6 +222,35 @@ TEST_F(TestInodeCacheManager, GetInodeAttr) {
     ASSERT_EQ(inodeId + 1, out.inodeid());
     ASSERT_EQ(fsId_ + 1, out.fsid());
     ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
+
+    // enable cto will get from metaserver
+    curvefs::client::common::FLAGS_enableCto = true;
+    EXPECT_CALL(*metaClient_, BatchGetInodeAttr(fsId_, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(attrs), Return(MetaStatusCode::OK)));
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(inodeId, out.inodeid());
+    ASSERT_EQ(fsId_, out.fsid());
+    ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
+
+    // set this inode open
+    curvefs::client::common::FLAGS_enableCto = false;
+    iCacheManager_->AddOpenedInode(inodeId + 1);
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(inodeId + 1, out.inodeid());
+    ASSERT_EQ(fsId_ + 1, out.fsid());
+    ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
+
+    // set this inode dirty
+    curvefs::client::common::FLAGS_enableCto = false;
+    iCacheManager_->RemoveOpenedInode(inodeId + 1);
+    inodeWrapper->MarkDirty();
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(inodeId + 1, out.inodeid());
+    ASSERT_EQ(fsId_ + 1, out.fsid());
+    ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
 }
 
 TEST_F(TestInodeCacheManager, CreateAndGetInode) {
@@ -294,7 +274,7 @@ TEST_F(TestInodeCacheManager, CreateAndGetInode) {
     ASSERT_EQ(CURVEFS_ERROR::UNKNOWN, ret);
 
     ret = iCacheManager_->CreateInode(param, inodeWrapper);
-    Inode out = inodeWrapper->GetInodeUnlocked();
+    Inode out = inodeWrapper->GetInode();
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
     ASSERT_EQ(inodeId, out.inodeid());
     ASSERT_EQ(fsId_, out.fsid());
@@ -303,7 +283,7 @@ TEST_F(TestInodeCacheManager, CreateAndGetInode) {
     ret = iCacheManager_->GetInode(inodeId, inodeWrapper);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 
-    out = inodeWrapper->GetInodeUnlocked();
+    out = inodeWrapper->GetInode();
     ASSERT_EQ(inodeId, out.inodeid());
     ASSERT_EQ(fsId_, out.fsid());
     ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
@@ -343,13 +323,15 @@ TEST_F(TestInodeCacheManager, ShipToFlushAndFlushAll) {
 
     iCacheManager_->ShipToFlush(inodeWrapper);
 
-    EXPECT_CALL(*metaClient_, UpdateInodeWithOutNlinkAsync_rvr(_, _, _, _))
-        .WillOnce(Invoke([](const Inode &inode, MetaServerClientDone *done,
-                            InodeOpenStatusChange statusChange,
-                            DataIndices /*indices*/) {
-            done->SetMetaStatusCode(MetaStatusCode::OK);
-            done->Run();
-        }));
+    EXPECT_CALL(*metaClient_,
+                UpdateInodeWithOutNlinkAsync_rvr(_, _, _, _, _))
+        .WillOnce(Invoke(
+            [](uint32_t /*fsId*/, uint64_t /*inodeId*/,
+               const InodeAttr& /*attr*/, MetaServerClientDone* done,
+               DataIndices /*indices*/) {
+                done->SetMetaStatusCode(MetaStatusCode::OK);
+                done->Run();
+            }));
 
     iCacheManager_->FlushAll();
 }
@@ -391,6 +373,58 @@ TEST_F(TestInodeCacheManager, BatchGetInodeAttr) {
     ASSERT_EQ(getAttrs.begin()->length(), fileLength);
 }
 
+TEST_F(TestInodeCacheManager, BatchGetInodeAttrAsync) {
+    uint64_t parentId = 1;
+    uint64_t inodeId1 = 100;
+    uint64_t inodeId2 = 200;
+
+    // in
+    std::set<uint64_t> inodeIds;
+    inodeIds.emplace(inodeId1);
+    inodeIds.emplace(inodeId2);
+
+    // out
+    Inode inode;
+    inode.set_inodeid(inodeId1);
+
+    std::vector<std::vector<uint64_t>> inodeGroups;
+    inodeGroups.emplace_back(std::vector<uint64_t>{inodeId2});
+
+    std::map<uint64_t, InodeAttr> attrs;
+
+    RepeatedPtrField<InodeAttr> inodeAttrs;
+    inodeAttrs.Add()->set_inodeid(inodeId2);
+
+    // fill icache
+    EXPECT_CALL(*metaClient_, GetInode(_, inodeId1, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(inode),
+                Return(MetaStatusCode::OK)));
+    std::shared_ptr<InodeWrapper> wrapper;
+    iCacheManager_->GetInode(inodeId1, wrapper);
+
+    EXPECT_CALL(*metaClient_, SplitRequestInodes(_, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(inodeGroups),
+                Return(true)));
+    EXPECT_CALL(*metaClient_, BatchGetInodeAttrAsync(_, _, _))
+        .WillOnce(Invoke([inodeAttrs](uint32_t fsId,
+            const std::vector<uint64_t> &inodeIds,
+            MetaServerClientDone *done) {
+                done->SetMetaStatusCode(MetaStatusCode::OK);
+                static_cast<BatchGetInodeAttrDone *>(done)
+                    ->SetInodeAttrs(inodeAttrs);
+                done->Run();
+                return MetaStatusCode::OK;
+            }));
+
+    CURVEFS_ERROR ret = iCacheManager_->BatchGetInodeAttrAsync(parentId,
+        &inodeIds, &attrs);
+
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(attrs.size(), 2);
+    ASSERT_TRUE(attrs.find(inodeId1) != attrs.end());
+    ASSERT_TRUE(attrs.find(inodeId2) != attrs.end());
+}
+
 TEST_F(TestInodeCacheManager, BatchGetXAttr) {
     uint64_t inodeId1 = 100;
     uint64_t inodeId2 = 200;
@@ -456,10 +490,11 @@ TEST_F(TestInodeCacheManager, TestFlushInodeBackground) {
         inodeMap.emplace(inodeId + i, inodeWrapper);
     }
 
-    EXPECT_CALL(*metaClient_, UpdateInodeWithOutNlinkAsync_rvr(_, _, _, _))
+    EXPECT_CALL(*metaClient_,
+                UpdateInodeWithOutNlinkAsync_rvr(_, _, _, _, _))
         .WillRepeatedly(
-            Invoke([](const Inode& inode, MetaServerClientDone* done,
-                      InodeOpenStatusChange statusChange,
+            Invoke([](uint32_t /*fsId*/, uint64_t /*inodeId*/,
+                      const InodeAttr& /*attr*/, MetaServerClientDone* done,
                       DataIndices /*dataIndices*/) {
                 // run closure in a separate thread
                 std::thread th{[done]() {

curvefs/test/client/utils.cpp

@@ -0,0 +1,71 @@
+/*
+ *  Copyright (c) 2022 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#include "curvefs/test/client/utils.h"
+
+#include "curvefs/proto/metaserver.pb.h"
+
+namespace curvefs {
+namespace client {
+
+InodeAttr ToInodeAttr(const Inode& inode) {
+    InodeAttr attr;
+
+    attr.set_inodeid(inode.inodeid());
+    attr.set_fsid(inode.fsid());
+    attr.set_length(inode.length());
+    attr.set_ctime(inode.ctime());
+    attr.set_ctime_ns(inode.ctime_ns());
+    attr.set_mtime(inode.mtime());
+    attr.set_mtime_ns(inode.mtime_ns());
+    attr.set_atime(inode.atime());
+    attr.set_atime_ns(inode.atime_ns());
+
+    attr.set_uid(inode.uid());
+    attr.set_gid(inode.gid());
+    attr.set_mode(inode.mode());
+    attr.set_nlink(inode.nlink());
+    attr.set_type(inode.type());
+
+    if (inode.has_symlink()) {
+        attr.set_symlink(inode.symlink());
+    }
+
+    if (inode.has_rdev()) {
+        attr.set_rdev(inode.rdev());
+    }
+
+    if (inode.has_dtime()) {
+        attr.set_dtime(inode.dtime());
+    }
+
+    if (inode.has_openmpcount()) {
+        attr.set_openmpcount(inode.openmpcount());
+    }
+
+    if (!inode.xattr().empty()) {
+        *attr.mutable_xattr() = inode.xattr();
+    }
+
+    if (!inode.parent().empty()) {
+        *attr.mutable_parent() = inode.parent();
+    }
+
+    return attr;
+}
+
+}  // namespace client
+}  // namespace curvefs

curvefs/test/client/utils.h

@@ -0,0 +1,33 @@
+/*
+ *  Copyright (c) 2022 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+#ifndef CURVEFS_TEST_CLIENT_UTILS_H_
+#define CURVEFS_TEST_CLIENT_UTILS_H_
+
+#include "curvefs/proto/metaserver.pb.h"
+
+namespace curvefs {
+namespace client {
+
+using ::curvefs::metaserver::InodeAttr;
+using ::curvefs::metaserver::Inode;
+
+InodeAttr ToInodeAttr(const Inode& inode);
+
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_TEST_CLIENT_UTILS_H_

curvefs/test/client/volume/default_volume_storage_test.cpp

@@ -53,7 +53,7 @@ class DefaultVolumeStorageTest : public ::testing::Test {
           metaServerCli_(std::make_shared<MockMetaServerClient>()) {}
 
     void SetUp() override {
-        ON_CALL(*metaServerCli_, UpdateInodeAttrWithOutNlink(_, _, _, _))
+        ON_CALL(*metaServerCli_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
             .WillByDefault(Return(MetaStatusCode::OK));
     }
 
@@ -340,8 +340,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) {
 
     ASSERT_EQ(CURVEFS_ERROR::OK, storage_.Write(ino, offset, len, data.get()));
 
-    auto internal = inodeWrapper->GetInodeUnlocked();
-    ASSERT_EQ(offset + len, internal.length());
+    ASSERT_EQ(offset + len, inodeWrapper->GetInode().length());
 }
 
 }  // namespace client

curvefs/test/mds/fs_manager_test.cpp

@@ -71,6 +71,7 @@ using ::curvefs::mds::topology::TopologyImpl;
 using ::curvefs::mds::topology::CreatePartitionRequest;
 using ::curvefs::mds::topology::CreatePartitionResponse;
 using ::curvefs::mds::topology::TopoStatusCode;
+using ::curvefs::mds::topology::FsIdType;
 using ::curvefs::metaserver::copyset::MockCliService2;
 using ::curvefs::metaserver::copyset::GetLeaderResponse2;
 using ::curve::common::MockS3Adapter;
@@ -735,5 +736,41 @@ TEST_F(FSManagerTest, test_refreshSession) {
     }
 }
 
+TEST_F(FSManagerTest, GetLatestTxId_ParamFsId) {
+    // CASE 1: GetLatestTxId without fsid param
+    {
+        GetLatestTxIdRequest request;
+        GetLatestTxIdResponse response;
+        fsManager_->GetLatestTxId(&request, &response);
+        ASSERT_EQ(response.statuscode(), FSStatusCode::PARAM_ERROR);
+    }
+
+    // CASE 2: GetLatestTxId with fsid
+    {
+        GetLatestTxIdRequest request;
+        GetLatestTxIdResponse response;
+        request.set_fsid(1);
+        EXPECT_CALL(*topoManager_, ListPartitionOfFs(_, _))
+            .WillOnce(Invoke([&](FsIdType fsId,
+                                 std::list<PartitionInfo>* list) {
+                if (fsId != 1) {
+                    return;
+                }
+                PartitionInfo partition;
+                partition.set_fsid(0);
+                partition.set_poolid(0);
+                partition.set_copysetid(0);
+                partition.set_partitionid(0);
+                partition.set_start(0);
+                partition.set_end(0);
+                partition.set_txid(0);
+                list->push_back(partition);
+            }));
+        fsManager_->GetLatestTxId(&request, &response);
+        ASSERT_EQ(response.statuscode(), FSStatusCode::OK);
+        ASSERT_EQ(response.txids_size(), 1);
+    }
+}
+
 }  // namespace mds
 }  // namespace curvefs

curvefs/test/mds/topology/test_topology_metric.cpp

@@ -123,6 +123,14 @@ class TestTopologyMetric : public ::testing::Test {
             << "should have PrepareAddPool()";
     }
 
+    void UpdateCopysetMembers(CopySetIdType copysetId, PoolIdType poolId,
+                           const std::set<MetaServerIdType> &members) {
+        CopySetInfo cs(poolId, copysetId);
+        cs.SetCopySetMembers(members);
+        TopoStatusCode ret = topology_->UpdateCopySetTopo(cs);
+        ASSERT_EQ(TopoStatusCode::TOPO_OK, ret);
+    }
+
     void PrepareAddPartition(PartitionIdType partitionId,
                              CopySetIdType copysetId, PoolIdType poolId) {
         Partition partition;
@@ -208,6 +216,42 @@ TEST_F(TestTopologyMetric, TestUpdateTopologyMetricsOnePool) {
     ASSERT_EQ(30, gPoolMetrics[poolId]->inodeNum.get_value());
     ASSERT_EQ(300, gPoolMetrics[poolId]->dentryNum.get_value());
     ASSERT_EQ(3, gPoolMetrics[poolId]->partitionNum.get_value());
+
+    // Simulate metaserver offline and migrate data to another metaserver
+    PrepareAddMetaServer(0x44, "host4", "token4", 0x33, "127.0.0.1", 8890);
+    std::set<MetaServerIdType> replicas2;
+    replicas2.insert(0x41);
+    replicas2.insert(0x42);
+    replicas2.insert(0x44);
+    UpdateCopysetMembers(copysetId, poolId, replicas2);
+
+    testObj_->UpdateTopologyMetrics();
+
+    ASSERT_EQ(0, gMetaServerMetrics[0x43]->scatterWidth.get_value());
+    ASSERT_EQ(0, gMetaServerMetrics[0x43]->copysetNum.get_value());
+    ASSERT_EQ(0, gMetaServerMetrics[0x43]->leaderNum.get_value());
+    ASSERT_EQ(100 * 1024, gMetaServerMetrics[0x43]->diskThreshold.get_value());
+    ASSERT_EQ(10 * 1024, gMetaServerMetrics[0x43]->diskUsed.get_value());
+    ASSERT_EQ(20 * 1024, gMetaServerMetrics[0x43]->memoryUsed.get_value());
+    ASSERT_EQ(0, gMetaServerMetrics[0x43]->partitionNum.get_value());
+
+    ASSERT_EQ(2, gMetaServerMetrics[0x44]->scatterWidth.get_value());
+    ASSERT_EQ(1, gMetaServerMetrics[0x44]->copysetNum.get_value());
+    ASSERT_EQ(0, gMetaServerMetrics[0x44]->leaderNum.get_value());
+    ASSERT_EQ(100 * 1024, gMetaServerMetrics[0x44]->diskThreshold.get_value());
+    ASSERT_EQ(10 * 1024, gMetaServerMetrics[0x44]->diskUsed.get_value());
+    ASSERT_EQ(20 * 1024, gMetaServerMetrics[0x44]->memoryUsed.get_value());
+    ASSERT_EQ(3, gMetaServerMetrics[0x44]->partitionNum.get_value());
+
+
+    ASSERT_EQ(1, gPoolMetrics.size());
+    ASSERT_EQ(4, gPoolMetrics[poolId]->metaServerNum.get_value());
+    ASSERT_EQ(1, gPoolMetrics[poolId]->copysetNum.get_value());
+    ASSERT_EQ(100 * 1024 * 4, gPoolMetrics[poolId]->diskThreshold.get_value());
+    ASSERT_EQ(10 * 1024 * 4, gPoolMetrics[poolId]->diskUsed.get_value());
+    ASSERT_EQ(30, gPoolMetrics[poolId]->inodeNum.get_value());
+    ASSERT_EQ(300, gPoolMetrics[poolId]->dentryNum.get_value());
+    ASSERT_EQ(3, gPoolMetrics[poolId]->partitionNum.get_value());
 }
 
 }  // namespace topology

curvefs/test/metaserver/BUILD

@@ -26,7 +26,6 @@ cc_test(
             "mock_metaserver_s3.h",
             "metaserver_s3_adaptor_test.h",
             "metaserver_s3_adaptor_test.cpp",
-            "mock_metaserver_s3_adaptor.h",
             "metaserver_s3_test.cpp",
             "mock_s3compact_inode.h",
             "s3compact_test.cpp",
@@ -45,6 +44,7 @@ cc_test(
             "//curvefs/src/metaserver:metaserver_s3_lib",
             "//curvefs/test/metaserver/storage:metaserver_storage_test_utils",
             "//curvefs/test/metaserver/mock:metaserver_test_mock",
+            "//curvefs/test/client/rpcclient:rpcclient_test_mock",
     ],
 )
 

curvefs/test/metaserver/mock_metaserver_s3_adaptor.h

@@ -24,6 +24,7 @@
 
 #include <gmock/gmock.h>
 
+#include <memory>
 #include <string>
 
 #include "curvefs/src/metaserver/s3/metaserver_s3_adaptor.h"
@@ -44,6 +45,10 @@ class MockS3ClientAdaptor : public S3ClientAdaptor {
         const std::string& ak, const std::string& sk,
         const std::string& endpoint, const std::string& bucketName));
     MOCK_METHOD1(GetS3ClientAdaptorOption, void(S3ClientAdaptorOption *option));
+    MOCK_METHOD0(GetS3Client, std::shared_ptr<S3Client>());
+    MOCK_METHOD0(GetBlockSize, uint64_t());
+    MOCK_METHOD0(GetChunkSize, uint64_t());
+    MOCK_METHOD0(HasDiskCache, bool());
 };
 
 }  // namespace metaserver

curvefs/test/metaserver/trash_test.cpp

@@ -28,6 +28,8 @@
 #include "curvefs/src/metaserver/storage/rocksdb_storage.h"
 #include "curvefs/test/metaserver/storage/utils.h"
 #include "src/fs/ext4_filesystem_impl.h"
+#include "curvefs/test/client/rpcclient/mock_mds_client.h"
+#include "curvefs/test/metaserver/mock_metaserver_s3_adaptor.h"
 
 using ::testing::AtLeast;
 using ::testing::StrEq;
@@ -49,6 +51,7 @@ using ::curvefs::metaserver::storage::KVStorage;
 using ::curvefs::metaserver::storage::StorageOptions;
 using ::curvefs::metaserver::storage::RocksDBStorage;
 using ::curvefs::metaserver::storage::RandomStoragePath;
+using ::curvefs::client::rpcclient::MockMdsClient;
 
 class TestTrash : public ::testing::Test {
  protected:
@@ -103,7 +106,29 @@ class TestTrash : public ::testing::Test {
         inode.set_gid(0);
         inode.set_mode(0);
         inode.set_nlink(0);
-        inode.set_type(FsFileType::TYPE_FILE);
+        inode.set_type(FsFileType::TYPE_S3);
+        return inode;
+    }
+
+    Inode GenInodeHasChunks(uint32_t fsId, uint64_t inodeId) {
+        Inode inode;
+        inode.set_fsid(fsId);
+        inode.set_inodeid(inodeId);
+        inode.set_length(4096);
+        inode.set_ctime(0);
+        inode.set_ctime_ns(0);
+        inode.set_mtime(0);
+        inode.set_mtime_ns(0);
+        inode.set_atime(0);
+        inode.set_atime_ns(0);
+        inode.set_uid(0);
+        inode.set_gid(0);
+        inode.set_mode(0);
+        inode.set_nlink(0);
+        inode.set_type(FsFileType::TYPE_S3);
+
+        S3ChunkInfoList s3ChunkInfoList;
+        inode.mutable_s3chunkinfomap()->insert({0, s3ChunkInfoList});
         return inode;
     }
 
@@ -119,17 +144,18 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) {
     option.scanPeriodSec = 1;
     option.expiredAfterSec = 1;
 
+    option.mdsClient = std::make_shared<MockMdsClient>();
+    option.s3Adaptor = std::make_shared<MockS3ClientAdaptor>();
     trashManager_->Init(option);
     trashManager_->Run();
-
     auto trash1 = std::make_shared<TrashImpl>(inodeStorage_);
     auto trash2 = std::make_shared<TrashImpl>(inodeStorage_);
     trashManager_->Add(1, trash1);
     trashManager_->Add(2, trash2);
 
-    inodeStorage_->Insert(GenInode(1, 1));
-    inodeStorage_->Insert(GenInode(1, 2));
-    inodeStorage_->Insert(GenInode(2, 1));
+    inodeStorage_->Insert(GenInodeHasChunks(1, 1));
+    inodeStorage_->Insert(GenInodeHasChunks(1, 2));
+    inodeStorage_->Insert(GenInodeHasChunks(2, 1));
 
     ASSERT_EQ(inodeStorage_->Size(), 3);
 
@@ -138,17 +164,42 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) {
     trash2->Add(2, 1, 0);
 
     std::this_thread::sleep_for(std::chrono::seconds(5));
-
     std::list<TrashItem> list;
 
     trashManager_->ListItems(&list);
 
     ASSERT_EQ(0, list.size());
-
     ASSERT_EQ(inodeStorage_->Size(), 0);
 
     trashManager_->Fini();
 }
 
+TEST_F(TestTrash, testAdd3ItemAndNoDelete) {
+    TrashOption option;
+    option.scanPeriodSec = 1;
+    option.expiredAfterSec = 1;
+    option.mdsClient = std::make_shared<MockMdsClient>();
+    option.s3Adaptor = std::make_shared<MockS3ClientAdaptor>();
+    trashManager_->Init(option);
+    trashManager_->Run();
+
+    auto trash1 = std::make_shared<TrashImpl>(inodeStorage_);
+    trashManager_->Add(1, trash1);
+
+    inodeStorage_->Insert(GenInode(1, 1));
+    inodeStorage_->Insert(GenInode(1, 2));
+    inodeStorage_->Insert(GenInode(2, 1));
+    ASSERT_EQ(inodeStorage_->Size(), 3);
+    trash1->Add(1, 1, 0);
+    trash1->Add(1, 2, 0);
+    std::this_thread::sleep_for(std::chrono::seconds(5));
+    std::list<TrashItem> list;
+
+    trashManager_->ListItems(&list);
+    ASSERT_EQ(0, list.size());
+    ASSERT_EQ(inodeStorage_->Size(), 3);
+    trashManager_->Fini();
+}
+
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/util/build.sh

@@ -157,7 +157,7 @@ build_target() {
     # set version
     g_build_opts+=("--copt -DCURVEVERSION=${curve_version}")
 
-    if [ "$g_os" == "debian10" -o "$g_os" == "debian11" ]; then
+    if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -gt 6 ]; then
         g_build_opts+=("--config=gcc7-later")
     fi
 
@@ -191,8 +191,6 @@ build_requirements() {
     fi
     g_rocksdb_root="$(dirname ${PWD})/thirdparties/rocksdb"
     (cd ${g_rocksdb_root} && make build from_source=${g_build_rocksdb} && make install prefix=${g_rocksdb_root})
-    g_aws_sdk_root="$(dirname ${PWD})/thirdparties/aws"
-    (cd ${g_aws_sdk_root} && make)
 }
 
 main() {

curvefs/util/deploy.sh

@@ -1,127 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-############################  GLOBAL VARIABLES
-g_hosts=""
-g_only=""
-g_tags=""
-g_projects=(etcd mds metaserver tools client)
-g_devops_dir="$PWD/devops"
-
-g_color_yellow=`printf '\033[33m'`
-g_color_red=`printf '\033[31m'`
-g_color_normal=`printf '\033[0m'`
-
-############################  BASIC FUNCTIONS
-msg() {
-    printf '%b' "$1" >&2
-}
-
-success() {
-    msg "$g_color_yellow[✔]$g_color_normal ${1}${2}"
-}
-
-die() {
-    msg "$g_color_red[✘]$g_color_normal ${1}${2}"
-    exit 1
-}
-
-program_must_exist() {
-    command -v $1 >/dev/null 2>&1
-
-    if [ $? -ne 0 ]; then
-        die "You must have '$1' installed to continue\n"
-    fi
-}
-
-############################  FUNCTIONS
-usage () {
-    cat << _EOC_
-Usage:
-    deploy.sh --host=HOST --only=ONLY --tags=TAGS...
-
-Examples:
-    deploy.sh --host=* --only=* --tags=core,config
-    deploy.sh --host=mds --only=* --tags=status
-    deploy.sh --host=* --only=mds:metaserver --tags=config,restart
-_EOC_
-}
-
-get_options() {
-    local long_opts="hosts:,only:,tags:"
-    local args=`getopt -o eoth --long $long_opts -n "$0" -- "$@"`
-    eval set -- "${args}"
-    while true
-    do
-        case "$1" in
-            -e|--hosts)
-                g_hosts=$2
-                shift 2
-                ;;
-            -o|--only)
-                g_only=$2
-                shift 2
-                ;;
-            -t|--tags)
-                g_tags=$2
-                shift 2
-                ;;
-            -h|--help)
-                usage
-                exit 1
-                ;;
-            --)
-                shift
-                break
-                ;;
-            *)
-                exit 1
-                ;;
-        esac
-    done
-}
-
-begin() {
-	printf "$g_color_yellow>%.0s" $(seq 1 $(tput cols))
-	printf "$g_color_normal\n"
-}
-
-end() {
-	printf "$g_color_yellow<%.0s" $(seq 1 $(tput cols))
-	printf "$g_color_normal\n"
-}
-
-exec_ansible() {
-    cd $g_devops_dir
-    for project in ${g_projects[@]};
-    do
-        local hosts=$project
-        if [[ $g_only != "*" && $g_only != $project ]]; then
-            continue
-        elif [ "$g_hosts" != "*" ]; then
-            hosts=$g_hosts
-        fi
-
-        echo "ansible-playbook $project.yml -e hosts=$hosts -t $g_tags"
-        ansible-playbook $project.yml -e hosts=$hosts -t $g_tags
-    done
-}
-
-main() {
-    get_options "$@"
-
-    if [ $1 == "begin" ]; then
-        begin
-    elif [ $1 == "end" ]; then
-        end
-    elif [[ $g_hosts == "" || $g_only == "" || $g_tags == "" ]]; then
-        usage
-        exit 1
-    fi
-
-    exec_ansible
-}
-
-############################  MAIN()
-main "$@"

curvefs/util/image.sh

@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-
-# tmpl.sh = /usr/local/metaserver.conf /tmp/metaserver.conf
-# $1: tag $2: os
-function tmpl() {
-    dsv=$1
-    src=$2
-    dst=$3
-    regex="^([^$dsv]+$dsv[[:space:]]*)(.+)__CURVEADM_TEMPLATE__[[:space:]]+(.+)[[:space:]]+__CURVEADM_TEMPLATE__(.*)$"
-    while IFS= read -r line; do
-        if [[ ! $line =~ $regex ]]; then
-            echo $line
-        else
-            echo ${BASH_REMATCH[1]}${BASH_REMATCH[3]}
-        fi
-    done < $src > $dst
-}
-
-prefix="$(pwd)/docker/$2/curvefs"
-mkdir -p $prefix $prefix/conf
-make install prefix="$prefix"
-make install prefix="$prefix" only=etcd
-make install prefix="$prefix" only=monitor
-cp -f ../thirdparties/aws/aws-sdk-cpp/build/aws-cpp-sdk-core/libaws-cpp-sdk-core.so docker/$2
-cp -f ../thirdparties/aws/aws-sdk-cpp/build/aws-cpp-sdk-s3-crt/libaws-cpp-sdk-s3-crt.so docker/$2
-for file in `ls conf`;
-do
-    dsv="="
-    if [ $file = "etcd.conf" ]; then
-        dsv=": "
-    fi
-
-    tmpl $dsv "conf/$file" "$prefix/conf/$file"
-done
-
-cp ../conf/client.conf "$prefix/conf/curvebs-client.conf"
-
-docker pull opencurvedocker/curve-base:$2
-docker build -t "$1" "$(pwd)/docker/$2"

curvefs/util/install.sh

@@ -1,269 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-# metaserevr
-# ├── conf
-# │   └── metaserver.conf
-# ├── logs
-# │   └── metaserevr.pid
-# ├── sbin
-# │   ├── servicectl
-# │   └── curvefs-metaserver
-# └── lib
-#     └── metaserver_lib.so
-
-############################  GLOBAL VARIABLES
-g_prefix=""
-g_only=""
-g_project_name=""
-g_etcd_version="v3.4.10"
-g_util_dir="$(dirname $(realpath $0))"
-g_curvefs_dir="$(dirname $g_util_dir)"
-g_curve_dir="$(dirname $g_curvefs_dir)"
-g_build_release=0
-
-g_color_yellow=`printf '\033[33m'`
-g_color_red=`printf '\033[31m'`
-g_color_normal=`printf '\033[0m'`
-
-############################  BASIC FUNCTIONS
-msg() {
-    printf '%b' "$1" >&2
-}
-
-success() {
-    msg "$g_color_yellow[✔]$g_color_normal [$g_project_name] ${1}${2}"
-}
-
-die() {
-    msg "$g_color_red[✘]$g_color_normal [$g_project_name] ${1}${2}"
-    exit 1
-}
-
-program_must_exist() {
-    command -v $1 >/dev/null 2>&1
-
-    if [ $? -ne 0 ]; then
-        die "You must have '$1' installed to continue\n"
-    fi
-}
-
-############################ FUNCTIONS
-usage () {
-    cat << _EOC_
-Usage:
-    install.sh --prefix=PREFIX --only=TARGET
-
-Examples:
-    install.sh --prefix=/usr/local/curvefs --only=*
-    install.sh --prefix=/usr/local/curvefs --only=metaserver
-    install.sh --prefix=/usr/local/curvefs --only=etcd --etcd_version="v3.4.0"
-_EOC_
-}
-
-get_options() {
-    local long_opts="prefix:,only:,etcd_version:,help"
-    local args=`getopt -o povh --long $long_opts -n "$0" -- "$@"`
-    eval set -- "${args}"
-    while true
-    do
-        case "$1" in
-            -p|--prefix)
-                g_prefix=$2
-                shift 2
-                ;;
-            -o|--only)
-                g_only=$2
-                shift 2
-                ;;
-            -v|--etcd_version)
-                g_etcd_version=$2
-                shift 2
-                ;;
-            -h|--help)
-                usage
-                exit 1
-                ;;
-            --)
-                shift
-                break
-                ;;
-            *)
-                exit 1
-                ;;
-        esac
-    done
-}
-
-get_build_mode() {
-    grep "release" BUILD_MODE > /dev/null 2>&1
-    if [ $? -eq 0 ]; then
-        g_build_release=1
-    fi
-}
-
-strip_debug_info() {
-    if [ $g_build_release -eq 1 ]; then
-        # binary file generated by bazel isn't writable by default
-        chmod +w $1
-        objcopy --only-keep-debug $1 $1.dbg-sym
-        objcopy --strip-debug $1
-        objcopy --add-gnu-debuglink=$1.dbg-sym $1
-        chmod -w $1
-    fi
-}
-
-create_project_dir() {
-    mkdir -p $1/{conf,logs,sbin,lib,data}
-    if [ $? -eq 0 ]; then
-        success "create project directory $1 success\n"
-    else
-        die "create directory $1 failed\n"
-    fi
-}
-
-copy_file() {
-    cp -f "$1" "$2"
-    if [ $? -eq 0 ]; then
-        success "copy file $1 to $2 success\n"
-    else
-        die "copy file $1 to $2 failed\n"
-    fi
-}
-
-get_targets() {
-    bazel query 'kind("cc_binary", //curvefs/src/...)' | grep -E "$g_only"
-}
-
-gen_servicectl() {
-    local src="$g_util_dir/servicectl.sh"
-    local dst="$4/servicectl"
-    sed -e "s|__PROJECT__|$1|g" \
-        -e "s|__BIN_FILENAME__|$2|g" \
-        -e "s|__START_ARGS__|$3|g" \
-        $src > $dst \
-    && chmod a+x $dst
-
-    if [ $? -eq 0 ]; then
-        success "generate servicectl success\n"
-    else
-        die "generate servicectl failed\n"
-    fi
-}
-
-install_curvefs() {
-    for target in `get_targets`
-    do
-        local regex_target="//(curvefs/src/([^:]+)):(.+)"
-        if [[ ! $target =~ $regex_target ]]; then
-            die "unknown target: $target\n"
-        fi
-
-        local project_name="${BASH_REMATCH[2]}"  # ex: metaserver
-        g_project_name=$project_name
-        local project_prefix="$g_prefix/$project_name"  # ex: /usr/local/curvefs/metaserver
-
-        local project_bin_filename=${BASH_REMATCH[3]}  # ex: curvefs_metaserver
-        local bazel_dir="$g_curve_dir/bazel-bin"
-        local binary="$bazel_dir/${BASH_REMATCH[1]}/$project_bin_filename"
-
-        # The below actions:
-        #   1) create project directory
-        #   2) copy binary to project directory
-        #   3) generate servicectl script into project directory.
-        create_project_dir $project_prefix
-        copy_file "$binary" "$project_prefix/sbin"
-        strip_debug_info "$project_prefix/sbin/$project_bin_filename"
-        gen_servicectl \
-            $project_name \
-            $project_bin_filename \
-            '--confPath=$g_conf_file' \
-            "$project_prefix/sbin"
-
-        success "install $project_name success\n"
-    done
-}
-
-download_etcd() {
-    local now=`date +"%s%6N"`
-    local nos_url="https://curve-build.nos-eastchina1.126.net"
-    local src="${nos_url}/etcd-${g_etcd_version}-linux-amd64.tar.gz"
-    local tmpfile="/tmp/$now-etcd-${g_etcd_version}-linux-amd64.tar.gz"
-    local dst="$1"
-
-    msg "download etcd: $src to $dst\n"
-
-    # download etcd tarball and decompress to dst directory
-    mkdir -p $dst &&
-        curl -L $src -o $tmpfile &&
-        tar -zxvf $tmpfile -C $dst --strip-components=1 >/dev/null 2>&1
-
-    local ret=$?
-    rm -rf $tmpfile
-    if [ $ret -ne 0 ]; then
-        die "download etcd-$g_etcd_version failed\n"
-    else
-        success "download etcd-$g_etcd_version success\n"
-    fi
-}
-
-install_etcd() {
-    local project_name="etcd"
-    g_project_name=$project_name
-
-    # The below actions:
-    #   1) download etcd tarball from github
-    #   2) create project directory
-    #   3) copy binary to project directory
-    #   4) generate servicectl script into project directory.
-    local now=`date +"%s%6N"`
-    local dst="/tmp/$now/etcd-$g_etcd_version"
-    download_etcd $dst
-    local project_prefix="$g_prefix/etcd"
-    create_project_dir $project_prefix
-    copy_file "$dst/etcd" "$project_prefix/sbin"
-    copy_file "$dst/etcdctl" "$project_prefix/sbin"
-    gen_servicectl \
-        $project_name \
-        "etcd" \
-        '--config-file=$g_project_prefix/conf/etcd.conf' \
-        "$project_prefix/sbin"
-
-    rm -rf "$dst"
-    success "install $project_name success\n"
-}
-
-install_monitor() {
-    local project_name="monitor"
-    g_project_name=$project_name
-
-    local project_prefix="$g_prefix/monitor"
-    local dst="monitor"
-    mkdir -p $project_prefix
-    mkdir -p "$project_prefix/prometheus"
-    mkdir -p "$project_prefix/data"
-    copy_file "$dst/target_json.py" "$project_prefix"
-    copy_file "$dst/target.ini" "$project_prefix"
-
-    success "install $project_name success\n"
-}
-
-main() {
-    get_options "$@"
-    get_build_mode
-
-    if [[ $g_prefix == "" || $g_only == "" ]]; then
-        usage
-        exit 1
-    elif [ "$g_only" == "etcd" ]; then
-        install_etcd
-    elif [ "$g_only" == "monitor" ]; then
-        install_monitor
-    else
-        install_curvefs
-    fi
-}
-
-############################  MAIN()
-main "$@"

docker/debian10/Dockerfile

@@ -4,7 +4,5 @@ RUN mkdir -p /curvebs /etc/curve /etc/nebd /core
 COPY curvebs /curvebs
 COPY entrypoint.sh /
 COPY curvebs/tools/sbin/curve_ops_tool curvebs/nbd/sbin/curve-nbd /usr/bin/
-COPY libaws-cpp-sdk-core.so /usr/lib/
-COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

docker/debian11/Dockerfile

@@ -4,7 +4,5 @@ RUN mkdir -p /curvebs /etc/curve /etc/nebd /core
 COPY curvebs /curvebs
 COPY entrypoint.sh /
 COPY curvebs/tools/sbin/curve_ops_tool curvebs/nbd/sbin/curve-nbd /usr/bin/
-COPY libaws-cpp-sdk-core.so /usr/lib/
-COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
 RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

docker/debian9/Dockerfile

@@ -3,8 +3,9 @@ ENV TZ=Asia/Shanghai
 RUN mkdir -p /curvebs /etc/curve /etc/nebd /core
 COPY curvebs /curvebs
 COPY entrypoint.sh /
-COPY curvebs/tools/sbin/curve_ops_tool curvebs/nbd/sbin/curve-nbd /usr/bin/
-COPY libaws-cpp-sdk-core.so /usr/lib/
-COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
-RUN chmod a+x /entrypoint.sh
+RUN mkdir -p /etc/curve /etc/nebd /curve/init.d/ \
+    && chmod a+x /entrypoint.sh \
+    && cp curvebs/nbd/sbin/curve-nbd /usr/bin/ \
+    && cp curvebs/tools/sbin/curve_ops_tool /usr/bin/ \
+    && cp curvebs/tools-v2/sbin/curve /usr/bin/
 ENTRYPOINT ["/entrypoint.sh"]

docker/debian9/compile/Dockerfile

@@ -1,5 +1,6 @@
 FROM opencurvedocker/curve-base:debian9
 
+<<<<<<< HEAD
 RUN apt-get clean \
     && apt-get -y update \
     && apt-get -y install \
@@ -22,6 +23,68 @@ RUN apt-get clean \
         python3-setuptools \
         python3-wheel \
     && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
+=======
+ENV GITHUB_PROXY=https://ghproxy.com/
+ENV ETCD_VERSION=v3.4.0 \
+    DOWNLOAD_ETCD_URL=${GITHUB_PROXY}https://github.com/etcd-io/etcd/releases/download \
+    GOPROXY=https://goproxy.io,direct \
+    PROTOC_VERSION=21.8 \
+    GO_VERSION=1.19.6 \
+    PROTOC_GEN_GO_VERSION=v1.28 \
+    PROTOC_GEN_GO_GRPC_VERSION=v1.2
+RUN echo "deb http://mirrors.163.com/debian/ stretch main\n" \
+    "deb http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
+    "deb-src http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
+    "deb http://mirrors.163.com/debian-security/ stretch/updates main non-free contrib\n" \
+    "deb http://httpredir.debian.org/debian stretch-backports main contrib non-free\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+    g++ \
+    wget \
+    clang \
+    git \
+    libnl-genl-3-dev \
+    libssl-dev \
+    uuid-dev \
+    libcurl4-gnutls-dev \
+    libfiu-dev \
+    openjdk-8-jdk \
+    sudo \
+    python-dev \
+    python-setuptools \
+    python-wheel \
+    python3-dev \
+    python3-setuptools \
+    python3-wheel \
+    zlib1g-dev \
+    libsnappy-dev \
+    liblz4-dev \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release \
+    curl \
+    apt-transport-https \
+    git \
+    libbison-dev \
+    flex \
+    libhashkit-dev \
+    libevent-dev \
+    procps \
+    lsof \
+    gdb \
+    unzip \
+    musl-tools \
+    && mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+    $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get update \
+    && apt-get install -y docker-ce docker-ce-cli containerd.io \
+    && wget ${GITHUB_PROXY}https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
+>>>>>>> 51f85310... [feat]toolsv2: build to image
     && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
     && ln -s /usr/bin/bazelisk /usr/bin/bazel \
     && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
@@ -30,5 +93,43 @@ RUN apt-get clean \
     && wget https://github.com/bazelbuild/bazel/releases/download/0.17.2/bazel-0.17.2-linux-x86_64 \
     && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin \
     && mv bazel-0.17.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel \
+<<<<<<< HEAD
     && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk
 COPY fuse3/* /usr/include/fuse3/
+=======
+    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk \
+    && wget ${GITHUB_PROXY}https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.tar.gz \
+    && tar Cxzf /usr/share/ cmake-3.13.4-Linux-x86_64.tar.gz \
+    && rm cmake-3.13.4-Linux-x86_64.tar.gz \
+    && ln -s /usr/share/cmake-3.13.4-Linux-x86_64/bin/cmake /usr/bin/cmake \
+    && wget https://curve-build.nos-eastchina1.126.net/fuse3.tar.gz \
+    && tar Cxzf /usr/include/ fuse3.tar.gz && rm fuse3.tar.gz \
+    && rm -f /usr/local/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \
+    && rm -rf /usr/local/etcd && mkdir -p /usr/local/etcd \
+    && curl -L ${DOWNLOAD_ETCD_URL}/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -o /usr/local/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \
+    && tar xzvf /usr/local/etcd-${ETCD_VERSION}-linux-amd64.tar.gz -C /usr/local/etcd --strip-components=1 \
+    && rm -f /usr/local/etcd-${ETCD_VERSION}-linux-amd64.tar.gz \
+    && ln -sf /usr/local/etcd/etcd /usr/bin/etcd && ln -sf /usr/local/etcd/etcdctl /usr/bin/etcdctl \
+    && wget https://curve-build.nos-eastchina1.126.net/memcache/memcached-1.6.17.tar.gz \
+    && tar xvf memcached-1.6.17.tar.gz && cd memcached-1.6.17/ \
+    && ./configure && make && make install \
+    && cd / && rm -rf memcached-1.6.17/ memcached-1.6.17.tar.gz \
+    && wget https://curve-build.nos-eastchina1.126.net/curve-base.tar.gz \
+    && tar -xzf curve-base.tar.gz \
+    && mv fusermount3 /usr/local/bin \
+    && mv libetcdclient.so /usr/lib/ \
+    && mv libfuse3.so libfuse3.so.3 libfuse3.so.3.10.5 libnl-3.so.200 libnl-genl-3.so.200 /usr/lib/x86_64-linux-gnu/ \
+    && mv libjemalloc.so libjemalloc.so.1 libjemalloc.so.2 /usr/local/lib/ \
+    && rm curve-base.tar.gz \
+    && wget https://golang.google.cn/dl/go${GO_VERSION}.linux-amd64.tar.gz \
+    && tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz \
+    && rm go${GO_VERSION}.linux-amd64.tar.gz \
+    && export PATH=$PATH:/usr/local/go/bin \
+    && go env -w GO111MODULE=on && go env -w GOPROXY=https://goproxy.cn,direct \
+    && wget ${GITHUB_PROXY}https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip \
+    && unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip "bin/protoc" -d /usr/ \
+    && rm protoc-${PROTOC_VERSION}-linux-x86_64.zip \
+    && go install google.golang.org/protobuf/cmd/protoc-gen-go@${PROTOC_GEN_GO_VERSION} \
+    && go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@${PROTOC_GEN_GO_GRPC_VERSION}
+ENV PATH=$PATH:/usr/local/go/bin:/root/go/bin
+>>>>>>> 51f85310... [feat]toolsv2: build to image

docker/debian9/entrypoint.sh

@@ -6,7 +6,7 @@
 g_role=""
 g_args=""
 g_prefix=""
-g_preexec=""
+g_preexec="/curvebs/tools-v2/sbin/daemon"
 g_binary=""
 g_start_args=""
 
@@ -123,7 +123,7 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
-    [[ ! -z $g_preexec ]] && $g_preexec
+    [[ ! -z $g_preexec ]] && $g_preexec &
     if [ $g_role == "etcd" ]; then
         exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
     elif [ $g_role == "monitor" ]; then

docs/cn/build_and_run.md

@@ -41,8 +41,8 @@ git clone https://github.com/opencurve/curve.git 或者 git clone https://gitee.
 bash mk-tar.sh （编译 curvebs 并打tar包）
 bash mk-deb.sh （编译 curvebs 并打debian包）
 # curve v2.0 及之后
-编译 curvebs: cd curve && make build
-编译 curvefs: cd curve/curvefs && make build dep=1
+编译 curvebs: cd curve && make build stor=bs dep=1
+编译 curvefs: cd curve && make build stor=fs dep=1
 ```
 
 
@@ -70,8 +70,8 @@ git clone https://github.com/opencurve/curve.git 或者 git clone https://gitee.
 bash mk-tar.sh （编译 curvebs 并打tar包）
 bash mk-deb.sh （编译 curvebs 并打debian包）
 # curve v2.0 及之后
-编译 curvebs: cd curve && make build
-编译 curvefs: cd curve/curvefs && make build dep=1
+编译 curvebs: cd curve && make build stor=bs dep=1
+编译 curvefs: cd curve && make build stor=fs dep=1
 ```
 
 ## 测试用例编译及执行

docs/en/build_and_run_en.md

@@ -37,8 +37,8 @@ git clone https://github.com/opencurve/curve.git or git clone https://gitee.com/
 bash mk-tar.sh （compile curvebs and make tar package）
 bash mk-deb.sh （compile curvebs and make debian package）
 # after curve v2.0
-compile curvebs: cd curve && make build
-compile curvefs: cd curve/curvefs && make build dep=1
+compile curvebs: cd curve && make build stor=bs dep=1
+compile curvefs: cd curve && make build stor=fs dep=1
 ```
 
 ## Compile on a physical machine
@@ -65,8 +65,8 @@ git clone https://github.com/opencurve/curve.git or git clone https://gitee.com/
 bash mk-tar.sh （compile curvebs and make tar package）
 bash mk-deb.sh （compile curvebs and make debian package）
 # after curve v2.0
-compile curvebs: cd curve && make build
-compile curvefs: cd curve/curvefs && make build dep=1
+compile curvebs: cd curve && make build stor=bs dep=1
+compile curvefs: cd curve && make build stor=fs dep=1
 ```
 
 ## Test case compilation and execution

nebd/src/part2/file_entity.cpp

@@ -33,6 +33,8 @@
 namespace nebd {
 namespace server {
 
+extern const char* kOpenFlagsAttrKey;
+
 bool IsOpenFlagsExactlySame(const OpenFlags* lhs, const OpenFlags* rhs) {
     if (lhs == nullptr && rhs == nullptr) {
         return true;
@@ -145,6 +147,15 @@ int NebdFileEntity::Reopen(const ExtendAttribute& xattr) {
                    << "filename: " << fileName_;
         return -1;
     }
+
+    OpenFlags flags;
+    bool hasOpenFlags =
+        fileInstance->xattr.count(kOpenFlagsAttrKey) &&
+        flags.ParseFromString(fileInstance->xattr.at(kOpenFlagsAttrKey));
+    if (hasOpenFlags) {
+        openFlags_.reset(new OpenFlags{flags});
+    }
+
     LOG(INFO) << "Reopen file success. "
               << "fd: " << fd_
               << ", filename: " << fileName_;