curvefs/client: fix s3 object will not be removed

Signed-off-by: wanghai01 <seanhaizi@163.com>

Makefile

@@ -1,22 +1,27 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build install image
+.PHONY: list build dep install image
 
+stor?=""
 prefix?= "$(PWD)/projects"
 release?= 0
+dep?= 0
 only?= "*"
 tag?= "curvebs:unknown"
 case?= "*"
 os?= "debian9"
 
 list:
-	@bash util/build.sh --list
+	@bash util/build.sh --stor=${stor} --list
 
 build:
-	@bash util/build.sh --only=$(only) --release=$(release) --os=$(os)
+	@bash util/build.sh --stor=${stor} --only=$(only) --dep=$(dep) --release=$(release) --os=$(os)
+
+dep:
+	@bash util/build.sh --stor=${stor} --only="" --dep=1
 
 install:
-	@bash util/install.sh --prefix=$(prefix) --only=$(only)
+	@bash util/install.sh --stor=${stor} --prefix=$(prefix) --only=$(only)
 
 image:
-	@bash util/image.sh $(tag) $(os)
+	@bash util/image.sh ${stor} $(tag) $(os)

curvefs/Makefile

@@ -1,23 +0,0 @@
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-.PHONY: build dep install image
-
-prefix?= "$(PWD)/devops/projects"
-only?= "*"
-dep?= 0
-release?= 0
-os?= "debian9"
-build_rocksdb?= 0
-tag?= "curvefs:unknown"
-
-build:
-	@bash util/build.sh --only=$(only) --dep=$(dep) --release=$(release) --os=$(os) --build_rocksdb=$(build_rocksdb)
-
-dep:
-	@bash util/build.sh --only="" --dep=1 --build_rocksdb=$(build_rocksdb)
-
-install:
-	@bash util/install.sh --prefix=$(prefix) --only=$(only)
-
-image:
-	@bash util/image.sh $(tag) $(os)

curvefs/conf/client.conf

@@ -90,7 +90,7 @@ fuseClient.iCacheLruSize=65536
 fuseClient.dCacheLruSize=65536
 fuseClient.enableICacheMetrics=true
 fuseClient.enableDCacheMetrics=true
-fuseClient.cto=false
+fuseClient.cto=true
 # you shoudle enable it when mount one filesystem to multi mountpoints,
 # it gurantee the consistent of file after rename, otherwise you should
 # disable it for performance.

curvefs/conf/metaserver.conf

@@ -149,7 +149,7 @@ bthread.worker_count=auto
 # Call fsync when need
 # braft default is True. Setting to false can greatly improve performance.
 # We can select according to the specified scene.
-braft.raft_sync=True
+braft.raft_sync=False
 # Sync log meta, snapshot meta and raft meta
 # braft default is False
 braft.raft_sync_meta=True
@@ -224,6 +224,12 @@ storage.rocksdb.write_buffer_manager_capacity=6442450944
 storage.rocksdb.WBM_cost_block_cache=false
 # Maximum number of concurrent background jobs (compactions and flushes)
 storage.rocksdb.max_background_jobs=16
+# Maxinum number of threads to perform a compaction job by simultaneously (default: 4)
+storage.rocksdb.max_subcompactions=4
+# Number of files to trigger level-0 compaction (default: 1)
+storage.rocksdb.level0_file_num_compaction_trigger=1
+# Control maximum total data size for a level (default: 1GB)
+storage.rocksdb.max_bytes_for_level_base=1073741824
 # rocksdb column family's write_buffer_size
 # for store inode which exclude its s3chunkinfo list (unit: bytes, default: 64MB)
 storage.rocksdb.unordered_write_buffer_size=67108864
@@ -236,6 +242,8 @@ storage.rocksdb.ordered_write_buffer_size=67108864
 # rocksdb column family's max_write_buffer_number
 # for store dentry and inode's s3chunkinfo list (default: 3)
 storage.rocksdb.ordered_max_write_buffer_number=3
+# The target number of write history bytes to hold in memory (default: 20MB)
+storage.rocksdb.max_write_buffer_size_to_maintain=20971520
 # rocksdb memtable prefix bloom size ratio (size=write_buffer_size*memtable_prefix_bloom_size_ratio)
 storage.rocksdb.memtable_prefix_bloom_size_ratio=0.1
 # dump rocksdb.stats to LOG every stats_dump_period_sec

curvefs/proto/mds.proto

@@ -53,6 +53,8 @@ enum FSStatusCode {
     LOCK_FAILED = 28;
     LOCK_TIMEOUT = 29;
     COMMIT_TX_SEQUENCE_MISMATCH = 30;
+    VOLUME_INFO_ERROR = 31;
+    MOUNT_POINT_CONFLICT = 32;
 }
 
 // fs interface
@@ -76,6 +78,7 @@ message Mountpoint {
     required string hostname = 1;
     required uint32 port = 2;
     required string path = 3;
+    optional bool cto = 4;
 }
 
 message FsInfo {

curvefs/proto/metaserver.proto

@@ -48,6 +48,7 @@ enum MetaStatusCode {
     STORAGE_INTERNAL_ERROR = 24;
     RPC_STREAM_ERROR = 25;
     INODE_S3_META_TOO_LARGE = 26;
+    STORAGE_CLOSED = 27;
 }
 
 // dentry interface
@@ -217,7 +218,7 @@ message Inode {
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index
     optional uint32 dtime = 19;
     optional uint32 openmpcount = 20; // openmpcount mount points had the file open
-    map<string, string> xattr = 21;
+    map<string, bytes> xattr = 21;
     repeated uint64 parent = 22;
 }
 
@@ -290,7 +291,7 @@ message UpdateInodeRequest {
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 17;
     optional uint32 nlink = 18;
     optional InodeOpenStatusChange inodeOpenstatusChange = 19;
-    map<string, string> xattr = 20;
+    map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
 }
 
@@ -381,7 +382,7 @@ message InodeAttr {
     optional uint64 rdev = 16;
     optional uint32 dtime = 17;
     optional uint32 openmpcount = 18;
-    map<string, string> xattr = 19;
+    map<string, bytes> xattr = 19;
     repeated uint64 parent = 20;
 }
 
@@ -412,7 +413,7 @@ message BatchGetXAttrRequest {
 message XAttr {
     required uint64 inodeId = 1;
     required uint32 fsId = 2;
-    map<string, string> xAttrInfos = 3;
+    map<string, bytes> xAttrInfos = 3;
 }
 
 message BatchGetXAttrResponse {

curvefs/src/client/BUILD

@@ -72,5 +72,6 @@ cc_library(
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",
+        "@com_google_googletest//:gtest_prod",
     ],
 )

curvefs/src/client/client_operator.cpp

@@ -279,7 +279,7 @@ CURVEFS_ERROR RenameOperator::LinkInode(uint64_t inodeId, uint64_t parent) {
         return rc;
     }
 
-    rc = inodeWrapper->LinkLocked(parent);
+    rc = inodeWrapper->Link(parent);
     if (rc != CURVEFS_ERROR::OK) {
         LOG_ERROR("Link", rc);
         return rc;
@@ -297,7 +297,7 @@ CURVEFS_ERROR RenameOperator::UnLinkInode(uint64_t inodeId, uint64_t parent) {
         return rc;
     }
 
-    rc = inodeWrapper->UnLinkLocked(parent);
+    rc = inodeWrapper->UnLink(parent);
     if (rc != CURVEFS_ERROR::OK) {
         LOG_ERROR("UnLink", rc);
         return rc;
@@ -316,10 +316,7 @@ CURVEFS_ERROR RenameOperator::UpdateMCTime(uint64_t inodeId) {
     }
 
     curve::common::UniqueLock lk = inodeWrapper->GetUniqueLock();
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    inodeWrapper->SetMTime(now.tv_sec, now.tv_nsec);
-    inodeWrapper->SetCTime(now.tv_sec, now.tv_nsec);
+    inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
 
     rc = inodeWrapper->SyncAttr();
     if (rc != CURVEFS_ERROR::OK) {
@@ -388,7 +385,7 @@ CURVEFS_ERROR RenameOperator::UpdateInodeParent() {
         return rc;
     }
 
-    rc = inodeWrapper->UpdateParentLocked(parentId_, newParentId_);
+    rc = inodeWrapper->UpdateParent(parentId_, newParentId_);
     if (rc != CURVEFS_ERROR::OK) {
         LOG_ERROR("UpdateInodeParent", rc);
         return rc;

curvefs/src/client/client_operator.h

@@ -68,9 +68,9 @@ class RenameOperator {
         *oldInodeType = oldInodeType_;
     }
 
- private:
     std::string DebugString();
 
+ private:
     CURVEFS_ERROR CheckOverwrite();
 
     CURVEFS_ERROR GetLatestTxIdWithLock();

curvefs/src/client/common/common.h

@@ -62,7 +62,13 @@ enum class MetaServerOpType {
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);
 
-const uint32_t MAXXATTRLENGTH = 256;
+const uint32_t MAX_XATTR_NAME_LENGTH = 255;
+const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024;
+
+enum class FileHandle : uint64_t {
+    kDefaultValue = 0,
+    kKeepCache = 1,
+};
 
 }  // namespace common
 }  // namespace client

curvefs/src/client/common/config.cpp

@@ -264,19 +264,25 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
                               &clientOption->enableICacheMetrics);
     conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
                               &clientOption->enableDCacheMetrics);
-    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
     conf->GetValueFatalIfFail("client.dummyserver.startport",
                               &clientOption->dummyServerStartPort);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                               &clientOption->enableMultiMountPointRename);
     conf->GetValueFatalIfFail("fuseClient.disableXattr",
                               &clientOption->disableXattr);
+    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
 
     LOG_IF(WARNING, conf->GetBoolValue("fuseClient.enableSplice",
                                        &clientOption->enableFuseSplice))
         << "Not found `fuseClient.enableSplice` in conf, use default value `"
         << std::boolalpha << clientOption->enableFuseSplice << '`';
 
+    // if enableCto, attr and entry cache must invalid
+    if (FLAGS_enableCto) {
+        clientOption->attrTimeOut = 0;
+        clientOption->entryTimeOut = 0;
+    }
+
     SetBrpcOpt(conf);
 }
 

curvefs/src/client/curve_fuse_op.cpp

@@ -46,12 +46,12 @@ using ::curvefs::client::FuseClient;
 using ::curvefs::client::FuseS3Client;
 using ::curvefs::client::FuseVolumeClient;
 using ::curvefs::client::common::FuseClientOption;
-using ::curvefs::client::common::MAXXATTRLENGTH;
 using ::curvefs::client::rpcclient::MdsClientImpl;
 using ::curvefs::client::rpcclient::MDSBaseClient;
 using ::curvefs::client::metric::ClientOpMetric;
 using ::curvefs::common::LatencyUpdater;
 using ::curvefs::client::metric::InflightGuard;
+using ::curvefs::client::common::FileHandle;
 
 using ::curvefs::common::FLAGS_vlog_level;
 
@@ -241,6 +241,9 @@ void FuseReplyErrByErrCode(fuse_req_t req, CURVEFS_ERROR errcode) {
     case CURVEFS_ERROR::NODATA:
         fuse_reply_err(req, ENODATA);
         break;
+    case CURVEFS_ERROR::EXISTS:
+        fuse_reply_err(req, EEXIST);
+        break;
     default:
         fuse_reply_err(req, EIO);
         break;
@@ -273,13 +276,26 @@ void FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) {
     fuse_reply_attr(req, &attr, g_fuseClientOption->attrTimeOut);
 }
 
+void FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino, const char* name,
+                    const char* value, size_t size, int flags) {
+    std::string xattrValue(value, size);
+    VLOG(9) << "FuseOpSetXattr"
+            << " ino " << ino << " name " << name << " value " << xattrValue
+            << " flags " << flags;
+    // set xattr
+    CURVEFS_ERROR ret = g_ClientInstance->FuseOpSetXattr(req, ino, name,
+        value, size, flags);
+    FuseReplyErrByErrCode(req, ret);
+    VLOG(9) << "FuseOpSetXattr done";
+}
+
 void FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino, const char *name,
     size_t size) {
     InflightGuard guard(&g_clientOpMetric->opGetXattr.inflightOpNum);
     LatencyUpdater updater(&g_clientOpMetric->opGetXattr.latency);
-    char buf[MAXXATTRLENGTH] = {0};
+    std::string buf;
     CURVEFS_ERROR ret = g_ClientInstance->FuseOpGetXattr(req, ino, name,
-                                                         buf, size);
+                                                         &buf, size);
     if (ret != CURVEFS_ERROR::OK) {
         g_clientOpMetric->opGetXattr.ecount << 1;
         FuseReplyErrByErrCode(req, ret);
@@ -287,9 +303,9 @@ void FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino, const char *name,
     }
 
     if (size == 0) {
-        fuse_reply_xattr(req, strlen(buf));
+        fuse_reply_xattr(req, buf.length());
     } else {
-        fuse_reply_buf(req, buf, strlen(buf));
+        fuse_reply_buf(req, buf.data(), buf.length());
     }
 }
 

curvefs/src/client/dentry_cache_manager.cpp

@@ -22,6 +22,7 @@
  */
 #include "curvefs/src/client/dentry_cache_manager.h"
 
+#include <cstdint>
 #include <string>
 #include <list>
 #include <vector>
@@ -123,10 +124,18 @@ CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent,
 CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                                                  std::list<Dentry> *dentryList,
                                                  uint32_t limit,
-                                                 bool onlyDir) {
-    bool perceed = true;
-    MetaStatusCode ret = MetaStatusCode::OK;
+                                                 bool onlyDir,
+                                                 uint32_t nlink) {
     dentryList->clear();
+    // means no dir under this dir
+    if (onlyDir && nlink == 2) {
+        LOG(INFO) << "ListDentry parent = " << parent
+                  << ", onlyDir = 1 and nlink = 2, return directly";
+        return CURVEFS_ERROR::OK;
+    }
+
+    MetaStatusCode ret = MetaStatusCode::OK;
+    bool perceed = true;
     std::string last = "";
     do {
         std::list<Dentry> part;
@@ -147,12 +156,30 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                        << ", count = " << limit << ", onlyDir = " << onlyDir;
             return MetaStatusCodeToCurvefsErrCode(ret);
         }
-        if (part.size() < limit) {
-            perceed = false;
-        }
-        if (!part.empty()) {
-            last = part.back().name();
-            dentryList->splice(dentryList->end(), part);
+
+        if (!onlyDir) {
+            if (part.size() < limit) {
+                perceed = false;
+            }
+            if (!part.empty()) {
+                last = part.back().name();
+                dentryList->splice(dentryList->end(), part);
+            }
+        } else {
+            // means iterate over the range
+            if (part.empty()) {
+                perceed = false;
+            } else {
+                last = part.back().name();
+                if (part.back().type() != FsFileType::TYPE_DIRECTORY) {
+                    part.pop_back();
+                }
+                dentryList->splice(dentryList->end(), part);
+                // means already get all the dir under this dir
+                if (nlink - dentryList->size() == 2) {
+                    perceed = false;
+                }
+            }
         }
     } while (perceed);
 

curvefs/src/client/dentry_cache_manager.h

@@ -76,7 +76,7 @@ class DentryCacheManager {
 
     virtual CURVEFS_ERROR ListDentry(uint64_t parent,
         std::list<Dentry> *dentryList, uint32_t limit,
-        bool onlyDir = false) = 0;
+        bool onlyDir = false, uint32_t nlink = 0) = 0;
 
  protected:
     uint32_t fsId_;
@@ -120,7 +120,7 @@ class DentryCacheManagerImpl : public DentryCacheManager {
 
     CURVEFS_ERROR ListDentry(uint64_t parent,
         std::list<Dentry> *dentryList, uint32_t limit,
-        bool dirOnly = false) override;
+        bool dirOnly = false, uint32_t nlink = 0) override;
 
     std::string GetDentryCacheKey(uint64_t parent, const std::string &name) {
         return std::to_string(parent) + kDentryKeyDelimiter + name;

curvefs/src/client/fuse_client.cpp

@@ -33,8 +33,11 @@
 #include <utility>
 
 #include "curvefs/proto/mds.pb.h"
+#include "curvefs/src/client/common/common.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/fuse_common.h"
 #include "curvefs/src/client/client_operator.h"
+#include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/xattr_manager.h"
 #include "src/common/net_common.h"
 #include "src/common/dummyserver.h"
@@ -46,7 +49,9 @@ using ::curvefs::common::S3Info;
 using ::curvefs::common::Volume;
 using ::curvefs::mds::topology::PartitionTxId;
 using ::curvefs::mds::FSStatusCode_Name;
-using ::curvefs::client::common::MAXXATTRLENGTH;
+using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH;
+using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
+using ::curvefs::client::common::FileHandle;
 
 #define RETURN_IF_UNSUCCESS(action)                                            \
     do {                                                                       \
@@ -159,6 +164,8 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                                        : mOpts->mountPoint);
     std::string fsName = (mOpts->fsName == nullptr) ? "" : mOpts->fsName;
 
+    mountpoint_.set_cto(FLAGS_enableCto);
+
     int retVal = SetHostPortInMountPoint(&mountpoint_);
     if (retVal < 0) {
         LOG(ERROR) << "Set Host and Port in MountPoint failed, ret = "
@@ -166,17 +173,6 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
         return CURVEFS_ERROR::INTERNAL;
     }
 
-    auto find = std::find_if(fsInfo_->mountpoints().begin(),
-                             fsInfo_->mountpoints().end(),
-                             [this](const Mountpoint& mp) {
-                                 return mp.path() == mountpoint_.path() &&
-                                        mp.hostname() == mountpoint_.hostname();
-                             });
-    if (find != fsInfo_->mountpoints().end()) {
-        LOG(ERROR) << "MountFs found mountPoint exist";
-        return CURVEFS_ERROR::MOUNT_POINT_EXIST;
-    }
-
     auto ret = mdsClient_->MountFs(fsName, mountpoint_, fsInfo_.get());
     if (ret != FSStatusCode::OK && ret != FSStatusCode::MOUNT_POINT_EXIST) {
         LOG(ERROR) << "MountFs failed, FSStatusCode = " << ret
@@ -191,8 +187,7 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
     dentryManager_->SetFsId(fsInfo_->fsid());
     enableSumInDir_ = fsInfo_->enablesumindir() && !FLAGS_enableCto;
     LOG(INFO) << "Mount " << fsName << " on " << mountpoint_.ShortDebugString()
-              << " success!"
-              << " enableSumInDir = " << enableSumInDir_;
+              << " success!" << " enableSumInDir = " << enableSumInDir_;
 
     fsMetric_ = std::make_shared<FSMetric>(fsName);
 
@@ -243,7 +238,7 @@ void FuseClient::FuseOpDestroy(void *userdata) {
     return;
 }
 
-void InodeAttr2ParamAttr(InodeAttr inodeAttr, struct stat *attr) {
+void InodeAttr2ParamAttr(const InodeAttr &inodeAttr, struct stat *attr) {
     attr->st_ino = inodeAttr.inodeid();
     attr->st_mode = inodeAttr.mode();
     attr->st_nlink = inodeAttr.nlink();
@@ -271,12 +266,12 @@ void InodeAttr2ParamAttr(InodeAttr inodeAttr, struct stat *attr) {
 
 void GetDentryParamFromInodeAttr(
     const FuseClientOption &option,
-    InodeAttr inodeAttr,
+    const InodeAttr &inodeAttr,
     fuse_entry_param *param) {
     memset(param, 0, sizeof(fuse_entry_param));
     param->ino = inodeAttr.inodeid();
     param->generation = 0;
-    InodeAttr2ParamAttr(std::move(inodeAttr), &param->attr);
+    InodeAttr2ParamAttr(inodeAttr, &param->attr);
     param->attr_timeout = option.attrTimeOut;
     param->entry_timeout = option.entryTimeOut;
 }
@@ -302,21 +297,20 @@ CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
 
     fuse_ino_t ino = dentry.inodeid();
     InodeAttr attr;
-    ret = inodeManager_->GetInodeAttr(ino, &attr, parent);
+    ret = inodeManager_->GetInodeAttr(ino, &attr);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                   << ", parentId = " << parent
                    << ", inodeid = " << ino;
         return ret;
     }
 
-    GetDentryParamFromInodeAttr(option_, std::move(attr), e);
+    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
                                      struct fuse_file_info *fi) {
-    LOG(INFO) << "FuseOpOpen, ino: " << ino;
+    VLOG(1) << "FuseOpOpen, ino: " << ino;
     std::shared_ptr<InodeWrapper> inodeWrapper;
     CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
@@ -326,29 +320,17 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
     }
 
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-
-    ret = inodeWrapper->Open();
-    if (ret != CURVEFS_ERROR::OK) {
-        return ret;
-    }
-
     if (fi->flags & O_TRUNC) {
         if (fi->flags & O_WRONLY || fi->flags & O_RDWR) {
-            Inode *inode = inodeWrapper->GetMutableInodeUnlocked();
-            uint64_t length = inode->length();
-            CURVEFS_ERROR tRet = Truncate(inode, 0);
+            uint64_t length = inodeWrapper->GetLengthLocked();
+            CURVEFS_ERROR tRet = Truncate(inodeWrapper.get(), 0);
             if (tRet != CURVEFS_ERROR::OK) {
                 LOG(ERROR) << "truncate file fail, ret = " << ret
                            << ", inodeid = " << ino;
                 return CURVEFS_ERROR::INTERNAL;
             }
-            inode->set_length(0);
-            struct timespec now;
-            clock_gettime(CLOCK_REALTIME, &now);
-            inode->set_ctime(now.tv_sec);
-            inode->set_ctime_ns(now.tv_nsec);
-            inode->set_mtime(now.tv_sec);
-            inode->set_mtime_ns(now.tv_nsec);
+            inodeWrapper->SetLengthLocked(0);
+            inodeWrapper->UpdateTimestampLocked(kChangeTime | kModifyTime);
             if (length != 0) {
                 ret = inodeWrapper->Sync();
                 if (ret != CURVEFS_ERROR::OK) {
@@ -360,6 +342,7 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
 
             if (enableSumInDir_ && length != 0) {
                 // update parent summary info
+                const Inode *inode = inodeWrapper->GetInodeLocked();
                 XAttr xattr;
                 xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
                     std::to_string(length)});
@@ -377,7 +360,9 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
             return CURVEFS_ERROR::NOPERMISSION;
         }
     }
-
+    if (FLAGS_enableCto) {
+        inodeManager_->AddOpenedInode(ino);
+    }
     return ret;
 }
 
@@ -393,10 +378,7 @@ CURVEFS_ERROR FuseClient::UpdateParentInodeMCTimeAndInvalidNlink(
 
     {
         curve::common::UniqueLock lk = parentInodeWrapper->GetUniqueLock();
-        struct timespec now;
-        clock_gettime(CLOCK_REALTIME, &now);
-        parentInodeWrapper->SetMTime(now.tv_sec, now.tv_nsec);
-        parentInodeWrapper->SetCTime(now.tv_sec, now.tv_nsec);
+        parentInodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
 
         if (FsFileType::TYPE_DIRECTORY == type) {
             parentInodeWrapper->InvalidateNlink();
@@ -499,25 +481,23 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
     }
 
     InodeAttr attr;
-    inodeWrapper->GetInodeAttrLocked(&attr);
-    GetDentryParamFromInodeAttr(option_, std::move(attr), e);
+    inodeWrapper->GetInodeAttr(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
                                       const char *name, mode_t mode,
                                       fuse_entry_param *e) {
-    LOG(INFO) << "FuseOpMkDir, parent: " << parent
-              << ", name: " << name
-              << ", mode: " << mode;
+    VLOG(1) << "FuseOpMkDir, parent: " << parent << ", name: " << name
+            << ", mode: " << mode;
     return MakeNode(req, parent, name, S_IFDIR | mode,
                     FsFileType::TYPE_DIRECTORY, 0, e);
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
                                       const char *name) {
-    LOG(INFO) << "FuseOpRmDir, parent: " << parent
-              << ", name: " << name;
+    VLOG(1) << "FuseOpRmDir, parent: " << parent << ", name: " << name;
     return RemoveNode(req, parent, name, FsFileType::TYPE_DIRECTORY);
 }
 
@@ -575,7 +555,7 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
         return ret;
     }
 
-    ret = inodeWrapper->UnLinkLocked(parent);
+    ret = inodeWrapper->UnLink(parent);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "UnLink failed, ret = " << ret << ", inodeid = " << ino
                    << ", parent = " << parent << ", name = " << name;
@@ -594,9 +574,9 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
             std::to_string(inodeWrapper->GetLength())});
         auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, false);
         if (tret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "UpdateParentInodeXattr failed,"
-                       << " inodeId = " << parent
-                       << ", xattr = " << xattr.DebugString();
+            LOG(WARNING) << "UpdateParentInodeXattr failed,"
+                         << " inodeId = " << parent
+                         << ", xattr = " << xattr.DebugString();
         }
     }
 
@@ -654,7 +634,7 @@ static void dirbuf_add(fuse_req_t req, struct DirBufferHead *b,
         b->size += fuse_add_direntry_plus(req, NULL, 0, dentry.name().c_str(),
                                           NULL, 0);
         b->p = static_cast<char *>(realloc(b->p, b->size));
-        GetDentryParamFromInodeAttr(option, std::move(*attr), &param);
+        GetDentryParamFromInodeAttr(option, *attr, &param);
         fuse_add_direntry_plus(req, b->p + oldsize, b->size - oldsize,
                                dentry.name().c_str(), &param, b->size);
     }
@@ -675,8 +655,6 @@ CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
         return ret;
     }
 
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-
     uint64_t dindex = fi->fh;
     DirBufferHead *bufHead = dirBuf_->DirBufferGet(dindex);
     if (!bufHead->wasRead) {
@@ -700,7 +678,7 @@ CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
                 inodeIds.emplace(dentry.inodeid());
             }
             VLOG(3) << "batch get inode size = " << inodeIds.size();
-            ret = inodeManager_->BatchGetInodeAttrAsync(ino, inodeIds,
+            ret = inodeManager_->BatchGetInodeAttrAsync(ino, &inodeIds,
                                                         &inodeAttrMap);
             if (ret != CURVEFS_ERROR::OK) {
                 LOG(ERROR) << "BatchGetInodeAttr failed when FuseOpReadDir"
@@ -735,8 +713,8 @@ CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
                                        const char *name, fuse_ino_t newparent,
                                        const char *newname) {
-    LOG(INFO) << "FuseOpRename from (" << parent << ", " << name << ") to ("
-              << newparent << ", " << newname << ")";
+    VLOG(1) << "FuseOpRename from (" << parent << ", " << name << ") to ("
+            << newparent << ", " << newname << ")";
     if (strlen(name) > option_.maxNameLength ||
         strlen(newname) > option_.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
@@ -750,6 +728,7 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
 
     curve::common::LockGuard lg(renameMutex_);
     CURVEFS_ERROR rc = CURVEFS_ERROR::OK;
+    VLOG(3) << "FuseOpRename [start]: " << renameOp.DebugString();
     RETURN_IF_UNSUCCESS(GetTxId);
     RETURN_IF_UNSUCCESS(Precheck);
     RETURN_IF_UNSUCCESS(RecordOldInodeInfo);
@@ -758,6 +737,7 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
     RETURN_IF_UNSUCCESS(LinkDestParentInode);
     RETURN_IF_UNSUCCESS(PrepareTx);
     RETURN_IF_UNSUCCESS(CommitTx);
+    VLOG(3) << "FuseOpRename [success]: " << renameOp.DebugString();
     // Do not check UnlinkSrcParentInode, beause rename is already success
     renameOp.UnlinkSrcParentInode();
     renameOp.UnlinkOldInode();
@@ -777,13 +757,14 @@ CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct stat *attr) {
     VLOG(1) << "FuseOpGetAttr ino = " << ino;
     InodeAttr inodeAttr;
-    CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
+    CURVEFS_ERROR ret =
+        inodeManager_->GetInodeAttr(ino, &inodeAttr);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
                    << ", inodeid = " << ino;
         return ret;
     }
-    InodeAttr2ParamAttr(std::move(inodeAttr), attr);
+    InodeAttr2ParamAttr(inodeAttr, attr);
     return ret;
 }
 
@@ -791,9 +772,8 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct stat *attr, int to_set,
                                         struct fuse_file_info *fi,
                                         struct stat *attrOut) {
-    LOG(INFO) << "FuseOpSetAttr to_set: " << to_set
-              << ", ino: " << ino
-              << ", attr: " << *attr;
+    VLOG(1) << "FuseOpSetAttr to_set: " << to_set << ", ino: " << ino
+            << ", attr: " << *attr;
     std::shared_ptr<InodeWrapper> inodeWrapper;
     CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
@@ -803,62 +783,59 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
     }
 
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-    Inode *inode = inodeWrapper->GetMutableInodeUnlocked();
+    if (to_set & FUSE_SET_ATTR_MODE) {
+        inodeWrapper->SetMode(attr->st_mode);
+    }
+    if (to_set & FUSE_SET_ATTR_UID) {
+        inodeWrapper->SetUid(attr->st_uid);
+    }
+    if (to_set & FUSE_SET_ATTR_GID) {
+        inodeWrapper->SetGid(attr->st_gid);
+    }
 
     struct timespec now;
     clock_gettime(CLOCK_REALTIME, &now);
 
-    if (to_set & FUSE_SET_ATTR_MODE) {
-        inode->set_mode(attr->st_mode);
-    }
-    if (to_set & FUSE_SET_ATTR_UID) {
-        inode->set_uid(attr->st_uid);
-    }
-    if (to_set & FUSE_SET_ATTR_GID) {
-        inode->set_gid(attr->st_gid);
-    }
     if (to_set & FUSE_SET_ATTR_ATIME) {
-        inode->set_atime(attr->st_atim.tv_sec);
-        inode->set_atime_ns(attr->st_atim.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(attr->st_atim, kAccessTime);
     }
     if (to_set & FUSE_SET_ATTR_ATIME_NOW) {
-        inode->set_atime(now.tv_sec);
-        inode->set_atime_ns(now.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(now, kAccessTime);
     }
     if (to_set & FUSE_SET_ATTR_MTIME) {
-        inode->set_mtime(attr->st_mtim.tv_sec);
-        inode->set_mtime_ns(attr->st_mtim.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(attr->st_mtim, kModifyTime);
     }
     if (to_set & FUSE_SET_ATTR_MTIME_NOW) {
-        inode->set_mtime(now.tv_sec);
-        inode->set_mtime_ns(now.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(now, kModifyTime);
     }
     if (to_set & FUSE_SET_ATTR_CTIME) {
-        inode->set_ctime(attr->st_ctim.tv_sec);
-        inode->set_ctime_ns(attr->st_ctim.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(attr->st_ctim, kChangeTime);
     } else {
-        inode->set_ctime(now.tv_sec);
-        inode->set_ctime_ns(now.tv_nsec);
+        inodeWrapper->UpdateTimestampLocked(now, kChangeTime);
     }
+
     if (to_set & FUSE_SET_ATTR_SIZE) {
-        int64_t changeSize =  attr->st_size - inode->length();
-        CURVEFS_ERROR tRet = Truncate(inode, attr->st_size);
+        int64_t changeSize =
+            attr->st_size -
+            static_cast<int64_t>(inodeWrapper->GetLengthLocked());
+        CURVEFS_ERROR tRet = Truncate(inodeWrapper.get(), attr->st_size);
         if (tRet != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "truncate file fail, ret = " << ret
                        << ", inodeid = " << ino;
             return tRet;
         }
-        inode->set_length(attr->st_size);
+        inodeWrapper->SetLengthLocked(attr->st_size);
         ret = inodeWrapper->Sync();
         if (ret != CURVEFS_ERROR::OK) {
             return ret;
         }
         InodeAttr inodeAttr;
-        inodeWrapper->GetInodeAttrUnlocked(&inodeAttr);
-        InodeAttr2ParamAttr(std::move(inodeAttr), attrOut);
+        inodeWrapper->GetInodeAttrLocked(&inodeAttr);
+        InodeAttr2ParamAttr(inodeAttr, attrOut);
 
         if (enableSumInDir_ && changeSize != 0) {
             // update parent summary info
+            const Inode* inode = inodeWrapper->GetInodeLocked();
             XAttr xattr;
             xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
                 std::to_string(std::abs(changeSize))});
@@ -880,27 +857,13 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
         return ret;
     }
     InodeAttr inodeAttr;
-    inodeWrapper->GetInodeAttrUnlocked(&inodeAttr);
-    InodeAttr2ParamAttr(std::move(inodeAttr), attrOut);
+    inodeWrapper->GetInodeAttrLocked(&inodeAttr);
+    InodeAttr2ParamAttr(inodeAttr, attrOut);
     return ret;
 }
 
-bool IsSummaryInfo(const char *name) {
-    return std::strstr(name, SUMMARYPREFIX);
-}
-
-bool IsOneLayer(const char *name) {
-    if (std::strcmp(name, XATTRFILES) == 0 ||
-        std::strcmp(name, XATTRSUBDIRS) == 0 ||
-        std::strcmp(name, XATTRENTRIES) == 0 ||
-        std::strcmp(name, XATTRFBYTES) == 0) {
-        return true;
-    }
-    return false;
-}
-
 CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                          size_t size) {
     VLOG(9) << "FuseOpGetXattr, ino: " << ino
             << ", name: " << name << ", size = " << size;
@@ -908,51 +871,28 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
         return CURVEFS_ERROR::NOTSUPPORT;
     }
 
-    std::string xValue;
-    // get summary info
-    if (IsSummaryInfo(name)) {
-        InodeAttr inodeAttr;
-        CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                       << ", inodeid = " << ino;
-            return ret;
-        }
-
-        if (inodeAttr.type() == FsFileType::TYPE_DIRECTORY) {
-            // not enable record summary info in dir xattr,
-            // need recursive computation all files
-            if (!enableSumInDir_) {
-                if (IsOneLayer(name)) {
-                    ret = xattrManager_->CalOneLayerSumInfo(&inodeAttr);
-                } else {
-                    ret = xattrManager_->CalAllLayerSumInfo(&inodeAttr);
-                }
-            } else {
-                if (IsOneLayer(name)) {
-                    ret = xattrManager_->FastCalOneLayerSumInfo(&inodeAttr);
-                } else {
-                    ret = xattrManager_->FastCalAllLayerSumInfo(&inodeAttr);
-                }
-            }
-
-            if (CURVEFS_ERROR::OK != ret) {
-                return ret;
-            }
-            LOG(INFO) << "After calculate summary info:\n"
-                      << inodeAttr.DebugString();
-            auto it = inodeAttr.xattr().find(name);
-            if (it != inodeAttr.xattr().end()) {
-                xValue = it->second;
-            }
-        }
+    InodeAttr inodeAttr;
+    CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
+                    << ", inodeid = " << ino;
+        return ret;
     }
 
-    CURVEFS_ERROR ret = CURVEFS_ERROR::NODATA;
-    if (xValue.length() > 0) {
-        if ((size == 0 && xValue.length() <= MAXXATTRLENGTH) ||
-            (size >= xValue.length() && size <= MAXXATTRLENGTH)) {
-            memcpy(value, xValue.c_str(), xValue.length());
+    ret = xattrManager_->GetXattr(name, value, &inodeAttr, enableSumInDir_);
+    if (CURVEFS_ERROR::OK != ret) {
+        LOG(ERROR) << "xattrManager get xattr failed, name = " << name;
+        return ret;
+    }
+
+    ret = CURVEFS_ERROR::NODATA;
+    if (value->length() > 0) {
+        if ((size == 0 && value->length() <= MAX_XATTR_VALUE_LENGTH) ||
+            (size >= value->length() &&
+                value->length() <= MAX_XATTR_VALUE_LENGTH)) {
+            VLOG(1) << "FuseOpGetXattr name = " << name
+                    << ", length = " << value->length()
+                    << ", value = " << *value;
             ret = CURVEFS_ERROR::OK;
         } else {
             ret = CURVEFS_ERROR::OUT_OF_RANGE;
@@ -961,10 +901,49 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
     return ret;
 }
 
+CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
+                                         const char* name, const char* value,
+                                         size_t size, int flags) {
+    if (option_.disableXattr) {
+        return CURVEFS_ERROR::NOTSUPPORT;
+    }
+
+    std::string strname(name);
+    std::string strvalue(value, size);
+    VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
+            << ", size = " << size
+            << ", strvalue: " << strvalue;
+    if (strname.length() > MAX_XATTR_NAME_LENGTH  ||
+        size > MAX_XATTR_VALUE_LENGTH) {
+        LOG(ERROR) << "xattr length is too long, name = " << name
+                   << ", name length = " << strname.length()
+                   << ", value length = " << size;
+        return CURVEFS_ERROR::OUT_OF_RANGE;
+    }
+
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
+    }
+
+    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+    inodeWrapper->SetXattrLocked(strname, strvalue);
+    ret = inodeWrapper->SyncAttr();
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "set xattr fail, ret = " << ret << ", inodeid = " << ino
+                   << ", name = " << strname << ", value = " << strvalue;
+        return ret;
+    }
+    VLOG(1) << "FuseOpSetXattr end";
+    return CURVEFS_ERROR::OK;
+}
+
 CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize) {
-    LOG(INFO) << "FuseOpListXattr, ino: " << ino
-              << ", size = " << size;
+    VLOG(1) << "FuseOpListXattr, ino: " << ino << ", size = " << size;
     InodeAttr inodeAttr;
     CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
     if (ret != CURVEFS_ERROR::OK) {
@@ -1083,8 +1062,8 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
     }
 
     InodeAttr attr;
-    inodeWrapper->GetInodeAttrLocked(&attr);
-    GetDentryParamFromInodeAttr(option_, std::move(attr), e);
+    inodeWrapper->GetInodeAttr(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
@@ -1102,7 +1081,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
                    << ", inodeid = " << ino;
         return ret;
     }
-    ret = inodeWrapper->LinkLocked(newparent);
+    ret = inodeWrapper->Link(newparent);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "Link Inode fail, ret = " << ret << ", inodeid = " << ino
                    << ", newparent = " << newparent
@@ -1120,7 +1099,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
         LOG(ERROR) << "dentryManager_ CreateDentry fail, ret = " << ret
                    << ", parent = " << newparent << ", name = " << newname;
 
-        CURVEFS_ERROR ret2 = inodeWrapper->UnLinkLocked(newparent);
+        CURVEFS_ERROR ret2 = inodeWrapper->UnLink(newparent);
         if (ret2 != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "Also unlink inode failed, ret = " << ret2
                        << ", inodeid = " << inodeWrapper->GetInodeId();
@@ -1154,15 +1133,14 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
     }
 
     InodeAttr attr;
-    inodeWrapper->GetInodeAttrLocked(&attr);
-    GetDentryParamFromInodeAttr(option_, std::move(attr), e);
+    inodeWrapper->GetInodeAttr(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string *linkStr) {
-    LOG(INFO) << "FuseOpReadLink, ino: " << ino
-              << ", linkStr: " << linkStr;
+    VLOG(1) << "FuseOpReadLink, ino: " << ino << ", linkStr: " << linkStr;
     std::shared_ptr<InodeWrapper> inodeWrapper;
     CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
@@ -1176,27 +1154,11 @@ CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
 
 CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
-    LOG(INFO) << "FuseOpRelease, ino: " << ino;
-    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    ret = inodeManager_->GetInode(ino, inodeWrapper);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                   << ", ino: " << ino;
-        return ret;
+    VLOG(1) << "FuseOpRelease, ino: " << ino;
+    if (FLAGS_enableCto) {
+        inodeManager_->RemoveOpenedInode(ino);
     }
-
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-
-    ret = inodeWrapper->Release();
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager release inode fail, ret = " << ret
-                   << ", ino: " << ino;
-        return ret;
-    }
-
-    LOG(INFO) << "FuseOpRelease, ino: " << ino << " success";
-    return ret;
+    return CURVEFS_ERROR::OK;
 }
 
 void FuseClient::FlushInode() { inodeManager_->FlushInodeOnce(); }

curvefs/src/client/fuse_client.h

@@ -180,9 +180,13 @@ class FuseClient {
                                         struct stat* attrOut);
 
     virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                          size_t size);
 
+    virtual CURVEFS_ERROR FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
+                                         const char* name, const char* value,
+                                         size_t size, int flags);
+
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 
@@ -272,7 +276,7 @@ class FuseClient {
     }
 
  private:
-    virtual CURVEFS_ERROR Truncate(Inode* inode, uint64_t length) = 0;
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
 
     virtual void FlushData() = 0;
 

curvefs/src/client/fuse_s3_client.cpp

@@ -51,7 +51,9 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
     SetFuseClientS3Option(&opt, fsS3Option);
 
     auto s3Client = std::make_shared<S3ClientImpl>();
+    auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
     s3Client->Init(opt.s3Opt.s3AdaptrOpt);
+    s3DiskCacheClient->Init(opt.s3Opt.s3AdaptrOpt);
     auto fsCacheManager = std::make_shared<FsCacheManager>(
         dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
         opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
@@ -64,7 +66,7 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
         auto diskCacheManager = std::make_shared<DiskCacheManager>(
             wrapper, diskCacheWrite, diskCacheRead);
         auto diskCacheManagerImpl = std::make_shared<DiskCacheManagerImpl>(
-            diskCacheManager, s3Client.get());
+            diskCacheManager, s3DiskCacheClient);
         ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
                                inodeManager_, mdsClient_, fsCacheManager,
                                diskCacheManagerImpl, true);
@@ -126,21 +128,16 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
     }
 
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-    Inode *inode = inodeWrapper->GetMutableInodeUnlocked();
 
     *wSize = wRet;
     size_t changeSize = 0;
     // update file len
-    if (inode->length() < off + *wSize) {
-        changeSize = off + *wSize - inode->length();
-        inode->set_length(off + *wSize);
+    if (inodeWrapper->GetLengthLocked() < off + *wSize) {
+        changeSize = off + *wSize - inodeWrapper->GetLengthLocked();
+        inodeWrapper->SetLengthLocked(off + *wSize);
     }
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    inode->set_mtime(now.tv_sec);
-    inode->set_mtime_ns(now.tv_nsec);
-    inode->set_ctime(now.tv_sec);
-    inode->set_ctime_ns(now.tv_nsec);
+
+    inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
 
     inodeManager_->ShipToFlush(inodeWrapper);
 
@@ -149,6 +146,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
     }
 
     if (enableSumInDir_ && changeSize != 0) {
+        const Inode* inode = inodeWrapper->GetInodeLocked();
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
             std::to_string(changeSize)});
@@ -212,13 +210,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
     }
 
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-    Inode *newInode = inodeWrapper->GetMutableInodeUnlocked();
-
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    newInode->set_atime(now.tv_sec);
-    newInode->set_atime_ns(now.tv_nsec);
-
+    inodeWrapper->UpdateTimestampLocked(kAccessTime);
     inodeManager_->ShipToFlush(inodeWrapper);
 
     VLOG(6) << "read end, read size = " << *rSize;
@@ -229,9 +221,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                          const char *name, mode_t mode,
                                          struct fuse_file_info *fi,
                                          fuse_entry_param *e) {
-    LOG(INFO) << "FuseOpCreate, parent: " << parent
-              << ", name: " << name
-              << ", mode: " << mode;
+    VLOG(1) << "FuseOpCreate, parent: " << parent << ", name: " << name
+            << ", mode: " << mode;
     CURVEFS_ERROR ret =
         MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, e);
     if (ret != CURVEFS_ERROR::OK) {
@@ -243,34 +234,30 @@ CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
 CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
                                         const char *name, mode_t mode,
                                         dev_t rdev, fuse_entry_param *e) {
-    LOG(INFO) << "FuseOpMkNod, parent: " << parent
-              << ", name: " << name
-              << ", mode: " << mode
-              << ", rdev: " << rdev;
+    VLOG(1) << "FuseOpMkNod, parent: " << parent << ", name: " << name
+            << ", mode: " << mode << ", rdev: " << rdev;
     return MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, rdev, e);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
                                      fuse_ino_t newparent, const char *newname,
                                      fuse_entry_param *e) {
-    LOG(INFO) << "FuseOpLink, ino: " << ino
-              << ", newparent: " << newparent
-              << ", newname: " << newname;
+    VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
+            << ", newname: " << newname;
     return FuseClient::FuseOpLink(
         req, ino, newparent, newname, FsFileType::TYPE_S3, e);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                          const char *name) {
-    LOG(INFO) << "FuseOpUnlink, parent: " << parent
-              << ", name: " << name;
+    VLOG(1) << "FuseOpUnlink, parent: " << parent << ", name: " << name;
     return RemoveNode(req, parent, name, FsFileType::TYPE_S3);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                         int datasync,
                                         struct fuse_file_info *fi) {
-    LOG(INFO) << "FuseOpFsync, ino: " << ino << ", datasync: " << datasync;
+    VLOG(1) << "FuseOpFsync, ino: " << ino << ", datasync: " << datasync;
 
     CURVEFS_ERROR ret = s3Adaptor_->Flush(ino);
     if (ret != CURVEFS_ERROR::OK) {
@@ -292,13 +279,13 @@ CURVEFS_ERROR FuseS3Client::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
     return inodeWrapper->Sync();
 }
 
-CURVEFS_ERROR FuseS3Client::Truncate(Inode *inode, uint64_t length) {
+CURVEFS_ERROR FuseS3Client::Truncate(InodeWrapper *inode, uint64_t length) {
     return s3Adaptor_->Truncate(inode, length);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
-    LOG(INFO) << "FuseOpFlush, ino: " << ino;
+    VLOG(1) << "FuseOpFlush, ino: " << ino;
     CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
 
     // if enableCto, flush all write cache both in memory cache and disk cache
@@ -336,7 +323,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
         }
     }
 
-    LOG(INFO) << "FuseOpFlush, ino: " << ino << " flush ok";
+    VLOG(1) << "FuseOpFlush, ino: " << ino << " flush ok";
     return CURVEFS_ERROR::OK;
 }
 

curvefs/src/client/fuse_s3_client.h

@@ -85,7 +85,7 @@ class FuseS3Client : public FuseClient {
         struct fuse_file_info *fi) override;
 
  private:
-    CURVEFS_ERROR Truncate(Inode *inode, uint64_t length) override;
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
 
     void FlushData() override;
 

curvefs/src/client/fuse_volume_client.cpp

@@ -255,17 +255,15 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
 CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
                                      fuse_ino_t newparent, const char *newname,
                                      fuse_entry_param *e) {
-    LOG(INFO) << "FuseOpLink, ino: " << ino
-              << ", newparent: " << newparent
-              << ", newname: " << newname;
+    VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
+            << ", newname: " << newname;
     return FuseClient::FuseOpLink(
         req, ino, newparent, newname, FsFileType::TYPE_FILE, e);
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                              const char *name) {
-    LOG(INFO) << "FuseOpUnlink, parent: " << parent
-              << ", name: " << name;
+    VLOG(1) << "FuseOpUnlink, parent: " << parent << ", name: " << name;
     return RemoveNode(req, parent, name, FsFileType::TYPE_FILE);
 }
 
@@ -297,7 +295,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
     return inodeWrapper->Sync();
 }
 
-CURVEFS_ERROR FuseVolumeClient::Truncate(Inode *inode, uint64_t length) {
+CURVEFS_ERROR FuseVolumeClient::Truncate(InodeWrapper *inode, uint64_t length) {
     // Todo: call volume truncate
     return CURVEFS_ERROR::OK;
 }

curvefs/src/client/fuse_volume_client.h

@@ -97,7 +97,7 @@ class FuseVolumeClient : public FuseClient {
     void SetVolumeStorageForTesting(VolumeStorage *storage);
 
  private:
-    CURVEFS_ERROR Truncate(Inode *inode, uint64_t length) override;
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
 
     void FlushData() override;
 

curvefs/src/client/inode_cache_manager.cpp

@@ -46,102 +46,88 @@ namespace curvefs {
 namespace client {
 
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
+using curvefs::client::common::FLAGS_enableCto;
 
 bool IsNotDirtyInode(const std::shared_ptr<InodeWrapper> &inode) {
     return !inode->IsDirty() && inode->S3ChunkInfoEmpty();
 }
 
-CURVEFS_ERROR InodeCacheManagerImpl::GetInode(uint64_t inodeId,
-    std::shared_ptr<InodeWrapper> &out) {
+#define GET_INODE_REMOTE(FSID, INODEID, OUT, STREAMING)                        \
+    MetaStatusCode ret = metaClient_->GetInode(FSID, INODEID, OUT, STREAMING); \
+    if (ret != MetaStatusCode::OK) {                                           \
+        LOG_IF(ERROR, ret != MetaStatusCode::NOT_FOUND)                        \
+            << "metaClient_ GetInode failed, MetaStatusCode = " << ret         \
+            << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)          \
+            << ", inodeid = " << INODEID;                                      \
+        return MetaStatusCodeToCurvefsErrCode(ret);                            \
+    }
+
+#define PUT_INODE_CACHE(INODEID, INODEWRAPPER)                                 \
+    std::shared_ptr<InodeWrapper> eliminatedOne;                               \
+    bool eliminated = iCache_->Put(INODEID, INODEWRAPPER, &eliminatedOne);     \
+    if (eliminated) {                                                          \
+        VLOG(3) << "GetInode eliminate one inode, ino: "                       \
+                << eliminatedOne->GetInodeId()                                 \
+                << ", iCache does not evict inodes via put interface ";        \
+        assert(0);                                                             \
+    }
+
+#define REFRESH_DATA_REMOTE(OUT, STREAMING)                                    \
+    CURVEFS_ERROR rc = RefreshData(OUT, STREAMING);                            \
+    if (rc != CURVEFS_ERROR::OK) {                                             \
+        return rc;                                                             \
+    }
+
+CURVEFS_ERROR
+InodeCacheManagerImpl::GetInode(uint64_t inodeId,
+                                std::shared_ptr<InodeWrapper> &out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
+    // get inode from cache
     bool ok = iCache_->Get(inodeId, &out);
-    if (ok) {
-        // if enableCto, we need and is unopen, we need reload from metaserver
-        if (curvefs::client::common::FLAGS_enableCto && !out->IsOpen()) {
-            VLOG(6) << "InodeCacheManagerImpl, GetInode: enableCto and inode: "
-                    << inodeId << " opencount is 0";
-            iCache_->Remove(inodeId);
-        } else {
-            return CURVEFS_ERROR::OK;
-        }
+    if (ok && NeedUseCahce(inodeId, out->IsDirty())) {
+        return CURVEFS_ERROR::OK;
     }
 
+    // get inode from metaserver
     Inode inode;
-    bool streaming;
+    bool streaming = false;
+    GET_INODE_REMOTE(fsId_, inodeId, &inode, &streaming);
+    out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_);
 
-    MetaStatusCode ret2 = metaClient_->GetInode(
-        fsId_, inodeId, &inode, &streaming);
+    // refresh data
+    REFRESH_DATA_REMOTE(out, streaming);
 
-    if (ret2 != MetaStatusCode::OK) {
-        LOG_IF(ERROR, ret2 != MetaStatusCode::NOT_FOUND)
-            << "metaClient_ GetInode failed, MetaStatusCode = " << ret2
-            << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret2)
-            << ", inodeid = " << inodeId;
-        return MetaStatusCodeToCurvefsErrCode(ret2);
-    }
+    // put to cache
+    PUT_INODE_CACHE(inodeId, out);
 
-    auto type = inode.type();
-    out = std::make_shared<InodeWrapper>(
-        std::move(inode), metaClient_);
-
-    // NOTE: if the s3chunkinfo inside inode is too large,
-    // we should invoke RefreshS3ChunkInfo() to receive s3chunkinfo
-    // by streaming and padding its into inode.
-    if (type == FsFileType::TYPE_S3 && streaming) {
-        CURVEFS_ERROR rc = out->RefreshS3ChunkInfo();
-        if (rc != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "RefreshS3ChunkInfo() failed, retCode = " << rc;
-            return rc;
-        }
-    } else if (type == FsFileType::TYPE_FILE) {
-        auto rc = out->RefreshVolumeExtent();
-        if (rc != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "RefreshVolumeExtent failed, error: " << rc;
-            return rc;
-        }
-    }
-
-    std::shared_ptr<InodeWrapper> eliminatedOne;
-    bool eliminated = iCache_->Put(inodeId, out, &eliminatedOne);
-    if (eliminated) {
-        VLOG(3) << "GetInode eliminate one inode, ino: "
-                << eliminatedOne->GetInodeId();
-        eliminatedOne->FlushAsync();
-    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
-    InodeAttr *out, uint64_t parentId) {
+                                                  InodeAttr *out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
     // 1. find in icache
     std::shared_ptr<InodeWrapper> inodeWrapper;
     bool ok = iCache_->Get(inodeId, &inodeWrapper);
-    if (ok) {
-        if (curvefs::client::common::FLAGS_enableCto &&
-            !inodeWrapper->IsOpen()) {
-            iCache_->Remove(inodeId);
-        } else {
-            inodeWrapper->GetInodeAttrLocked(out);
-            return CURVEFS_ERROR::OK;
-        }
+    if (ok && NeedUseCahce(inodeId, inodeWrapper->IsDirty())) {
+        inodeWrapper->GetInodeAttr(out);
+        return CURVEFS_ERROR::OK;
     }
 
     // 2. get form metaserver
     std::set<uint64_t> inodeIds;
     std::list<InodeAttr> attrs;
     inodeIds.emplace(inodeId);
-    MetaStatusCode ret = metaClient_->BatchGetInodeAttr(
-        fsId_, inodeIds, &attrs);
+    MetaStatusCode ret =
+        metaClient_->BatchGetInodeAttr(fsId_, inodeIds, &attrs);
     if (MetaStatusCode::OK != ret) {
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed"
-                   << ", inodeId = " << inodeId
-                   << ", MetaStatusCode = " << ret
+                   << ", inodeId = " << inodeId << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
         return MetaStatusCodeToCurvefsErrCode(ret);
     }
 
-    if (attrs.size() !=  1) {
+    if (attrs.size() != 1) {
         LOG(ERROR) << "metaClient BatchGetInodeAttr error,"
                    << " getSize is 1, inodeId = " << inodeId
                    << "but real size = " << attrs.size();
@@ -154,27 +140,28 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
     std::set<uint64_t> *inodeIds,
-    std::list<InodeAttr> *attr) {
-    // get some inode in icache
+    std::list<InodeAttr> *attrs) {
+    // get some inode attr in icache
     for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok) {
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
             InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttrLocked(&tmpAttr);
-            attr->emplace_back(tmpAttr);
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace_back(std::move(tmpAttr));
             iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
+            continue;
         }
+        ++iter;
     }
 
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
 
-    MetaStatusCode ret = metaClient_->BatchGetInodeAttr(fsId_, *inodeIds, attr);
+    MetaStatusCode ret = metaClient_->BatchGetInodeAttr(fsId_, *inodeIds,
+        attrs);
     if (MetaStatusCode::OK != ret) {
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed, MetaStatusCode = "
                    << ret << ", MetaStatusCode_Name = "
@@ -185,21 +172,37 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     uint64_t parentId,
-    const std::set<uint64_t> &inodeIds,
+    std::set<uint64_t> *inodeIds,
     std::map<uint64_t, InodeAttr> *attrs) {
-    if (inodeIds.empty()) {
-        return CURVEFS_ERROR::OK;
+    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
+    std::map<uint64_t, InodeAttr> cachedAttr;
+    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
+
+    // get some inode attr in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+            InodeAttr tmpAttr;
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace(*iter, std::move(tmpAttr));
+            iter = inodeIds->erase(iter);
+        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
+            attrs->emplace(*iter, cachedAttr[*iter]);
+            iter = inodeIds->erase(iter);
+        } else {
+            ++iter;
+        }
     }
 
-    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    bool ok  = iAttrCache_->Get(parentId, attrs);
-    if (ok) {
+    if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
 
     // split inodeIds by partitionId and batch limit
     std::vector<std::vector<uint64_t>> inodeGroups;
-    if (!metaClient_->SplitRequestInodes(fsId_, inodeIds, &inodeGroups)) {
+    if (!metaClient_->SplitRequestInodes(fsId_, *inodeIds, &inodeGroups)) {
         return CURVEFS_ERROR::NOTEXIST;
     }
 
@@ -222,7 +225,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     // wait for all sudrequest finished
     cond->Wait();
 
-    ok  = iAttrCache_->Get(parentId, attrs);
+    bool ok  = iAttrCache_->Get(parentId, attrs);
     if (!ok) {
         LOG(WARNING) << "get attrs form iAttrCache_ failed.";
     }
@@ -237,10 +240,8 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok) {
-            XAttr tmpXattr;
-            inodeWrapper->GetXattrLocked(&tmpXattr);
-            xattr->emplace_back(tmpXattr);
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+            xattr->emplace_back(inodeWrapper->GetXattr());
             iter = inodeIds->erase(iter);
         } else {
             ++iter;
@@ -372,6 +373,12 @@ void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
                 iCache_->Remove(inodeId);
                 trimSize--;
             }
+            trimSize--;
+            // remove the attr of the inode in iattrcache
+            auto parents = inodeWrapper->GetParentLocked();
+            for (uint64_t parent : parents) {
+                iAttrCache_->Remove(parent, inodeId);
+            }
         } else {
             VLOG(9) << "iCache size " << iCache_->Size() << " wait inode flush";
             break;
@@ -379,5 +386,64 @@ void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
     }
 }
 
+CURVEFS_ERROR
+InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
+                                   bool streaming) {
+    auto type = inode->GetType();
+    CURVEFS_ERROR rc = CURVEFS_ERROR::OK;
+
+    switch (type) {
+    case FsFileType::TYPE_S3:
+        if (streaming) {
+            // NOTE: if the s3chunkinfo inside inode is too large,
+            // we should invoke RefreshS3ChunkInfo() to receive s3chunkinfo
+            // by streaming and padding its into inode.
+            rc = inode->RefreshS3ChunkInfo();
+            LOG_IF(ERROR, rc != CURVEFS_ERROR::OK)
+                << "RefreshS3ChunkInfo() failed, retCode = " << rc;
+        }
+        break;
+
+    case FsFileType::TYPE_FILE:
+        rc = inode->RefreshVolumeExtent();
+        LOG_IF(ERROR, rc != CURVEFS_ERROR::OK)
+            << "RefreshVolumeExtent failed, error: " << rc;
+        break;
+
+    default:
+        rc = CURVEFS_ERROR::OK;
+    }
+
+    return rc;
+}
+
+void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    openedInodes_.emplace(inodeId);
+}
+
+void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    if (iter != openedInodes_.end()) {
+        openedInodes_.erase(iter);
+    }
+}
+
+bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    return iter != openedInodes_.end();
+}
+
+bool InodeCacheManagerImpl::NeedUseCahce(uint64_t inodeId, bool IsDirty) {
+    if (!FLAGS_enableCto || OpenInodeCached(inodeId) || IsDirty) {
+        return true;
+    }
+    return false;
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_cache_manager.h

@@ -71,7 +71,7 @@ class InodeAttrCache {
         curve::common::LockGuard lg(iAttrCacheMutex_);
         auto iter = iAttrCache_.find(parentId);
         if (iter != iAttrCache_.end()) {
-            *imap = iter->second;
+            imap->insert(iter->second.begin(), iter->second.end());
             return true;
         }
         return false;
@@ -99,6 +99,14 @@ class InodeAttrCache {
                 << size << ", after = " << iAttrCache_.size();
     }
 
+    void Remove(uint64_t parentId, uint64_t inodeId) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            iter->second.erase(inodeId);
+        }
+    }
+
  private:
     // inodeAttr cache; <parentId, <inodeId, inodeAttr>>
     std::map<uint64_t, std::map<uint64_t, InodeAttr>> iAttrCache_;
@@ -122,11 +130,11 @@ class InodeCacheManager {
 
     virtual void Stop() = 0;
 
-    virtual CURVEFS_ERROR GetInode(uint64_t inodeId,
-        std::shared_ptr<InodeWrapper> &out) = 0;   // NOLINT
+    virtual CURVEFS_ERROR
+    GetInode(uint64_t inodeId,
+             std::shared_ptr<InodeWrapper> &out) = 0;  // NOLINT
 
-    virtual CURVEFS_ERROR GetInodeAttr(uint64_t inodeId,
-        InodeAttr *out, uint64_t parentId = 0) = 0;
+    virtual CURVEFS_ERROR GetInodeAttr(uint64_t inodeId, InodeAttr *out) = 0;
 
     virtual CURVEFS_ERROR BatchGetInodeAttr(
         std::set<uint64_t> *inodeIds,
@@ -134,7 +142,7 @@ class InodeCacheManager {
 
     virtual CURVEFS_ERROR BatchGetInodeAttrAsync(
         uint64_t parentId,
-        const std::set<uint64_t> &inodeIds,
+        std::set<uint64_t> *inodeIds,
         std::map<uint64_t, InodeAttr> *attrs) = 0;
 
     virtual CURVEFS_ERROR BatchGetXAttr(std::set<uint64_t> *inodeIds,
@@ -159,6 +167,10 @@ class InodeCacheManager {
 
     virtual void ReleaseCache(uint64_t parentId) = 0;
 
+    virtual void AddOpenedInode(uint64_t inodeId) = 0;
+
+    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
+
  protected:
     uint32_t fsId_;
 };
@@ -214,16 +226,15 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     }
 
     CURVEFS_ERROR GetInode(uint64_t inodeId,
-        std::shared_ptr<InodeWrapper> &out) override;
+                           std::shared_ptr<InodeWrapper> &out) override;
 
-    CURVEFS_ERROR GetInodeAttr(uint64_t inodeId,
-        InodeAttr *out, uint64_t parentId) override;
+    CURVEFS_ERROR GetInodeAttr(uint64_t inodeId, InodeAttr *out) override;
 
     CURVEFS_ERROR BatchGetInodeAttr(std::set<uint64_t> *inodeIds,
         std::list<InodeAttr> *attrs) override;
 
     CURVEFS_ERROR BatchGetInodeAttrAsync(uint64_t parentId,
-        const std::set<uint64_t> &inodeIds,
+        std::set<uint64_t> *inodeIds,
         std::map<uint64_t, InodeAttr> *attrs = nullptr) override;
 
     CURVEFS_ERROR BatchGetXAttr(std::set<uint64_t> *inodeIds,
@@ -248,9 +259,18 @@ class InodeCacheManagerImpl : public InodeCacheManager,
 
     void ReleaseCache(uint64_t parentId) override;
 
+    void AddOpenedInode(uint64_t inodeId) override;
+
+    void RemoveOpenedInode(uint64_t inodeId) override;
+
+    bool NeedUseCahce(uint64_t inodeId, bool IsDirty);
+
  private:
     virtual void FlushInodeBackground();
     void TrimIcache(uint64_t trimSize);
+    CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
+                              bool streaming = true);
+    bool OpenInodeCached(uint64_t inodeId);
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
@@ -262,6 +282,10 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
     curve::common::Mutex dirtyMapMutex_;
 
+    // record opened inode
+    std::multiset<uint64_t> openedInodes_;
+    curve::common::Mutex openInodesMutex_;
+
     curve::common::GenericNameLock<Mutex> nameLock_;
 
     curve::common::GenericNameLock<Mutex> asyncNameLock_;

curvefs/src/client/inode_wrapper.cpp

@@ -24,6 +24,8 @@
 
 #include <glog/logging.h>
 
+#include <cstddef>
+#include <ctime>
 #include <memory>
 #include <mutex>
 #include <sstream>
@@ -128,18 +130,20 @@ class GetOrModifyS3ChunkInfoAsyncDone : public MetaServerClientDone {
 CURVEFS_ERROR InodeWrapper::SyncAttr(bool internal) {
     curve::common::UniqueLock lock = GetSyncingInodeUniqueLock();
     if (dirty_) {
-        dirty_ = false;
         MetaStatusCode ret = metaClient_->UpdateInodeAttrWithOutNlink(
-            inode_, InodeOpenStatusChange::NOCHANGE, internal);
+            inode_.fsid(), inode_.inodeid(), dirtyAttr_,
+            InodeOpenStatusChange::NOCHANGE, internal);
 
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "metaClient_ UpdateInodeAttrWithOutNlink failed, "
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            dirty_ = true;
             return MetaStatusCodeToCurvefsErrCode(ret);
         }
+
+        dirty_ = false;
+        dirtyAttr_.Clear();
     }
     return CURVEFS_ERROR::OK;
 }
@@ -166,8 +170,10 @@ void InodeWrapper::FlushAttrAsync() {
     if (dirty_) {
         LockSyncingInode();
         auto *done = new UpdateInodeAsyncDone(shared_from_this());
-        metaClient_->UpdateInodeAttrWithOutNlinkAsync(inode_, done);
+        metaClient_->UpdateInodeAttrWithOutNlinkAsync(
+            inode_.fsid(), inode_.inodeid(), dirtyAttr_, done);
         dirty_ = false;
+        dirtyAttr_.Clear();
     }
 }
 
@@ -229,24 +235,23 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR InodeWrapper::LinkLocked(uint64_t parent) {
+CURVEFS_ERROR InodeWrapper::Link(uint64_t parent) {
     curve::common::UniqueLock lg(mtx_);
     REFRESH_NLINK_IF_NEED;
     uint32_t old = inode_.nlink();
     inode_.set_nlink(old + 1);
+    dirtyAttr_.set_nlink(inode_.nlink());
     VLOG(3) << "LinkLocked, inodeid = " << inode_.inodeid()
             << ", newnlink = " << inode_.nlink();
 
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    inode_.set_ctime(now.tv_sec);
-    inode_.set_ctime_ns(now.tv_nsec);
-    inode_.set_mtime(now.tv_sec);
-    inode_.set_mtime_ns(now.tv_nsec);
+    UpdateTimestampLocked(kChangeTime | kModifyTime);
     if (inode_.type() != FsFileType::TYPE_DIRECTORY && parent != 0) {
         inode_.add_parent(parent);
+        dirtyAttr_.add_parent(parent);
     }
-    MetaStatusCode ret = metaClient_->UpdateInodeAttr(inode_);
+
+    MetaStatusCode ret = metaClient_->UpdateInodeAttr(
+        inode_.fsid(), inode_.inodeid(), dirtyAttr_);
     if (ret != MetaStatusCode::OK) {
         inode_.set_nlink(old);
         LOG(ERROR) << "metaClient_ UpdateInodeAttr failed"
@@ -255,11 +260,13 @@ CURVEFS_ERROR InodeWrapper::LinkLocked(uint64_t parent) {
                    << ", inodeid = " << inode_.inodeid();
         return MetaStatusCodeToCurvefsErrCode(ret);
     }
+
     dirty_ = false;
+    dirtyAttr_.Clear();
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
+CURVEFS_ERROR InodeWrapper::UnLink(uint64_t parent) {
     curve::common::UniqueLock lg(mtx_);
     REFRESH_NLINK_IF_NEED;
     uint32_t old = inode_.nlink();
@@ -273,17 +280,12 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
         VLOG(3) << "UnLinkLocked, inodeid = " << inode_.inodeid()
                 << ", newnlink = " << inode_.nlink()
                 << ", type = " << inode_.type();
-        struct timespec now;
-        clock_gettime(CLOCK_REALTIME, &now);
-        inode_.set_ctime(now.tv_sec);
-        inode_.set_ctime_ns(now.tv_nsec);
-        inode_.set_mtime(now.tv_sec);
-        inode_.set_mtime_ns(now.tv_nsec);
-        // newlink == 0 will be deleted at metasever
+        UpdateTimestampLocked(kChangeTime | kModifyTime);
+        // newnlink == 0 will be deleted at metaserver
         // dir will not update parent
         // parent = 0; is useless
-        if (newnlink != 0 && inode_.type() != FsFileType::TYPE_DIRECTORY
-            && parent != 0) {
+        if (newnlink != 0 && inode_.type() != FsFileType::TYPE_DIRECTORY &&
+            parent != 0) {
             auto parents = inode_.mutable_parent();
             for (auto iter = parents->begin(); iter != parents->end(); iter++) {
                 if (*iter == parent) {
@@ -313,7 +315,12 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
                 return err;
             }
         }
-        MetaStatusCode ret = metaClient_->UpdateInodeAttr(inode_);
+
+        dirtyAttr_.set_nlink(inode_.nlink());
+        *dirtyAttr_.mutable_parent() = inode_.parent();
+
+        MetaStatusCode ret = metaClient_->UpdateInodeAttr(
+            inode_.fsid(), inode_.inodeid(), dirtyAttr_);
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "metaClient_ UpdateInodeAttr failed"
                        << ", MetaStatusCode = " << ret
@@ -322,6 +329,7 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
             return MetaStatusCodeToCurvefsErrCode(ret);
         }
         dirty_ = false;
+        dirtyAttr_.Clear();
         return CURVEFS_ERROR::OK;
     }
     LOG(ERROR) << "Unlink find nlink <= 0, nlink = " << old
@@ -329,49 +337,7 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
     return CURVEFS_ERROR::INTERNAL;
 }
 
-CURVEFS_ERROR InodeWrapper::Open() {
-    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
-    if (0 == openCount_) {
-        ret = UpdateInodeStatus(InodeOpenStatusChange::OPEN);
-        if (ret != CURVEFS_ERROR::OK) {
-            return ret;
-        }
-    }
-    openCount_++;
-
-    return CURVEFS_ERROR::OK;
-}
-
-bool InodeWrapper::IsOpen() { return openCount_ > 0; }
-
-CURVEFS_ERROR InodeWrapper::Release() {
-    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
-    if (1 == openCount_) {
-        ret = UpdateInodeStatus(InodeOpenStatusChange::CLOSE);
-        if (ret != CURVEFS_ERROR::OK) {
-            return ret;
-        }
-    }
-    openCount_--;
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR
-InodeWrapper::UpdateInodeStatus(InodeOpenStatusChange statusChange) {
-    MetaStatusCode ret =
-        metaClient_->UpdateInodeAttrWithOutNlink(inode_, statusChange);
-    if (ret != MetaStatusCode::OK) {
-        LOG(ERROR) << "metaClient_ UpdateInodeAttrWithOutNlink failed"
-                   << ", MetaStatusCode = " << ret
-                   << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
-                   << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
-    }
-    dirty_ = false;
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR InodeWrapper::UpdateParentLocked(
+CURVEFS_ERROR InodeWrapper::UpdateParent(
     uint64_t oldParent, uint64_t newParent) {
     curve::common::UniqueLock lg(mtx_);
     auto parents = inode_.mutable_parent();
@@ -382,8 +348,10 @@ CURVEFS_ERROR InodeWrapper::UpdateParentLocked(
         }
     }
     inode_.add_parent(newParent);
+    *dirtyAttr_.mutable_parent() = inode_.parent();
 
-    MetaStatusCode ret = metaClient_->UpdateInodeAttrWithOutNlink(inode_);
+    MetaStatusCode ret = metaClient_->UpdateInodeAttrWithOutNlink(
+        inode_.fsid(), inode_.inodeid(), dirtyAttr_);
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ UpdateInodeAttrWithOutNlink failed"
                    << ", MetaStatusCode = " << ret
@@ -392,6 +360,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParentLocked(
         return MetaStatusCodeToCurvefsErrCode(ret);
     }
     dirty_ = false;
+    dirtyAttr_.Clear();
     return CURVEFS_ERROR::OK;
 }
 
@@ -461,5 +430,52 @@ CURVEFS_ERROR InodeWrapper::RefreshNlink() {
     return CURVEFS_ERROR::OK;
 }
 
+void InodeWrapper::MergeXAttrLocked(
+    const google::protobuf::Map<std::string, std::string>& xattrs) {
+    auto helper =
+        [](const google::protobuf::Map<std::string, std::string>& incoming,
+           google::protobuf::Map<std::string, std::string>* xattrs) {
+            for (const auto& attr : incoming) {
+                (*xattrs)[attr.first] = attr.second;
+            }
+        };
+
+    helper(xattrs, inode_.mutable_xattr());
+    helper(xattrs, dirtyAttr_.mutable_xattr());
+    dirty_ = true;
+}
+
+void InodeWrapper::UpdateTimestampLocked(int flags) {
+    struct timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+    return UpdateTimestampLocked(now, flags);
+}
+
+void InodeWrapper::UpdateTimestampLocked(const timespec& now, int flags) {
+    if (flags & kAccessTime) {
+        inode_.set_atime(now.tv_sec);
+        inode_.set_atime_ns(now.tv_nsec);
+        dirtyAttr_.set_atime(now.tv_sec);
+        dirtyAttr_.set_atime_ns(now.tv_nsec);
+        dirty_ = true;
+    }
+
+    if (flags & kChangeTime) {
+        inode_.set_ctime(now.tv_sec);
+        inode_.set_ctime_ns(now.tv_nsec);
+        dirtyAttr_.set_ctime(now.tv_sec);
+        dirtyAttr_.set_ctime_ns(now.tv_nsec);
+        dirty_ = true;
+    }
+
+    if (flags & kModifyTime) {
+        inode_.set_mtime(now.tv_sec);
+        inode_.set_mtime_ns(now.tv_nsec);
+        dirtyAttr_.set_mtime(now.tv_sec);
+        dirtyAttr_.set_mtime_ns(now.tv_nsec);
+        dirty_ = true;
+    }
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_wrapper.h

@@ -26,6 +26,9 @@
 
 #include <sys/stat.h>
 
+#include <gtest/gtest_prod.h>
+#include <climits>
+#include <cstdint>
 #include <utility>
 #include <memory>
 #include <string>
@@ -45,6 +48,10 @@ using ::curvefs::metaserver::S3ChunkInfo;
 namespace curvefs {
 namespace client {
 
+constexpr int kAccessTime = 1 << 0;
+constexpr int kChangeTime = 1 << 1;
+constexpr int kModifyTime = 1 << 2;
+
 #define REFRESH_NLINK_IF_NEED               \
 do {                                        \
     if (!isNlinkValid_) {                   \
@@ -57,9 +64,9 @@ do {                                        \
 
 using ::curvefs::metaserver::VolumeExtentList;
 
-enum InodeStatus {
-    Normal = 0,
-    Error = -1,
+enum class InodeStatus {
+    kNormal = 0,
+    kError = -1,
 };
 
 // TODO(xuchaojie) : get from conf maybe?
@@ -74,22 +81,11 @@ void AppendS3ChunkInfoToMap(uint64_t chunkIndex, const S3ChunkInfo &info,
 
 class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
  public:
-    InodeWrapper(const Inode &inode,
-                 const std::shared_ptr<MetaServerClient> &metaClient)
-        : inode_(inode),
-          status_(InodeStatus::Normal),
-          isNlinkValid_(true),
-          metaClient_(metaClient),
-          openCount_(0),
-          dirty_(false) {}
-
-    InodeWrapper(Inode &&inode,
-                 const std::shared_ptr<MetaServerClient> &metaClient)
+    InodeWrapper(Inode inode, std::shared_ptr<MetaServerClient> metaClient)
         : inode_(std::move(inode)),
-          status_(InodeStatus::Normal),
+          status_(InodeStatus::kNormal),
           isNlinkValid_(true),
-          metaClient_(metaClient),
-          openCount_(0),
+          metaClient_(std::move(metaClient)),
           dirty_(false) {}
 
     uint64_t GetInodeId() const {
@@ -110,15 +106,26 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     }
 
     void SetLength(uint64_t len) {
+        curve::common::UniqueLock lg(mtx_);
+        SetLengthLocked(len);
+    }
+
+    void SetLengthLocked(uint64_t len) {
         inode_.set_length(len);
+        dirtyAttr_.set_length(len);
         dirty_ = true;
     }
 
     void SetType(FsFileType type) {
         inode_.set_type(type);
+        dirtyAttr_.set_type(type);
         dirty_ = true;
     }
 
+    uint64_t GetLengthLocked() const {
+        return inode_.length();
+    }
+
     uint64_t GetLength() const {
         curve::common::UniqueLock lg(mtx_);
         return inode_.length();
@@ -126,52 +133,48 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
 
     void SetUid(uint32_t uid) {
         inode_.set_uid(uid);
+        dirtyAttr_.set_uid(uid);
         dirty_ = true;
     }
 
     void SetGid(uint32_t gid) {
         inode_.set_gid(gid);
+        dirtyAttr_.set_gid(gid);
         dirty_ = true;
     }
 
     void SetMode(uint32_t mode) {
         inode_.set_mode(mode);
+        dirtyAttr_.set_mode(mode);
         dirty_ = true;
     }
 
-    void SetMTime(uint64_t mtime, uint32_t mtime_ns) {
-        inode_.set_mtime(mtime);
-        inode_.set_mtime_ns(mtime_ns);
-        dirty_ = true;
-    }
-
-    void SetCTime(uint64_t ctime, uint32_t ctime_ns) {
-        inode_.set_ctime(ctime);
-        inode_.set_ctime_ns(ctime_ns);
-        dirty_ = true;
-    }
-
-    void SetATime(uint64_t atime, uint32_t atime_ns) {
-        inode_.set_atime(atime);
-        inode_.set_atime_ns(atime_ns);
-        dirty_ = true;
-    }
-
-    Inode GetInodeUnlocked() const {
-        return inode_;
-    }
-
-    Inode GetInodeLocked() const {
+    Inode GetInode() const {
         curve::common::UniqueLock lg(mtx_);
         return inode_;
     }
 
-    Inode* GetMutableInodeUnlocked() {
-        dirty_ = true;
+    // Get an immutable inode.
+    //
+    // The const return value is used to forbid modify inode through this
+    // interface, all modification operations should using `SetXXX()`.
+    const Inode* GetInodeLocked() const {
         return &inode_;
     }
 
-    CURVEFS_ERROR GetInodeAttrUnlocked(InodeAttr *attr) {
+    // Update timestamp of inode.
+    //
+    // flags can be any combination of kAccessTime/kModifyTime/kChangeTime
+    void UpdateTimestampLocked(int flags);
+    void UpdateTimestampLocked(const timespec& now, int flags);
+
+    // Merge incoming extended attributes.
+    //
+    // Existing attributes will be overwritten, new attributes well be inserted.
+    void MergeXAttrLocked(
+        const google::protobuf::Map<std::string, std::string>& xattrs);
+
+    CURVEFS_ERROR GetInodeAttrLocked(InodeAttr *attr) {
         REFRESH_NLINK_IF_NEED;
 
         attr->set_inodeid(inode_.inodeid());
@@ -198,34 +201,29 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         if (inode_.has_dtime()) {
             attr->set_dtime(inode_.dtime());
         }
-        if (inode_.has_openmpcount()) {
-            attr->set_openmpcount(inode_.openmpcount());
-        }
         if (inode_.xattr_size() > 0) {
             *(attr->mutable_xattr()) = inode_.xattr();
         }
         return CURVEFS_ERROR::OK;
     }
 
-    void GetInodeAttrLocked(InodeAttr *attr) {
+    void GetInodeAttr(InodeAttr *attr) {
         curve::common::UniqueLock lg(mtx_);
-        GetInodeAttrUnlocked(attr);
+        GetInodeAttrLocked(attr);
     }
 
-    void GetXattrLocked(XAttr *xattr) {
+    XAttr GetXattr() const {
+        XAttr ret;
         curve::common::UniqueLock lg(mtx_);
-        xattr->set_fsid(inode_.fsid());
-        xattr->set_inodeid(inode_.inodeid());
-        *(xattr->mutable_xattrinfos()) = inode_.xattr();
+        ret.set_fsid(inode_.fsid());
+        ret.set_inodeid(inode_.inodeid());
+        *(ret.mutable_xattrinfos()) = inode_.xattr();
+        return ret;
     }
 
-    void UpdateInode(const Inode &inode) {
-        inode_ = inode;
-        dirty_ = true;
-    }
-
-    void SwapInode(Inode *other) {
-        inode_.Swap(other);
+    void SetXattrLocked(const std::string &key, const std::string &value) {
+        (*inode_.mutable_xattr())[key] = value;
+        (*dirtyAttr_.mutable_xattr()) = inode_.xattr();
         dirty_ = true;
     }
 
@@ -233,12 +231,16 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         return curve::common::UniqueLock(mtx_);
     }
 
-    CURVEFS_ERROR UpdateParentLocked(uint64_t oldParent, uint64_t newParent);
+    const google::protobuf::RepeatedField<uint64_t>& GetParentLocked() {
+        return inode_.parent();
+    }
+
+    CURVEFS_ERROR UpdateParent(uint64_t oldParent, uint64_t newParent);
 
     // dir will not update parent
-    CURVEFS_ERROR LinkLocked(uint64_t parent = 0);
+    CURVEFS_ERROR Link(uint64_t parent = 0);
 
-    CURVEFS_ERROR UnLinkLocked(uint64_t parent = 0);
+    CURVEFS_ERROR UnLink(uint64_t parent = 0);
 
     // mark nlink invalid, need to refresh from metaserver
     void InvalidateNlink() {
@@ -301,9 +303,13 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
             inode_.mutable_s3chunkinfomap());
     }
 
+    google::protobuf::Map<uint64_t, S3ChunkInfoList>* GetChunkInfoMap() {
+        return inode_.mutable_s3chunkinfomap();
+    }
+
     void MarkInodeError() {
         // TODO(xuchaojie) : when inode is marked error, prevent futher write.
-        status_ = InodeStatus::Error;
+        status_ = InodeStatus::kError;
     }
 
     void LockSyncingInode() const {
@@ -330,8 +336,6 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         return curve::common::UniqueLock(syncingS3ChunkInfoMtx_);
     }
 
-    void SetOpenCount(uint32_t openCount) { openCount_ = openCount; }
-
     ExtentCache* GetMutableExtentCache() {
         curve::common::UniqueLock lk(mtx_);
         return &extentCache_;
@@ -340,8 +344,6 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     CURVEFS_ERROR RefreshVolumeExtent();
 
  private:
-    CURVEFS_ERROR UpdateInodeStatus(InodeOpenStatusChange statusChange);
-
     CURVEFS_ERROR SyncS3ChunkInfo(bool internal = false);
 
  private:
@@ -351,8 +353,13 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     void FlushVolumeExtentAsync();
 
  private:
+    FRIEND_TEST(TestInodeWrapper, TestUpdateInodeAttrIncrementally);
+
     Inode inode_;
-    uint32_t openCount_;
+
+    // dirty attributes, and needs update to metaserver
+    InodeAttr dirtyAttr_;
+
     InodeStatus status_;
 
     bool isNlinkValid_;

curvefs/src/client/main.cpp

@@ -47,15 +47,17 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     release : FuseOpRelease,
     fsync : FuseOpFsync,
     opendir : FuseOpOpenDir,
-    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    readdir : 0,
-    #else
+    // TODO(wuhongsong): readdirplus is problematic,
+    // resulting in inconsistent metadata
+    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    // readdir : 0,
+    // #else
     readdir : FuseOpReadDir,
-    #endif
+    // #endif
     releasedir : FuseOpReleaseDir,
     fsyncdir : 0,
     statfs : FuseOpStatFs,
-    setxattr : 0,
+    setxattr : FuseOpSetXattr,
     getxattr : FuseOpGetXattr,
     listxattr : FuseOpListXattr,
     removexattr : 0,
@@ -75,9 +77,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     flock : 0,
     fallocate : 0,
     #endif
-    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    readdirplus : FuseOpReadDirPlus,
-    #endif
+    // TODO(wuhongsong): The current implementation is problematic,
+    // resulting in inconsistent metadata
+    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    readdirplus : 0,
+    // #endif
     #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 4)
     copy_file_range : 0,
     #endif

curvefs/src/client/metric/client_metric.h

@@ -41,23 +41,32 @@ struct MDSClientMetric {
     InterfaceMetric mountFs;
     InterfaceMetric umountFs;
     InterfaceMetric getFsInfo;
-    InterfaceMetric allocateS3Chunk;
-
+    InterfaceMetric getMetaServerInfo;
+    InterfaceMetric getMetaServerListInCopysets;
+    InterfaceMetric createPartition;
+    InterfaceMetric getCopysetOfPartitions;
+    InterfaceMetric listPartition;
+    InterfaceMetric allocS3ChunkId;
+    InterfaceMetric refreshSession;
     InterfaceMetric getLatestTxId;
     InterfaceMetric commitTx;
 
-    InterfaceMetric getMetaserverInfo;
-
     explicit MDSClientMetric(const std::string &prefix_ = "")
         : prefix(!prefix_.empty() ? prefix_
                                   : "curvefs_mds_client_" +
-                                        curve::common::ToHexString(this)),
-          mountFs(prefix, "mountFs"), umountFs(prefix, "unmountFs"),
+                                    curve::common::ToHexString(this)),
+          mountFs(prefix, "mountFs"),
+          umountFs(prefix, "umountFs"),
           getFsInfo(prefix, "getFsInfo"),
-          allocateS3Chunk(prefix, "allocateS3Chunk"),
+          getMetaServerInfo(prefix, "getMetaServerInfo"),
+          getMetaServerListInCopysets(prefix, "getMetaServerListInCopysets"),
+          createPartition(prefix, "createPartition"),
+          getCopysetOfPartitions(prefix, "getCopysetOfPartitions"),
+          listPartition(prefix, "listPartition"),
+          allocS3ChunkId(prefix, "allocS3ChunkId"),
+          refreshSession(prefix, "refreshSession"),
           getLatestTxId(prefix, "getLatestTxId"),
-          commitTx(prefix, "commitTx"),
-          getMetaserverInfo(prefix, "getMetaserverInfo") {}
+          commitTx(prefix, "commitTx") {}
 };
 
 struct MetaServerClientMetric {
@@ -76,15 +85,11 @@ struct MetaServerClientMetric {
     InterfaceMetric createInode;
     InterfaceMetric updateInode;
     InterfaceMetric deleteInode;
-    InterfaceMetric createRootInode;
     InterfaceMetric appendS3ChunkInfo;
 
     // tnx
     InterfaceMetric prepareRenameTx;
 
-    // partition
-    InterfaceMetric createPartition;
-
     // volume extent
     InterfaceMetric updateVolumeExtent;
     InterfaceMetric getVolumeExtent;
@@ -103,10 +108,8 @@ struct MetaServerClientMetric {
           createInode(prefix, "createInode"),
           updateInode(prefix, "updateInode"),
           deleteInode(prefix, "deleteInode"),
-          createRootInode(prefix, "createRootInode"),
           appendS3ChunkInfo(prefix, "appendS3ChunkInfo"),
           prepareRenameTx(prefix, "prepareRenameTx"),
-          createPartition(prefix, "createPartition"),
           updateVolumeExtent(prefix, "updateVolumeExtent"),
           getVolumeExtent(prefix, "getVolumeExtent") {}
 };
@@ -123,10 +126,11 @@ struct InflightGuard {
 
     bvar::Adder<int64_t>* inflight_;
 };
+
 struct OpMetric {
-    bvar::Adder<uint64_t> ecount;
     bvar::LatencyRecorder latency;
     bvar::Adder<int64_t> inflightOpNum;
+    bvar::Adder<uint64_t> ecount;
 
     explicit OpMetric(const std::string& prefix, const std::string& name)
         : latency(prefix, name + "_lat"),
@@ -261,10 +265,13 @@ struct DiskCacheMetric {
 
     std::string fsName;
     InterfaceMetric writeS3;
+    bvar::Status<uint64_t> diskUsedBytes;
+
     explicit DiskCacheMetric(const std::string &name = "")
         : fsName(!name.empty() ? name
                                : prefix + curve::common::ToHexString(this)),
-          writeS3(prefix, fsName + "_write_s3") {}
+          writeS3(prefix, fsName + "_write_s3"),
+          diskUsedBytes(prefix, fsName + "_diskcache_usedbytes", 0) {}
 };
 
 }  // namespace metric

curvefs/src/client/rpcclient/mds_client.cpp

@@ -20,13 +20,13 @@
  * Author: lixiaocui
  */
 
-#include "curvefs/src/client/rpcclient/mds_client.h"
-
 #include <map>
 #include <utility>
 #include <vector>
 
 #include "curvefs/proto/space.pb.h"
+#include "curvefs/src/client/rpcclient/mds_client.h"
+#include "curvefs/src/common/metric_utils.h"
 
 namespace curvefs {
 namespace client {
@@ -34,6 +34,7 @@ namespace rpcclient {
 
 using ::curvefs::mds::space::SpaceErrCode;
 using ::curvefs::mds::space::SpaceErrCode_Name;
+using ::curvefs::common::LatencyUpdater;
 
 FSStatusCode
 MdsClientImpl::Init(const ::curve::client::MetaServerOption &mdsOpt,
@@ -59,6 +60,7 @@ FSStatusCode MdsClientImpl::MountFs(const std::string& fsName,
                                     const Mountpoint& mountPt, FsInfo* fsInfo) {
     auto task = RPCTask {
         mdsClientMetric_.mountFs.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.mountFs.latency);
         MountFsResponse response;
         mdsbasecli_->MountFs(fsName, mountPt, &response, cntl, channel);
         if (cntl->Failed()) {
@@ -87,6 +89,7 @@ FSStatusCode MdsClientImpl::UmountFs(const std::string& fsName,
                                      const Mountpoint& mountPt) {
     auto task = RPCTask {
         mdsClientMetric_.umountFs.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.umountFs.latency);
         UmountFsResponse response;
         mdsbasecli_->UmountFs(fsName, mountPt, &response, cntl, channel);
         if (cntl->Failed()) {
@@ -111,6 +114,7 @@ FSStatusCode MdsClientImpl::GetFsInfo(const std::string &fsName,
                                       FsInfo *fsInfo) {
     auto task = RPCTask {
         mdsClientMetric_.getFsInfo.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.getFsInfo.latency);
         GetFsInfoResponse response;
         mdsbasecli_->GetFsInfo(fsName, &response, cntl, channel);
 
@@ -140,6 +144,7 @@ FSStatusCode MdsClientImpl::GetFsInfo(const std::string &fsName,
 FSStatusCode MdsClientImpl::GetFsInfo(uint32_t fsId, FsInfo *fsInfo) {
     auto task = RPCTask {
         mdsClientMetric_.getFsInfo.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.getFsInfo.latency);
         GetFsInfoResponse response;
         mdsbasecli_->GetFsInfo(fsId, &response, cntl, channel);
         if (cntl->Failed()) {
@@ -198,9 +203,12 @@ bool MdsClientImpl::GetMetaServerInfo(
     ::curve::common::StringToUll(strs[1], &port);
 
     auto task = RPCTask {
+        mdsClientMetric_.getMetaServerInfo.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.getMetaServerInfo.latency);
         GetMetaServerInfoResponse response;
         mdsbasecli_->GetMetaServerInfo(port, ip, &response, cntl, channel);
         if (cntl->Failed()) {
+            mdsClientMetric_.getMetaServerInfo.eps.count << 1;
             LOG(WARNING) << "GetMetaServerInfo Failed, errorcode = "
                          << cntl->ErrorCode()
                          << ", error content:" << cntl->ErrorText()
@@ -231,10 +239,14 @@ bool MdsClientImpl::GetMetaServerListInCopysets(
     const LogicPoolID &logicalpooid, const std::vector<CopysetID> &copysetidvec,
     std::vector<CopysetInfo<MetaserverID>> *cpinfoVec) {
     auto task = RPCTask {
+        mdsClientMetric_.getMetaServerListInCopysets.qps.count << 1;
+        LatencyUpdater updater(
+            &mdsClientMetric_.getMetaServerListInCopysets.latency);
         GetMetaServerListInCopySetsResponse response;
         mdsbasecli_->GetMetaServerListInCopysets(logicalpooid, copysetidvec,
                                                  &response, cntl, channel);
         if (cntl->Failed()) {
+            mdsClientMetric_.getMetaServerListInCopysets.eps.count << 1;
             LOG(WARNING) << "get metaserver list from mds failed, error is "
                          << cntl->ErrorText()
                          << ", log id = " << cntl->log_id();
@@ -278,9 +290,12 @@ bool MdsClientImpl::GetMetaServerListInCopysets(
 bool MdsClientImpl::CreatePartition(
     uint32_t fsID, uint32_t count, std::vector<PartitionInfo> *partitionInfos) {
     auto task = RPCTask {
+        mdsClientMetric_.createPartition.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.createPartition.latency);
         CreatePartitionResponse response;
         mdsbasecli_->CreatePartition(fsID, count, &response, cntl, channel);
         if (cntl->Failed()) {
+            mdsClientMetric_.createPartition.eps.count << 1;
             LOG(WARNING) << "CreatePartition from mds failed, error is "
                          << cntl->ErrorText()
                          << ", log id = " << cntl->log_id();
@@ -304,10 +319,11 @@ bool MdsClientImpl::CreatePartition(
             return TopoStatusCode::TOPO_CREATE_PARTITION_FAIL;
         }
 
+        partitionInfos->reserve(count);
         partitionInfos->clear();
-        for (int i = 0; i < partitionNum; i++) {
-            partitionInfos->push_back(response.partitioninfolist(i));
-        }
+        std::move(response.mutable_partitioninfolist()->begin(),
+                  response.mutable_partitioninfolist()->end(),
+                  std::back_inserter(*partitionInfos));
 
         return TopoStatusCode::TOPO_OK;
     };
@@ -319,10 +335,14 @@ bool MdsClientImpl::GetCopysetOfPartitions(
     const std::vector<uint32_t> &partitionIDList,
     std::map<uint32_t, Copyset> *copysetMap) {
     auto task = RPCTask {
+        mdsClientMetric_.getCopysetOfPartitions.qps.count << 1;
+        LatencyUpdater updater(
+            &mdsClientMetric_.getCopysetOfPartitions.latency);
         GetCopysetOfPartitionResponse response;
         mdsbasecli_->GetCopysetOfPartitions(partitionIDList, &response, cntl,
                                             channel);
         if (cntl->Failed()) {
+            mdsClientMetric_.getCopysetOfPartitions.eps.count << 1;
             LOG(WARNING) << "GetCopysetOfPartition from mds failed, error is "
                          << cntl->ErrorText()
                          << ", log id = " << cntl->log_id();
@@ -359,9 +379,12 @@ bool MdsClientImpl::GetCopysetOfPartitions(
 bool MdsClientImpl::ListPartition(uint32_t fsID,
                                   std::vector<PartitionInfo> *partitionInfos) {
     auto task = RPCTask {
+        mdsClientMetric_.listPartition.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.listPartition.latency);
         ListPartitionResponse response;
         mdsbasecli_->ListPartition(fsID, &response, cntl, channel);
         if (cntl->Failed()) {
+            mdsClientMetric_.listPartition.eps.count << 1;
             LOG(WARNING) << "ListPartition from mds failed, error is "
                          << cntl->ErrorText()
                          << ", log id = " << cntl->log_id();
@@ -392,9 +415,12 @@ bool MdsClientImpl::ListPartition(uint32_t fsID,
 
 FSStatusCode MdsClientImpl::AllocS3ChunkId(uint32_t fsId, uint64_t *chunkId) {
     auto task = RPCTask {
+        mdsClientMetric_.allocS3ChunkId.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.allocS3ChunkId.latency);
         AllocateS3ChunkResponse response;
         mdsbasecli_->AllocS3ChunkId(fsId, &response, cntl, channel);
         if (cntl->Failed()) {
+            mdsClientMetric_.allocS3ChunkId.eps.count << 1;
             LOG(WARNING) << "AllocS3ChunkId Failed, errorcode = "
                          << cntl->ErrorCode()
                          << ", error content:" << cntl->ErrorText()
@@ -422,6 +448,8 @@ MdsClientImpl::RefreshSession(const std::vector<PartitionTxId> &txIds,
                               const std::string& fsName,
                               const Mountpoint& mountpoint) {
     auto task = RPCTask {
+        mdsClientMetric_.refreshSession.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.refreshSession.latency);
         RefreshSessionRequest request;
         RefreshSessionResponse response;
         *request.mutable_txids() = {txIds.begin(), txIds.end()};
@@ -429,6 +457,7 @@ MdsClientImpl::RefreshSession(const std::vector<PartitionTxId> &txIds,
         *request.mutable_mountpoint() = mountpoint;
         mdsbasecli_->RefreshSession(request, &response, cntl, channel);
         if (cntl->Failed()) {
+            mdsClientMetric_.refreshSession.eps.count << 1;
             LOG(WARNING) << "RefreshSession fail, errcode = "
                          << cntl->ErrorCode()
                          << ", error content: " << cntl->ErrorText()
@@ -456,9 +485,12 @@ MdsClientImpl::RefreshSession(const std::vector<PartitionTxId> &txIds,
 FSStatusCode MdsClientImpl::GetLatestTxId(const GetLatestTxIdRequest& request,
                                           GetLatestTxIdResponse* response) {
     auto task = RPCTask {
+        VLOG(3) << "GetLatestTxId [request]: " << request.DebugString();
         mdsClientMetric_.getLatestTxId.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.getLatestTxId.latency);
         mdsbasecli_->GetLatestTxId(request, response, cntl, channel);
         if (cntl->Failed()) {
+            mdsClientMetric_.getLatestTxId.eps.count << 1;
             LOG(WARNING) << "GetLatestTxId fail, errCode = "
                          << cntl->ErrorCode()
                          << ", errorText = " << cntl->ErrorText()
@@ -477,6 +509,8 @@ FSStatusCode MdsClientImpl::GetLatestTxId(const GetLatestTxIdRequest& request,
             LOG(WARNING) << "GetLatestTxId fail, errcode = " << rc
                          << ", errmsg = " << FSStatusCode_Name(rc);
         }
+
+        VLOG(3) << "GetLatestTxId [response]: " << response->DebugString();
         return rc;
     };
 
@@ -486,7 +520,9 @@ FSStatusCode MdsClientImpl::GetLatestTxId(const GetLatestTxIdRequest& request,
 
 FSStatusCode MdsClientImpl::CommitTx(const CommitTxRequest& request) {
     auto task = RPCTask {
+        VLOG(3) << "CommitTx [request]: " << request.DebugString();
         mdsClientMetric_.commitTx.qps.count << 1;
+        LatencyUpdater updater(&mdsClientMetric_.commitTx.latency);
         CommitTxResponse response;
         mdsbasecli_->CommitTx(request, &response, cntl, channel);
 
@@ -509,15 +545,18 @@ FSStatusCode MdsClientImpl::CommitTx(const CommitTxRequest& request) {
             LOG(WARNING) << "CommitTx: retCode = " << rc
                          << ", message = " << FSStatusCode_Name(rc);
         }
+        VLOG(3) << "CommitTx [response]: " << response.DebugString();
         return rc;
     };
     // for rpc error or get lock failed/timeout, we will retry until success
     return ReturnError(rpcexcutor_.DoRPCTask(task, 0));
 }
 
-FSStatusCode MdsClientImpl::GetLatestTxId(std::vector<PartitionTxId>* txIds) {
+FSStatusCode MdsClientImpl::GetLatestTxId(uint32_t fsId,
+                                          std::vector<PartitionTxId>* txIds) {
     GetLatestTxIdRequest request;
     GetLatestTxIdResponse response;
+    request.set_fsid(fsId);
     FSStatusCode rc = GetLatestTxId(request, &response);
     if (rc == FSStatusCode::OK) {
         *txIds = { response.txids().begin(), response.txids().end() };

curvefs/src/client/rpcclient/mds_client.h

@@ -111,7 +111,8 @@ class MdsClient {
                    const std::string& fsName,
                    const Mountpoint& mountpoint) = 0;
 
-    virtual FSStatusCode GetLatestTxId(std::vector<PartitionTxId>* txIds) = 0;
+    virtual FSStatusCode GetLatestTxId(uint32_t fsId,
+                                       std::vector<PartitionTxId>* txIds) = 0;
 
     virtual FSStatusCode
     GetLatestTxIdWithLock(uint32_t fsId,
@@ -194,7 +195,8 @@ class MdsClientImpl : public MdsClient {
                                 const std::string& fsName,
                                 const Mountpoint& mountpoint) override;
 
-    FSStatusCode GetLatestTxId(std::vector<PartitionTxId>* txIds) override;
+    FSStatusCode GetLatestTxId(uint32_t fsId,
+                               std::vector<PartitionTxId>* txIds) override;
 
     FSStatusCode
     GetLatestTxIdWithLock(uint32_t fsId,

curvefs/src/client/rpcclient/metacache.cpp

@@ -79,7 +79,7 @@ void MetaCache::GetAllTxIds(std::vector<PartitionTxId> *txIds) {
 
 bool MetaCache::RefreshTxId() {
     std::vector<PartitionTxId> txIds;
-    FSStatusCode rc = mdsClient_->GetLatestTxId(&txIds);
+    FSStatusCode rc = mdsClient_->GetLatestTxId(fsID_, &txIds);
     if (rc != FSStatusCode::OK) {
         LOG(ERROR) << "Get latest txid failed, retCode=" << rc;
         return false;
@@ -129,7 +129,7 @@ bool MetaCache::SelectTarget(uint32_t fsID, CopysetTarget *target,
 
         if (!SelectPartition(target)) {
             LOG(ERROR) << "select target for {fsid:" << fsID
-                       << "} fail,  select paritiotn fail";
+                       << "} fail,  select partition fail";
             return false;
         }
     }
@@ -254,7 +254,7 @@ bool MetaCache::ListPartitions(uint32_t fsID) {
     WriteLockGuard wl4CopysetMap(rwlock4copysetInfoMap_);
 
     fsID_ = fsID;
-    PatitionInfoList partitionInfos;
+    PartitionInfoList partitionInfos;
     std::map<PoolIDCopysetID, CopysetInfo<MetaserverID>> copysetMap;
     if (!DoListOrCreatePartitions(true, &partitionInfos, &copysetMap)) {
         return false;
@@ -266,13 +266,17 @@ bool MetaCache::ListPartitions(uint32_t fsID) {
 }
 
 bool MetaCache::CreatePartitions(int currentNum,
-                                 PatitionInfoList *newPartitions) {
+                                 PartitionInfoList *newPartitions) {
     std::lock_guard<Mutex> lg(createMutex_);
 
     // already create
     {
         ReadLockGuard rl(rwlock4Partitions_);
         if (partitionInfos_.size() > currentNum) {
+            newPartitions->reserve(partitionInfos_.size() - currentNum);
+            newPartitions->insert(newPartitions->end(),
+                                  partitionInfos_.begin() + currentNum,
+                                  partitionInfos_.end());
             return true;
         }
     }
@@ -293,7 +297,7 @@ bool MetaCache::CreatePartitions(int currentNum,
 }
 
 bool MetaCache::DoListOrCreatePartitions(
-    bool list, PatitionInfoList *partitionInfos,
+    bool list, PartitionInfoList *partitionInfos,
     std::map<PoolIDCopysetID, CopysetInfo<MetaserverID>> *copysetMap) {
     // TODO(@lixiaocui): list or get partition need too many rpc,
     // it's better to return all infos once.
@@ -374,7 +378,7 @@ bool MetaCache::DoListOrCreatePartitions(
 }
 
 void MetaCache::DoAddOrResetPartitionAndCopyset(
-    PatitionInfoList partitionInfos,
+    PartitionInfoList partitionInfos,
     std::map<PoolIDCopysetID, CopysetInfo<MetaserverID>> copysetMap,
     bool reset) {
     if (reset) {
@@ -498,18 +502,20 @@ bool MetaCache::SelectPartition(CopysetTarget *target) {
     }
 
     if (candidate.empty()) {
-        // create parition for fs
+        // create partition for fs
         LOG(INFO) << "no partition can be select for fsid:" << fsID_
                   << ", need create new partitions";
-        PatitionInfoList newPartitions;
+        PartitionInfoList newPartitions;
         if (!CreatePartitions(currentNum, &newPartitions)) {
             LOG(ERROR) << "create partition for fsid:" << fsID_ << " fail";
             return false;
         }
-        target->groupID = CopysetGroupID(newPartitions[0].poolid(),
-                                         newPartitions[0].copysetid());
-        target->partitionID = newPartitions[0].partitionid();
-        target->txId = newPartitions[0].txid();
+        CHECK(!newPartitions.empty());
+        const auto index = butil::fast_rand() % newPartitions.size();
+        auto iter = newPartitions.begin() + index;
+        target->groupID = CopysetGroupID(iter->poolid(), iter->copysetid());
+        target->partitionID = iter->partitionid();
+        target->txId = iter->txid();
     } else {
         // random select a partition
         const auto index = butil::fast_rand() % candidate.size();

curvefs/src/client/rpcclient/metacache.h

@@ -111,15 +111,14 @@ class MetaCache {
  public:
     void Init(MetaCacheOpt opt, std::shared_ptr<Cli2Client> cli2Client,
               std::shared_ptr<MdsClient> mdsClient) {
-        metacacheopt_ = opt;
-        cli2Client_ = cli2Client;
-        mdsClient_ = mdsClient;
+        metacacheopt_ = std::move(opt);
+        cli2Client_ = std::move(cli2Client);
+        mdsClient_ = std::move(mdsClient);
         init_ = false;
     }
 
     using PoolIDCopysetID = uint64_t;
-    using PatitionInfoList = std::vector<PartitionInfo>;
-    using FS2PatitionInfoMap = std::unordered_map<uint32_t, PatitionInfoList>;
+    using PartitionInfoList = std::vector<PartitionInfo>;
     using CopysetInfoMap =
         std::unordered_map<PoolIDCopysetID, CopysetInfo<MetaserverID>>;
 
@@ -162,12 +161,12 @@ class MetaCache {
 
  private:
     void GetTxId(uint32_t partitionId, uint64_t *txId);
-    bool CreatePartitions(int currentNum, PatitionInfoList *newPartitions);
+    bool CreatePartitions(int currentNum, PartitionInfoList *newPartitions);
     bool DoListOrCreatePartitions(
-        bool list, PatitionInfoList *partitionInfos,
+        bool list, PartitionInfoList *partitionInfos,
         std::map<PoolIDCopysetID, CopysetInfo<MetaserverID>> *copysetMap);
     void DoAddOrResetPartitionAndCopyset(
-        PatitionInfoList partitionInfos,
+        PartitionInfoList partitionInfos,
         std::map<PoolIDCopysetID, CopysetInfo<MetaserverID>> copysetMap,
         bool reset);
 
@@ -206,7 +205,7 @@ class MetaCache {
     std::unordered_map<uint32_t, uint64_t> partitionTxId_;
 
     RWLock rwlock4Partitions_;
-    PatitionInfoList partitionInfos_;
+    PartitionInfoList partitionInfos_;
     RWLock rwlock4copysetInfoMap_;
     CopysetInfoMap copysetInfoMap_;
 

curvefs/src/client/rpcclient/metaserver_client.cpp

@@ -221,7 +221,7 @@ MetaStatusCode MetaServerClientImpl::ListDentry(uint32_t fsId, uint64_t inodeid,
                 << ", onlyDir = " << onlyDir
                 << ", errcode = " << ret
                 << ", errmsg = " << MetaStatusCode_Name(ret);
-        } else if (response.has_appliedindex() && response.dentrys_size() > 0) {
+        } else if (response.has_appliedindex()) {
             metaCache_->UpdateApplyIndex(CopysetGroupID(poolID, copysetID),
                                          response.appliedindex());
 
@@ -233,7 +233,7 @@ MetaStatusCode MetaServerClientImpl::ListDentry(uint32_t fsId, uint64_t inodeid,
                 << "ListDentry: fsId = " << fsId << ", inodeid = " << inodeid
                 << ", last = " << last << ", count = " << count
                 << ", onlyDir = " << onlyDir
-                << " ok, but dentry and applyIndex not set in response:"
+                << " ok, but applyIndex not set in response:"
                 << response.DebugString();
             return -1;
         }
@@ -828,51 +828,72 @@ MetaServerClientImpl::UpdateInode(const UpdateInodeRequest &request,
     return ConvertToMetaStatusCode(excutor.DoRPCTask());
 }
 
+#define SET_REQUEST_FIELD_IF_HAS(request, attr, field) \
+    do {                                               \
+        if ((attr).has_##field()) {                    \
+            (request).set_##field((attr).field());    \
+        }                                              \
+    } while (false)
+
 UpdateInodeRequest
 MetaServerClientImpl::BuileUpdateInodeAttrWithOutNlinkRequest(
-    const Inode &inode,
+    uint32_t fsId,
+    uint64_t inodeId,
+    const InodeAttr& attr,
     InodeOpenStatusChange statusChange) {
     UpdateInodeRequest request;
-    request.set_inodeid(inode.inodeid());
-    request.set_fsid(inode.fsid());
-    request.set_length(inode.length());
-    request.set_ctime(inode.ctime());
-    request.set_mtime(inode.mtime());
-    request.set_atime(inode.atime());
-    request.set_uid(inode.uid());
-    request.set_gid(inode.gid());
-    request.set_mode(inode.mode());
+    request.set_fsid(fsId);
+    request.set_inodeid(inodeId);
+
+    SET_REQUEST_FIELD_IF_HAS(request, attr, length);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, atime);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, atime_ns);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, ctime);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, ctime_ns);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, mtime);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, mtime_ns);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, uid);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, gid);
+    SET_REQUEST_FIELD_IF_HAS(request, attr, mode);
+
     request.set_inodeopenstatuschange(statusChange);
-    *(request.mutable_parent()) = inode.parent();
-    if (inode.xattr_size() > 0) {
-        *(request.mutable_xattr()) = inode.xattr();
+    *(request.mutable_parent()) = attr.parent();
+    if (attr.xattr_size() > 0) {
+        *(request.mutable_xattr()) = attr.xattr();
     }
     return request;
 }
 
-UpdateInodeRequest
-MetaServerClientImpl::BuildeUpdateInodeAttrRequest(const Inode &inode,
+UpdateInodeRequest MetaServerClientImpl::BuildeUpdateInodeAttrRequest(
+    uint32_t fsId,
+    uint64_t inodeId,
+    const InodeAttr& attr,
     InodeOpenStatusChange statusChange) {
     UpdateInodeRequest request = BuileUpdateInodeAttrWithOutNlinkRequest(
-        inode, statusChange);
-    request.set_nlink(inode.nlink());
+        fsId, inodeId, attr, statusChange);
+    assert(attr.has_nlink());
+    request.set_nlink(attr.nlink());
     return request;
 }
 
-MetaStatusCode
-MetaServerClientImpl::UpdateInodeAttr(const Inode &inode,
-                                      InodeOpenStatusChange statusChange) {
+MetaStatusCode MetaServerClientImpl::UpdateInodeAttr(
+    uint32_t fsId,
+    uint64_t inodeId,
+    const InodeAttr& attr,
+    InodeOpenStatusChange statusChange) {
     UpdateInodeRequest request =
-        BuildeUpdateInodeAttrRequest(inode, statusChange);
+        BuildeUpdateInodeAttrRequest(fsId, inodeId, attr, statusChange);
     return UpdateInode(request);
 }
 
-MetaStatusCode
-MetaServerClientImpl::UpdateInodeAttrWithOutNlink(const Inode &inode,
-                                     InodeOpenStatusChange statusChange,
-                                     bool internal) {
+MetaStatusCode MetaServerClientImpl::UpdateInodeAttrWithOutNlink(
+    uint32_t fsId,
+    uint64_t inodeId,
+    const InodeAttr& attr,
+    InodeOpenStatusChange statusChange,
+    bool internal) {
     UpdateInodeRequest request = BuileUpdateInodeAttrWithOutNlinkRequest(
-        inode, statusChange);
+        fsId, inodeId, attr, statusChange);
     return UpdateInode(request, internal);
 }
 
@@ -921,7 +942,6 @@ void UpdateInodeRpcDone::Run() {
     VLOG(6) << "UpdateInode done, "
             << "response: " << response.DebugString();
     done_->SetRetCode(ret);
-    return;
 }
 
 void MetaServerClientImpl::UpdateInodeAsync(const UpdateInodeRequest &request,
@@ -950,18 +970,24 @@ void MetaServerClientImpl::UpdateInodeAsync(const UpdateInodeRequest &request,
 }
 
 void MetaServerClientImpl::UpdateInodeAttrAsync(
-    const Inode &inode, MetaServerClientDone *done,
+    uint32_t fsId,
+    uint64_t inodeId,
+    const InodeAttr& attr,
+    MetaServerClientDone* done,
     InodeOpenStatusChange statusChange) {
     UpdateInodeRequest request =
-        BuildeUpdateInodeAttrRequest(inode, statusChange);
+        BuildeUpdateInodeAttrRequest(fsId, inodeId, attr, statusChange);
     UpdateInodeAsync(request, done);
 }
 
 void MetaServerClientImpl::UpdateInodeAttrWithOutNlinkAsync(
-    const Inode &inode, MetaServerClientDone *done,
+    uint32_t fsId,
+    uint64_t inodeId,
+    const InodeAttr& attr,
+    MetaServerClientDone* done,
     InodeOpenStatusChange statusChange) {
-    UpdateInodeRequest request =
-        BuileUpdateInodeAttrWithOutNlinkRequest(inode, statusChange);
+    UpdateInodeRequest request = BuileUpdateInodeAttrWithOutNlinkRequest(
+        fsId, inodeId, attr, statusChange);
     UpdateInodeAsync(request, done);
 }
 

curvefs/src/client/rpcclient/metaserver_client.h

@@ -107,22 +107,32 @@ class MetaServerClient {
         const std::set<uint64_t> &inodeIds,
         std::list<XAttr> *xattr) = 0;
 
-    virtual MetaStatusCode UpdateInodeAttr(const Inode &inode,
+    virtual MetaStatusCode UpdateInodeAttr(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
         InodeOpenStatusChange statusChange =
             InodeOpenStatusChange::NOCHANGE) = 0;
 
-    virtual MetaStatusCode UpdateInodeAttrWithOutNlink(const Inode &inode,
-        InodeOpenStatusChange statusChange =
-            InodeOpenStatusChange::NOCHANGE,
-            bool internal = false) = 0;
+    virtual MetaStatusCode UpdateInodeAttrWithOutNlink(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
+        InodeOpenStatusChange statusChange = InodeOpenStatusChange::NOCHANGE,
+        bool internal = false) = 0;
 
-    virtual void UpdateInodeAttrAsync(const Inode &inode,
-        MetaServerClientDone *done,
-        InodeOpenStatusChange statusChange =
-            InodeOpenStatusChange::NOCHANGE) = 0;
+    virtual void UpdateInodeAttrAsync(uint32_t fsId,
+                                      uint64_t inodeId,
+                                      const InodeAttr& attr,
+                                      MetaServerClientDone* done,
+                                      InodeOpenStatusChange statusChange =
+                                          InodeOpenStatusChange::NOCHANGE) = 0;
 
-    virtual void UpdateInodeAttrWithOutNlinkAsync(const Inode &inode,
-        MetaServerClientDone *done,
+    virtual void UpdateInodeAttrWithOutNlinkAsync(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
+        MetaServerClientDone* done,
         InodeOpenStatusChange statusChange =
             InodeOpenStatusChange::NOCHANGE) = 0;
 
@@ -209,21 +219,32 @@ class MetaServerClientImpl : public MetaServerClient {
         const std::set<uint64_t> &inodeIds,
         std::list<XAttr> *xattr) override;
 
-    MetaStatusCode UpdateInodeAttr(const Inode &inode,
+    MetaStatusCode UpdateInodeAttr(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
         InodeOpenStatusChange statusChange =
             InodeOpenStatusChange::NOCHANGE) override;
 
-    MetaStatusCode UpdateInodeAttrWithOutNlink(const Inode &inode,
-        InodeOpenStatusChange statusChange =
-            InodeOpenStatusChange::NOCHANGE,
-            bool internal = false) override;
+    MetaStatusCode UpdateInodeAttrWithOutNlink(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
+        InodeOpenStatusChange statusChange = InodeOpenStatusChange::NOCHANGE,
+        bool internal = false) override;
 
-    void UpdateInodeAttrAsync(const Inode &inode, MetaServerClientDone *done,
-                          InodeOpenStatusChange statusChange =
-                              InodeOpenStatusChange::NOCHANGE) override;
+    void UpdateInodeAttrAsync(uint32_t fsId,
+                              uint64_t inodeId,
+                              const InodeAttr& attr,
+                              MetaServerClientDone* done,
+                              InodeOpenStatusChange statusChange =
+                                  InodeOpenStatusChange::NOCHANGE) override;
 
-    void UpdateInodeAttrWithOutNlinkAsync(const Inode &inode,
-        MetaServerClientDone *done,
+    void UpdateInodeAttrWithOutNlinkAsync(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
+        MetaServerClientDone* done,
         InodeOpenStatusChange statusChange =
             InodeOpenStatusChange::NOCHANGE) override;
 
@@ -267,11 +288,16 @@ class MetaServerClientImpl : public MetaServerClient {
     void UpdateInodeAsync(const UpdateInodeRequest &request,
                           MetaServerClientDone *done);
 
-    UpdateInodeRequest BuildeUpdateInodeAttrRequest(const Inode &inode,
+    UpdateInodeRequest BuildeUpdateInodeAttrRequest(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
         InodeOpenStatusChange statusChange);
 
     UpdateInodeRequest BuileUpdateInodeAttrWithOutNlinkRequest(
-        const Inode &inode,
+        uint32_t fsId,
+        uint64_t inodeId,
+        const InodeAttr& attr,
         InodeOpenStatusChange statusChange);
 
     bool ParseS3MetaStreamBuffer(butil::IOBuf* buffer,

curvefs/src/client/s3/client_s3_adaptor.cpp

@@ -176,7 +176,9 @@ int S3ClientAdaptorImpl::Read(uint64_t inodeId, uint64_t offset,
     return ret;
 }
 
-CURVEFS_ERROR S3ClientAdaptorImpl::Truncate(Inode *inode, uint64_t size) {
+CURVEFS_ERROR S3ClientAdaptorImpl::Truncate(InodeWrapper *inodeWrapper,
+                                            uint64_t size) {
+    const auto *inode = inodeWrapper->GetInodeLocked();
     uint64_t fileSize = inode->length();
 
     if (size < fileSize) {
@@ -212,7 +214,7 @@ CURVEFS_ERROR S3ClientAdaptorImpl::Truncate(Inode *inode, uint64_t size) {
                 n = len;
             }
             S3ChunkInfo *tmp;
-            auto s3ChunkInfoMap = inode->mutable_s3chunkinfomap();
+            auto* s3ChunkInfoMap = inodeWrapper->GetChunkInfoMap();
             auto s3chunkInfoListIter = s3ChunkInfoMap->find(index);
             if (s3chunkInfoListIter == s3ChunkInfoMap->end()) {
                 S3ChunkInfoList s3chunkInfoList;

curvefs/src/client/s3/client_s3_adaptor.h

@@ -80,7 +80,8 @@ class S3ClientAdaptor {
                       const char *buf) = 0;
     virtual int Read(uint64_t inodeId, uint64_t offset, uint64_t length,
                      char *buf) = 0;
-    virtual CURVEFS_ERROR Truncate(Inode *inode, uint64_t size) = 0;
+    virtual CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper,
+                                   uint64_t size) = 0;
     virtual void ReleaseCache(uint64_t inodeId) = 0;
     virtual CURVEFS_ERROR Flush(uint64_t inodeId) = 0;
     virtual CURVEFS_ERROR FlushAllCache(uint64_t inodeId) = 0;
@@ -129,7 +130,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
     int Write(uint64_t inodeId, uint64_t offset, uint64_t length,
               const char *buf);
     int Read(uint64_t inodeId, uint64_t offset, uint64_t length, char *buf);
-    CURVEFS_ERROR Truncate(Inode *inode, uint64_t size);
+    CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper, uint64_t size);
     void ReleaseCache(uint64_t inodeId);
     CURVEFS_ERROR Flush(uint64_t inodeId);
     CURVEFS_ERROR FlushAllCache(uint64_t inodeId);

curvefs/src/client/s3/client_s3_cache_manager.cpp

@@ -356,7 +356,8 @@ int FileCacheManager::Read(uint64_t inodeId, uint64_t offset, uint64_t length,
             {
                 ::curve::common::UniqueLock lgGuard =
                     inodeWrapper->GetUniqueLock();
-                Inode* inode = inodeWrapper->GetMutableInodeUnlocked();
+                const Inode* inode = inodeWrapper->GetInodeLocked();
+                const auto* s3chunkinfo = inodeWrapper->GetChunkInfoMap();
                 VLOG(9) << "FileCacheManager::Read Inode: "
                           << inode->DebugString();
                 fileLen = inode->length();
@@ -366,8 +367,8 @@ int FileCacheManager::Read(uint64_t inodeId, uint64_t offset, uint64_t length,
                             << ",len:" << iter->len
                             << ",bufOffset:" << iter->bufOffset;
                     auto s3InfoListIter =
-                        inode->s3chunkinfomap().find(iter->index);
-                    if (s3InfoListIter == inode->s3chunkinfomap().end()) {
+                        s3chunkinfo->find(iter->index);
+                    if (s3InfoListIter == s3chunkinfo->end()) {
                         VLOG(6)
                             << "s3infolist is not found.index:" << iter->index;
                         memset(dataBuf + iter->bufOffset, 0, iter->len);
@@ -497,23 +498,31 @@ int FileCacheManager::ReadFromS3(const std::vector<S3ReadRequest> &requests,
                 << iter->chunkId << ",fsid" << iter->fsId
                 << ",inodeId:" << iter->inodeId;
         // prefetch read
+        // TODO(huyao): The read-ahead trigger logic needs to be refactored and
+        // supplemented with unit tests
         if (s3ClientAdaptor_->HasDiskCache()) {
             uint64_t blockIndexTmp = blockIndex;
             // the counts of blocks that need prefetch
             uint32_t prefetchBlocks = s3ClientAdaptor_->GetPrefetchBlocks();
-            std::vector<std::string> prefetchObjs;
+            std::vector<std::pair<std::string, uint64_t>> prefetchObjs;
             for (int count = 0; count < prefetchBlocks; count++) {
                 std::string name = curvefs::common::s3util::GenObjName(
                     iter->chunkId, blockIndexTmp, iter->compaction, iter->fsId,
                     iter->inodeId);
-                prefetchObjs.push_back(name);
-                blockIndexTmp++;
-                uint64_t readLen = blockIndexTmp * blockSize;
-                if ((readLen > fileLen) ||
-                    (blockIndexTmp >= chunkSize / blockSize)) {
+                uint64_t readLen = (blockIndexTmp + 1) * blockSize;
+                if (readLen > fileLen) {
                     VLOG(6) << "end, redLen :" << readLen
                             << ", fileLen: " << fileLen << ", blockIndexTmp "
                             << blockIndexTmp;
+                    readLen = fileLen - blockIndexTmp * blockSize;
+                    prefetchObjs.push_back(std::make_pair(name, readLen));
+                    break;
+                } else {
+                    prefetchObjs.push_back(std::make_pair(name, blockSize));
+                }
+
+                blockIndexTmp++;
+                if (blockIndexTmp >= chunkSize / blockSize) {
                     break;
                 }
             }
@@ -653,10 +662,11 @@ class AsyncPrefetchCallback {
     S3ClientAdaptorImpl *s3Client_;
 };
 
-void FileCacheManager::PrefetchS3Objs(std::vector<std::string> prefetchObjs) {
-    uint64_t blockSize = s3ClientAdaptor_->GetBlockSize();
+void FileCacheManager::PrefetchS3Objs(
+    const std::vector<std::pair<std::string, uint64_t>> &prefetchObjs) {
     for (auto &obj : prefetchObjs) {
-        std::string name = obj;
+        std::string name = obj.first;
+        uint64_t readLen = obj.second;
         curve::common::LockGuard lg(downloadMtx_);
         if (downloadingObj_.find(name) != downloadingObj_.end()) {
             VLOG(9) << "obj is already in downloading: " << name
@@ -674,13 +684,13 @@ void FileCacheManager::PrefetchS3Objs(std::vector<std::string> prefetchObjs) {
 
         auto inode = inode_;
         auto s3ClientAdaptor = s3ClientAdaptor_;
-        auto task = [name, inode, s3ClientAdaptor, blockSize]() {
-            char *dataCacheS3 = new char[blockSize];
+        auto task = [name, inode, s3ClientAdaptor, readLen]() {
+            char *dataCacheS3 = new char[readLen];
             auto context = std::make_shared<GetObjectAsyncContext>();
             context->key = name;
             context->buf = dataCacheS3;
             context->offset = 0;
-            context->len = blockSize;
+            context->len = readLen;
             context->cb = AsyncPrefetchCallback{inode, s3ClientAdaptor};
             VLOG(9) << "prefetch start: " << context->key
                     << ", len: " << context->len;
@@ -1467,10 +1477,11 @@ void ChunkCacheManager::AddReadDataCache(DataCachePtr dataCache) {
     for (auto key : deleteKeyVec) {
         auto iter = dataRCacheMap_.find(key);
         std::list<DataCachePtr>::iterator dcpIter = iter->second;
+        uint64_t actualLen = (*dcpIter)->GetActualLen();
         if (s3ClientAdaptor_->GetFsCacheManager()->Delete(dcpIter)) {
             g_s3MultiManagerMetric->readDataCacheNum << -1;
             g_s3MultiManagerMetric->readDataCacheByte
-                << -1 * (*dcpIter)->GetActualLen();
+                << -1 * actualLen;
             dataRCacheMap_.erase(iter);
         }
     }

curvefs/src/client/s3/client_s3_cache_manager.h

@@ -315,7 +315,8 @@ class FileCacheManager {
     int ReadFromS3(const std::vector<S3ReadRequest> &requests,
                             std::vector<S3ReadResponse> *responses,
                             uint64_t fileLen);
-    void PrefetchS3Objs(std::vector<std::string> prefetchObjs);
+    void PrefetchS3Objs(
+        const std::vector<std::pair<std::string, uint64_t>> &prefetchObjs);
     void HandleReadRequest(const ReadRequest &request,
                            const S3ChunkInfo &s3ChunkInfo,
                            std::vector<ReadRequest> *addReadRequests,

curvefs/src/client/s3/disk_cache_base.cpp

@@ -93,7 +93,7 @@ int DiskCacheBase::LoadAllCacheFile(std::set<std::string> *cachedObj) {
         return -1;
     }
 
-    LOG(INFO) << "LoadAllCacheFile start, dir: " << cachePath;
+    VLOG(3) << "LoadAllCacheFile start, dir: " << cachePath;
     DIR *cacheDir = NULL;
     struct dirent *cacheDirent = NULL;
     cacheDir = posixWrapper_->opendir(cachePath.c_str());
@@ -115,7 +115,7 @@ int DiskCacheBase::LoadAllCacheFile(std::set<std::string> *cachedObj) {
         LOG(ERROR) << "LoadAllCacheFile, opendir error, errno = " << errno;
         return rc;
     }
-    LOG(INFO) << "LoadAllCacheReadFile end, dir: " << cachePath;
+    VLOG(3) << "LoadAllCacheReadFile end, dir: " << cachePath;
     return 0;
 }
 

curvefs/src/client/s3/disk_cache_manager.cpp

@@ -67,11 +67,11 @@ DiskCacheManager::DiskCacheManager(std::shared_ptr<PosixWrapper> posixWrapper,
     // cannot limit the size,
     // because cache is been delete must after upload to s3
     cachedObjName_ = std::make_shared<
-      LRUCache<std::string, bool>>(0,
+      SglLRUCache<std::string>>(0,
         std::make_shared<CacheMetrics>("diskcache"));
 }
 
-int DiskCacheManager::Init(S3Client *client,
+int DiskCacheManager::Init(std::shared_ptr<S3Client> client,
                            const S3ClientAdaptorOption option) {
     LOG(INFO) << "DiskCacheManager init start.";
     client_ = client;
@@ -111,7 +111,6 @@ int DiskCacheManager::Init(S3Client *client,
     // start trim thread
     TrimRun();
 
-    SetDiskInitUsedBytes();
     SetDiskFsUsedRatio();
 
     FLAGS_avgFlushIops = option_.diskCacheOpt.avgFlushIops;
@@ -159,13 +158,12 @@ int DiskCacheManager::ClearReadCache(const std::list<std::string> &files) {
 
 void DiskCacheManager::AddCache(const std::string name,
   bool cacheWriteExist) {
-    cachedObjName_->Put(name, cacheWriteExist);
+    cachedObjName_->Put(name);
     VLOG(9) << "cache size is: " << cachedObjName_->Size();
 }
 
 bool DiskCacheManager::IsCached(const std::string name) {
-    bool exist;
-    if (!cachedObjName_->Get(name, &exist)) {
+    if (!cachedObjName_->IsCached(name)) {
         VLOG(9) << "not cached, name = " << name;
         return false;
     }
@@ -263,7 +261,8 @@ int DiskCacheManager::LinkWriteToRead(const std::string fileName,
 int64_t DiskCacheManager::SetDiskFsUsedRatio() {
     struct statfs stat;
     if (posixWrapper_->statfs(cacheDir_.c_str(), &stat) == -1) {
-        LOG(ERROR) << "get cache disk space error.";
+        LOG_EVERY_N(WARNING, 100)
+            << "get cache disk space error, errno is: " << errno;
         return -1;
     }
 
@@ -272,8 +271,12 @@ int64_t DiskCacheManager::SetDiskFsUsedRatio() {
     int64_t freeBytes = stat.f_bfree * frsize;
     int64_t availableBytes = stat.f_bavail * frsize;
     int64_t usedBytes = totalBytes - freeBytes;
+    if ((usedBytes == 0) &&
+      (availableBytes == 0)) {
+        LOG_EVERY_N(WARNING, 100) << "get cache disk space zero.";
+        return -1;
+    }
     int64_t usedPercent = 100 * usedBytes / (usedBytes + availableBytes) + 1;
-
     diskFsUsedRatio_.store(usedPercent, std::memory_order_seq_cst);
     return usedPercent;
 }
@@ -284,15 +287,19 @@ void DiskCacheManager::SetDiskInitUsedBytes() {
     SysUtils sysUtils;
     std::string result = sysUtils.RunSysCmd(cmd);
     if (result.empty()) {
-        LOG(ERROR) << "get disk used size failed.";
+        LOG_EVERY_N(WARNING, 100)
+            << "get disk used size failed.";
         return;
     }
     uint64_t usedBytes = 0;
     if (!curve::common::StringToUll(result, &usedBytes)) {
-        LOG(ERROR) << "get disk used size failed.";
+        LOG_EVERY_N(WARNING, 100)
+            << "get disk used size failed.";
         return;
     }
     usedBytes_.fetch_add(usedBytes, std::memory_order_seq_cst);
+    if (metric_.get() != nullptr)
+        metric_->diskUsedBytes.set_value(usedBytes_);
     VLOG(9) << "cache disk used size is: " << result;
     return;
 }
@@ -355,6 +362,7 @@ void DiskCacheManager::TrimCache() {
     cacheReadFullDir = GetCacheReadFullDir();
     cacheWriteFullDir = GetCacheWriteFullDir();
     while (true) {
+        SetDiskFsUsedRatio();
         waitIntervalSec_.WaitForNextExcution();
         if (!isRunning_) {
             LOG(INFO) << "trim thread end.";
@@ -362,51 +370,53 @@ void DiskCacheManager::TrimCache() {
         }
         VLOG(9) << "trim thread wake up.";
         InitQosParam();
-        SetDiskFsUsedRatio();
-            while (!IsDiskCacheSafe()) {
-                if (!cachedObjName_->GetLast(false, &cacheKey)) {
-                    VLOG(9) << "obj is empty";
-                    break;
-                }
-
-                VLOG(6) << "obj will be removed01: " << cacheKey;
-                cacheReadFile = cacheReadFullDir + "/" + cacheKey;
-                cacheWriteFile = cacheWriteFullDir + "/" + cacheKey;
-                struct stat statFile;
-                int ret;
-                ret = posixWrapper_->stat(cacheWriteFile.c_str(), &statFile);
-                // if file has not been uploaded to S3,
-                // but remove the cache read file,
-                // then read will fail when do cache read,
-                // and then it cannot load the file from S3.
-                // so read is fail.
-                if (ret == 0) {
-                    VLOG(1) << "do not remove this disk file"
-                            << ", file has not been uploaded to S3."
-                            << ", file is: " << cacheKey;
-                    continue;
-                }
-                cachedObjName_->Remove(cacheKey);
-                struct stat statReadFile;
-                ret = posixWrapper_->stat(cacheReadFile.c_str(), &statReadFile);
-                if (ret != 0) {
-                    VLOG(0) << "stat disk file error"
-                            << ", file is: " << cacheKey;
-                    continue;
-                }
-                // if remove disk file before delete cache,
-                // then read maybe fail.
-                const char *toDelFile;
-                toDelFile = cacheReadFile.c_str();
-                ret = posixWrapper_->remove(toDelFile);
-                if (ret < 0) {
-                    LOG(ERROR)
-                        << "remove disk file error, file is: " << cacheKey;
-                    continue;
-                }
-                DecDiskUsedBytes(statReadFile.st_size);
-                VLOG(6) << "remove disk file success, file is: " << cacheKey;
+        while (!IsDiskCacheSafe()) {
+            SetDiskFsUsedRatio();
+            if (!cachedObjName_->GetBack(&cacheKey)) {
+                VLOG(9) << "obj is empty";
+                break;
             }
+
+            VLOG(6) << "obj will be removed01: " << cacheKey;
+            cacheReadFile = cacheReadFullDir + "/" + cacheKey;
+            cacheWriteFile = cacheWriteFullDir + "/" + cacheKey;
+            struct stat statFile;
+            int ret = 0;
+            ret = posixWrapper_->stat(cacheWriteFile.c_str(), &statFile);
+            // if file has not been uploaded to S3,
+            // but remove the cache read file,
+            // then read will fail when do cache read,
+            // and then it cannot load the file from S3.
+            // so read is fail.
+            if (ret == 0) {
+                VLOG(1) << "do not remove this disk file"
+                        << ", file has not been uploaded to S3."
+                        << ", file is: " << cacheKey;
+                usleep(1000);
+                continue;
+            }
+            cachedObjName_->Remove(cacheKey);
+            struct stat statReadFile;
+            ret = posixWrapper_->stat(cacheReadFile.c_str(), &statReadFile);
+            if (ret != 0) {
+                VLOG(0) << "stat disk file error"
+                        << ", file is: " << cacheKey;
+                continue;
+            }
+            // if remove disk file before delete cache,
+            // then read maybe fail.
+            const char *toDelFile;
+            toDelFile = cacheReadFile.c_str();
+            ret = posixWrapper_->remove(toDelFile);
+            if (ret < 0) {
+                LOG(ERROR)
+                    << "remove disk file error, file is: " << cacheKey
+                    << "error is: " << errno;
+                continue;
+            }
+            DecDiskUsedBytes(statReadFile.st_size);
+            VLOG(6) << "remove disk file success, file is: " << cacheKey;
+        }
     }
     LOG(INFO) << "trim function end.";
 }
@@ -439,6 +449,9 @@ void DiskCacheManager::InitMetrics(const std::string &fsName) {
     metric_ = std::make_shared<DiskCacheMetric>(fsName);
     cacheWrite_->InitMetrics(metric_);
     cacheRead_->InitMetrics(metric_);
+    // this function move to here from init，
+    // Otherwise, you can't get the original metric
+    SetDiskInitUsedBytes();
 }
 
 }  // namespace client

curvefs/src/client/s3/disk_cache_manager.h

@@ -61,7 +61,8 @@ class DiskCacheManager {
     DiskCacheManager() {}
     virtual ~DiskCacheManager() { TrimStop(); }
 
-    virtual int Init(S3Client *client, const S3ClientAdaptorOption option);
+    virtual int Init(std::shared_ptr<S3Client> client,
+        const S3ClientAdaptorOption option);
 
     virtual int UmountDiskCache();
     virtual bool IsCached(const std::string name);
@@ -114,9 +115,11 @@ class DiskCacheManager {
      * @brief add the used bytes of disk cache.
      */
     void AddDiskUsedBytes(uint64_t length) {
-        usedBytes_.fetch_add(length, std::memory_order_seq_cst);
+        usedBytes_.fetch_add(length);
+        if (metric_.get() != nullptr)
+            metric_->diskUsedBytes.set_value(usedBytes_/1024/1024);
         VLOG(9) << "add disk used size is: " << length
-                << ", now is: " << usedBytes_.load(std::memory_order_seq_cst);
+                << ", now is: " << usedBytes_.load();
         return;
     }
     /**
@@ -125,17 +128,17 @@ class DiskCacheManager {
      * because there are link in read cache
      */
     void DecDiskUsedBytes(uint64_t length) {
-        int64_t usedBytes;
-        usedBytes = usedBytes_.fetch_sub(length, std::memory_order_seq_cst);
-        assert(usedBytes >= 0);
-        (void)usedBytes;
+        usedBytes_.fetch_sub(length);
+        assert(usedBytes_ >= 0);
+        if (metric_.get() != nullptr)
+            metric_->diskUsedBytes.set_value(usedBytes_);
         VLOG(9) << "dec disk used size is: " << length
-                << ", now is: " << usedBytes_.load(std::memory_order_seq_cst);
+                << ", now is: " << usedBytes_.load();
         return;
     }
     void SetDiskInitUsedBytes();
     uint64_t GetDiskUsedbytes() {
-        return usedBytes_.load(std::memory_order_seq_cst);
+        return usedBytes_.load();
     }
 
     void InitQosParam();
@@ -167,9 +170,9 @@ class DiskCacheManager {
     std::shared_ptr<DiskCacheWrite> cacheWrite_;
     std::shared_ptr<DiskCacheRead> cacheRead_;
 
-    std::shared_ptr<LRUCache<std::string, bool>> cachedObjName_;
+    std::shared_ptr<SglLRUCache<std::string>> cachedObjName_;
 
-    S3Client *client_;
+    std::shared_ptr<S3Client> client_;
     std::shared_ptr<PosixWrapper> posixWrapper_;
     std::shared_ptr<DiskCacheMetric> metric_;
 

curvefs/src/client/s3/disk_cache_manager_impl.cpp

@@ -34,7 +34,8 @@ namespace curvefs {
 namespace client {
 
 DiskCacheManagerImpl::DiskCacheManagerImpl(
-    std::shared_ptr<DiskCacheManager> diskCacheManager, S3Client *client) {
+    std::shared_ptr<DiskCacheManager> diskCacheManager,
+    std::shared_ptr<S3Client> client) {
     diskCacheManager_ = diskCacheManager;
     client_ = client;
 }

curvefs/src/client/s3/disk_cache_manager_impl.h

@@ -68,7 +68,7 @@ struct DiskCacheOption {
 class DiskCacheManagerImpl {
  public:
     DiskCacheManagerImpl(std::shared_ptr<DiskCacheManager> diskCacheManager,
-                         S3Client *client);
+        std::shared_ptr<S3Client> client);
     DiskCacheManagerImpl() {}
     virtual ~DiskCacheManagerImpl() {}
     /**
@@ -125,7 +125,7 @@ class DiskCacheManagerImpl {
     std::shared_ptr<DiskCacheManager> diskCacheManager_;
 
     bool forceFlush_;
-    S3Client *client_;
+    std::shared_ptr<S3Client> client_;
 
     int WriteClosure(std::shared_ptr<PutObjectAsyncContext> context);
     // threads for disk cache

curvefs/src/client/s3/disk_cache_read.cpp

@@ -107,7 +107,7 @@ int DiskCacheRead::LinkWriteToRead(const std::string fileName,
 }
 
 int DiskCacheRead::LoadAllCacheReadFile(
-    std::shared_ptr<LRUCache<std::string, bool>> cachedObj) {
+    std::shared_ptr<SglLRUCache<std::string>> cachedObj) {
     std::set<std::string> tmp;
     int ret = LoadAllCacheFile(&tmp);
     if (ret < 0) {
@@ -118,7 +118,7 @@ int DiskCacheRead::LoadAllCacheReadFile(
     }
 
     for (auto iter = tmp.begin(); iter != tmp.end(); iter++) {
-        cachedObj->Put(std::move(*iter), false);
+        cachedObj->Put(std::move(*iter));
     }
 
     return ret;

curvefs/src/client/s3/disk_cache_read.h

@@ -37,7 +37,7 @@
 namespace curvefs {
 namespace client {
 
-using curve::common::LRUCache;
+using curve::common::SglLRUCache;
 using curvefs::common::PosixWrapper;
 
 class DiskCacheRead : public DiskCacheBase {
@@ -58,8 +58,8 @@ class DiskCacheRead : public DiskCacheBase {
      * @brief after reboot，load all files that store in read cache.
      */
     virtual int
-    LoadAllCacheReadFile(std::shared_ptr<LRUCache<
-      std::string, bool>> cachedObj);
+    LoadAllCacheReadFile(std::shared_ptr<SglLRUCache<
+      std::string>> cachedObj);
     virtual int ClearReadCache(const std::list<std::string> &files);
     virtual void InitMetrics(std::shared_ptr<DiskCacheMetric> metric) {
         metric_ = metric;

curvefs/src/client/s3/disk_cache_write.cpp

@@ -35,12 +35,12 @@ namespace curvefs {
 
 namespace client {
 
-void DiskCacheWrite::Init(S3Client *client,
+void DiskCacheWrite::Init(std::shared_ptr<S3Client> client,
                           std::shared_ptr<PosixWrapper> posixWrapper,
                           const std::string cacheDir,
                           uint64_t asyncLoadPeriodMs,
-                          std::shared_ptr<LRUCache<
-                            std::string, bool>> cachedObjName) {
+                          std::shared_ptr<SglLRUCache<
+                            std::string>> cachedObjName) {
     client_ = client;
     posixWrapper_ = posixWrapper;
     asyncLoadPeriodMs_ = asyncLoadPeriodMs;
@@ -260,7 +260,7 @@ int DiskCacheWrite::UploadFileByInode(const std::string &inode) {
         if (ret <= 0) {
             return ret;
         }
-        LOG(INFO) << "UploadFileByInode, need wait file on disk uopload ok";
+        VLOG(3) << "UploadFileByInode, need wait file on disk upload ok";
         sleeper_.wait_for(std::chrono::milliseconds(asyncLoadPeriodMs_));
     }
 
@@ -392,7 +392,6 @@ int DiskCacheWrite::UploadAllCacheWriteFile() {
     }
     for (auto iter = uploadObjs.begin(); iter != uploadObjs.end(); iter++) {
         RemoveFile(*iter);
-        cachedObjName_->Put(*iter, false);
     }
     VLOG(3) << "upload all cached write file end.";
     return 0;
@@ -409,7 +408,7 @@ int DiskCacheWrite::RemoveFile(const std::string fileName) {
                    << ", errno = " << errno;
         return -1;
     }
-    cachedObjName_->Put(fileName, false);
+    cachedObjName_->MoveBack(fileName);
     VLOG(9) << "remove file success, file = " << fileName;
     return 0;
 }

curvefs/src/client/s3/disk_cache_write.h

@@ -49,7 +49,7 @@ namespace client {
 
 using curvefs::common::PosixWrapper;
 using curve::common::InterruptibleSleeper;
-using ::curve::common::LRUCache;
+using ::curve::common::SglLRUCache;
 using curve::common::PutObjectAsyncCallBack;
 
 class DiskCacheWrite : public DiskCacheBase {
@@ -81,9 +81,10 @@ class DiskCacheWrite : public DiskCacheBase {
     virtual ~DiskCacheWrite() {
        AsyncUploadStop();
     }
-    void Init(S3Client *client, std::shared_ptr<PosixWrapper> posixWrapper,
+    void Init(std::shared_ptr<S3Client> client,
+              std::shared_ptr<PosixWrapper> posixWrapper,
               const std::string cacheDir, uint64_t asyncLoadPeriodMs,
-              std::shared_ptr<LRUCache<std::string, bool>> cachedObjName);
+              std::shared_ptr<SglLRUCache<std::string>> cachedObjName);
     /**
      * @brief write obj to write cahce disk
      * @param[in] client S3Client
@@ -148,12 +149,12 @@ class DiskCacheWrite : public DiskCacheBase {
     bthread::Mutex mtx_;
     InterruptibleSleeper sleeper_;
     uint64_t asyncLoadPeriodMs_;
-    S3Client *client_;
+    std::shared_ptr<S3Client> client_;
     // file system operation encapsulation
     std::shared_ptr<PosixWrapper> posixWrapper_;
     std::shared_ptr<DiskCacheMetric> metric_;
 
-    std::shared_ptr<LRUCache<std::string, bool>> cachedObjName_;
+    std::shared_ptr<SglLRUCache<std::string>> cachedObjName_;
 };
 
 }  // namespace client

curvefs/src/client/volume/default_volume_storage.cpp

@@ -112,12 +112,11 @@ ssize_t DefaultVolumeStorage::Write(uint64_t ino,
 
     {
         auto lk = inodeWrapper->GetUniqueLock();
-        auto* inode = inodeWrapper->GetMutableInodeUnlocked();
-        UpdateInodeTimestamp(inode, kModifyTime | kChangeTime);
-
-        if (offset + len > inode->length()) {
-            inode->set_length(offset + len);
+        if (offset + len > inodeWrapper->GetLengthLocked()) {
+            inodeWrapper->SetLengthLocked(offset + len);
         }
+
+        inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
     }
 
     inodeCacheManager_->ShipToFlush(inodeWrapper);
@@ -165,8 +164,10 @@ ssize_t DefaultVolumeStorage::Read(uint64_t ino,
     }
 
     // TODO(all): check whether inode is opened with 'NO_ATIME'
-    auto* inode = inodeWrapper->GetMutableInodeUnlocked();
-    UpdateInodeTimestamp(inode, kAccessTime);
+    {
+        auto lock = inodeWrapper->GetUniqueLock();
+        inodeWrapper->UpdateTimestampLocked(kAccessTime);
+    }
     inodeCacheManager_->ShipToFlush(inodeWrapper);
 
     return len;

curvefs/src/client/volume/utils.cpp

@@ -36,26 +36,6 @@ namespace client {
 using ::curvefs::metaserver::Inode;
 using ::curvefs::volume::AllocateHint;
 
-void UpdateInodeTimestamp(Inode* inode, int flags) {
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-
-    if (flags & kAccessTime) {
-        inode->set_atime(now.tv_sec);
-        inode->set_atime_ns(now.tv_nsec);
-    }
-
-    if (flags & kChangeTime) {
-        inode->set_ctime(now.tv_sec);
-        inode->set_ctime_ns(now.tv_nsec);
-    }
-
-    if (flags & kModifyTime) {
-        inode->set_mtime(now.tv_sec);
-        inode->set_mtime_ns(now.tv_nsec);
-    }
-}
-
 bool AllocSpace(SpaceManager* space,
                 const AllocPart& part,
                 std::map<uint64_t, WritePart>* writes,

curvefs/src/client/volume/utils.h

@@ -43,14 +43,6 @@ using ::curvefs::volume::Extent;
 using ::curvefs::volume::SpaceManager;
 using ::curvefs::volume::WritePart;
 
-enum {
-    kAccessTime = 1 << 0,
-    kChangeTime = 1 << 1,
-    kModifyTime = 1 << 2,
-};
-
-void UpdateInodeTimestamp(Inode* inode, int flags);
-
 /**
  * @brief Allocate space for a write request
  * @param space space manager that used for allocating space

curvefs/src/client/xattr_manager.cpp

@@ -31,6 +31,21 @@ namespace client {
 using ::curve::common::StringToUll;
 using ::curve::common::Thread;
 
+
+bool IsSummaryInfo(const char *name) {
+    return std::strstr(name, SUMMARYPREFIX);
+}
+
+bool IsOneLayer(const char *name) {
+    if (std::strcmp(name, XATTRFILES) == 0 ||
+        std::strcmp(name, XATTRSUBDIRS) == 0 ||
+        std::strcmp(name, XATTRENTRIES) == 0 ||
+        std::strcmp(name, XATTRFBYTES) == 0) {
+        return true;
+    }
+    return false;
+}
+
 // if direction is true means '+', false means '-'
 bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction) {
     uint64_t firstNum = 0;
@@ -41,8 +56,9 @@ bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction) {
         } else {
             if (firstNum < secondNum) {
                 *first = std::to_string(0);
-                LOG(ERROR) << "AddUllStringToFirst failed when minus, first = "
-                           << firstNum << ", second = " << secondNum;
+                LOG(WARNING) << "AddUllStringToFirst failed when minus,"
+                             << " first = " << firstNum
+                             << ", second = " << secondNum;
                 return false;
             }
             *first = std::to_string(firstNum - secondNum);
@@ -157,8 +173,23 @@ bool XattrManager::ConcurrentListDentry(
             continue;
         }
 
-        auto tret = dentryManager_->ListDentry(ino, dentrys,
-                                              listDentryLimit_, dirOnly);
+        // if onlydir, can get parent nlink to know dir number under this dir
+        uint32_t nlink = 0;
+        if (dirOnly) {
+            InodeAttr attr;
+            auto retCode = inodeManager_->GetInodeAttr(ino, &attr);
+            if (retCode != CURVEFS_ERROR::OK) {
+                LOG(ERROR) << "inodeManager get inodeAttr fail, ret = "
+                           << retCode << ", inodeid = " << ino;
+                ret->store(false);
+                inflightNum->fetch_sub(1);
+                return false;
+            }
+            nlink = attr.nlink();
+        }
+
+        auto tret = dentryManager_->ListDentry(ino, dentrys, listDentryLimit_,
+                                               dirOnly, nlink);
         if (CURVEFS_ERROR::OK != tret) {
             LOG(ERROR) << "ListDentry failed, inodeId = " << ino
                        << ", limit = " << listDentryLimit_ << ", onlyDir = "
@@ -425,6 +456,43 @@ CURVEFS_ERROR XattrManager::FastCalAllLayerSumInfo(InodeAttr *attr) {
     return CURVEFS_ERROR::OK;
 }
 
+CURVEFS_ERROR XattrManager::GetXattr(const char* name, std::string *value,
+    InodeAttr *attr, bool enableSumInDir) {
+    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
+    // get summary info if the xattr name is summary type
+    if (IsSummaryInfo(name) && attr->type() == FsFileType::TYPE_DIRECTORY) {
+        // if not enable record summary info in dir xattr,
+        // need recursive computation all files;
+        // otherwise only recursive computation all dirs.
+        if (!enableSumInDir) {
+            if (IsOneLayer(name)) {
+                ret = CalOneLayerSumInfo(attr);
+            } else {
+                ret = CalAllLayerSumInfo(attr);
+            }
+        } else {
+            if (IsOneLayer(name)) {
+                ret = FastCalOneLayerSumInfo(attr);
+            } else {
+                ret = FastCalAllLayerSumInfo(attr);
+            }
+        }
+
+        if (CURVEFS_ERROR::OK != ret) {
+            return ret;
+        }
+        LOG(INFO) << "After calculate summary info:\n"
+                  << attr->DebugString();
+    }
+
+    auto it = attr->xattr().find(name);
+    if (it != attr->xattr().end()) {
+        *value = it->second;
+    }
+    return ret;
+}
+
+
 CURVEFS_ERROR XattrManager::UpdateParentInodeXattr(uint64_t parentId,
     const XAttr &xattr, bool direction) {
     VLOG(9) << "UpdateParentInodeXattr inodeId = " << parentId
@@ -439,10 +507,11 @@ CURVEFS_ERROR XattrManager::UpdateParentInodeXattr(uint64_t parentId,
     }
 
     ::curve::common::UniqueLock lgGuard = pInodeWrapper->GetUniqueLock();
-    auto inode = pInodeWrapper->GetMutableInodeUnlocked();
+    auto inodeXAttr = pInodeWrapper->GetInodeLocked()->xattr();
+    bool update = false;
     for (const auto &it : xattr.xattrinfos()) {
-        auto iter = inode->mutable_xattr()->find(it.first);
-        if (iter != inode->mutable_xattr()->end()) {
+        auto iter = inodeXAttr.find(it.first);
+        if (iter != inodeXAttr.end()) {
             uint64_t dat = 0;
             if (StringToUll(it.second, &dat)) {
                 if (!AddUllStringToFirst(&(iter->second), dat, direction)) {
@@ -452,9 +521,15 @@ CURVEFS_ERROR XattrManager::UpdateParentInodeXattr(uint64_t parentId,
                 LOG(ERROR) << "StringToUll failed, first = " << it.second;
                 return CURVEFS_ERROR::INTERNAL;
             }
+            update = true;
         }
     }
-    inodeManager_->ShipToFlush(pInodeWrapper);
+
+    if (update) {
+        pInodeWrapper->MergeXAttrLocked(inodeXAttr);
+        inodeManager_->ShipToFlush(pInodeWrapper);
+    }
+
     return CURVEFS_ERROR::OK;
 }
 

curvefs/src/client/xattr_manager.h

@@ -74,13 +74,8 @@ class XattrManager {
         isStop_.store(true);
     }
 
-    CURVEFS_ERROR CalOneLayerSumInfo(InodeAttr *attr);
-
-    CURVEFS_ERROR CalAllLayerSumInfo(InodeAttr *attr);
-
-    CURVEFS_ERROR FastCalOneLayerSumInfo(InodeAttr *attr);
-
-    CURVEFS_ERROR FastCalAllLayerSumInfo(InodeAttr *attr);
+    CURVEFS_ERROR GetXattr(const char* name, std::string *value,
+        InodeAttr *attr, bool enableSumInDir);
 
     CURVEFS_ERROR UpdateParentInodeXattr(uint64_t parentId,
         const XAttr &xattr, bool direction);
@@ -116,6 +111,14 @@ class XattrManager {
         Atomic<uint32_t> *inflightNum,
         Atomic<bool> *ret);
 
+    CURVEFS_ERROR CalOneLayerSumInfo(InodeAttr *attr);
+
+    CURVEFS_ERROR CalAllLayerSumInfo(InodeAttr *attr);
+
+    CURVEFS_ERROR FastCalOneLayerSumInfo(InodeAttr *attr);
+
+    CURVEFS_ERROR FastCalAllLayerSumInfo(InodeAttr *attr);
+
  private:
     // inode cache manager
     std::shared_ptr<InodeCacheManager> inodeManager_;

curvefs/src/mds/fs_info_wrapper.cpp

@@ -37,15 +37,35 @@ using ::curvefs::common::S3Info;
 using ::curvefs::common::Volume;
 using google::protobuf::util::MessageDifferencer;
 
-bool FsInfoWrapper::IsMountPointExist(const Mountpoint& mp) const {
+bool FsInfoWrapper::IsMountPointExist(const Mountpoint &mp) const {
     return std::find_if(fsInfo_.mountpoints().begin(),
                         fsInfo_.mountpoints().end(),
-                        [mp](const Mountpoint& mountPoint) {
+                        [mp](const Mountpoint &mountPoint) {
                             return mp.path() == mountPoint.path() &&
                                    mp.hostname() == mountPoint.hostname();
                         }) != fsInfo_.mountpoints().end();
 }
 
+bool FsInfoWrapper::IsMountPointConflict(const Mountpoint &mp) const {
+    bool cto = (fsInfo_.mountpoints_size() ? false : mp.cto());
+
+    bool exist =
+        std::find_if(fsInfo_.mountpoints().begin(), fsInfo_.mountpoints().end(),
+                     [&](const Mountpoint &mountPoint) {
+                         if (mountPoint.has_cto() && mountPoint.cto()) {
+                             cto = true;
+                         }
+
+                         return mp.path() == mountPoint.path() &&
+                                mp.hostname() == mountPoint.hostname();
+                     }) != fsInfo_.mountpoints().end();
+
+    // NOTE:
+    // 1. if mount point exist (exist = true), conflict
+    // 2. if existing mount point enableCto is diffrent from newcomer, conflict
+    return exist || (cto != mp.cto());
+}
+
 void FsInfoWrapper::AddMountPoint(const Mountpoint& mp) {
     // TODO(wuhanqing): sort after add ?
     auto* p = fsInfo_.add_mountpoints();
@@ -57,8 +77,10 @@ void FsInfoWrapper::AddMountPoint(const Mountpoint& mp) {
 FSStatusCode FsInfoWrapper::DeleteMountPoint(const Mountpoint& mp) {
     auto iter =
         std::find_if(fsInfo_.mountpoints().begin(), fsInfo_.mountpoints().end(),
-                     [mp](const Mountpoint& mountPoint) {
-                         return MessageDifferencer::Equals(mp, mountPoint);
+                     [mp](const Mountpoint &mountPoint) {
+                         return mp.path() == mountPoint.path() &&
+                                mp.hostname() == mountPoint.hostname() &&
+                                mp.port() == mountPoint.port();
                      });
 
     bool found = iter != fsInfo_.mountpoints().end();

curvefs/src/mds/fs_info_wrapper.h

@@ -153,6 +153,8 @@ class FsInfoWrapper {
 
     bool IsMountPointExist(const Mountpoint& mp) const;
 
+    bool IsMountPointConflict(const Mountpoint &mp) const;
+
     void AddMountPoint(const Mountpoint& mp);
 
     FSStatusCode DeleteMountPoint(const Mountpoint& mp);

curvefs/src/mds/fs_manager.cpp

@@ -447,7 +447,7 @@ FSStatusCode FsManager::MountFs(const std::string& fsName,
                                 const Mountpoint& mountpoint, FsInfo* fsInfo) {
     NameLockGuard lock(nameLock_, fsName);
 
-    // 1. query fs
+    // query fs
     FsInfoWrapper wrapper;
     FSStatusCode ret = fsStorage_->Get(fsName, &wrapper);
     if (ret != FSStatusCode::OK) {
@@ -456,7 +456,7 @@ FSStatusCode FsManager::MountFs(const std::string& fsName,
         return ret;
     }
 
-    // 2. check fs status
+    // check fs status
     FsStatus status = wrapper.GetStatus();
     switch (status) {
         case FsStatus::NEW:
@@ -474,14 +474,22 @@ FSStatusCode FsManager::MountFs(const std::string& fsName,
             return FSStatusCode::UNKNOWN_ERROR;
     }
 
-    // 3. if mount point exist, return MOUNT_POINT_EXIST
-    if (wrapper.IsMountPointExist(mountpoint)) {
-        LOG(WARNING) << "MountFs fail, mount point exist, fsName = " << fsName
-                     << ", mountpoint = " << mountpoint.ShortDebugString();
-        return FSStatusCode::MOUNT_POINT_EXIST;
+    // check param
+    if (!mountpoint.has_cto()) {
+        LOG(WARNING) << "MountFs fail, mount point miss cto param, fsName = "
+                     << fsName << ", fs status = " << FsStatus_Name(status);
+        return FSStatusCode::PARAM_ERROR;
     }
 
-    // 4. If this is the first mountpoint, init space,
+    // mount point conflict
+    if (wrapper.IsMountPointConflict(mountpoint)) {
+        LOG(WARNING) << "MountFs fail, mount point conflict, fsName = "
+                     << fsName
+                     << ", mountpoint = " << mountpoint.ShortDebugString();
+        return FSStatusCode::MOUNT_POINT_CONFLICT;
+    }
+
+    // If this is the first mountpoint, init space,
     if (wrapper.GetFsType() == FSType::TYPE_VOLUME &&
         wrapper.IsMountPointEmpty()) {
         FsInfo tempFsInfo = wrapper.ProtoFsInfo();
@@ -494,7 +502,7 @@ FSStatusCode FsManager::MountFs(const std::string& fsName,
         }
     }
 
-    // 5. insert mountpoint
+    // insert mountpoint
     wrapper.AddMountPoint(mountpoint);
     // for persistence consider
     ret = fsStorage_->Update(wrapper);
@@ -509,7 +517,6 @@ FSStatusCode FsManager::MountFs(const std::string& fsName,
 
     // 6. convert fs info
     *fsInfo = wrapper.ProtoFsInfo();
-
     FsMetric::GetInstance().OnMount(wrapper.GetFsName(), mountpoint);
 
     return FSStatusCode::OK;
@@ -758,6 +765,13 @@ FSStatusCode FsManager::GetFsTxSequence(const std::string& fsName,
 void FsManager::GetLatestTxId(const GetLatestTxIdRequest* request,
                               GetLatestTxIdResponse* response) {
     std::vector<PartitionTxId> txIds;
+    if (!request->has_fsid()) {
+        response->set_statuscode(FSStatusCode::PARAM_ERROR);
+        LOG(ERROR) << "Bad GetLatestTxId request which missing fsid"
+                   << ", request=" << request->DebugString();
+        return;
+    }
+
     uint32_t fsId = request->fsid();
     if (!request->lock()) {
         GetLatestTxId(fsId, &txIds);

curvefs/src/mds/fs_manager.h

@@ -134,8 +134,8 @@ class FsManager {
      * @param[out] fsInfo: return the fsInfo
      *
      * @return If success return OK;
-     *         if fs has same mount point, return MOUNT_POINT_EXIST;
-     *         else return error code
+     *         if fs has same mount point or cto not consistent, return
+     *         MOUNT_POINT_CONFLICT; else return error code
      */
     FSStatusCode MountFs(const std::string& fsName,
                          const Mountpoint& mountpoint, FsInfo* fsInfo);

curvefs/src/mds/main.cpp

@@ -56,7 +56,6 @@ int main(int argc, char **argv) {
     conf->GetValueFatalIfFail("mds.loglevel", &FLAGS_v);
     LoadConfigFromCmdline(conf.get());
     FLAGS_vlog_level = FLAGS_v;
-    conf->PrintConfig();
     if (FLAGS_log_dir.empty()) {
         if (!conf->GetStringValue("mds.common.logDir", &FLAGS_log_dir)) {
             LOG(WARNING) << "no mds.common.logDir in " << confPath
@@ -67,6 +66,8 @@ int main(int argc, char **argv) {
     // initialize logging module
     google::InitGoogleLogging(argv[0]);
 
+    conf->PrintConfig();
+
     curvefs::mds::MDS mds;
 
     // initialize MDS options

curvefs/src/mds/mds.cpp

@@ -297,7 +297,7 @@ void MDS::Stop() {
 }
 
 void MDS::StartDummyServer() {
-    conf_->ExposeMetric("curvefs_mds");
+    conf_->ExposeMetric("curvefs_mds_config");
     status_.expose("curvefs_mds_status");
     status_.set_value("follower");
 

curvefs/src/mds/mds_service.cpp

@@ -273,7 +273,7 @@ void MdsServiceImpl::AllocateS3Chunk(
     ::curvefs::mds::AllocateS3ChunkResponse* response,
     ::google::protobuf::Closure* done) {
     brpc::ClosureGuard guard(done);
-    VLOG(0) << "start to allocate chunkId.";
+    VLOG(9) << "start to allocate chunkId.";
 
     uint64_t chunkId = 0;
     int stat = chunkIdAllocator_->GenChunkId(&chunkId);
@@ -292,7 +292,7 @@ void MdsServiceImpl::AllocateS3Chunk(
                    << ", error: " << FSStatusCode_Name(resStat);
     } else {
         response->set_chunkid(chunkId);
-        VLOG(0) << "AllocateS3Chunk success, request: "
+        VLOG(9) << "AllocateS3Chunk success, request: "
                 << request->ShortDebugString()
                 << ", response: " << response->ShortDebugString();
     }
@@ -327,7 +327,9 @@ void MdsServiceImpl::GetLatestTxId(
     GetLatestTxIdResponse* response,
     ::google::protobuf::Closure* done) {
     brpc::ClosureGuard guard(done);
+    VLOG(3) << "GetLatestTxId [request]: " << request->DebugString();
     fsManager_->GetLatestTxId(request, response);
+    VLOG(3) << "GetLatestTxId [response]: " << response->DebugString();
 }
 
 void MdsServiceImpl::CommitTx(::google::protobuf::RpcController* controller,
@@ -335,7 +337,9 @@ void MdsServiceImpl::CommitTx(::google::protobuf::RpcController* controller,
                               CommitTxResponse* response,
                               ::google::protobuf::Closure* done) {
     brpc::ClosureGuard guard(done);
+    VLOG(3) << "CommitTx [request]: " << request->DebugString();
     fsManager_->CommitTx(request, response);
+    VLOG(3) << "CommitTx [response]: " << request->DebugString();
 }
 
 }  // namespace mds

curvefs/src/mds/metaserverclient/metaserver_client.cpp

@@ -108,6 +108,7 @@ FSStatusCode MetaserverClient::GetLeader(const LeaderCtx &ctx,
     request.set_copysetid(ctx.copysetId);
 
     for (const std::string &item : ctx.addrs) {
+        LOG(INFO) << "GetLeader from " << item;
         if (channel_.Init(item.c_str(), nullptr) != 0) {
             LOG(ERROR) << "Init channel to metaserver: " << item << " failed!";
             continue;
@@ -119,20 +120,24 @@ FSStatusCode MetaserverClient::GetLeader(const LeaderCtx &ctx,
         stub.GetLeader(&cntl, &request, &response, nullptr);
 
         uint32_t maxRetry = options_.rpcRetryTimes;
-        while (cntl.Failed() && maxRetry > 0) {
+        while (cntl.Failed() && (maxRetry > 0)) {
+            int32_t retCode = cntl.ErrorCode();
+            LOG(WARNING) << "GetLeader failed"
+                         << ", poolid = " << ctx.poolId
+                         << ", copysetId = " << ctx.copysetId
+                         << ", errorCode = " << retCode
+                         << ", Rpc error = " << cntl.ErrorText();
+            if (retCode == EHOSTDOWN || retCode == ECONNRESET ||
+                retCode == ECONNREFUSED || retCode == brpc::ELOGOFF) {
+                break;
+            }
             maxRetry--;
             bthread_usleep(options_.rpcRetryIntervalUs);
             cntl.Reset();
             cntl.set_timeout_ms(options_.rpcTimeoutMs);
             stub.GetLeader(&cntl, &request, &response, nullptr);
         }
-        if (cntl.Failed()) {
-            LOG(WARNING) << "GetLeader failed"
-                         << ", poolid = " << ctx.poolId
-                         << ", copysetId = " << ctx.copysetId
-                         << ", Rpc error = " << cntl.ErrorText();
-            continue;
-        }
+
         if (response.has_leader()) {
             std::string ip;
             uint32_t port;
@@ -267,6 +272,8 @@ FSStatusCode MetaserverClient::CreatePartition(
     } else {
         switch (response.statuscode()) {
             case MetaStatusCode::OK:
+                LOG(INFO) << "CreatePartition success, partitionId = "
+                          << partitionId;
                 return FSStatusCode::OK;
             case MetaStatusCode::PARTITION_EXIST:
                 LOG(ERROR) << "CreatePartition failed, partition exist.";

curvefs/src/mds/topology/topology_manager.cpp

@@ -57,7 +57,8 @@ void TopologyManager::RegistMetaServer(const MetaServerRegistRequest *request,
     std::vector<MetaServerIdType> list = topology_->GetMetaServerInCluster(
         [&hostIp, &port](const MetaServer &ms) {
             return (ms.GetInternalIp() == hostIp) &&
-                   (ms.GetInternalPort() == port);
+                   (ms.GetInternalPort() == port) &&
+                   (ms.GetOnlineState() != OnlineState::OFFLINE);
         });
     if (1 == list.size()) {
         // report duplicated register (already a metaserver with same ip and

curvefs/src/metaserver/copyset/copyset_node.cpp

@@ -174,7 +174,7 @@ void CopysetNode::Stop() {
     }
 
     if (metaStore_) {
-        LOG_IF(ERROR, metaStore_->Clear() != true)
+        LOG_IF(ERROR, metaStore_->Destroy() != true)
             << "Failed to clear metastore, copyset: " << name_;
     }
 }

curvefs/src/metaserver/dentry_storage.cpp

@@ -20,6 +20,8 @@
  * Author: chenwei
  */
 
+#include <butil/time.h>
+#include <cstdint>
 #include <vector>
 #include <memory>
 #include <algorithm>
@@ -133,13 +135,19 @@ void DentryList::PushBack(DentryVec* vec) {
         return;
     } else if (dentrys.size() == 0 || HasDeleteMarkFlag(*last)) {
         return;
-    } else if (onlyDir_ && last->type() != FsFileType::TYPE_DIRECTORY) {
-        return;
     } else if (last->name() == exclude_) {
         return;
     }
 
     size_++;
+
+    if (onlyDir_ && last->type() != FsFileType::TYPE_DIRECTORY) {
+        // record the last even if it is not dir(will deal in client)
+        if (IsFull()) {
+            list_->push_back(*last);
+        }
+        return;
+    }
     list_->push_back(*last);
     VLOG(9) << "Push dentry, dentry = (" << last->ShortDebugString() << ")";
 }
@@ -338,6 +346,7 @@ MetaStatusCode DentryStorage::List(const Dentry& dentry,
                                    std::vector<Dentry>* dentrys,
                                    uint32_t limit,
                                    bool onlyDir) {
+    // TODO(all): consider store dir dentry and file dentry separately
     ReadLockGuard lg(rwLock_);
 
     // 1. precheck for dentry vector
@@ -356,11 +365,6 @@ MetaStatusCode DentryStorage::List(const Dentry& dentry,
     Key4Dentry key(fsId, parentInodeId, name);
     std::string lower = conv_.SerializeToString(key);  // "1:1:", "1:1:/a/b/c"
 
-    VLOG(3) << "ListDentry request: dentry = ("
-            << dentry.ShortDebugString() << ")"
-            << ", limit = " << limit << ", onlyDir = " << onlyDir
-            << ", lower key = " << lower;
-
     // 3. iterator key/value pair one by one
     auto iterator = kvStorage_->SSeek(table4Dentry_, lower);
     iterator->DisablePrefixChecking();
@@ -370,7 +374,11 @@ MetaStatusCode DentryStorage::List(const Dentry& dentry,
 
     DentryVec current;
     DentryList list(dentrys, limit, name, dentry.txid(), onlyDir);
+    butil::Timer time;
+    uint32_t seekTimes = 0;
+    time.start();
     for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
+        seekTimes++;
         std::string skey = iterator->Key();
         std::string svalue = iterator->Value();
         if (!StringStartWith(skey, sprefix)) {
@@ -384,6 +392,15 @@ MetaStatusCode DentryStorage::List(const Dentry& dentry,
             break;
         }
     }
+    time.stop();
+    VLOG(1) << "ListDentry request: dentry = ("
+            << dentry.ShortDebugString() << ")"
+            << ", onlyDir = " << onlyDir
+            << ", limit = " << limit
+            << ", lower key = " << lower
+            << ", seekTimes = " << seekTimes
+            << ", dentrySize = " << dentrys->size()
+            << ", costUs = " << time.u_elapsed();
 
     if (list.Size() == 0) {
         return MetaStatusCode::NOT_FOUND;

curvefs/src/metaserver/inode_manager.cpp

@@ -120,7 +120,6 @@ void InodeManager::GenerateInodeInternal(uint64_t inodeId,
     inode->set_ctime(now.tv_sec);
     inode->set_ctime_ns(now.tv_nsec);
 
-    inode->set_openmpcount(0);
     if (FsFileType::TYPE_DIRECTORY == param.type) {
         inode->set_nlink(2);
         // set summary xattr
@@ -147,9 +146,15 @@ MetaStatusCode InodeManager::GetInode(uint32_t fsId,
     }
 
     if (rc != MetaStatusCode::OK) {
-        LOG(ERROR) << "GetInode fail, fsId = " << fsId
-                   << ", inodeId = " << inodeId
-                   << ", retCode = " << MetaStatusCode_Name(rc);
+        std::ostringstream oss;
+        oss << "GetInode fail, fsId = " << fsId << ", inodeId = " << inodeId
+            << ", retCode = " << MetaStatusCode_Name(rc);
+
+        if (rc == MetaStatusCode::STORAGE_CLOSED) {
+            LOG(WARNING) << oss.str();
+        } else {
+            LOG(ERROR) << oss.str();
+        }
         return rc;
     }
 
@@ -272,30 +277,6 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request,
         needUpdate = true;
     }
 
-    // TODO(@one): openmpcount is incorrect in exceptional cases
-    // 1. rpc retry: metaserver update ok but client do not have response, it
-    // will retry.
-    // 2. client exits unexpectedly: openmpcount will not be updated forerver.
-    // if inode is in delete status, operation of DeleteIndoe will be performed
-    // incorrectly.
-    if (request.has_inodeopenstatuschange() && old->has_openmpcount() &&
-        InodeOpenStatusChange::NOCHANGE != request.inodeopenstatuschange()) {
-        VLOG(9) << "update inode open status, fsid: " << request.fsid()
-                << ", inodeid: " << request.inodeid();
-        int32_t oldcount = old->openmpcount();
-        int32_t newcount =
-            request.inodeopenstatuschange() == InodeOpenStatusChange::OPEN
-                ? oldcount + 1
-                : oldcount - 1;
-        if (newcount < 0) {
-            LOG(ERROR) << "open mount point for inode: " << request.inodeid()
-                       << " is " << newcount;
-        } else {
-            old->set_openmpcount(newcount);
-            needUpdate = true;
-        }
-    }
-
     if (needUpdate) {
         ret = inodeStorage_->Update(*old);
         if (ret != MetaStatusCode::OK) {

curvefs/src/metaserver/inode_storage.cpp

@@ -96,6 +96,8 @@ MetaStatusCode InodeStorage::Get(const Key4Inode& key, Inode* inode) {
         return MetaStatusCode::OK;
     } else if (s.IsNotFound()) {
         return MetaStatusCode::NOT_FOUND;
+    } else if (s.IsDBClosed()) {
+        return MetaStatusCode::STORAGE_CLOSED;
     }
 
     LOG(ERROR) << "Get inode failed, status = " << s.ToString();
@@ -139,9 +141,6 @@ MetaStatusCode InodeStorage::GetAttr(const Key4Inode& key,
     if (inode.has_dtime()) {
         attr->set_dtime(inode.dtime());
     }
-    if (inode.has_openmpcount()) {
-        attr->set_openmpcount(inode.openmpcount());
-    }
     if (inode.xattr_size() > 0) {
         *(attr->mutable_xattr()) = inode.xattr();
     }
@@ -188,17 +187,7 @@ MetaStatusCode InodeStorage::Update(const Inode& inode) {
     Key4Inode key(inode.fsid(), inode.inodeid());
     std::string skey = conv_.SerializeToString(key);
 
-    // FIXME: we need't to HGet() before HSet(), because every time
-    // we try to update inode we will get inode first in inode manager.
-    Inode out;
-    Status s = kvStorage_->HGet(table4Inode_, skey, &out);
-    if (s.IsNotFound()) {
-        return MetaStatusCode::NOT_FOUND;
-    } else if (!s.ok()) {
-        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
-    }
-
-    s = kvStorage_->HSet(table4Inode_, skey, inode);
+    Status s = kvStorage_->HSet(table4Inode_, skey, inode);
     if (s.ok()) {
         return MetaStatusCode::OK;
     }

curvefs/src/metaserver/metaserver.cpp

@@ -188,6 +188,7 @@ void Metaserver::Init() {
     InitPartitionOption(s3Adaptor_, mdsClient_, &partitionCleanOption);
     PartitionCleanManager::GetInstance().Init(partitionCleanOption);
 
+    conf_->ExposeMetric("curvefs_metaserver_config");
     inited_ = true;
 }
 

curvefs/src/metaserver/metastore.cpp

@@ -183,8 +183,7 @@ bool MetaStoreImpl::Save(const std::string& dir,
     return true;
 }
 
-bool MetaStoreImpl::Clear() {
-    WriteLockGuard writeLockGuard(rwLock_);
+bool MetaStoreImpl::ClearInternal() {
     for (auto it = partitionMap_.begin(); it != partitionMap_.end(); it++) {
         TrashManager::GetInstance().Remove(it->first);
         it->second->ClearS3Compact();
@@ -200,6 +199,27 @@ bool MetaStoreImpl::Clear() {
     return true;
 }
 
+bool MetaStoreImpl::Clear() {
+    WriteLockGuard writeLockGuard(rwLock_);
+    return ClearInternal();
+}
+
+bool MetaStoreImpl::Destroy() {
+    WriteLockGuard writeLockGuard(rwLock_);
+    if (!ClearInternal()) {
+        LOG(WARNING) << "Failed to clear metastore";
+        return false;
+    }
+
+    if (kvStorage_ != nullptr && !kvStorage_->Close()) {
+        LOG(WARNING) << "Failed to close storage";
+        return false;
+    }
+
+    kvStorage_.reset();
+    return true;
+}
+
 MetaStatusCode MetaStoreImpl::CreatePartition(
     const CreatePartitionRequest* request, CreatePartitionResponse* response) {
     WriteLockGuard writeLockGuard(rwLock_);

curvefs/src/metaserver/metastore.h

@@ -93,6 +93,7 @@ class MetaStore {
     virtual bool Save(const std::string& dir,
                       OnSnapshotSaveDoneClosure* done) = 0;
     virtual bool Clear() = 0;
+    virtual bool Destroy() = 0;
     virtual MetaStatusCode CreatePartition(
         const CreatePartitionRequest* request,
         CreatePartitionResponse* response) = 0;
@@ -175,6 +176,7 @@ class MetaStoreImpl : public MetaStore {
     bool Save(const std::string& dir,
               OnSnapshotSaveDoneClosure* done) override;
     bool Clear() override;
+    bool Destroy() override;
 
     MetaStatusCode CreatePartition(const CreatePartitionRequest* request,
                                    CreatePartitionResponse* response) override;
@@ -271,6 +273,10 @@ class MetaStoreImpl : public MetaStore {
 
     bool InitStorage();
 
+    // Clear data and stop background tasks
+    // REQUIRES: rwLock_ is held with write permission
+    bool ClearInternal();
+
  private:
     RWLock rwLock_;  // protect partitionMap_
     std::shared_ptr<KVStorage> kvStorage_;

curvefs/src/metaserver/partition_cleaner.cpp

@@ -19,16 +19,15 @@
  * @Date: 2021-12-15 10:54:37
  * @Author: chenwei
  */
-
 #include "curvefs/src/metaserver/partition_cleaner.h"
+
 #include <list>
+
 #include "curvefs/src/metaserver/copyset/meta_operator.h"
-#include "curvefs/proto/mds.pb.h"
 
 namespace curvefs {
 namespace metaserver {
 
-using ::curvefs::mds::FsInfo;
 using ::curvefs::mds::FSStatusCode;
 using ::curvefs::mds::FSStatusCode_Name;
 
@@ -89,18 +88,24 @@ MetaStatusCode PartitionCleaner::CleanDataAndDeleteInode(const Inode& inode) {
     if (FsFileType::TYPE_S3 == inode.type()) {
          // get s3info from mds
         FsInfo fsInfo;
-        auto ret = mdsClient_->GetFsInfo(inode.fsid(), &fsInfo);
-        if (ret != FSStatusCode::OK) {
-            if (FSStatusCode::NOT_FOUND == ret) {
-                LOG(ERROR) << "The fsName not exist, fsId = " << inode.fsid();
-                return MetaStatusCode::S3_DELETE_ERR;
-            } else {
-                LOG(ERROR) << "GetFsInfo failed, FSStatusCode = " << ret
-                        << ", FSStatusCode_Name = "
-                        << FSStatusCode_Name(ret)
+        if (fsInfoMap_.find(inode.fsid()) == fsInfoMap_.end()) {
+            auto ret = mdsClient_->GetFsInfo(inode.fsid(), &fsInfo);
+            if (ret != FSStatusCode::OK) {
+                if (FSStatusCode::NOT_FOUND == ret) {
+                    LOG(ERROR)
+                        << "The fsName not exist, fsId = " << inode.fsid();
+                    return MetaStatusCode::S3_DELETE_ERR;
+                } else {
+                    LOG(ERROR)
+                        << "GetFsInfo failed, FSStatusCode = " << ret
+                        << ", FSStatusCode_Name = " << FSStatusCode_Name(ret)
                         << ", fsId = " << inode.fsid();
-                return MetaStatusCode::S3_DELETE_ERR;
+                    return MetaStatusCode::S3_DELETE_ERR;
+                }
             }
+            fsInfoMap_.insert({inode.fsid(), fsInfo});
+        } else {
+            fsInfo = fsInfoMap_.find(inode.fsid())->second;
         }
         const auto& s3Info = fsInfo.detail().s3info();
         // reinit s3 adaptor

curvefs/src/metaserver/partition_cleaner.h

@@ -24,15 +24,19 @@
 #define CURVEFS_SRC_METASERVER_PARTITION_CLEANER_H_
 
 #include <memory>
+#include <unordered_map>
+
+#include "curvefs/proto/mds.pb.h"
+#include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
 #include "curvefs/src/metaserver/partition.h"
 #include "curvefs/src/metaserver/s3/metaserver_s3_adaptor.h"
-#include "curvefs/src/client/rpcclient/mds_client.h"
 
 namespace curvefs {
 namespace metaserver {
 
 using ::curvefs::client::rpcclient::MdsClient;
+using ::curvefs::mds::FsInfo;
 
 class PartitionCleaner {
  public:
@@ -79,6 +83,7 @@ class PartitionCleaner {
     std::shared_ptr<MdsClient> mdsClient_;
     bool isStop_;
     uint32_t inodeDeletePeriodMs_;
+    std::unordered_map<uint32_t, FsInfo> fsInfoMap_;
 };
 
 class PartitionCleanerClosure : public google::protobuf::Closure {

curvefs/src/metaserver/register.cpp

@@ -99,14 +99,15 @@ int Register::RegisterToMDS(MetaServerMetadata *metadata) {
         if (!cntl.Failed() && resp.statuscode() == TopoStatusCode::TOPO_OK) {
             break;
         } else {
-            LOG(INFO) << ops_.metaserverInternalIp << ":"
-                      << ops_.metaserverInternalPort
-                      << " Fail to register to MDS "
-                      << mdsEps_[inServiceIndex_]
-                      << ", cntl errorCode: " << cntl.ErrorCode() << ","
-                      << " cntl error: " << cntl.ErrorText() << ","
-                      << " statusCode: " << resp.statuscode() << ","
-                      << " going to sleep and try again.";
+            LOG(WARNING) << ops_.metaserverInternalIp << ":"
+                         << ops_.metaserverInternalPort
+                         << " Fail to register to MDS "
+                         << mdsEps_[inServiceIndex_]
+                         << ", cntl errorCode: " << cntl.ErrorCode() << ","
+                         << " cntl error: " << cntl.ErrorText() << ","
+                         << " statusCode: "
+                         << TopoStatusCode_Name(resp.statuscode()) << ","
+                         << " going to sleep and try again.";
             if (cntl.ErrorCode() == EHOSTDOWN ||
                 cntl.ErrorCode() == brpc::ELOGOFF) {
                 inServiceIndex_ = (inServiceIndex_ + 1) % mdsEps_.size();

curvefs/src/metaserver/storage/converter.cpp

@@ -254,7 +254,7 @@ bool Key4Dentry::ParseFromString(const std::string& value) {
         return false;
     }
 
-    size_t prefixLength = sizeof(keyType_) +
+    size_t prefixLength = items[0].size() +
                           items[1].size() +
                           items[2].size() +
                           3 * strlen(kDelimiter);

curvefs/src/metaserver/storage/rocksdb_options.cpp

@@ -58,12 +58,32 @@ DEFINE_int32(
     16,
     "Maximum number of concurrent background jobs (compactions and flushes)");
 
+DEFINE_int32(
+    rocksdb_max_subcompactions,
+    4,
+    "Maxinum number of threads to perform a compaction job by simultaneously");
+
+DEFINE_int32(rocksdb_level0_file_num_compaction_trigger,
+             1,
+             "Number of files to trigger level-0 compaction");
+
+// NOTE: now we enable `level_compaction_dynamic_level_bytes`,
+// the `max_bytes_for_level_base` represent level-6's capacity,
+// and in this model, all level-0 SST will compacted into L6 directly,
+// so we should increase level-6's max_bytes, it helped to reduce
+// range scan performance skews caused by scanning excessed number of tombstones
+// for this reason, `rocksdb_level0_file_num_compaction_trigger`
+// should also been set to 1
+DEFINE_int64(rocksdb_max_bytes_for_level_base,
+             1ULL << 30,
+             "Control maximum total data size for a level");
+
 DEFINE_double(rocksdb_memtable_prefix_bloom_size_ratio,
               0.1,
               "Rocksdb memtable prefix bloom size ratio");
 
 DEFINE_int64(rocksdb_unordered_cf_write_buffer_size,
-             64 << 20,
+             64ULL << 20,
              "Writer buffer size for unordered column family");
 
 DEFINE_int32(rocksdb_unordered_cf_max_write_buffer_number,
@@ -71,13 +91,17 @@ DEFINE_int32(rocksdb_unordered_cf_max_write_buffer_number,
              "Number of writer buffer for unordered column family");
 
 DEFINE_int64(rocksdb_ordered_cf_write_buffer_size,
-             64 << 20,
+             64ULL << 20,
              "Writer buffer size for ordered column family");
 
 DEFINE_int32(rocksdb_ordered_cf_max_write_buffer_number,
              2,
              "Number of writer buffer for ordered column family");
 
+DEFINE_int32(rocksdb_max_write_buffer_size_to_maintain,
+             20ULL << 20,
+             "The target number of write history bytes to hold in memory");
+
 DEFINE_int32(rocksdb_stats_dump_period_sec,
              180,
              "Dump rocksdb.stats to LOG every stats_dump_period_sec");
@@ -140,6 +164,7 @@ void InitRocksdbOptions(
     options->listeners.push_back(GetMetricEventListener());
     options->statistics = rocksdb::CreateDBStatistics();
     options->stats_dump_period_sec = FLAGS_rocksdb_stats_dump_period_sec;
+    options->max_subcompactions = FLAGS_rocksdb_max_subcompactions;
 
     rocksdb::BlockBasedTableOptions tableOptions;
     tableOptions.block_size = 16ULL << 10;  // 16KiB
@@ -149,15 +174,27 @@ void InitRocksdbOptions(
     tableOptions.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false));
 
     rocksdb::ColumnFamilyOptions defaultCfOptions;
+    defaultCfOptions.max_write_buffer_size_to_maintain =
+        FLAGS_rocksdb_max_write_buffer_size_to_maintain;
     defaultCfOptions.enable_blob_files = true;
     defaultCfOptions.level_compaction_dynamic_level_bytes = true;
     defaultCfOptions.compaction_pri = rocksdb::kMinOverlappingRatio;
+    defaultCfOptions.level0_file_num_compaction_trigger =
+        FLAGS_rocksdb_level0_file_num_compaction_trigger;
+    defaultCfOptions.max_bytes_for_level_base =
+        FLAGS_rocksdb_max_bytes_for_level_base;
     defaultCfOptions.prefix_extractor.reset(
         rocksdb::NewFixedPrefixTransform(RocksDBStorage::GetKeyPrefixLength()));
     defaultCfOptions.memtable_prefix_bloom_size_ratio =
         FLAGS_rocksdb_memtable_prefix_bloom_size_ratio;
     defaultCfOptions.table_factory.reset(
         rocksdb::NewBlockBasedTableFactory(tableOptions));
+    const size_t slidingWindowSize = 10000;
+    const size_t deletionTrigger = 1000;
+    const double deletionRatio = 0.2;
+    defaultCfOptions.table_properties_collector_factories.emplace_back(
+        rocksdb::NewCompactOnDeletionCollectorFactory(
+            slidingWindowSize, deletionTrigger, deletionRatio));
 
     rocksdb::ColumnFamilyOptions orderedCfOptions = defaultCfOptions;
     orderedCfOptions.write_buffer_size =
@@ -192,6 +229,17 @@ void ParseRocksdbOptions(curve::common::Configuration* conf) {
     dummy.Load(conf, "rocksdb_max_background_jobs",
                "storage.rocksdb.max_background_jobs",
                &FLAGS_rocksdb_max_background_jobs, /*fatalIfMissing*/ false);
+    dummy.Load(conf, "rocksdb_max_subcompactions",
+               "storage.rocksdb.max_subcompactions",
+               &FLAGS_rocksdb_max_subcompactions, /*fatalIfMissing*/ false);
+    dummy.Load(conf, "rocksdb_level0_file_num_compaction_trigger",
+               "storage.rocksdb.level0_file_num_compaction_trigger",
+               &FLAGS_rocksdb_level0_file_num_compaction_trigger,
+               /*fatalIfMissing*/ false);
+    dummy.Load(conf, "rocksdb_max_bytes_for_level_base",
+               "storage.rocksdb.max_bytes_for_level_base",
+               &FLAGS_rocksdb_max_bytes_for_level_base,
+               /*fatalIfMissing*/ false);
     dummy.Load(conf, "rocksdb_memtable_prefix_bloom_size_ratio",
                "storage.rocksdb.memtable_prefix_bloom_size_ratio",
                &FLAGS_rocksdb_memtable_prefix_bloom_size_ratio,
@@ -212,6 +260,10 @@ void ParseRocksdbOptions(curve::common::Configuration* conf) {
                "storage.rocksdb.ordered_max_write_buffer_number",
                &FLAGS_rocksdb_ordered_cf_max_write_buffer_number,
                /*fatalIfMissing*/ false);
+    dummy.Load(conf, "rocksdb_max_write_buffer_size_to_maintain",
+               "storage.rocksdb.max_write_buffer_size_to_maintain",
+               &FLAGS_rocksdb_max_write_buffer_size_to_maintain,
+               /*fatalIfMissing*/ false);
     dummy.Load(conf, "rocksdb_stats_dump_period_sec",
                "storage.rocksdb.stats_dump_period_sec",
                &FLAGS_rocksdb_stats_dump_period_sec, /*fatalIfMissing*/ false);

curvefs/src/metaserver/storage/rocksdb_storage.cpp

@@ -102,6 +102,9 @@ bool RocksDBStorage::Open() {
         return true;
     }
 
+    assert(txnDB_ == nullptr);
+    assert(db_ == nullptr);
+
     if (cleanOpen_ && options_.localFileSystem->DirExists(options_.dataDir)) {
         int ret = options_.localFileSystem->Delete(options_.dataDir);
         if (ret != 0) {
@@ -151,6 +154,11 @@ bool RocksDBStorage::Close() {
 
     handles_.clear();
     inited_ = false;
+
+    delete txnDB_;
+    db_ = nullptr;
+    txnDB_ = nullptr;
+
     return true;
 }
 
@@ -387,41 +395,6 @@ bool DoCheckpoint(rocksdb::DB* db, const std::string& dest) {
     return true;
 }
 
-}  // namespace
-
-bool RocksDBStorage::Checkpoint(const std::string& dir,
-                                std::vector<std::string>* files) {
-    rocksdb::FlushOptions options;
-    options.wait = true;
-    options.allow_write_stall = true;
-    auto status = db_->Flush(options, handles_);
-    if (!status.ok()) {
-        LOG(ERROR) << "Failed to flush DB, " << status.ToString();
-        return false;
-    }
-
-    const std::string dest = dir + "/" + kRocksdbCheckpointPath;
-    if (!DoCheckpoint(db_, dest)) {
-        return false;
-    }
-
-    std::vector<std::string> filenames;
-    int ret = options_.localFileSystem->List(dest, &filenames);
-
-    if (ret != 0) {
-        LOG(ERROR) << "Failed to list checkpoint files at `" << dest << "`, "
-                   << berror();
-        return false;
-    }
-
-    files->reserve(filenames.size());
-    for (const auto& f : filenames) {
-        files->push_back(std::string(kRocksdbCheckpointPath) + "/" + f);
-    }
-
-    return true;
-}
-
 bool DuplicateRocksdbCheckpoint(const std::string& from,
                                 const std::string& to) {
     LOG(INFO) << "Duplicating rocksdb storage from `" << from << "` to `" << to
@@ -457,6 +430,41 @@ bool DuplicateRocksdbCheckpoint(const std::string& from,
     return DoCheckpoint(db, to);
 }
 
+}  // namespace
+
+bool RocksDBStorage::Checkpoint(const std::string& dir,
+                                std::vector<std::string>* files) {
+    rocksdb::FlushOptions options;
+    options.wait = true;
+    options.allow_write_stall = true;
+    auto status = db_->Flush(options, handles_);
+    if (!status.ok()) {
+        LOG(ERROR) << "Failed to flush DB, " << status.ToString();
+        return false;
+    }
+
+    const std::string dest = dir + "/" + kRocksdbCheckpointPath;
+    if (!DoCheckpoint(db_, dest)) {
+        return false;
+    }
+
+    std::vector<std::string> filenames;
+    int ret = options_.localFileSystem->List(dest, &filenames);
+
+    if (ret != 0) {
+        LOG(ERROR) << "Failed to list checkpoint files at `" << dest << "`, "
+                   << berror();
+        return false;
+    }
+
+    files->reserve(filenames.size());
+    for (const auto& f : filenames) {
+        files->push_back(std::string(kRocksdbCheckpointPath) + "/" + f);
+    }
+
+    return true;
+}
+
 bool RocksDBStorage::Recover(const std::string& dir) {
     LOG(INFO) << "Recovering storage from `" << dir << "`";
 

curvefs/src/metaserver/storage/rocksdb_storage.h

@@ -35,8 +35,10 @@
 #include "rocksdb/options.h"
 #include "rocksdb/filter_policy.h"
 #include "rocksdb/slice_transform.h"
+#include "rocksdb/table_properties.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
+#include "rocksdb/utilities/table_properties_collectors.h"
 #include "src/common/concurrent/rw_lock.h"
 #include "curvefs/src/metaserver/storage/utils.h"
 #include "curvefs/src/metaserver/storage/storage.h"
@@ -183,10 +185,10 @@ class RocksDBStorage : public KVStorage, public StorageTransaction {
     void InitDbOptions();
 
  private:
-    bool inited_;
+    bool inited_ = false;
     StorageOptions options_;
-    DB* db_;
-    TransactionDB* txnDB_;
+    DB* db_ = nullptr;
+    TransactionDB* txnDB_ = nullptr;
     std::vector<ColumnFamilyHandle*> handles_;
     static const std::string kDelimiter_;
 
@@ -195,7 +197,7 @@ class RocksDBStorage : public KVStorage, public StorageTransaction {
 
     // only for transaction
     bool InTransaction_;
-    Transaction* txn_;
+    Transaction* txn_ = nullptr;
 
     // db options
     rocksdb::DBOptions dbOptions_;

curvefs/src/metaserver/trash.cpp

@@ -123,11 +123,7 @@ bool TrashImpl::NeedDelete(const TrashItem &item) {
                      << ", ret = " << MetaStatusCode_Name(ret);
         return false;
     }
-    if (inode.has_openmpcount() && inode.openmpcount() > 0) {
-            return false;
-    } else {
-        return ((now - item.dtime) >= options_.expiredAfterSec);
-    }
+    return ((now - item.dtime) >= options_.expiredAfterSec);
 }
 
 MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
@@ -146,18 +142,23 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
     } else if (FsFileType::TYPE_S3 == inode.type()) {
         // get s3info from mds
         FsInfo fsInfo;
-        auto ret = mdsClient_->GetFsInfo(item.fsId, &fsInfo);
-        if (ret != FSStatusCode::OK) {
-            if (FSStatusCode::NOT_FOUND == ret) {
-                LOG(ERROR) << "The fsName not exist, fsId = " << item.fsId;
-                return MetaStatusCode::S3_DELETE_ERR;
-            } else {
-                LOG(ERROR) << "GetFsInfo failed, FSStatusCode = " << ret
-                        << ", FSStatusCode_Name = "
-                        << FSStatusCode_Name(ret)
+        if (fsInfoMap_.find(item.fsId) == fsInfoMap_.end()) {
+            auto ret = mdsClient_->GetFsInfo(item.fsId, &fsInfo);
+            if (ret != FSStatusCode::OK) {
+                if (FSStatusCode::NOT_FOUND == ret) {
+                    LOG(ERROR) << "The fsName not exist, fsId = " << item.fsId;
+                    return MetaStatusCode::S3_DELETE_ERR;
+                } else {
+                    LOG(ERROR)
+                        << "GetFsInfo failed, FSStatusCode = " << ret
+                        << ", FSStatusCode_Name = " << FSStatusCode_Name(ret)
                         << ", fsId = " << item.fsId;
-                return MetaStatusCode::S3_DELETE_ERR;
+                    return MetaStatusCode::S3_DELETE_ERR;
+                }
             }
+            fsInfoMap_.insert({item.fsId, fsInfo});
+        } else {
+            fsInfo = fsInfoMap_.find(item.fsId)->second;
         }
         const auto& s3Info = fsInfo.detail().s3info();
         // reinit s3 adaptor
@@ -167,6 +168,21 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
         clientAdaptorOption.chunkSize = s3Info.chunksize();
         s3Adaptor_->Reinit(clientAdaptorOption, s3Info.ak(), s3Info.sk(),
             s3Info.endpoint(), s3Info.bucketname());
+        ret = inodeStorage_->PaddingInodeS3ChunkInfo(item.fsId,
+          item.inodeId, inode.mutable_s3chunkinfomap());
+        if (ret != MetaStatusCode::OK) {
+            LOG(ERROR) << "GetInode chunklist fail, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId
+                << ", retCode = " << MetaStatusCode_Name(ret);
+            return ret;
+        }
+        if (inode.s3chunkinfomap().empty()) {
+            LOG(WARNING) << "GetInode chunklist empty, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId;
+            return MetaStatusCode::NOT_FOUND;
+        }
+        VLOG(9) << "DeleteInodeAndData, inode: "
+            << inode.ShortDebugString();
         int retVal = s3Adaptor_->Delete(inode);
         if (retVal != 0) {
             LOG(ERROR) << "S3ClientAdaptor delete s3 data failed"
@@ -175,7 +191,6 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
             return MetaStatusCode::S3_DELETE_ERR;
         }
     }
-
     ret = inodeStorage_->Delete(Key4Inode(item.fsId, item.inodeId));
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "Delete Inode fail, fsId = " << item.fsId

curvefs/src/metaserver/trash.h

@@ -26,6 +26,7 @@
 #include <cstdint>
 #include <memory>
 #include <list>
+#include <unordered_map>
 
 #include "src/common/configuration.h"
 #include "src/common/concurrent/concurrent.h"
@@ -116,6 +117,7 @@ class TrashImpl : public Trash {
     std::shared_ptr<InodeStorage> inodeStorage_;
     std::shared_ptr<S3ClientAdaptor>  s3Adaptor_;
     std::shared_ptr<MdsClient> mdsClient_;
+    std::unordered_map<uint32_t, FsInfo> fsInfoMap_;
 
     std::list<TrashItem> trashItems_;
 

curvefs/test/client/BUILD

@@ -36,6 +36,21 @@ cc_library(
     linkopts = ["-L/usr/local/lib/x86_64-linux-gnu"],
 )
 
+cc_library(
+    name = "client_test_utils",
+    srcs = [
+        "utils.cpp",
+    ],
+    hdrs = [
+        "utils.h",
+    ],
+    copts = CURVE_TEST_COPTS,
+    visibility = ["//visibility:public"],
+    deps = [
+        "//curvefs/proto:metaserver_cc_proto",
+    ],
+)
+
 cc_test(
     name = "client_s3_test",
     srcs = glob([

curvefs/test/client/client_s3_adaptor_test.cpp

@@ -26,6 +26,7 @@
 #include <google/protobuf/util/message_differencer.h>
 #include <gtest/gtest.h>
 
+#include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/test/client/mock_client_s3.h"
 #include "curvefs/test/client/mock_client_s3_cache_manager.h"
 #include "curvefs/test/client/mock_disk_cache_manager.h"
@@ -33,6 +34,7 @@
 #include "curvefs/test/client/mock_metaserver_service.h"
 #include "curvefs/test/client/rpcclient/mock_mds_client.h"
 #include "src/common/curve_define.h"
+#include "absl/memory/memory.h"
 
 namespace curvefs {
 namespace client {
@@ -87,19 +89,23 @@ class ClientS3AdaptorTest : public testing::Test {
 };
 
 uint64_t gInodeId = 1;
-void InitInode(Inode *inode) {
-    inode->set_inodeid(gInodeId);
-    inode->set_fsid(2);
-    inode->set_length(0);
-    inode->set_ctime(1623835517);
-    inode->set_mtime(1623835517);
-    inode->set_atime(1623835517);
-    inode->set_uid(1);
-    inode->set_gid(1);
-    inode->set_mode(1);
-    inode->set_nlink(1);
-    inode->set_type(curvefs::metaserver::FsFileType::TYPE_S3);
+std::unique_ptr<InodeWrapper> InitInode() {
+    Inode inode;
+
+    inode.set_inodeid(gInodeId);
+    inode.set_fsid(2);
+    inode.set_length(0);
+    inode.set_ctime(1623835517);
+    inode.set_mtime(1623835517);
+    inode.set_atime(1623835517);
+    inode.set_uid(1);
+    inode.set_gid(1);
+    inode.set_mode(1);
+    inode.set_nlink(1);
+    inode.set_type(curvefs::metaserver::FsFileType::TYPE_S3);
     gInodeId++;
+
+    return absl::make_unique<InodeWrapper>(std::move(inode), nullptr);
 }
 
 TEST_F(ClientS3AdaptorTest, test_init) {
@@ -154,43 +160,39 @@ TEST_F(ClientS3AdaptorTest, read_fail) {
 }
 
 TEST_F(ClientS3AdaptorTest, truncate_small) {
-    curvefs::metaserver::Inode inode;
-    InitInode(&inode);
-    inode.set_length(1000);
+    auto inode = InitInode();
+    inode->SetLength(1000);
 
     auto fileCache = std::make_shared<MockFileCacheManager>();
     EXPECT_CALL(*mockFsCacheManager_, FindOrCreateFileCacheManager(_, _))
         .WillOnce(Return(fileCache));
     EXPECT_CALL(*fileCache, TruncateCache(_, _)).WillOnce(Return());
-    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(&inode, 100));
+    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(inode.get(), 100));
 }
 
 TEST_F(ClientS3AdaptorTest, truncate_unchange) {
-    curvefs::metaserver::Inode inode;
-    InitInode(&inode);
-    inode.set_length(1000);
+    auto inode = InitInode();
+    inode->SetLength(1000);
 
-    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(&inode, 1000));
+    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(inode.get(), 1000));
 }
 
 TEST_F(ClientS3AdaptorTest, truncate_big_alloc_chunkId_fail) {
-    curvefs::metaserver::Inode inode;
-    InitInode(&inode);
+    auto inode = InitInode();
 
     EXPECT_CALL(*mockMdsClient_, AllocS3ChunkId(_, _))
         .WillOnce(Return(FSStatusCode::UNKNOWN_ERROR));
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL,
-              s3ClientAdaptor_->Truncate(&inode, 1000));
+              s3ClientAdaptor_->Truncate(inode.get(), 1000));
 }
 
 TEST_F(ClientS3AdaptorTest, truncate_big_success) {
-    curvefs::metaserver::Inode inode;
-    InitInode(&inode);
+    auto inode = InitInode();
 
     uint64_t chunkId = 999;
     EXPECT_CALL(*mockMdsClient_, AllocS3ChunkId(_, _))
         .WillOnce(DoAll(SetArgPointee<1>(chunkId), Return(FSStatusCode::OK)));
-    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(&inode, 1000));
+    ASSERT_EQ(CURVEFS_ERROR::OK, s3ClientAdaptor_->Truncate(inode.get(), 1000));
 }
 
 TEST_F(ClientS3AdaptorTest, flush_no_file_cache) {

curvefs/test/client/mock_client_s3_adaptor.h

@@ -26,6 +26,7 @@
 #include <memory>
 #include <string>
 
+#include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
 
 namespace curvefs {
@@ -33,21 +34,6 @@ namespace client {
 
 class MockS3ClientAdaptor : public S3ClientAdaptor {
  public:
-    MockS3ClientAdaptor() {}
-    ~MockS3ClientAdaptor() {}
-/*
-    MOCK_METHOD5(Init,
-                 CURVEFS_ERROR(const S3ClientAdaptorOption &option,
-                               std::shared_ptr<S3Client> client,
-                               std::shared_ptr<InodeCacheManager> inodeManager,
-                               std::shared_ptr<MdsClient> mdsClient,
-                               std::shared_ptr<DiskCacheManagerImpl>
-                                   diskcacheManagerImpl));
-    MOCK_METHOD4(Init,
-                 CURVEFS_ERROR(const S3ClientAdaptorOption& option,
-                               std::shared_ptr<S3Client> client,
-                               std::shared_ptr<InodeCacheManager> inodeManager,
-                               std::shared_ptr<MdsClient> mdsClient));*/
     MOCK_METHOD7(Init,
                  CURVEFS_ERROR(const S3ClientAdaptorOption &option,
                                std::shared_ptr<S3Client> client,
@@ -68,7 +54,7 @@ class MockS3ClientAdaptor : public S3ClientAdaptor {
     MOCK_METHOD1(FlushAllCache, CURVEFS_ERROR(uint64_t inodeId));
     MOCK_METHOD0(FsSync, CURVEFS_ERROR());
     MOCK_METHOD0(Stop, int());
-    MOCK_METHOD2(Truncate, CURVEFS_ERROR(Inode* inode, uint64_t size));
+    MOCK_METHOD2(Truncate, CURVEFS_ERROR(InodeWrapper* inode, uint64_t size));
     MOCK_METHOD2(AllocS3ChunkId,
                  FSStatusCode(uint32_t fsId, uint64_t* chunkId));
     MOCK_METHOD1(SetFsId, void(uint32_t fsId));

curvefs/test/client/mock_dentry_cache_mamager.h

@@ -54,10 +54,11 @@ class MockDentryCacheManager : public DentryCacheManager {
                                              const std::string &name,
                                              FsFileType type));
 
-    MOCK_METHOD4(ListDentry, CURVEFS_ERROR(uint64_t parent,
+    MOCK_METHOD5(ListDentry, CURVEFS_ERROR(uint64_t parent,
                                            std::list<Dentry> *dentryList,
                                            uint32_t limit,
-                                           bool onlyDir));
+                                           bool onlyDir,
+                                           uint32_t nlink));
 };
 
 

curvefs/test/client/mock_disk_cache_manager.h

@@ -47,7 +47,8 @@ class MockDiskCacheManager : public DiskCacheManager {
     ~MockDiskCacheManager() {}
 
     MOCK_METHOD2(Init,
-                 int(S3Client *client, const S3ClientAdaptorOption option));
+                 int(std::shared_ptr<S3Client> client,
+                 const S3ClientAdaptorOption option));
     MOCK_METHOD0(IsDiskCacheFull,
                  bool());
     MOCK_METHOD3(WriteReadDirect,

curvefs/test/client/mock_inode_cache_manager.h

@@ -51,14 +51,14 @@ class MockInodeCacheManager : public InodeCacheManager {
                  CURVEFS_ERROR(uint64_t inodeId,
                                std::shared_ptr<InodeWrapper> &out));  // NOLINT
 
-    MOCK_METHOD3(GetInodeAttr, CURVEFS_ERROR(
-        uint64_t inodeId, InodeAttr *out, uint64_t parentId));
+    MOCK_METHOD2(GetInodeAttr, CURVEFS_ERROR(
+        uint64_t inodeId, InodeAttr *out));
 
     MOCK_METHOD2(BatchGetInodeAttr, CURVEFS_ERROR(
         std::set<uint64_t> *inodeIds, std::list<InodeAttr> *attrs));
 
     MOCK_METHOD3(BatchGetInodeAttrAsync,
-        CURVEFS_ERROR(uint64_t parentId, const std::set<uint64_t> &inodeIds,
+        CURVEFS_ERROR(uint64_t parentId, std::set<uint64_t> *inodeIds,
         std::map<uint64_t, InodeAttr> *attrs));
 
     MOCK_METHOD2(BatchGetXAttr, CURVEFS_ERROR(
@@ -84,6 +84,10 @@ class MockInodeCacheManager : public InodeCacheManager {
     MOCK_METHOD0(FlushInodeOnce, void());
 
     MOCK_METHOD1(ReleaseCache, void(uint64_t parentId));
+
+    MOCK_METHOD1(AddOpenedInode, void(uint64_t inodeId));
+
+    MOCK_METHOD1(RemoveOpenedInode, void(uint64_t inodeId));
 };
 
 }  // namespace client

curvefs/test/client/mock_metaserver_client.h

@@ -34,18 +34,12 @@
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 
-using ::testing::Return;
-using ::testing::_;
-
 namespace curvefs {
 namespace client {
 namespace rpcclient {
 
 class MockMetaServerClient : public MetaServerClient {
  public:
-    MockMetaServerClient() {}
-    ~MockMetaServerClient() {}
-
     MOCK_METHOD4(Init,
                  MetaStatusCode(const ExcutorOpt &excutorOpt,
                                 const ExcutorOpt &excutorInternalOpt,
@@ -94,21 +88,31 @@ class MockMetaServerClient : public MetaServerClient {
         uint32_t fsId, const std::set<uint64_t> &inodeIds,
         std::list<XAttr> *xattr));
 
-    MOCK_METHOD2(UpdateInodeAttr,
-                 MetaStatusCode(const Inode &inode,
+    MOCK_METHOD4(UpdateInodeAttr,
+                 MetaStatusCode(uint32_t,
+                                uint64_t,
+                                const InodeAttr&,
                                 InodeOpenStatusChange statusChange));
 
-    MOCK_METHOD3(UpdateInodeAttrWithOutNlink,
-                 MetaStatusCode(const Inode &inode,
+    MOCK_METHOD5(UpdateInodeAttrWithOutNlink,
+                 MetaStatusCode(uint32_t,
+                                uint64_t,
+                                const InodeAttr&,
                                 InodeOpenStatusChange statusChange,
                                 bool internal));
 
-    MOCK_METHOD3(UpdateInodeAttrAsync,
-                 void(const Inode &inode, MetaServerClientDone *done,
+    MOCK_METHOD5(UpdateInodeAttrAsync,
+                 void(uint32_t,
+                      uint64_t,
+                      const InodeAttr&,
+                      MetaServerClientDone* done,
                       InodeOpenStatusChange statusChange));
 
-    MOCK_METHOD3(UpdateInodeAttrWithOutNlinkAsync,
-                 void(const Inode &inode, MetaServerClientDone *done,
+    MOCK_METHOD5(UpdateInodeAttrWithOutNlinkAsync,
+                 void(uint32_t,
+                      uint64_t,
+                      const InodeAttr&,
+                      MetaServerClientDone* done,
                       InodeOpenStatusChange statusChange));
 
     MOCK_METHOD2(UpdateXattrAsync, void(const Inode &inode,

curvefs/test/client/rpcclient/BUILD

@@ -30,6 +30,7 @@ cc_test(
        "//curvefs/proto:curvefs_topology_cc_proto",
        "//curvefs/src/common:curvefs_common",
        "//curvefs/test/utils:curvefs_test_utils",
+       "//curvefs/test/client:client_test_utils"
    ],
    visibility = ["//visibility:public"],
 )

curvefs/test/client/rpcclient/mds_client_test.cpp

@@ -440,22 +440,40 @@ TEST_F(MdsClientImplTest, CommitTxWithLock) {
 
 TEST_F(MdsClientImplTest, GetLatestTxId) {
     std::vector<PartitionTxId> txIds;
+    uint32_t fsId = 1;
 
     // CASE 1: GetLatestTxId success
-    GetLatestTxIdResponse response;
-    response.set_statuscode(FSStatusCode::OK);
     EXPECT_CALL(mockmdsbasecli_, GetLatestTxId(_, _, _, _))
-        .WillOnce(SetArgPointee<1>(response));
+        .WillOnce(
+            Invoke([&](const GetLatestTxIdRequest& request,
+                       GetLatestTxIdResponse* response,
+                       brpc::Controller *cntl,
+                       brpc::Channel *channel) {
+                if (request.fsid() != fsId) {
+                    response->set_statuscode(FSStatusCode::PARAM_ERROR);
+                } else {
+                    response->set_statuscode(FSStatusCode::OK);
+                }
+            }));
 
-    auto rc = mdsclient_.GetLatestTxId(&txIds);
+    auto rc = mdsclient_.GetLatestTxId(fsId, &txIds);
     ASSERT_EQ(rc, FSStatusCode::OK);
 
     // CASE 2: GetLatestTxId fail
-    response.set_statuscode(FSStatusCode::UNKNOWN_ERROR);
     EXPECT_CALL(mockmdsbasecli_, GetLatestTxId(_, _, _, _))
-        .WillOnce(SetArgPointee<1>(response));
+        .WillOnce(
+            Invoke([&](const GetLatestTxIdRequest& request,
+                       GetLatestTxIdResponse* response,
+                       brpc::Controller *cntl,
+                       brpc::Channel *channel) {
+                if (request.fsid() != fsId) {
+                    response->set_statuscode(FSStatusCode::PARAM_ERROR);
+                } else {
+                    response->set_statuscode(FSStatusCode::UNKNOWN_ERROR);
+                }
+            }));
 
-    rc = mdsclient_.GetLatestTxId(&txIds);
+    rc = mdsclient_.GetLatestTxId(fsId, &txIds);
     ASSERT_EQ(rc, FSStatusCode::UNKNOWN_ERROR);
 
     // CASE 3: RPC error, retry until success
@@ -467,14 +485,16 @@ TEST_F(MdsClientImplTest, GetLatestTxId) {
                        GetLatestTxIdResponse* response,
                        brpc::Controller *cntl,
                        brpc::Channel *channel) {
-                if (++count <= 5) {
+                if (request.fsid() != fsId) {
+                    response->set_statuscode(FSStatusCode::PARAM_ERROR);
+                } else if (++count <= 5) {
                     cntl->SetFailed(112, "Not connected to");
                 } else {
                     response->set_statuscode(FSStatusCode::OK);
                 }
             }));
 
-    rc = mdsclient_.GetLatestTxId(&txIds);
+    rc = mdsclient_.GetLatestTxId(fsId, &txIds);
     ASSERT_EQ(rc, FSStatusCode::OK);
 }
 

curvefs/test/client/rpcclient/metacache_test.cpp

@@ -133,8 +133,8 @@ class MetaCacheTest : public testing::Test {
     std::shared_ptr<MockCli2Client> mockCli2Client_;
 
     curve::client::CopysetInfo<MetaserverID> metaServerList_;
-    MetaCache::PatitionInfoList pInfoList_;
-    MetaCache::PatitionInfoList pInfoList2_;
+    MetaCache::PartitionInfoList pInfoList_;
+    MetaCache::PartitionInfoList pInfoList2_;
     std::map<PartitionID, Copyset> copysetMap_;
 
     CopysetTarget expect;

curvefs/test/client/rpcclient/metaserver_client_test.cpp

@@ -43,6 +43,7 @@
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/common/define.h"
 #include "curvefs/src/common/process.h"
+#include "curvefs/test/client/utils.h"
 
 namespace curvefs {
 namespace client {
@@ -743,7 +744,8 @@ TEST_F(MetaServerClientImplTest, test_UpdateInodeAttr) {
         .WillRepeatedly(DoAll(SetArgPointee<2>(target_),
                               SetArgPointee<3>(applyIndex), Return(true)));
 
-    MetaStatusCode status = metaserverCli_.UpdateInodeAttr(inode);
+    MetaStatusCode status = metaserverCli_.UpdateInodeAttr(
+        inode.fsid(), inode.inodeid(), ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 
     // test1: update inode ok
@@ -757,7 +759,8 @@ TEST_F(MetaServerClientImplTest, test_UpdateInodeAttr) {
         .WillRepeatedly(DoAll(SetArgPointee<2>(target_),
                               SetArgPointee<3>(applyIndex), Return(true)));
     EXPECT_CALL(*mockMetacache_.get(), UpdateApplyIndex(_, _));
-    status = metaserverCli_.UpdateInodeAttr(inode);
+    status = metaserverCli_.UpdateInodeAttr(
+        inode.fsid(), inode.inodeid(), ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::OK, status);
 
     // test2: update inode with overload
@@ -766,7 +769,8 @@ TEST_F(MetaServerClientImplTest, test_UpdateInodeAttr) {
         .WillRepeatedly(DoAll(
             SetArgPointee<2>(response),
             Invoke(SetRpcService<UpdateInodeRequest, UpdateInodeResponse>)));
-    status = metaserverCli_.UpdateInodeAttr(inode);
+    status = metaserverCli_.UpdateInodeAttr(inode.fsid(), inode.inodeid(),
+                                            ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::OVERLOAD, status);
 
     // test3: response has no applyindex
@@ -777,13 +781,15 @@ TEST_F(MetaServerClientImplTest, test_UpdateInodeAttr) {
             SetArgPointee<2>(response),
             Invoke(SetRpcService<UpdateInodeRequest, UpdateInodeResponse>)));
 
-    status = metaserverCli_.UpdateInodeAttr(inode);
+    status = metaserverCli_.UpdateInodeAttr(inode.fsid(), inode.inodeid(),
+                                            ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 
     // test4: get target always fail
     EXPECT_CALL(*mockMetacache_.get(), GetTarget(_, _, _, _, _))
         .WillRepeatedly(Return(false));
-    status = metaserverCli_.UpdateInodeAttr(inode);
+    status = metaserverCli_.UpdateInodeAttr(inode.fsid(), inode.inodeid(),
+                                            ToInodeAttr(inode));
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 }
 

curvefs/test/client/rpcclient/mock_mds_client.h

@@ -63,8 +63,9 @@ class MockMdsClient : public MdsClient {
     MOCK_METHOD2(AllocS3ChunkId,
                  FSStatusCode(uint32_t fsId, uint64_t* chunkId));
 
-    MOCK_METHOD1(GetLatestTxId,
-                 FSStatusCode(std::vector<PartitionTxId>* txIds));
+    MOCK_METHOD2(GetLatestTxId,
+                 FSStatusCode(uint32_t fsId,
+                              std::vector<PartitionTxId>* txIds));
 
     MOCK_METHOD5(GetLatestTxIdWithLock,
                  FSStatusCode(uint32_t fsId,

curvefs/test/client/test_dentry_cache_manager.cpp

@@ -211,6 +211,14 @@ TEST_F(TestDentryCacheManager, ListDentryEmpty) {
     ASSERT_EQ(0, out.size());
 }
 
+TEST_F(TestDentryCacheManager, ListDentryOnlyDir) {
+    uint64_t parent = 99;
+    std::list<Dentry> out;
+    CURVEFS_ERROR ret = dCacheManager_->ListDentry(parent, &out, 0, 1, 2);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(0, out.size());
+}
+
 TEST_F(TestDentryCacheManager, ListDentryFailed) {
     uint64_t parent = 99;
 

curvefs/test/client/test_disk_cache_manager.cpp

@@ -22,6 +22,7 @@
 
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
+#include "curvefs/test/client/mock_client_s3.h"
 
 #include "curvefs/test/client/mock_disk_cache_write.h"
 #include "curvefs/test/client/mock_disk_cache_read.h"
@@ -57,17 +58,17 @@ class TestDiskCacheManager : public ::testing::Test {
     ~TestDiskCacheManager() {}
 
     virtual void SetUp() {
-        S3Client *client = nullptr;
+        client_ = std::make_shared<MockS3Client>();
         wrapper = std::make_shared<MockPosixWrapper>();
         diskCacheWrite_ = std::make_shared<MockDiskCacheWrite>();
         diskCacheRead_ = std::make_shared<MockDiskCacheRead>();
         diskCacheManager_ = std::make_shared<DiskCacheManager>(
             wrapper, diskCacheWrite_, diskCacheRead_);
         diskCacheRead_->Init(wrapper, "/mnt/test");
-         std::shared_ptr<LRUCache<std::string, bool>> cachedObjName
-          = std::make_shared<LRUCache<std::string, bool>>
+         std::shared_ptr<SglLRUCache<std::string>> cachedObjName
+          = std::make_shared<SglLRUCache<std::string>>
               (0, std::make_shared<CacheMetrics>("diskcache"));
-        diskCacheWrite_->Init(client, wrapper, "/mnt/test", 1, cachedObjName);
+        diskCacheWrite_->Init(client_, wrapper, "/mnt/test", 1, cachedObjName);
     }
 
     virtual void TearDown() {
@@ -80,33 +81,33 @@ class TestDiskCacheManager : public ::testing::Test {
     std::shared_ptr<MockDiskCacheWrite> diskCacheWrite_;
     std::shared_ptr<DiskCacheManager> diskCacheManager_;
     std::shared_ptr<MockPosixWrapper> wrapper;
+    std::shared_ptr<MockS3Client> client_;
 };
 
 TEST_F(TestDiskCacheManager, Init) {
     S3ClientAdaptorOption s3AdaptorOption;
-    S3Client *client = nullptr;
     EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(-1));
     EXPECT_CALL(*wrapper, mkdir(_, _)).WillOnce(Return(-1));
-    int ret = diskCacheManager_->Init(client, s3AdaptorOption);
+    int ret = diskCacheManager_->Init(client_, s3AdaptorOption);
     ASSERT_EQ(-1, ret);
 
     EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(-1));
     EXPECT_CALL(*wrapper, mkdir(_, _)).WillOnce(Return(0));
     EXPECT_CALL(*diskCacheWrite_, CreateIoDir("/mnt/test"))
         .WillOnce(Return(-1));
-    ret = diskCacheManager_->Init(client, s3AdaptorOption);
+    ret = diskCacheManager_->Init(client_, s3AdaptorOption);
     ASSERT_EQ(-1, ret);
 
     EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(0));
     EXPECT_CALL(*diskCacheWrite_, CreateIoDir("/mnt/test"))
         .WillOnce(Return(-1));
-    ret = diskCacheManager_->Init(client, s3AdaptorOption);
+    ret = diskCacheManager_->Init(client_, s3AdaptorOption);
     ASSERT_EQ(-1, ret);
 
     EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillOnce(Return(0));
     EXPECT_CALL(*diskCacheWrite_, CreateIoDir(_)).WillOnce(Return(0));
     EXPECT_CALL(*diskCacheRead_, CreateIoDir(_)).WillOnce(Return(-1));
-    ret = diskCacheManager_->Init(client, s3AdaptorOption);
+    ret = diskCacheManager_->Init(client_, s3AdaptorOption);
     ASSERT_EQ(-1, ret);
 }
 
@@ -211,6 +212,15 @@ TEST_F(TestDiskCacheManager, SetDiskFsUsedRatio) {
     ASSERT_EQ(-1, ret);
 
     struct statfs stat;
+    stat.f_frsize = 0;
+    stat.f_blocks = 0;
+    stat.f_bfree = 0;
+    stat.f_bavail = 0;
+    EXPECT_CALL(*wrapper, statfs(NotNull(), _))
+        .WillOnce(DoAll(SetArgPointee<1>(stat), Return(0)));
+    ret = diskCacheManager_->SetDiskFsUsedRatio();
+    ASSERT_EQ(-1, ret);
+
     stat.f_frsize = 1;
     stat.f_blocks = 1;
     stat.f_bfree = 0;
@@ -244,15 +254,14 @@ TEST_F(TestDiskCacheManager, IsDiskCacheSafe) {
     option.diskCacheOpt.maxUsableSpaceBytes = 0;
     option.diskCacheOpt.cmdTimeoutSec = 5;
     option.diskCacheOpt.asyncLoadPeriodMs = 10;
-    S3Client *client = nullptr;
-    diskCacheManager_->Init(client, option);
+    diskCacheManager_->Init(client_, option);
     bool ret = diskCacheManager_->IsDiskCacheSafe();
     ASSERT_EQ(false, ret);
 
     option.diskCacheOpt.fullRatio = 100;
     option.diskCacheOpt.safeRatio = 99;
     option.diskCacheOpt.maxUsableSpaceBytes = 100000000;
-    diskCacheManager_->Init(client, option);
+    diskCacheManager_->Init(client_, option);
     ret = diskCacheManager_->IsDiskCacheSafe();
     ASSERT_EQ(true, ret);
 }
@@ -289,7 +298,11 @@ TEST_F(TestDiskCacheManager, TrimCache_2) {
         .WillRepeatedly(Return(buf));
     EXPECT_CALL(*diskCacheRead_, GetCacheIoFullDir())
         .WillRepeatedly(Return(buf));
-    EXPECT_CALL(*wrapper, stat(NotNull(), NotNull())).WillRepeatedly(Return(0));
+    EXPECT_CALL(*wrapper, stat(NotNull(), NotNull()))
+        .Times(3)
+        .WillOnce(Return(0))
+        .WillOnce(Return(-1))
+        .WillOnce(Return(0));
     diskCacheManager_->AddCache("test");
     int ret = diskCacheManager_->TrimRun();
     sleep(6);
@@ -356,8 +369,6 @@ TEST_F(TestDiskCacheManager, TrimCache_5) {
 TEST_F(TestDiskCacheManager, TrimCache_noexceed) {
     S3ClientAdaptorOption option;
     option.diskCacheOpt.maxFileNums = 5;
-    S3Client *client = nullptr;
-
     option.diskCacheOpt.diskCacheType = (DiskCacheType)2;
     option.diskCacheOpt.cacheDir = "/mnt/test_unit";
     option.diskCacheOpt.trimCheckIntervalSec = 1;
@@ -367,7 +378,7 @@ TEST_F(TestDiskCacheManager, TrimCache_noexceed) {
     option.diskCacheOpt.cmdTimeoutSec = 5;
     option.diskCacheOpt.asyncLoadPeriodMs = 10;
 
-    diskCacheManager_->Init(client, option);
+    diskCacheManager_->Init(client_, option);
     diskCacheManager_->AddCache("test");
 
     std::string buf = "test";
@@ -381,8 +392,11 @@ TEST_F(TestDiskCacheManager, TrimCache_noexceed) {
     stat.f_blocks = 1;
     stat.f_bfree = 0;
     stat.f_bavail = 0;
-    EXPECT_CALL(*wrapper, statfs(NotNull(), _))
-        .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(0)));
+    EXPECT_CALL(*wrapper, stat(NotNull(), NotNull()))
+        .Times(3)
+        .WillOnce(Return(0))
+        .WillOnce(Return(-1))
+        .WillOnce(Return(0));
 
     int ret = diskCacheManager_->TrimRun();
     sleep(6);
@@ -390,20 +404,19 @@ TEST_F(TestDiskCacheManager, TrimCache_noexceed) {
 }
 
 TEST_F(TestDiskCacheManager, TrimCache_exceed) {
-     S3ClientAdaptorOption option;
+    S3ClientAdaptorOption option;
     option.diskCacheOpt.maxFileNums = 5;
-    S3Client *client = nullptr;
-
     option.diskCacheOpt.diskCacheType = (DiskCacheType)2;
     option.diskCacheOpt.cacheDir = "/mnt/test_unit";
     option.diskCacheOpt.trimCheckIntervalSec = 1;
     option.diskCacheOpt.fullRatio = 0;
-    option.diskCacheOpt.safeRatio = 0;
-    option.diskCacheOpt.maxUsableSpaceBytes = 0;
+    option.diskCacheOpt.safeRatio = 5;
+    option.diskCacheOpt.maxUsableSpaceBytes = 100;
     option.diskCacheOpt.cmdTimeoutSec = 5;
     option.diskCacheOpt.asyncLoadPeriodMs = 10;
-
-    diskCacheManager_->Init(client, option);
+    option.diskCacheOpt.maxFileNums =
+      std::numeric_limits<uint64_t>::max();
+    diskCacheManager_->Init(client_, option);
 
     std::string buf = "test";
     EXPECT_CALL(*diskCacheWrite_, GetCacheIoFullDir())
@@ -416,7 +429,7 @@ TEST_F(TestDiskCacheManager, TrimCache_exceed) {
     stat.f_bfree = 0;
     stat.f_bavail = 0;
     EXPECT_CALL(*wrapper, statfs(NotNull(), _))
-        .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(0)));
+        .WillRepeatedly(DoAll(SetArgPointee<1>(stat), Return(-1)));
 
     diskCacheManager_->AddCache("test00");
     diskCacheManager_->AddCache("test01");

curvefs/test/client/test_disk_cache_manager_impl.cpp

@@ -60,7 +60,7 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
 
     virtual void SetUp() {
         Aws::InitAPI(awsOptions_);
-        client_ = new MockS3Client();
+        client_ = std::make_shared<MockS3Client>();
         wrapper_ = std::make_shared<MockPosixWrapper>();
         diskCacheWrite_ = std::make_shared<MockDiskCacheWrite>();
         diskCacheRead_ = std::make_shared<MockDiskCacheRead>();
@@ -68,8 +68,8 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
             wrapper_, diskCacheWrite_, diskCacheRead_);
         diskCacheRead_->Init(wrapper_, "/mnt/test");
 
-        std::shared_ptr<LRUCache<std::string, bool>> cachedObjName
-          = std::make_shared<LRUCache<std::string, bool>>
+        std::shared_ptr<SglLRUCache<std::string>> cachedObjName
+          = std::make_shared<SglLRUCache<std::string>>
               (0, std::make_shared<CacheMetrics>("diskcache"));
         diskCacheWrite_->Init(client_, wrapper_, "/mnt/test", 1, cachedObjName);
         diskCacheManagerImpl_ =
@@ -77,7 +77,6 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
     }
 
     virtual void TearDown() {
-        delete client_;
         Mock::VerifyAndClear(wrapper_.get());
         Mock::VerifyAndClear(diskCacheManagerImpl_.get());
         Mock::VerifyAndClear(diskCacheWrite_.get());
@@ -90,7 +89,7 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
     std::shared_ptr<MockDiskCacheManager> diskCacheManager_;
     std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl_;
     std::shared_ptr<MockPosixWrapper> wrapper_;
-    MockS3Client *client_;
+    std::shared_ptr<MockS3Client> client_;
     Aws::SDKOptions awsOptions_;
 };
 
@@ -100,7 +99,6 @@ TEST_F(TestDiskCacheManagerImpl, Init) {
     EXPECT_CALL(*diskCacheManager_, Init(_, _)).WillOnce(Return(-1));
     int ret = diskCacheManagerImpl_->Init(s3AdaptorOption);
     ASSERT_EQ(-1, ret);
-
     EXPECT_CALL(*diskCacheManager_, Init(_, _)).WillOnce(Return(0));
     ret = diskCacheManagerImpl_->Init(s3AdaptorOption);
     ASSERT_EQ(0, ret);

curvefs/test/client/test_disk_cache_read.cpp

@@ -130,8 +130,8 @@ TEST_F(TestDiskCacheRead, LinkWriteToRead) {
 
 TEST_F(TestDiskCacheRead, LoadAllCacheFile) {
     EXPECT_CALL(*wrapper_, stat(NotNull(), NotNull())).WillOnce(Return(-1));
-    std::shared_ptr<LRUCache<std::string, bool>> cachedObj;
-    cachedObj =  std::make_shared<LRUCache<std::string, bool>>
+    std::shared_ptr<SglLRUCache<std::string>> cachedObj;
+    cachedObj =  std::make_shared<SglLRUCache<std::string>>
         (0, std::make_shared<CacheMetrics>("diskcache"));;
     int ret = diskCacheRead_->LoadAllCacheReadFile(cachedObj);
     ASSERT_EQ(-1, ret);
@@ -226,4 +226,3 @@ TEST_F(TestDiskCacheRead, ClearReadCache) {
 
 }  // namespace client
 }  // namespace curvefs
-

curvefs/test/client/test_disk_cache_write.cpp

@@ -57,14 +57,15 @@ class TestDiskCacheWrite : public ::testing::Test {
 
     virtual void SetUp() {
         Aws::InitAPI(awsOptions_);
-        client_ = new MockS3Client();
+//        client_ = new MockS3Client();
+        client_ = std::make_shared<MockS3Client>();
         diskCacheWrite_ = std::make_shared<DiskCacheWrite>();
         wrapper_ = std::make_shared<MockPosixWrapper>();
 
         std::shared_ptr<PosixWrapper> wrapper =
             std::make_shared<PosixWrapper>();
-         std::shared_ptr<LRUCache<std::string, bool>> cachedObjName
-          = std::make_shared<LRUCache<std::string, bool>>
+         std::shared_ptr<SglLRUCache<std::string>> cachedObjName
+          = std::make_shared<SglLRUCache<std::string>>
               (0, std::make_shared<CacheMetrics>("diskcache"));
         diskCacheWrite_->Init(client_, wrapper_, "test", 1, cachedObjName);
     }
@@ -72,13 +73,14 @@ class TestDiskCacheWrite : public ::testing::Test {
     virtual void TearDown() {
         // allows the destructor of lfs_ to be invoked correctly
         Mock::VerifyAndClear(wrapper_.get());
-        delete client_;
+//        delete client_;
         Mock::VerifyAndClear(diskCacheWrite_.get());
         Aws::ShutdownAPI(awsOptions_);
     }
     std::shared_ptr<DiskCacheWrite> diskCacheWrite_;
     std::shared_ptr<MockPosixWrapper> wrapper_;
-    MockS3Client *client_;
+    std::shared_ptr<MockS3Client> client_;
+//    MockS3Client *client_;
     Aws::SDKOptions awsOptions_;
 };
 

curvefs/test/client/test_fuse_client.cpp

@@ -24,6 +24,9 @@
 #include <gtest/gtest.h>
 
 #include "absl/memory/memory.h"
+#include "curvefs/src/client/common/common.h"
+#include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/fuse_s3_client.h"
 #include "curvefs/src/client/fuse_volume_client.h"
 #include "curvefs/src/common/define.h"
@@ -59,12 +62,14 @@ using ::testing::Invoke;
 using ::testing::Return;
 using ::testing::SetArgPointee;
 using ::testing::SetArgReferee;
+using ::testing::AtLeast;
 
 using rpcclient::MockMdsClient;
 using rpcclient::MockMetaServerClient;
 using rpcclient::MetaServerClientDone;
 using ::curvefs::volume::MockBlockDeviceClient;
 using ::curvefs::volume::MockSpaceManager;
+using ::curvefs::client::common::FileHandle;
 
 #define EQUAL(a) (lhs.a() == rhs.a())
 
@@ -180,7 +185,6 @@ TEST_F(TestFuseVolumeClient, FuseOpInit_when_fs_exist) {
 
     EXPECT_CALL(*blockDeviceClient_, Open(_, _))
         .WillOnce(Return(true));
-
     CURVEFS_ERROR ret = client_->FuseOpInit(&mOpts, nullptr);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 
@@ -223,7 +227,7 @@ TEST_F(TestFuseVolumeClient, FuseOpLookup) {
         .WillOnce(DoAll(SetArgPointee<2>(dentry), Return(CURVEFS_ERROR::OK)));
 
     InodeAttr inode;
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(inodeid, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(inodeid, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
 
@@ -249,7 +253,7 @@ TEST_F(TestFuseVolumeClient, FuseOpLookupFail) {
         .WillOnce(Return(CURVEFS_ERROR::INTERNAL))
         .WillOnce(DoAll(SetArgPointee<2>(dentry), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(inodeid, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(inodeid, _))
         .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
 
     fuse_entry_param e;
@@ -347,13 +351,8 @@ TEST_F(TestFuseVolumeClient, FuseOpOpen) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
-        .WillOnce(Return(MetaStatusCode::OK));
-
     CURVEFS_ERROR ret = client_->FuseOpOpen(req, ino, &fi);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-
-    ASSERT_TRUE(inodeWrapper->IsOpen());
 }
 
 TEST_F(TestFuseVolumeClient, FuseOpOpenFailed) {
@@ -370,18 +369,10 @@ TEST_F(TestFuseVolumeClient, FuseOpOpenFailed) {
     auto inodeWrapper = std::make_shared<InodeWrapper>(inode, metaClient_);
 
     EXPECT_CALL(*inodeManager_, GetInode(ino, _))
-        .WillOnce(Return(CURVEFS_ERROR::INTERNAL))
-        .WillOnce(
-            DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
-
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
-        .WillOnce(Return(MetaStatusCode::UNKNOWN_ERROR));
+        .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
 
     CURVEFS_ERROR ret = client_->FuseOpOpen(req, ino, &fi);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
-
-    ret = client_->FuseOpOpen(req, ino, &fi);
-    ASSERT_EQ(CURVEFS_ERROR::UNKNOWN, ret);
 }
 
 TEST_F(TestFuseVolumeClient, FuseOpCreate) {
@@ -423,14 +414,9 @@ TEST_F(TestFuseVolumeClient, FuseOpCreate) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
-        .WillOnce(Return(MetaStatusCode::OK));
-
     fuse_entry_param e;
     CURVEFS_ERROR ret = client_->FuseOpCreate(req, parent, name, mode, &fi, &e);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-
-    ASSERT_TRUE(inodeWrapper->IsOpen());
 }
 
 TEST_F(TestFuseVolumeClient, FuseOpMkDir) {
@@ -569,14 +555,14 @@ TEST_F(TestFuseVolumeClient, FuseOpUnlink) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .WillOnce(Return(MetaStatusCode::OK));
 
     EXPECT_CALL(*inodeManager_, ClearInodeCache(inodeid)).Times(1);
 
     CURVEFS_ERROR ret = client_->FuseOpUnlink(req, parent, name.c_str());
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    Inode inode2 = inodeWrapper->GetInodeUnlocked();
+    Inode inode2 = inodeWrapper->GetInode();
     ASSERT_EQ(nlink - 1, inode2.nlink());
 }
 
@@ -623,14 +609,14 @@ TEST_F(TestFuseVolumeClient, FuseOpRmDir) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .WillOnce(Return(MetaStatusCode::OK));
 
     EXPECT_CALL(*inodeManager_, ClearInodeCache(inodeid)).Times(1);
 
     CURVEFS_ERROR ret = client_->FuseOpRmDir(req, parent, name.c_str());
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    Inode inode2 = inodeWrapper->GetInodeUnlocked();
+    Inode inode2 = inodeWrapper->GetInode();
     ASSERT_EQ(nlink - 1, inode2.nlink());
 
     ASSERT_EQ(false, parentInodeWrapper->IsNlinkValid());
@@ -687,7 +673,7 @@ TEST_F(TestFuseVolumeClient, FuseOpUnlinkFailed) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .WillOnce(Return(MetaStatusCode::UNKNOWN_ERROR));
 
     EXPECT_CALL(*inodeManager_, ClearInodeCache(inodeid)).Times(1);
@@ -788,7 +774,7 @@ TEST_F(TestFuseVolumeClient, FuseOpOpenAndFuseOpReadDir) {
     dentry.set_inodeid(2);
     dentryList.push_back(dentry);
 
-    EXPECT_CALL(*dentryManager_, ListDentry(ino, _, listDentryLimit_, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(ino, _, listDentryLimit_, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dentryList), Return(CURVEFS_ERROR::OK)));
     EXPECT_CALL(*inodeManager_, BatchGetInodeAttrAsync(ino, _, _))
@@ -834,7 +820,7 @@ TEST_F(TestFuseVolumeClient, FuseOpOpenAndFuseOpReadDirFailed) {
     dentry.set_inodeid(2);
     dentryList.push_back(dentry);
 
-    EXPECT_CALL(*dentryManager_, ListDentry(ino, _, listDentryLimit_, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(ino, _, listDentryLimit_, _, _))
         .WillOnce(DoAll(SetArgPointee<1>(dentryList),
                         Return(CURVEFS_ERROR::INTERNAL)));
 
@@ -873,17 +859,12 @@ TEST_F(TestFuseVolumeClient, FuseOpRenameBasic) {
         .WillOnce(DoAll(SetArgReferee<1>(inodeWrapper),
                   Return(CURVEFS_ERROR::OK)));
     // include below unlink operate and update inode parent
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .Times(2)
-        .WillRepeatedly(Invoke([&](const Inode& inode,
-                                   InodeOpenStatusChange statusChange) {
-            return MetaStatusCode::OK;
-        }));
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+        .WillRepeatedly(Return(MetaStatusCode::OK));
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .Times(1)
-        .WillRepeatedly(
-            Invoke([&](const Inode &inode, InodeOpenStatusChange statusChange,
-                       bool internal) { return MetaStatusCode::OK; }));
+        .WillRepeatedly(Return(MetaStatusCode::OK));
 
     // step3: precheck
     // dentry = { fsid, parentid, name, txid, inodeid, DELETE }
@@ -1040,17 +1021,16 @@ TEST_F(TestFuseVolumeClient, FuseOpRenameOverwrite) {
         .WillOnce(DoAll(SetArgReferee<1>(inodeWrapper),
                   Return(CURVEFS_ERROR::OK)));
     // include below unlink old inode and update inode parent
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .Times(1)
-        .WillRepeatedly(Invoke([&](const Inode& inode,
+        .WillRepeatedly(Invoke([&](uint32_t /*fsId*/, uint64_t /*inodeId*/,
+                                   const InodeAttr& /*attr*/,
                                    InodeOpenStatusChange statusChange) {
             return MetaStatusCode::OK;
         }));
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .Times(3)
-        .WillRepeatedly(
-            Invoke([&](const Inode &inode, InodeOpenStatusChange statusChange,
-                       bool internal) { return MetaStatusCode::OK; }));
+        .WillRepeatedly(Return(MetaStatusCode::OK));
 
     // step6: unlink old inode
     Inode inode;
@@ -1123,7 +1103,7 @@ TEST_F(TestFuseVolumeClient, FuseOpRenameOverwriteDir) {
     // step3: list directory
     auto dentrys = std::list<Dentry>();
     dentrys.push_back(Dentry());
-    EXPECT_CALL(*dentryManager_, ListDentry(oldInodeId, _, 1, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(oldInodeId, _, 1, _, _))
         .WillOnce(DoAll(SetArgPointee<1>(dentrys), Return(CURVEFS_ERROR::OK)));
 
     auto rc = client_->FuseOpRename(req, parent, name.c_str(), newparent,
@@ -1197,11 +1177,11 @@ TEST_F(TestFuseVolumeClient, FuseOpRenameParallel) {
         .WillRepeatedly(DoAll(SetArgReferee<1>(srcParentInodeWrapper),
                               Return(CURVEFS_ERROR::OK)));
     // include below operator which unlink old inode and update inode parent
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .Times(times * 2)
         .WillRepeatedly(Return(MetaStatusCode::OK));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .Times(times * 2)
         .WillRepeatedly(Return(MetaStatusCode::OK));
     // step6: unlink old inode
@@ -1264,7 +1244,7 @@ TEST_F(TestFuseVolumeClient, FuseOpGetAttr) {
     inode.set_inodeid(ino);
     inode.set_length(0);
 
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
 
@@ -1283,7 +1263,7 @@ TEST_F(TestFuseVolumeClient, FuseOpGetAttrFailed) {
     inode.set_inodeid(ino);
     inode.set_length(0);
 
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(DoAll(SetArgPointee<1>(inode),
                         Return(CURVEFS_ERROR::INTERNAL)));
 
@@ -1291,6 +1271,32 @@ TEST_F(TestFuseVolumeClient, FuseOpGetAttrFailed) {
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 }
 
+TEST_F(TestFuseVolumeClient, FuseOpGetAttrEnableCto) {
+    curvefs::client::common::FLAGS_enableCto = true;
+
+    fuse_req_t req;
+    fuse_ino_t ino = 1;
+    struct fuse_file_info fi;
+    memset(&fi, 0, sizeof(fi));
+    struct stat attr;
+
+    InodeAttr inode;
+    inode.set_inodeid(ino);
+    inode.set_length(0);
+
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
+        .WillOnce(DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
+
+    ASSERT_EQ(CURVEFS_ERROR::OK, client_->FuseOpGetAttr(req, ino, &fi, &attr));
+
+    // need not refresh inode
+    fi.fh = static_cast<uint64_t>(FileHandle::kKeepCache);
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
+        .WillOnce(DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
+
+    ASSERT_EQ(CURVEFS_ERROR::OK, client_->FuseOpGetAttr(req, ino, &fi, &attr));
+}
+
 TEST_F(TestFuseVolumeClient, FuseOpSetAttr) {
     fuse_req_t req;
     fuse_ino_t ino = 1;
@@ -1310,7 +1316,7 @@ TEST_F(TestFuseVolumeClient, FuseOpSetAttr) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .WillOnce(Return(MetaStatusCode::OK));
 
     attr.st_mode = 1;
@@ -1357,7 +1363,7 @@ TEST_F(TestFuseVolumeClient, FuseOpSetAttrFailed) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .WillOnce(Return(MetaStatusCode::UNKNOWN_ERROR));
 
     attr.st_mode = 1;
@@ -1516,13 +1522,13 @@ TEST_F(TestFuseVolumeClient, FuseOpLink) {
     EXPECT_CALL(*dentryManager_, CreateDentry(_))
         .WillOnce(Return(CURVEFS_ERROR::OK));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .WillOnce(Return(MetaStatusCode::OK));
 
     fuse_entry_param e;
     CURVEFS_ERROR ret = client_->FuseOpLink(req, ino, newparent, newname, &e);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    Inode inode2 = inodeWrapper->GetInodeUnlocked();
+    Inode inode2 = inodeWrapper->GetInode();
     ASSERT_EQ(nlink + 1, inode2.nlink());
 }
 
@@ -1549,7 +1555,7 @@ TEST_F(TestFuseVolumeClient, FuseOpLinkFailed) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .WillOnce(Return(MetaStatusCode::UNKNOWN_ERROR))   // link
         .WillOnce(Return(MetaStatusCode::OK))              // link
         .WillOnce(Return(MetaStatusCode::OK))              // link
@@ -1564,25 +1570,25 @@ TEST_F(TestFuseVolumeClient, FuseOpLinkFailed) {
     // get inode failed
     CURVEFS_ERROR ret = client_->FuseOpLink(req, ino, newparent, newname, &e);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
-    Inode inode2 = inodeWrapper->GetInodeUnlocked();
+    Inode inode2 = inodeWrapper->GetInode();
     ASSERT_EQ(nlink, inode2.nlink());
 
     // link failed
     ret = client_->FuseOpLink(req, ino, newparent, newname, &e);
     ASSERT_EQ(CURVEFS_ERROR::UNKNOWN, ret);
-    Inode inode3 = inodeWrapper->GetInodeUnlocked();
+    Inode inode3 = inodeWrapper->GetInode();
     ASSERT_EQ(nlink, inode3.nlink());
 
     // create dentry failed
     ret = client_->FuseOpLink(req, ino, newparent, newname, &e);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
-    Inode inode4 = inodeWrapper->GetInodeUnlocked();
+    Inode inode4 = inodeWrapper->GetInode();
     ASSERT_EQ(nlink, inode4.nlink());
 
     // also unlink failed
     ret = client_->FuseOpLink(req, ino, newparent, newname, &e);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
-    Inode inode5 = inodeWrapper->GetInodeUnlocked();
+    Inode inode5 = inodeWrapper->GetInode();
     ASSERT_EQ(nlink, inode5.nlink());
 }
 
@@ -1633,13 +1639,6 @@ TEST_F(TestFuseVolumeClient, FuseOpRelease) {
     inode.set_inodeid(ino);
 
     auto inodeWrapper = std::make_shared<InodeWrapper>(inode, metaClient_);
-    inodeWrapper->SetOpenCount(1);
-    EXPECT_CALL(*inodeManager_, GetInode(ino, _))
-        .WillOnce(
-            DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(
-        _, InodeOpenStatusChange::CLOSE, _))
-        .WillOnce(Return(MetaStatusCode::OK));
     ASSERT_EQ(CURVEFS_ERROR::OK, client_->FuseOpRelease(req, ino, &fi));
 }
 
@@ -1872,7 +1871,7 @@ TEST_F(TestFuseS3Client, FuseOpFsync) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .WillOnce(Return(MetaStatusCode::OK));
 
     CURVEFS_ERROR ret = client_->FuseOpFsync(req, ino, 0, fi);
@@ -1893,7 +1892,6 @@ TEST_F(TestFuseS3Client, FuseOpFlush) {
 
     auto inodeWrapper = std::make_shared<InodeWrapper>(inode, metaClient_);
     inodeWrapper->SetUid(32);
-    inodeWrapper->SetOpenCount(1);
 
     LOG(INFO) << "############ case1: test disable cto and s3 flush fail";
     curvefs::client::common::FLAGS_enableCto = false;
@@ -1928,11 +1926,9 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotSummaryInfo) {
     fuse_ino_t ino = 1;
     const char name[] = "security.selinux";
     size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;
 
-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::NODATA, ret);
 }
 
@@ -1943,8 +1939,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir) {
     const char rname[] = "curve.dir.rfbytes";
     const char name[] = "curve.dir.fbytes";
     size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;
 
     // out
     uint32_t fsId = 1;
@@ -2006,10 +2001,10 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir) {
     inode.mutable_xattr()->insert({XATTRENTRIES, "0"});
     inode.mutable_xattr()->insert({XATTRFBYTES, "0"});
 
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)))
         .WillOnce(
@@ -2020,23 +2015,21 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir) {
         .WillOnce(
             DoAll(SetArgPointee<1>(attrs1), Return(CURVEFS_ERROR::OK)));
 
-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
     ASSERT_EQ(std::string(value), "4596");
 
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)));
     EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));
 
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
     ASSERT_EQ(std::string(value), "4396");
 }
@@ -2048,8 +2041,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
     const char rname[] = "curve.dir.rfbytes";
     const char name[] = "curve.dir.fbytes";
     size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;
 
     // out
     uint32_t fsId = 1;
@@ -2083,96 +2075,89 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_NotEnableSumInDir_Failed) {
     inode.mutable_xattr()->insert({XATTRFBYTES, "100"});
 
     // get inode failed
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(DoAll(SetArgPointee<1>(inode),
                         Return(CURVEFS_ERROR::INTERNAL)));
-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // list dentry failed
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(Return(CURVEFS_ERROR::NOTEXIST));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // BatchGetInodeAttr failed
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)));
     EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
         .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // AddUllStringToFirst  XATTRFILES failed
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)));
     EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // AddUllStringToFirst  XATTRSUBDIRS failed
     inode.mutable_xattr()->find(XATTRFILES)->second = "0";
     inode.mutable_xattr()->find(XATTRSUBDIRS)->second = "aaa";
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)));
     EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // AddUllStringToFirst  XATTRENTRIES failed
     inode.mutable_xattr()->find(XATTRSUBDIRS)->second = "0";
     inode.mutable_xattr()->find(XATTRENTRIES)->second = "aaa";
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)));
     EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // AddUllStringToFirst  XATTRFBYTES failed
     inode.mutable_xattr()->find(XATTRENTRIES)->second = "0";
     inode.mutable_xattr()->find(XATTRFBYTES)->second = "aaa";
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)));
     EXPECT_CALL(*inodeManager_, BatchGetInodeAttr(_, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(attrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 }
 
@@ -2183,12 +2168,10 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir) {
     fuse_ino_t ino = 1;
     const char name[] = "curve.dir.rentries";
     size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;
 
     // out
     uint32_t fsId = 1;
-    uint64_t inodeId1 = 2;
     std::string name1 = "file1";
     uint64_t txId = 1;
 
@@ -2196,7 +2179,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir) {
     std::list<Dentry> dlist;
     Dentry dentry;
     dentry.set_fsid(fsId);
-    dentry.set_inodeid(inodeId1);
+    dentry.set_inodeid(ino);
     dentry.set_parentinodeid(ino);
     dentry.set_name(name1);
     dentry.set_txid(txId);
@@ -2206,7 +2189,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir) {
     std::list<XAttr> xattrs;
     XAttr xattr;
     xattr.set_fsid(fsId);
-    xattr.set_inodeid(inodeId1);
+    xattr.set_inodeid(ino);
     xattr.mutable_xattrinfos()->insert({XATTRFILES, "2"});
     xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "2"});
     xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "4"});
@@ -2215,15 +2198,24 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir) {
 
     InodeAttr inode;
     inode.set_inodeid(ino);
+    inode.set_nlink(3);
     inode.mutable_xattr()->insert({XATTRFILES, "1"});
     inode.mutable_xattr()->insert({XATTRSUBDIRS, "1"});
     inode.mutable_xattr()->insert({XATTRENTRIES, "2"});
     inode.mutable_xattr()->insert({XATTRFBYTES, "100"});
 
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    InodeAttr attr = inode;
+    attr.set_nlink(2);
+
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
+        .Times(3)
         .WillOnce(
-            DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+            DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)))
+        .WillOnce(
+            DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)))
+        .WillOnce(
+            DoAll(SetArgPointee<1>(attr), Return(CURVEFS_ERROR::OK)));
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)))
         .WillOnce(
@@ -2232,8 +2224,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir) {
         .WillOnce(
             DoAll(SetArgPointee<1>(xattrs), Return(CURVEFS_ERROR::OK)));
 
-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
     ASSERT_EQ(std::string(value), "6");
 }
@@ -2246,8 +2237,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
     const char name[] = "curve.dir.entries";
     const char rname[] = "curve.dir.rentries";
     size_t size = 100;
-    char value[100];
-    std::memset(value, 0, 100);
+    std::string value;
 
     // out
     uint32_t fsId = 1;
@@ -2278,6 +2268,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
 
     InodeAttr inode;
     inode.set_inodeid(ino);
+    inode.set_nlink(3);
     inode.set_type(FsFileType::TYPE_DIRECTORY);
     inode.mutable_xattr()->insert({XATTRFILES, "1"});
     inode.mutable_xattr()->insert({XATTRSUBDIRS, "1"});
@@ -2285,53 +2276,51 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
     inode.mutable_xattr()->insert({XATTRFBYTES, "aaa"});
 
     // get inode failed
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
-        .WillOnce(DoAll(SetArgPointee<1>(inode),
-                        Return(CURVEFS_ERROR::OK)));
-    CURVEFS_ERROR ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
+        .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
+    CURVEFS_ERROR ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // AddUllStringToFirst failed
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(DoAll(SetArgPointee<1>(inode),
-                        Return(CURVEFS_ERROR::INTERNAL)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, name, static_cast<void*>(value), size);
+                        Return(CURVEFS_ERROR::OK)));
+    ret = client_->FuseOpGetXattr(req, ino, name, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
     inode.mutable_xattr()->find(XATTRFBYTES)->second = "100";
 
     // list dentry failed
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
-        .WillOnce(
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
+        .Times(2)
+        .WillRepeatedly(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(Return(CURVEFS_ERROR::NOTEXIST));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // BatchGetInodeAttr failed
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
-        .WillOnce(
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
+        .Times(AtLeast(2))
+        .WillRepeatedly(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)))
         .WillRepeatedly(
             DoAll(SetArgPointee<1>(emptyDlist), Return(CURVEFS_ERROR::OK)));
     EXPECT_CALL(*inodeManager_, BatchGetXAttr(_, _))
         .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // AddUllStringToFirst  XATTRFILES failed
     inode.mutable_xattr()->find(XATTRFILES)->second = "aaa";
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
-        .WillOnce(
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
+        .Times(AtLeast(2))
+        .WillRepeatedly(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)))
         .WillRepeatedly(
@@ -2339,17 +2328,17 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
     EXPECT_CALL(*inodeManager_, BatchGetXAttr(_, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(xattrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // AddUllStringToFirst  XATTRSUBDIRS failed
     inode.mutable_xattr()->find(XATTRFILES)->second = "0";
     inode.mutable_xattr()->find(XATTRSUBDIRS)->second = "aaa";
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
-        .WillOnce(
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
+        .Times(AtLeast(2))
+        .WillRepeatedly(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)))
         .WillRepeatedly(
@@ -2357,17 +2346,17 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
     EXPECT_CALL(*inodeManager_, BatchGetXAttr(_, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(xattrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
     // AddUllStringToFirst  XATTRENTRIES failed
     inode.mutable_xattr()->find(XATTRSUBDIRS)->second = "0";
     inode.mutable_xattr()->find(XATTRENTRIES)->second = "aaa";
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
-        .WillOnce(
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
+        .Times(AtLeast(2))
+        .WillRepeatedly(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _))
+    EXPECT_CALL(*dentryManager_, ListDentry(_, _, _, _, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(dlist), Return(CURVEFS_ERROR::OK)))
         .WillRepeatedly(
@@ -2375,8 +2364,7 @@ TEST_F(TestFuseS3Client, FuseOpGetXattr_EnableSumInDir_Failed) {
     EXPECT_CALL(*inodeManager_, BatchGetXAttr(_, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(xattrs), Return(CURVEFS_ERROR::OK)));
-    ret = client_->FuseOpGetXattr(
-        req, ino, rname, static_cast<void*>(value), size);
+    ret = client_->FuseOpGetXattr(req, ino, rname, &value, size);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 }
 
@@ -2431,7 +2419,7 @@ TEST_F(TestFuseS3Client, FuseOpCreate_EnableSummary) {
         .WillOnce(
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .WillRepeatedly(Return(MetaStatusCode::OK));
 
     EXPECT_CALL(*inodeManager_, ShipToFlush(_))
@@ -2440,13 +2428,12 @@ TEST_F(TestFuseS3Client, FuseOpCreate_EnableSummary) {
     fuse_entry_param e;
     CURVEFS_ERROR ret = client_->FuseOpCreate(req, parent, name, mode, &fi, &e);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    ASSERT_TRUE(inodeWrapper->IsOpen());
 
     auto p = parentInodeWrapper->GetInodeLocked();
-    ASSERT_EQ(p.xattr().find(XATTRFILES)->second, "2");
-    ASSERT_EQ(p.xattr().find(XATTRSUBDIRS)->second, "1");
-    ASSERT_EQ(p.xattr().find(XATTRENTRIES)->second, "3");
-    ASSERT_EQ(p.xattr().find(XATTRFBYTES)->second, "4196");
+    ASSERT_EQ(p->xattr().find(XATTRFILES)->second, "2");
+    ASSERT_EQ(p->xattr().find(XATTRSUBDIRS)->second, "1");
+    ASSERT_EQ(p->xattr().find(XATTRENTRIES)->second, "3");
+    ASSERT_EQ(p->xattr().find(XATTRFBYTES)->second, "4196");
 }
 
 TEST_F(TestFuseS3Client, FuseOpWrite_EnableSummary) {
@@ -2499,10 +2486,10 @@ TEST_F(TestFuseS3Client, FuseOpWrite_EnableSummary) {
     ASSERT_EQ(size, wSize);
 
     auto p = parentInodeWrapper->GetInodeLocked();
-    ASSERT_EQ(p.xattr().find(XATTRFILES)->second, "1");
-    ASSERT_EQ(p.xattr().find(XATTRSUBDIRS)->second, "0");
-    ASSERT_EQ(p.xattr().find(XATTRENTRIES)->second, "1");
-    ASSERT_EQ(p.xattr().find(XATTRFBYTES)->second, std::to_string(size));
+    ASSERT_EQ(p->xattr().find(XATTRFILES)->second, "1");
+    ASSERT_EQ(p->xattr().find(XATTRSUBDIRS)->second, "0");
+    ASSERT_EQ(p->xattr().find(XATTRENTRIES)->second, "1");
+    ASSERT_EQ(p->xattr().find(XATTRFBYTES)->second, std::to_string(size));
 }
 
 TEST_F(TestFuseS3Client, FuseOpLink_EnableSummary) {
@@ -2534,7 +2521,7 @@ TEST_F(TestFuseS3Client, FuseOpLink_EnableSummary) {
             DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)))
         .WillRepeatedly(
             DoAll(SetArgReferee<1>(pinodeWrapper), Return(CURVEFS_ERROR::OK)));
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .WillRepeatedly(Return(MetaStatusCode::OK));
     EXPECT_CALL(*dentryManager_, CreateDentry(_))
         .WillOnce(Return(CURVEFS_ERROR::OK));
@@ -2543,7 +2530,7 @@ TEST_F(TestFuseS3Client, FuseOpLink_EnableSummary) {
     fuse_entry_param e;
     CURVEFS_ERROR ret = client_->FuseOpLink(req, ino, newparent, newname, &e);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    auto p = pinodeWrapper->GetInodeLocked();
+    auto p = pinodeWrapper->GetInode();
     ASSERT_EQ(p.xattr().find(XATTRFILES)->second, "1");
     ASSERT_EQ(p.xattr().find(XATTRSUBDIRS)->second, "0");
     ASSERT_EQ(p.xattr().find(XATTRENTRIES)->second, "1");
@@ -2604,7 +2591,7 @@ TEST_F(TestFuseS3Client, FuseOpUnlink_EnableSummary) {
             DoAll(SetArgReferee<1>(parentInodeWrapper),
                 Return(CURVEFS_ERROR::OK)));
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttr(_, _, _, _))
         .WillRepeatedly(Return(MetaStatusCode::OK));
 
     EXPECT_CALL(*inodeManager_, ShipToFlush(_))
@@ -2615,10 +2602,10 @@ TEST_F(TestFuseS3Client, FuseOpUnlink_EnableSummary) {
 
     CURVEFS_ERROR ret = client_->FuseOpUnlink(req, parent, name.c_str());
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    Inode inode2 = inodeWrapper->GetInodeUnlocked();
+    Inode inode2 = inodeWrapper->GetInode();
     ASSERT_EQ(nlink - 1, inode2.nlink());
 
-    auto p = parentInodeWrapper->GetInodeLocked();
+    auto p = parentInodeWrapper->GetInode();
     ASSERT_EQ(p.xattr().find(XATTRFILES)->second, "0");
     ASSERT_EQ(p.xattr().find(XATTRSUBDIRS)->second, "1");
     ASSERT_EQ(p.xattr().find(XATTRENTRIES)->second, "1");
@@ -2666,16 +2653,15 @@ TEST_F(TestFuseS3Client, FuseOpOpen_Trunc_EnableSummary) {
                 Return(CURVEFS_ERROR::OK)));
     EXPECT_CALL(*s3ClientAdaptor_, Truncate(_, _))
         .WillOnce(Return(CURVEFS_ERROR::OK));
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .WillRepeatedly(Return(MetaStatusCode::OK));
     EXPECT_CALL(*inodeManager_, ShipToFlush(_))
         .Times(1);
 
     CURVEFS_ERROR ret = client_->FuseOpOpen(req, ino, &fi);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-    ASSERT_TRUE(inodeWrapper->IsOpen());
 
-    auto p = parentInodeWrapper->GetInodeLocked();
+    auto p = parentInodeWrapper->GetInode();
     ASSERT_EQ(p.xattr().find(XATTRFILES)->second, "1");
     ASSERT_EQ(p.xattr().find(XATTRSUBDIRS)->second, "1");
     ASSERT_EQ(p.xattr().find(XATTRENTRIES)->second, "2");
@@ -2700,7 +2686,7 @@ TEST_F(TestFuseS3Client, FuseOpListXattr) {
     size_t realSize = 0;
 
     // failed when get inode
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode),
                 Return(CURVEFS_ERROR::INTERNAL)));
@@ -2708,7 +2694,7 @@ TEST_F(TestFuseS3Client, FuseOpListXattr) {
         req, ino, buf, size, &realSize);
     ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
 
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
     ret = client_->FuseOpListXattr(
@@ -2718,7 +2704,7 @@ TEST_F(TestFuseS3Client, FuseOpListXattr) {
 
     realSize = 0;
     inode.set_type(FsFileType::TYPE_DIRECTORY);
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
     ret = client_->FuseOpListXattr(
@@ -2730,7 +2716,7 @@ TEST_F(TestFuseS3Client, FuseOpListXattr) {
     ASSERT_EQ(realSize, expected);
 
     realSize = 0;
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
     ret = client_->FuseOpListXattr(
@@ -2738,7 +2724,7 @@ TEST_F(TestFuseS3Client, FuseOpListXattr) {
     ASSERT_EQ(CURVEFS_ERROR::OUT_OF_RANGE, ret);
 
     realSize = 0;
-    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _, _))
+    EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(
             DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
     ret = client_->FuseOpListXattr(
@@ -2746,5 +2732,58 @@ TEST_F(TestFuseS3Client, FuseOpListXattr) {
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 }
 
+TEST_F(TestFuseS3Client, FuseOpSetXattr_TooLong) {
+    // in
+    fuse_req_t req;
+    fuse_ino_t ino = 1;
+    const char name[] = "security.selinux";
+    size_t size = 64 * 1024 + 1;
+    char value[64 * 1024 + 1];
+    std::memset(value, 0, size);
+
+    CURVEFS_ERROR ret = client_->FuseOpSetXattr(
+        req, ino, name, value, size, 0);
+    ASSERT_EQ(CURVEFS_ERROR::OUT_OF_RANGE, ret);
+}
+
+TEST_F(TestFuseS3Client, FuseOpSetXattr) {
+    // in
+    fuse_req_t req;
+    fuse_ino_t ino = 1;
+    const char name[] = "security.selinux";
+    size_t size = 100;
+    char value[100];
+    std::memset(value, 0, 100);
+
+    // get inode failed
+    EXPECT_CALL(*inodeManager_, GetInode(ino, _))
+        .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
+    CURVEFS_ERROR ret = client_->FuseOpSetXattr(
+        req, ino, name, value, size, 0);
+    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
+
+    // updateInode failed
+    auto inodeWrapper = std::make_shared<InodeWrapper>(Inode(), metaClient_);
+    EXPECT_CALL(*inodeManager_, GetInode(ino, _))
+        .WillOnce(
+            DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
+        .WillOnce(Return(MetaStatusCode::NOT_FOUND));
+    ret = client_->FuseOpSetXattr(
+        req, ino, name, value, size, 0);
+    ASSERT_EQ(CURVEFS_ERROR::NOTEXIST, ret);
+
+    // success
+    EXPECT_CALL(*inodeManager_, GetInode(ino, _))
+        .WillOnce(
+            DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
+        .WillOnce(Return(MetaStatusCode::OK));
+    ret = client_->FuseOpSetXattr(
+        req, ino, name, value, size, 0);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+}
+
+
 }  // namespace client
 }  // namespace curvefs

curvefs/test/client/test_inodeWrapper.cpp

@@ -33,6 +33,7 @@
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/test/client/mock_metaserver_client.h"
 #include "curvefs/src/client/inode_wrapper.h"
+#include "src/common/timeutility.h"
 
 using ::google::protobuf::util::MessageDifferencer;
 
@@ -52,15 +53,12 @@ using rpcclient::MockMetaServerClient;
 
 class TestInodeWrapper : public ::testing::Test {
  protected:
-    TestInodeWrapper() {}
-    ~TestInodeWrapper() {}
-
-    virtual void SetUp() {
+    void SetUp() override {
         metaClient_ = std::make_shared<MockMetaServerClient>();
         inodeWrapper_ = std::make_shared<InodeWrapper>(Inode(), metaClient_);
     }
 
-    virtual void TearDown() {
+    void TearDown() override {
         metaClient_ = nullptr;
         inodeWrapper_ = nullptr;
     }
@@ -140,7 +138,7 @@ TEST_F(TestInodeWrapper, testSyncSuccess) {
     uint64_t chunkIndex1 = 1;
     inodeWrapper_->AppendS3ChunkInfo(chunkIndex1, info1);
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .WillOnce(Return(MetaStatusCode::OK));
 
     EXPECT_CALL(*metaClient_, GetOrModifyS3ChunkInfo(_, _, _, _, _, _))
@@ -165,7 +163,7 @@ TEST_F(TestInodeWrapper, testSyncFailed) {
     uint64_t chunkIndex1 = 1;
     inodeWrapper_->AppendS3ChunkInfo(chunkIndex1, info1);
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .WillOnce(Return(MetaStatusCode::NOT_FOUND))
         .WillOnce(Return(MetaStatusCode::OK));
 
@@ -184,7 +182,7 @@ TEST_F(TestInodeWrapper, TestFlushVolumeExtent_NoNeedFlush) {
 
     inodeWrapper_->SetType(FsFileType::TYPE_FILE);
     inodeWrapper_->ClearDirty();
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .Times(0);
     EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(_, _, _, _))
         .Times(0);
@@ -203,7 +201,7 @@ TEST_F(TestInodeWrapper, TestFlushVolumeExtent) {
     pext.pOffset = 0;
     pext.UnWritten = true;
     extentCache->Merge(0, pext);
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .Times(0);
     EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(_, _, _, _))
         .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentList&,
@@ -222,9 +220,51 @@ TEST_F(TestInodeWrapper, TestRefreshNlink) {
     EXPECT_CALL(*metaClient_, GetInodeAttr(_, _, _))
         .WillOnce(DoAll(SetArgPointee<2>(attr), Return(MetaStatusCode::OK)));
     inodeWrapper_->RefreshNlink();
-    Inode inode = inodeWrapper_->GetInodeUnlocked();
+    Inode inode = inodeWrapper_->GetInode();
     ASSERT_EQ(nlink, inode.nlink());
 }
 
+TEST_F(TestInodeWrapper, TestUpdateInodeAttrIncrementally) {
+    Inode inode;
+    inode.set_type(FsFileType::TYPE_S3);
+    inode.set_length(0);
+    inode.set_atime(0);
+    inode.set_atime_ns(0);
+    inode.set_ctime(0);
+    inode.set_ctime_ns(0);
+    inode.set_mtime(0);
+    inode.set_mtime_ns(0);
+
+    InodeWrapper wrapper(std::move(inode), metaClient_);
+
+    {
+        auto lock = wrapper.GetUniqueLock();
+        wrapper.UpdateTimestampLocked(kAccessTime);
+    }
+
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
+        .WillOnce(Invoke(
+            [](uint32_t /*fsId*/, uint64_t /*inodeId*/, const InodeAttr& attr,
+               InodeOpenStatusChange /*statusChange*/, bool /*internal*/
+            ) {
+                EXPECT_FALSE(attr.has_length());
+                return MetaStatusCode::OK;
+            }));
+
+    ASSERT_EQ(CURVEFS_ERROR::OK, wrapper.Sync());
+
+    ASSERT_FALSE(wrapper.dirty_);
+    ASSERT_FALSE(wrapper.dirtyAttr_.has_atime());
+    ASSERT_FALSE(wrapper.dirtyAttr_.has_atime_ns());
+}
+
+TEST_F(TestInodeWrapper, TestSetXattr) {
+    inodeWrapper_->SetXattrLocked("name", "value");
+    XAttr xattr = inodeWrapper_->GetXattr();
+    ASSERT_TRUE(xattr.xattrinfos().find("name") != xattr.xattrinfos().end());
+    ASSERT_EQ((*xattr.mutable_xattrinfos())["name"], "value");
+    ASSERT_TRUE(inodeWrapper_->IsDirty());
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/test/client/test_inode_cache_manager.cpp

@@ -20,10 +20,12 @@
  * Author: xuchaojie
  */
 
+#include <gmock/gmock-spec-builders.h>
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
 #include <cstdint>
-
+#include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/test/client/mock_metaserver_client.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/common/define.h"
@@ -83,46 +85,66 @@ TEST_F(TestInodeCacheManager, GetInode) {
     inode.set_inodeid(inodeId);
     inode.set_fsid(fsId_);
     inode.set_length(fileLength);
+    inode.set_type(FsFileType::TYPE_S3);
+    auto s3ChunkInfoMap = inode.mutable_s3chunkinfomap();
+    S3ChunkInfoList *s3ChunkInfoList = new S3ChunkInfoList();
+    S3ChunkInfo *s3ChunkInfo = s3ChunkInfoList->add_s3chunks();
+    s3ChunkInfo->set_chunkid(1);
+    s3ChunkInfo->set_compaction(1);
+    s3ChunkInfo->set_offset(0);
+    s3ChunkInfo->set_len(1024);
+    s3ChunkInfo->set_size(65536);
+    s3ChunkInfo->set_zero(true);
+    s3ChunkInfoMap->insert({1, *s3ChunkInfoList});
 
+    // miss cache and get inode failed
     EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(Return(MetaStatusCode::NOT_FOUND))
-        .WillOnce(DoAll(SetArgPointee<2>(inode), Return(MetaStatusCode::OK)));
-
+        .WillOnce(Return(MetaStatusCode::NOT_FOUND));
     std::shared_ptr<InodeWrapper> inodeWrapper;
     CURVEFS_ERROR ret = iCacheManager_->GetInode(inodeId, inodeWrapper);
     ASSERT_EQ(CURVEFS_ERROR::NOTEXIST, ret);
 
+    // miss cache and get inode ok, do not need streaming
+    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(inode), SetArgPointee<3>(false),
+                        Return(MetaStatusCode::OK)));
     ret = iCacheManager_->GetInode(inodeId, inodeWrapper);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 
-    Inode out = inodeWrapper->GetInodeUnlocked();
-    ASSERT_EQ(inodeId, out.inodeid());
-    ASSERT_EQ(fsId_, out.fsid());
-    ASSERT_EQ(fileLength, out.length());
-
-    ret = iCacheManager_->GetInode(inodeId, inodeWrapper);
-    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
-
-    out = inodeWrapper->GetInodeUnlocked();
-    ASSERT_EQ(inodeId, out.inodeid());
-    ASSERT_EQ(fsId_, out.fsid());
-    ASSERT_EQ(fileLength, out.length());
+    // miss cache and get inode ok, need streaming
+    uint64_t inodeId2 = 200;
+    Inode inode2 = inode;
+    inode2.set_inodeid(inodeId2);
+    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId2, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(inode2), SetArgPointee<3>(true),
+                        Return(MetaStatusCode::OK)));
+    EXPECT_CALL(*metaClient_,
+                GetOrModifyS3ChunkInfo(fsId_, inodeId2, _, true, _, _))
+        .WillOnce(Return(MetaStatusCode::OK));
+    ASSERT_EQ(CURVEFS_ERROR::OK,
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
 
+    // enable cto and not opened and not dirty
     curvefs::client::common::FLAGS_enableCto = true;
-    inodeWrapper->SetOpenCount(0);
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(Return(MetaStatusCode::NOT_FOUND));
-    ret = iCacheManager_->GetInode(inodeId, inodeWrapper);
-    ASSERT_EQ(CURVEFS_ERROR::NOTEXIST, ret);
-    ASSERT_EQ(inodeId, out.inodeid());
-    ASSERT_EQ(fsId_, out.fsid());
-    ASSERT_EQ(fileLength, out.length());
+    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId2, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(inode2), SetArgPointee<3>(true),
+                        Return(MetaStatusCode::OK)));
+    EXPECT_CALL(*metaClient_,
+                GetOrModifyS3ChunkInfo(fsId_, inodeId2, _, true, _, _))
+        .WillOnce(Return(MetaStatusCode::OK));
+    ASSERT_EQ(CURVEFS_ERROR::OK,
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
 
-    inodeWrapper->SetOpenCount(1);
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(Return(MetaStatusCode::NOT_FOUND));
-    ret = iCacheManager_->GetInode(inodeId, inodeWrapper);
-    ASSERT_EQ(CURVEFS_ERROR::NOTEXIST, ret);
+    // enable cto and opened will hit cache
+    iCacheManager_->AddOpenedInode(inodeId2);
+    ASSERT_EQ(CURVEFS_ERROR::OK,
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
+
+    // enable cto and not opened and inode dirty will hit cache
+    iCacheManager_->RemoveOpenedInode(inodeId2);
+    inodeWrapper->MarkDirty();
+    ASSERT_EQ(CURVEFS_ERROR::OK,
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
 }
 
 TEST_F(TestInodeCacheManager, GetInodeAttr) {
@@ -145,10 +167,10 @@ TEST_F(TestInodeCacheManager, GetInodeAttr) {
         .WillOnce(Return(MetaStatusCode::NOT_FOUND))
         .WillOnce(DoAll(SetArgPointee<2>(attrs), Return(MetaStatusCode::OK)));
 
-    CURVEFS_ERROR ret = iCacheManager_->GetInodeAttr(inodeId, &out, parentId);
+    CURVEFS_ERROR ret = iCacheManager_->GetInodeAttr(inodeId, &out);
     ASSERT_EQ(CURVEFS_ERROR::NOTEXIST, ret);
 
-    ret = iCacheManager_->GetInodeAttr(inodeId, &out, parentId);
+    ret = iCacheManager_->GetInodeAttr(inodeId, &out);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
     ASSERT_EQ(inodeId, out.inodeid());
     ASSERT_EQ(fsId_, out.fsid());
@@ -168,7 +190,36 @@ TEST_F(TestInodeCacheManager, GetInodeAttr) {
     ret = iCacheManager_->CreateInode(param, inodeWrapper);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 
-    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out, parentId);
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(inodeId + 1, out.inodeid());
+    ASSERT_EQ(fsId_ + 1, out.fsid());
+    ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
+
+    // enable cto will get from metaserver
+    curvefs::client::common::FLAGS_enableCto = true;
+    EXPECT_CALL(*metaClient_, BatchGetInodeAttr(fsId_, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(attrs), Return(MetaStatusCode::OK)));
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(inodeId, out.inodeid());
+    ASSERT_EQ(fsId_, out.fsid());
+    ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
+
+    // set this inode open
+    curvefs::client::common::FLAGS_enableCto = false;
+    iCacheManager_->AddOpenedInode(inodeId + 1);
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(inodeId + 1, out.inodeid());
+    ASSERT_EQ(fsId_ + 1, out.fsid());
+    ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
+
+    // set this inode dirty
+    curvefs::client::common::FLAGS_enableCto = false;
+    iCacheManager_->RemoveOpenedInode(inodeId + 1);
+    inodeWrapper->MarkDirty();
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
     ASSERT_EQ(inodeId + 1, out.inodeid());
     ASSERT_EQ(fsId_ + 1, out.fsid());
@@ -196,7 +247,7 @@ TEST_F(TestInodeCacheManager, CreateAndGetInode) {
     ASSERT_EQ(CURVEFS_ERROR::UNKNOWN, ret);
 
     ret = iCacheManager_->CreateInode(param, inodeWrapper);
-    Inode out = inodeWrapper->GetInodeUnlocked();
+    Inode out = inodeWrapper->GetInode();
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
     ASSERT_EQ(inodeId, out.inodeid());
     ASSERT_EQ(fsId_, out.fsid());
@@ -205,7 +256,7 @@ TEST_F(TestInodeCacheManager, CreateAndGetInode) {
     ret = iCacheManager_->GetInode(inodeId, inodeWrapper);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 
-    out = inodeWrapper->GetInodeUnlocked();
+    out = inodeWrapper->GetInode();
     ASSERT_EQ(inodeId, out.inodeid());
     ASSERT_EQ(fsId_, out.fsid());
     ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
@@ -245,12 +296,14 @@ TEST_F(TestInodeCacheManager, ShipToFlushAndFlushAll) {
 
     iCacheManager_->ShipToFlush(inodeWrapper);
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlinkAsync(_, _, _))
-        .WillOnce(Invoke([](const Inode &inode, MetaServerClientDone *done,
-                            InodeOpenStatusChange statusChange) {
-            done->SetMetaStatusCode(MetaStatusCode::OK);
-            done->Run();
-        }));
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlinkAsync(_, _, _, _, _))
+        .WillOnce(
+            Invoke([](uint32_t /*fsId*/, uint64_t /*inodeId*/,
+                      const InodeAttr& /*attr*/, MetaServerClientDone* done,
+                      InodeOpenStatusChange statusChange) {
+                done->SetMetaStatusCode(MetaStatusCode::OK);
+                done->Run();
+            }));
 
     EXPECT_CALL(*metaClient_, GetOrModifyS3ChunkInfoAsync(_, _, _, _))
         .WillOnce(
@@ -302,6 +355,58 @@ TEST_F(TestInodeCacheManager, BatchGetInodeAttr) {
     ASSERT_EQ(getAttrs.begin()->length(), fileLength);
 }
 
+TEST_F(TestInodeCacheManager, BatchGetInodeAttrAsync) {
+    uint64_t parentId = 1;
+    uint64_t inodeId1 = 100;
+    uint64_t inodeId2 = 200;
+
+    // in
+    std::set<uint64_t> inodeIds;
+    inodeIds.emplace(inodeId1);
+    inodeIds.emplace(inodeId2);
+
+    // out
+    Inode inode;
+    inode.set_inodeid(inodeId1);
+
+    std::vector<std::vector<uint64_t>> inodeGroups;
+    inodeGroups.emplace_back(std::vector<uint64_t>{inodeId2});
+
+    std::map<uint64_t, InodeAttr> attrs;
+
+    RepeatedPtrField<InodeAttr> inodeAttrs;
+    inodeAttrs.Add()->set_inodeid(inodeId2);
+
+    // fill icache
+    EXPECT_CALL(*metaClient_, GetInode(_, inodeId1, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(inode),
+                Return(MetaStatusCode::OK)));
+    std::shared_ptr<InodeWrapper> wrapper;
+    iCacheManager_->GetInode(inodeId1, wrapper);
+
+    EXPECT_CALL(*metaClient_, SplitRequestInodes(_, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(inodeGroups),
+                Return(true)));
+    EXPECT_CALL(*metaClient_, BatchGetInodeAttrAsync(_, _, _))
+        .WillOnce(Invoke([inodeAttrs](uint32_t fsId,
+            const std::vector<uint64_t> &inodeIds,
+            MetaServerClientDone *done) {
+                done->SetMetaStatusCode(MetaStatusCode::OK);
+                static_cast<BatchGetInodeAttrDone *>(done)
+                    ->SetInodeAttrs(inodeAttrs);
+                done->Run();
+                return MetaStatusCode::OK;
+            }));
+
+    CURVEFS_ERROR ret = iCacheManager_->BatchGetInodeAttrAsync(parentId,
+        &inodeIds, &attrs);
+
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(attrs.size(), 2);
+    ASSERT_TRUE(attrs.find(inodeId1) != attrs.end());
+    ASSERT_TRUE(attrs.find(inodeId2) != attrs.end());
+}
+
 TEST_F(TestInodeCacheManager, BatchGetXAttr) {
     uint64_t inodeId1 = 100;
     uint64_t inodeId2 = 200;
@@ -367,9 +472,10 @@ TEST_F(TestInodeCacheManager, TestFlushInodeBackground) {
         inodeMap.emplace(inodeId + i, inodeWrapper);
     }
 
-    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlinkAsync(_, _, _))
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlinkAsync(_, _, _, _, _))
         .WillRepeatedly(
-            Invoke([](const Inode &inode, MetaServerClientDone *done,
+            Invoke([](uint32_t /*fsId*/, uint64_t /*inodeId*/,
+                      const InodeAttr& /*attr*/, MetaServerClientDone *done,
                       InodeOpenStatusChange statusChange) {
                 done->SetMetaStatusCode(MetaStatusCode::OK);
                 done->Run();