curvebs : support spdk over nvme

Signed-off-by: jolly-sy <757050468@qq.com>

.bazelrc

@@ -2,4 +2,4 @@ build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
-build --incompatible_blacklisted_protos_requires_proto_info=false
+build --incompatible_blacklisted_protos_requires_proto_info=false

BUILD

@@ -37,3 +37,10 @@ config_setting(
         "@bazel_tools//tools/cpp:compiler": "clang",
     },
 )
+
+config_setting(
+    name = "with_spdk",
+    define_values = {"with_spdk": "true"},
+    visibility = ["//visibility:public"],
+)
+

README.md

@@ -37,7 +37,9 @@
     - [Deploy an All-in-one experience environment](#-deploy-an-all-in-one-experience-environment)
     - [FIO curve block storage engine](#fio-curve-block-storage-engine)
   - [CurveFS quick start](#-curvefs-quick-start)
+  - [Governance](#-Governance)
   - [Contribute us](#-contribute-us)
+  - [Code of Conduct](#-Code-of-Conduct)
   - [LICENSE](#-license)
   - [Release Cycle](#-release-cycle)
   - [Branch](#-branch)
@@ -201,11 +203,19 @@ $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest
 ##  CurveFS quick start
 Please use [CurveAdm](https://github.com/opencurve/curveadm/wiki) tool to deploy CurveFS，see [CurveFS Deployment Process](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), and the [CurveFS Command Instructions](curvefs/src/tools#readme).
 
+## Practical
+- [CurveBS+NFS Build NFS Server](docs/practical/curvebs_nfs.md)
+
+## Governance
+See [Governance](https://github.com/opencurve/community/blob/master/GOVERNANCE.md).
 
 ##  Contribute us
 Participation in the Curve project is described in the [Curve Open Source Community Guidelines](Community_Guidelines.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
 We welcome your contribution!
 
+## Code of Conduct
+Curve follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
+
 ## LICENSE
 Curve is distributed under the [**Apache 2.0 LICENSE**](LICENSE).
 

README_cn.md

@@ -37,7 +37,9 @@
     - [部署All-in-one体验环境](#-部署all-in-one体验环境)
     - [FIO curve块存储引擎](#fio-curve块存储引擎)
   - [CurveFS 快速体验](#-curvefs-快速体验)
+  - [社区治理](#-社区治理)
   - [贡献我们](#-贡献我们)
+  - [行为守则](#-行为守则)
   - [LICENSE](#-license)
   - [版本发布周期](#-版本发布周期)
   - [分支规则](#-分支规则)
@@ -198,11 +200,19 @@ $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest
 ##  CurveFS 快速体验
 请使用 [CurveAdm](https://github.com/opencurve/curveadm/wiki) 工具进行 CurveFS 的部署，具体流程见：[CurveFS部署流程](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), 以及[CurveFS命令行工具说明](curvefs/src/tools#readme)。
 
+##  社区治理
+请参考[社区治理](https://github.com/opencurve/community/blob/master/GOVERNANCE.md)。
 
 ##  贡献我们
 
 参与 Curve 项目开发详见[Curve 开源社区指南](Community_Guidelines_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
 
+## 最佳实践
+- [CurveBS+NFS搭建NFS存储](docs/practical/curvebs_nfs.md)
+
+##  行为守则
+Curve 的行为守则遵循[CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md)。
+
 ## LICENSE
 Curve 在 [Apache 2.0](LICENSE) 协议下进行分发。
 

WORKSPACE

@@ -128,10 +128,8 @@ bind(
 
 git_repository(
     name = "com_github_apache_brpc",
-    remote = "https://github.com/apache/incubator-brpc",
-    commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0",
-    patches = ["//:thirdparties/brpc/brpc.patch"],
-    patch_args = ["-p1"],
+    remote = "https://github.com/opencurve/incubator-brpc",
+    commit = "89ed709f71b2bab1ab4176609f28fc891222c508",
 )
 
 bind(
@@ -227,3 +225,55 @@ http_archive(
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()
+
+new_local_repository(
+    name = "pfs",
+    path = "/usr/local/curvestore/pfsd/include",
+    build_file_content = """
+package(default_visibility = ["//visibility:public"])
+cc_library(
+    name = "headers",
+    hdrs = glob(["**/*.h"])
+)
+"""
+)
+
+bind(
+    name = "pfs_headers",
+    actual = "@pfs//:headers",
+)
+
+new_local_repository(
+    name = "dpdk",
+    path = "/usr/local/dpdk/include",
+    build_file_content = """
+package(default_visibility = ["//visibility:public"])
+cc_library(
+    name = "headers",
+    hdrs = glob(["**/*.h"])
+)
+"""
+)
+
+bind(
+    name = "dpdk_headers",
+    actual = "@dpdk//:headers",
+)
+
+new_local_repository(
+    name = "spdk",
+    path = "/usr/local/spdk/include",
+    build_file_content = """
+package(default_visibility = ["//visibility:public"])
+cc_library(
+    name = "headers",
+    hdrs = glob(["**/*.h"])
+)
+"""
+)
+
+bind(
+    name = "spdk_headers",
+    actual = "@spdk//:headers",
+)
+

conf/chunkserver.conf

@@ -144,7 +144,16 @@ curve.curve_file_timeout_s=30
 #
 # 是否开启使用renameat2，ext4内核3.15以后开始支持
 fs.enable_renameat2=true
-
+# fs type: ext4, pfs
+fs.type=ext4
+#cluster name for pfs
+fs.pfs_cluster=spdk
+#pdb name for pfs
+fs.pfs_pbd_name=0000:81:00.0n1
+#host id for pfs
+fs.pfs_host_id=2
+#nvme controller for spdk
+fs.spdk_nvme_controller=0000:81:00.0
 #
 # metrics settings
 # true means on, false means off
@@ -194,8 +203,10 @@ chunkfilepool.clean.throttle_iops=500
 #
 # WAL file pool
 #
-# walpool是否共用chunkfilepool，如果为true，则以下配置无效
+# walpool是否共用chunkfilepool，如果为true，从第三条开始配置无效
 walfilepool.use_chunk_file_pool=true
+# WALpool和ChunkFilePool共用时启用，在容量分配时会预留walpool的空间
+walfilepool.use_chunk_file_pool_reserve=15
 # 是否开启从walfilepool获取chunk，一般是true
 walfilepool.enable_get_segment_from_pool=true
 # walpool目录

conf/chunkserver.conf.example

@@ -144,6 +144,16 @@ curve.curve_file_timeout_s=30
 #
 # 是否开启使用renameat2，ext4内核3.15以后开始支持
 fs.enable_renameat2=true
+# fs type: ext4, pfs
+fs.type=ext4
+#cluster name for pfs
+fs.pfs_cluster=spdk
+#pdb name for pfs
+fs.pfs_pbd_name=0000:81:00.0n1
+ #host id for pfs
+fs.pfs_host_id=2
+#nvme controller for spdk
+fs.spdk_nvme_controller=0000:81:00.0
 
 #
 # metrics settings
@@ -194,8 +204,10 @@ chunkfilepool.clean.throttle_iops=500
 #
 # WAL file pool
 #
-# walpool是否共用chunkfilepool，如果为true，则以下配置无效
+# walpool是否共用chunkfilepool，如果为true，从第三条开始配置无效
 walfilepool.use_chunk_file_pool=true
+# WALpool和ChunkFilePool共用时启用，在容量分配时会预留walpool的空间
+walfilepool.use_chunk_file_pool_reserve=15
 # 是否开启从walfilepool获取chunk，一般是true
 walfilepool.enable_get_segment_from_pool=true
 # walpool目录

conf/mds.conf

@@ -157,7 +157,9 @@ mds.topology.CreateCopysetRpcRetryTimes=20
 mds.topology.CreateCopysetRpcRetrySleepTimeMs=1000
 # Topology模块刷新metric时间间隔
 mds.topology.UpdateMetricIntervalSec=60
-# 物理池使用百分比，即使用量超过这个值即不再往这个池分配
+#和mds.chunkserver.failure.tolerance设置有关,一个zone 标准配置20台节点，如果允许3台节点failover,
+#那么剩余17台机器需要承载原先20台机器的空间,17/20=0.85,即使用量超过这个值即不再往这个池分配,
+#具体分为来两种情况, 当不使用chunkfilepool,物理池限制使用百分比,当使用 chunkfilepool 进行chunkfilepool分配时需预留failover空间,
 mds.topology.PoolUsagePercentLimit=85
 # 多pool选pool策略 0:Random, 1:Weight
 mds.topology.choosePoolPolicy=0

copts.bzl

@@ -162,3 +162,85 @@ CURVE_TEST_COPTS = CURVE_DEFAULT_COPTS + select({
     "//:clang_compiler": CURVE_LLVM_TEST_FLAGS,
     "//conditions:default": CURVE_GCC_TEST_FLAGS + CURVE_GCC_TEST_DISABLED_FLAGS,
 })
+
+SPDK_COPTS = ["-DWITH_SPDK", "-Iexternal/pfs", "-Iexternal/dpdk", "-Iexternal/spdk"]
+
+SPDK_LINK_OPTS = [
+    "-L/usr/local/curvestore/pfsd/lib",
+    "-L/usr/local/dpdk/lib",
+    "-L/usr/local/spdk/lib",
+    "-Wl,-rpath=/usr/local/curvestore/pfsd/lib",
+    "-Wl,--whole-archive",
+    "-lpfs",
+    "-lpfsd_svr",
+    "-Wl,--no-whole-archive",
+    "-Wl,-rpath=/usr/local/spdk/lib",
+    "-Wl,-rpath=/usr/local/dpdk/lib",
+    "-Wl,--disable-new-dtags",
+    "-Wl,--push-state",
+    "-Wl,--no-as-needed",
+    "-Wl,--whole-archive",
+    "-Wl,--start-group",
+    "-Bstatic",
+    "-lspdk_bdev_null",
+    "-lspdk_bdev_nvme",
+    "-lspdk_nvme",
+    "-lspdk_env_dpdk",
+    "-lspdk_sock_posix",
+    "-lspdk_event",
+    "-lspdk_event_bdev",
+    "-lspdk_bdev",
+    "-lspdk_notify",
+    "-lspdk_dma",
+    "-lspdk_event_accel",
+    "-lspdk_accel",
+    "-lspdk_event_vmd",
+    "-lspdk_vmd",
+    "-lspdk_event_sock",
+    "-lspdk_init",
+    "-lspdk_thread",
+    "-lspdk_trace",
+    "-lspdk_sock",
+    "-lspdk_rpc",
+    "-lspdk_jsonrpc",
+    "-lspdk_json",
+    "-lspdk_util",
+    "-lspdk_log",
+    "-Wl,--end-group",
+    "-Wl,--no-whole-archive",
+    "-Wl,--whole-archive",
+    "-Wl,--no-as-needed",
+    "-Wl,--start-group",
+    "-lrte_bus_pci",
+    "-lrte_cryptodev",
+    "-lrte_dmadev",
+    "-lrte_eal",
+    "-lrte_ethdev",
+    "-lrte_hash",
+    "-lrte_kvargs",
+    "-lrte_mbuf",
+    "-lrte_mempool",
+    "-lrte_mempool_ring",
+    "-lrte_net",
+    "-lrte_pci",
+    "-lrte_power",
+    "-lrte_rcu",
+    "-lrte_ring",
+    "-lrte_telemetry",
+    "-lrte_vhost",
+    "-lrte_meter",
+    "-lrte_timer",
+    "-lrte_stack",
+    "-Wl,--end-group",
+    "-Wl,--no-whole-archive",
+    "-Wl,--pop-state",
+]
+
+SPDK_DEPS = [
+    "//external:pfs_headers",
+    "//external:dpdk_headers",
+    "//external:spdk_headers",
+]
+
+
+

curvefs/conf/client.conf

@@ -85,7 +85,7 @@ fuseClient.iCacheLruSize=65536
 fuseClient.dCacheLruSize=1000000
 fuseClient.enableICacheMetrics=true
 fuseClient.enableDCacheMetrics=true
-fuseClient.cto=false
+fuseClient.cto=true
 # you shoudle enable it when mount one filesystem to multi mountpoints,
 # it gurantee the consistent of file after rename, otherwise you should
 # disable it for performance.

curvefs/src/client/curve_fuse_op.cpp

@@ -322,17 +322,21 @@ int Warmup(const std::string& name, const std::string& value) {
 
 void FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino, const char* name,
                     const char* value, size_t size, int flags) {
-    // only support "curvefs.warmup.op" xattr
     std::string xattrValue(value, size);
     VLOG(9) << "FuseOpSetXattr"
             << " ino " << ino << " name " << name << " value " << xattrValue
             << " flags " << flags;
-    int code = ENOTSUP;
     if (strcmp(name, curvefs::client::common::kCurveFsWarmupXAttr) == 0) {
         // warmup
-        code = Warmup(name, xattrValue);
+        int code = Warmup(name, xattrValue);
+        fuse_reply_err(req, code);
+    } else {
+        // set xattr
+        CURVEFS_ERROR ret = g_ClientInstance->FuseOpSetXattr(req, ino, name,
+            value, size, flags);
+        FuseReplyErrByErrCode(req, ret);
     }
-    fuse_reply_err(req, code);
+
     VLOG(9) << "FuseOpSetXattr done";
 }
 
@@ -418,8 +422,6 @@ void FuseOpOpen(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) {
         FuseReplyErrByErrCode(req, ret);
         return;
     }
-    // set fh to 1 to indicate needn't refresh inode when file is still opened
-    fi->fh = static_cast<uint64_t>(FileHandle::kKeepCache);
     fuse_reply_open(req, fi);
 }
 

curvefs/src/client/fuse_client.cpp

@@ -385,8 +385,7 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
     dentryManager_->SetFsId(fsInfo_->fsid());
     enableSumInDir_ = fsInfo_->enablesumindir() && !FLAGS_enableCto;
     LOG(INFO) << "Mount " << fsName << " on " << mountpoint_.ShortDebugString()
-              << " success!"
-              << " enableSumInDir = " << enableSumInDir_;
+              << " success!" << " enableSumInDir = " << enableSumInDir_;
 
     fsMetric_ = std::make_shared<FSMetric>(fsName);
 
@@ -561,7 +560,9 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
             return CURVEFS_ERROR::NOPERMISSION;
         }
     }
-
+    if (FLAGS_enableCto) {
+        inodeManager_->AddOpenedInode(ino);
+    }
     return ret;
 }
 
@@ -856,8 +857,6 @@ CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
         return ret;
     }
 
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-
     uint64_t dindex = fi->fh;
     DirBufferHead *bufHead = dirBuf_->DirBufferGet(dindex);
     if (!bufHead->wasRead) {
@@ -959,18 +958,6 @@ CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi,
                                         struct stat *attr) {
     VLOG(1) << "FuseOpGetAttr ino = " << ino;
-    if (FLAGS_enableCto) {
-        if (fi == nullptr ||
-            fi->fh != static_cast<uint64_t>(FileHandle::kKeepCache)) {
-            CURVEFS_ERROR ret = inodeManager_->RefreshInode(ino);
-            if (ret != CURVEFS_ERROR::OK) {
-                LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                        << ", inodeid = " << ino;
-                return ret;
-            }
-        }
-    }
-
     InodeAttr inodeAttr;
     CURVEFS_ERROR ret =
         inodeManager_->GetInodeAttr(ino, &inodeAttr);
@@ -1076,24 +1063,6 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
     return ret;
 }
 
-namespace {
-
-bool IsSummaryInfo(const char *name) {
-    return std::strstr(name, SUMMARYPREFIX);
-}
-
-bool IsOneLayer(const char *name) {
-    if (std::strcmp(name, XATTRFILES) == 0 ||
-        std::strcmp(name, XATTRSUBDIRS) == 0 ||
-        std::strcmp(name, XATTRENTRIES) == 0 ||
-        std::strcmp(name, XATTRFBYTES) == 0) {
-        return true;
-    }
-    return false;
-}
-
-}  // namespace
-
 CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, void* value,
                                          size_t size) {
@@ -1104,49 +1073,24 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
     }
 
     std::string xValue;
-    // get summary info
-    if (IsSummaryInfo(name)) {
-        InodeAttr inodeAttr;
-        CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                       << ", inodeid = " << ino;
-            return ret;
-        }
-
-        if (inodeAttr.type() == FsFileType::TYPE_DIRECTORY) {
-            // not enable record summary info in dir xattr,
-            // need recursive computation all files
-            if (!enableSumInDir_) {
-                if (IsOneLayer(name)) {
-                    ret = xattrManager_->CalOneLayerSumInfo(&inodeAttr);
-                } else {
-                    ret = xattrManager_->CalAllLayerSumInfo(&inodeAttr);
-                }
-            } else {
-                if (IsOneLayer(name)) {
-                    ret = xattrManager_->FastCalOneLayerSumInfo(&inodeAttr);
-                } else {
-                    ret = xattrManager_->FastCalAllLayerSumInfo(&inodeAttr);
-                }
-            }
-
-            if (CURVEFS_ERROR::OK != ret) {
-                return ret;
-            }
-            LOG(INFO) << "After calculate summary info:\n"
-                      << inodeAttr.DebugString();
-            auto it = inodeAttr.xattr().find(name);
-            if (it != inodeAttr.xattr().end()) {
-                xValue = it->second;
-            }
-        }
+    InodeAttr inodeAttr;
+    CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
+                    << ", inodeid = " << ino;
+        return ret;
     }
 
-    CURVEFS_ERROR ret = CURVEFS_ERROR::NODATA;
+    ret = xattrManager_->GetXattr(name, &xValue, &inodeAttr, enableSumInDir_);
+    if (CURVEFS_ERROR::OK != ret) {
+        LOG(ERROR) << "xattrManager get xattr failed, name = " << name;
+        return ret;
+    }
+
+    ret = CURVEFS_ERROR::NODATA;
     if (xValue.length() > 0) {
         if ((size == 0 && xValue.length() <= MAXXATTRLENGTH) ||
-            (size >= xValue.length() && size <= MAXXATTRLENGTH)) {
+            (size >= xValue.length() && xValue.length() <= MAXXATTRLENGTH)) {
             memcpy(value, xValue.c_str(), xValue.length());
             ret = CURVEFS_ERROR::OK;
         } else {
@@ -1156,6 +1100,44 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
     return ret;
 }
 
+CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
+                                         const char* name, const char* value,
+                                         size_t size, int flags) {
+    VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
+            << ", value: " << value;
+    if (option_.disableXattr) {
+        return CURVEFS_ERROR::NOTSUPPORT;
+    }
+
+    std::string strname(name);
+    std::string strvalue(value, size);
+    if (strname.length() > MAXXATTRLENGTH  || size > MAXXATTRLENGTH) {
+        LOG(ERROR) << "xattr length is too long, name = " << name
+                   << ", name length = " << strname.length()
+                   << ", value length = " << size;
+        return CURVEFS_ERROR::OUT_OF_RANGE;
+    }
+
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
+    }
+
+    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+    inodeWrapper->SetXattrLocked(strname, strvalue);
+    ret = inodeWrapper->SyncAttr();
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "set xattr fail, ret = " << ret << ", inodeid = " << ino
+                   << ", name = " << strname << ", value = " << strvalue;
+        return ret;
+    }
+    VLOG(1) << "FuseOpSetXattr end";
+    return CURVEFS_ERROR::OK;
+}
+
 CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize) {
     VLOG(1) << "FuseOpListXattr, ino: " << ino << ", size = " << size;
@@ -1370,6 +1352,9 @@ CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
     VLOG(1) << "FuseOpRelease, ino: " << ino;
+    if (FLAGS_enableCto) {
+        inodeManager_->RemoveOpenedInode(ino);
+    }
     return CURVEFS_ERROR::OK;
 }
 

curvefs/src/client/fuse_client.h

@@ -194,6 +194,10 @@ class FuseClient {
                                          const char* name, void* value,
                                          size_t size);
 
+    virtual CURVEFS_ERROR FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
+                                         const char* name, const char* value,
+                                         size_t size, int flags);
+
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 

curvefs/src/client/inode_cache_manager.cpp

@@ -47,6 +47,7 @@ namespace curvefs {
 namespace client {
 
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
+using curvefs::client::common::FLAGS_enableCto;
 
 class TrimICacheAsyncDone : public MetaServerClientDone {
  public:
@@ -107,7 +108,7 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
     // get inode from cache
     bool ok = iCache_->Get(inodeId, &out);
-    if (ok) {
+    if (ok && NeedUseCahce(inodeId, out->IsDirty())) {
         curve::common::UniqueLock lgGuard = out->GetUniqueLock();
         if (out->GetType() == FsFileType::TYPE_FILE) {
             return CURVEFS_ERROR::OK;
@@ -134,48 +135,13 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR
-InodeCacheManagerImpl::RefreshInode(uint64_t inodeId) {
-    NameLockGuard lock(nameLock_, std::to_string(inodeId));
-
-    // get inode from metaserver
-    Inode inode;
-    bool streaming = false;
-    GET_INODE_REMOTE(fsId_, inodeId, &inode, &streaming);
-
-    // get inode from cache
-    std::shared_ptr<InodeWrapper> out;
-    bool ok = iCache_->Get(inodeId, &out);
-    curve::common::UniqueLock lgGuard;
-    if (!ok) {
-        out = std::make_shared<InodeWrapper>(
-            std::move(inode), metaClient_, s3ChunkInfoMetric_,
-            option_.maxDataSize, option_.refreshDataIntervalSec);
-    } else {
-        lgGuard = out->GetUniqueLock();
-        streaming = true;
-    }
-
-    // refresh data
-    REFRESH_DATA_REMOTE(out, streaming);
-
-    // put to cache or refresh length
-    if (!ok) {
-        PUT_INODE_CACHE(inodeId, out);
-    } else {
-        out->SetLengthLocked(inode.length());
-    }
-
-    return CURVEFS_ERROR::OK;
-}
-
 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
                                                   InodeAttr *out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
     // 1. find in icache
     std::shared_ptr<InodeWrapper> inodeWrapper;
     bool ok = iCache_->Get(inodeId, &inodeWrapper);
-    if (ok) {
+    if (ok && NeedUseCahce(inodeId, inodeWrapper->IsDirty())) {
         inodeWrapper->GetInodeAttr(out);
         return CURVEFS_ERROR::OK;
     }
@@ -212,14 +178,14 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok) {
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
             InodeAttr tmpAttr;
             inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace_back(tmpAttr);
+            attrs->emplace_back(std::move(tmpAttr));
             iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
+            continue;
         }
+        ++iter;
     }
 
     if (inodeIds->empty()) {
@@ -242,19 +208,19 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     std::map<uint64_t, InodeAttr> *attrs) {
     NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
     std::map<uint64_t, InodeAttr> cachedAttr;
-    bool ok  = iAttrCache_->Get(parentId, &cachedAttr);
+    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
 
     // get some inode attr in icache
     for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok) {
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
             InodeAttr tmpAttr;
             inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace(*iter, tmpAttr);
+            attrs->emplace(*iter, std::move(tmpAttr));
             iter = inodeIds->erase(iter);
-        } else if (ok && cachedAttr.find(*iter) != cachedAttr.end()) {
+        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
             attrs->emplace(*iter, cachedAttr[*iter]);
             iter = inodeIds->erase(iter);
         } else {
@@ -291,7 +257,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     // wait for all sudrequest finished
     cond->Wait();
 
-    ok  = iAttrCache_->Get(parentId, attrs);
+    bool ok  = iAttrCache_->Get(parentId, attrs);
     if (!ok) {
         LOG(WARNING) << "get attrs form iAttrCache_ failed.";
     }
@@ -306,7 +272,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
         std::shared_ptr<InodeWrapper> inodeWrapper;
         NameLockGuard lock(nameLock_, std::to_string(*iter));
         bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok) {
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
             xattr->emplace_back(inodeWrapper->GetXattr());
             iter = inodeIds->erase(iter);
         } else {
@@ -524,5 +490,33 @@ InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
     return rc;
 }
 
+void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    openedInodes_.emplace(inodeId);
+}
+
+void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    if (iter != openedInodes_.end()) {
+        openedInodes_.erase(iter);
+    }
+}
+
+bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    return iter != openedInodes_.end();
+}
+
+bool InodeCacheManagerImpl::NeedUseCahce(uint64_t inodeId, bool IsDirty) {
+    if (!FLAGS_enableCto || OpenInodeCached(inodeId) || IsDirty) {
+        return true;
+    }
+    return false;
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_cache_manager.h

@@ -138,8 +138,6 @@ class InodeCacheManager {
     GetInode(uint64_t inodeId,
              std::shared_ptr<InodeWrapper> &out) = 0;  // NOLINT
 
-    virtual CURVEFS_ERROR RefreshInode(uint64_t inodeId) = 0;
-
     virtual CURVEFS_ERROR GetInodeAttr(uint64_t inodeId, InodeAttr *out) = 0;
 
     virtual CURVEFS_ERROR BatchGetInodeAttr(
@@ -173,6 +171,10 @@ class InodeCacheManager {
 
     virtual void ReleaseCache(uint64_t parentId) = 0;
 
+    virtual void AddOpenedInode(uint64_t inodeId) = 0;
+
+    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
+
  protected:
     uint32_t fsId_;
 };
@@ -233,8 +235,6 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     CURVEFS_ERROR GetInode(uint64_t inodeId,
                            std::shared_ptr<InodeWrapper> &out) override;
 
-    CURVEFS_ERROR RefreshInode(uint64_t inodeId) override;
-
     CURVEFS_ERROR GetInodeAttr(uint64_t inodeId, InodeAttr *out) override;
 
     CURVEFS_ERROR BatchGetInodeAttr(std::set<uint64_t> *inodeIds,
@@ -268,11 +268,18 @@ class InodeCacheManagerImpl : public InodeCacheManager,
 
     void RemoveICache(const std::shared_ptr<InodeWrapper> &inode);
 
+    void AddOpenedInode(uint64_t inodeId) override;
+
+    void RemoveOpenedInode(uint64_t inodeId) override;
+
+    bool NeedUseCahce(uint64_t inodeId, bool IsDirty);
+
  private:
     virtual void FlushInodeBackground();
     void TrimIcache(uint64_t trimSize);
     CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
                               bool streaming = true);
+    bool OpenInodeCached(uint64_t inodeId);
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
@@ -285,6 +292,10 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
     curve::common::Mutex dirtyMapMutex_;
 
+    // record opened inode
+    std::multiset<uint64_t> openedInodes_;
+    curve::common::Mutex openInodesMutex_;
+
     curve::common::GenericNameLock<Mutex> nameLock_;
 
     curve::common::GenericNameLock<Mutex> asyncNameLock_;

curvefs/src/client/inode_wrapper.h

@@ -199,6 +199,12 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         return ret;
     }
 
+    void SetXattrLocked(const std::string &key, const std::string &value) {
+        (*inode_.mutable_xattr())[key] = value;
+        (*dirtyAttr_.mutable_xattr()) = inode_.xattr();
+        dirty_ = true;
+    }
+
     curve::common::UniqueLock GetUniqueLock() {
         return curve::common::UniqueLock(mtx_);
     }

curvefs/src/client/xattr_manager.cpp

@@ -31,6 +31,21 @@ namespace client {
 using ::curve::common::StringToUll;
 using ::curve::common::Thread;
 
+
+bool IsSummaryInfo(const char *name) {
+    return std::strstr(name, SUMMARYPREFIX);
+}
+
+bool IsOneLayer(const char *name) {
+    if (std::strcmp(name, XATTRFILES) == 0 ||
+        std::strcmp(name, XATTRSUBDIRS) == 0 ||
+        std::strcmp(name, XATTRENTRIES) == 0 ||
+        std::strcmp(name, XATTRFBYTES) == 0) {
+        return true;
+    }
+    return false;
+}
+
 // if direction is true means '+', false means '-'
 bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction) {
     uint64_t firstNum = 0;
@@ -441,6 +456,43 @@ CURVEFS_ERROR XattrManager::FastCalAllLayerSumInfo(InodeAttr *attr) {
     return CURVEFS_ERROR::OK;
 }
 
+CURVEFS_ERROR XattrManager::GetXattr(const char* name, std::string *value,
+    InodeAttr *attr, bool enableSumInDir) {
+    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
+    // get summary info if the xattr name is summary type
+    if (IsSummaryInfo(name) && attr->type() == FsFileType::TYPE_DIRECTORY) {
+        // if not enable record summary info in dir xattr,
+        // need recursive computation all files;
+        // otherwise only recursive computation all dirs.
+        if (!enableSumInDir) {
+            if (IsOneLayer(name)) {
+                ret = CalOneLayerSumInfo(attr);
+            } else {
+                ret = CalAllLayerSumInfo(attr);
+            }
+        } else {
+            if (IsOneLayer(name)) {
+                ret = FastCalOneLayerSumInfo(attr);
+            } else {
+                ret = FastCalAllLayerSumInfo(attr);
+            }
+        }
+
+        if (CURVEFS_ERROR::OK != ret) {
+            return ret;
+        }
+        LOG(INFO) << "After calculate summary info:\n"
+                  << attr->DebugString();
+    }
+
+    auto it = attr->xattr().find(name);
+    if (it != attr->xattr().end()) {
+        *value = it->second;
+    }
+    return ret;
+}
+
+
 CURVEFS_ERROR XattrManager::UpdateParentInodeXattr(uint64_t parentId,
     const XAttr &xattr, bool direction) {
     VLOG(9) << "UpdateParentInodeXattr inodeId = " << parentId

curvefs/src/client/xattr_manager.h

@@ -74,13 +74,8 @@ class XattrManager {
         isStop_.store(true);
     }
 
-    CURVEFS_ERROR CalOneLayerSumInfo(InodeAttr *attr);
-
-    CURVEFS_ERROR CalAllLayerSumInfo(InodeAttr *attr);
-
-    CURVEFS_ERROR FastCalOneLayerSumInfo(InodeAttr *attr);
-
-    CURVEFS_ERROR FastCalAllLayerSumInfo(InodeAttr *attr);
+    CURVEFS_ERROR GetXattr(const char* name, std::string *value,
+        InodeAttr *attr, bool enableSumInDir);
 
     CURVEFS_ERROR UpdateParentInodeXattr(uint64_t parentId,
         const XAttr &xattr, bool direction);
@@ -116,6 +111,14 @@ class XattrManager {
         Atomic<uint32_t> *inflightNum,
         Atomic<bool> *ret);
 
+    CURVEFS_ERROR CalOneLayerSumInfo(InodeAttr *attr);
+
+    CURVEFS_ERROR CalAllLayerSumInfo(InodeAttr *attr);
+
+    CURVEFS_ERROR FastCalOneLayerSumInfo(InodeAttr *attr);
+
+    CURVEFS_ERROR FastCalAllLayerSumInfo(InodeAttr *attr);
+
  private:
     // inode cache manager
     std::shared_ptr<InodeCacheManager> inodeManager_;

curvefs/test/client/mock_inode_cache_manager.h

@@ -56,8 +56,6 @@ class MockInodeCacheManager : public InodeCacheManager {
     MOCK_METHOD2(GetInodeAttr, CURVEFS_ERROR(
         uint64_t inodeId, InodeAttr *out));
 
-    MOCK_METHOD1(RefreshInode, CURVEFS_ERROR(uint64_t inodeId));
-
     MOCK_METHOD2(BatchGetInodeAttr, CURVEFS_ERROR(
         std::set<uint64_t> *inodeIds, std::list<InodeAttr> *attrs));
 
@@ -88,6 +86,10 @@ class MockInodeCacheManager : public InodeCacheManager {
     MOCK_METHOD0(FlushInodeOnce, void());
 
     MOCK_METHOD1(ReleaseCache, void(uint64_t parentId));
+
+    MOCK_METHOD1(AddOpenedInode, void(uint64_t inodeId));
+
+    MOCK_METHOD1(RemoveOpenedInode, void(uint64_t inodeId));
 };
 
 }  // namespace client

curvefs/test/client/test_fuse_client.cpp

@@ -189,7 +189,6 @@ TEST_F(TestFuseVolumeClient, FuseOpInit_when_fs_exist) {
 
     EXPECT_CALL(*blockDeviceClient_, Open(_, _))
         .WillOnce(Return(true));
-
     CURVEFS_ERROR ret = client_->FuseOpInit(&mOpts, nullptr);
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 
@@ -1357,21 +1356,11 @@ TEST_F(TestFuseVolumeClient, FuseOpGetAttrEnableCto) {
     inode.set_inodeid(ino);
     inode.set_length(0);
 
-    // RefreshInode OK
-    EXPECT_CALL(*inodeManager_, RefreshInode(ino))
-        .WillOnce(Return(CURVEFS_ERROR::OK));
     EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
         .WillOnce(DoAll(SetArgPointee<1>(inode), Return(CURVEFS_ERROR::OK)));
 
     ASSERT_EQ(CURVEFS_ERROR::OK, client_->FuseOpGetAttr(req, ino, &fi, &attr));
 
-    // RefreshInode Fail
-    EXPECT_CALL(*inodeManager_, RefreshInode(ino))
-        .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
-
-    ASSERT_EQ(CURVEFS_ERROR::INTERNAL,
-              client_->FuseOpGetAttr(req, ino, &fi, &attr));
-
     // need not refresh inode
     fi.fh = static_cast<uint64_t>(FileHandle::kKeepCache);
     EXPECT_CALL(*inodeManager_, GetInodeAttr(ino, _))
@@ -3484,5 +3473,58 @@ TEST_F(TestFuseS3Client, FuseOpListXattr) {
     ASSERT_EQ(CURVEFS_ERROR::OK, ret);
 }
 
+TEST_F(TestFuseS3Client, FuseOpSetXattr_TooLong) {
+    // in
+    fuse_req_t req;
+    fuse_ino_t ino = 1;
+    const char name[] = "security.selinux";
+    size_t size = 300;
+    char value[300];
+    std::memset(value, 0, 300);
+
+    CURVEFS_ERROR ret = client_->FuseOpSetXattr(
+        req, ino, name, value, size, 0);
+    ASSERT_EQ(CURVEFS_ERROR::OUT_OF_RANGE, ret);
+}
+
+TEST_F(TestFuseS3Client, FuseOpSetXattr) {
+    // in
+    fuse_req_t req;
+    fuse_ino_t ino = 1;
+    const char name[] = "security.selinux";
+    size_t size = 100;
+    char value[100];
+    std::memset(value, 0, 100);
+
+    // get inode failed
+    EXPECT_CALL(*inodeManager_, GetInode(ino, _))
+        .WillOnce(Return(CURVEFS_ERROR::INTERNAL));
+    CURVEFS_ERROR ret = client_->FuseOpSetXattr(
+        req, ino, name, value, size, 0);
+    ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret);
+
+    // updateInode failed
+    auto inodeWrapper = std::make_shared<InodeWrapper>(Inode(), metaClient_);
+    EXPECT_CALL(*inodeManager_, GetInode(ino, _))
+        .WillOnce(
+            DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
+        .WillOnce(Return(MetaStatusCode::NOT_FOUND));
+    ret = client_->FuseOpSetXattr(
+        req, ino, name, value, size, 0);
+    ASSERT_EQ(CURVEFS_ERROR::NOTEXIST, ret);
+
+    // success
+    EXPECT_CALL(*inodeManager_, GetInode(ino, _))
+        .WillOnce(
+            DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK)));
+    EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
+        .WillOnce(Return(MetaStatusCode::OK));
+    ret = client_->FuseOpSetXattr(
+        req, ino, name, value, size, 0);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+}
+
+
 }  // namespace client
 }  // namespace curvefs

curvefs/test/client/test_inodeWrapper.cpp

@@ -333,5 +333,13 @@ TEST_F(TestInodeWrapper, TestUpdateInodeAttrIncrementally) {
     ASSERT_FALSE(wrapper.dirtyAttr_.has_atime_ns());
 }
 
+TEST_F(TestInodeWrapper, TestSetXattr) {
+    inodeWrapper_->SetXattrLocked("name", "value");
+    XAttr xattr = inodeWrapper_->GetXattr();
+    ASSERT_TRUE(xattr.xattrinfos().find("name") != xattr.xattrinfos().end());
+    ASSERT_EQ((*xattr.mutable_xattrinfos())["name"], "value");
+    ASSERT_TRUE(inodeWrapper_->IsDirty());
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/test/client/test_inode_cache_manager.cpp

@@ -150,88 +150,28 @@ TEST_F(TestInodeCacheManager, GetInode) {
     ASSERT_EQ(inodeId, out.inodeid());
     ASSERT_EQ(fsId_, out.fsid());
     ASSERT_EQ(fileLength, out.length());
-}
 
-TEST_F(TestInodeCacheManager, RefreshInode) {
-    uint64_t inodeId = 100;
-    uint64_t fileLength = 100;
-
-    Inode inode;
-    inode.set_inodeid(inodeId);
-    inode.set_fsid(fsId_);
-    inode.set_length(fileLength);
-    inode.set_type(FsFileType::TYPE_S3);
-    auto s3ChunkInfoMap = inode.mutable_s3chunkinfomap();
-    S3ChunkInfoList *s3ChunkInfoList = new S3ChunkInfoList();
-    S3ChunkInfo *s3ChunkInfo = s3ChunkInfoList->add_s3chunks();
-    s3ChunkInfo->set_chunkid(1);
-    s3ChunkInfo->set_compaction(1);
-    s3ChunkInfo->set_offset(0);
-    s3ChunkInfo->set_len(1024);
-    s3ChunkInfo->set_size(65536);
-    s3ChunkInfo->set_zero(true);
-    s3ChunkInfoMap->insert({1, *s3ChunkInfoList});
-
-    // cache miss, get s3-inode from metaserver failed
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(Return(MetaStatusCode::NOT_FOUND));
-    ASSERT_EQ(CURVEFS_ERROR::NOTEXIST, iCacheManager_->RefreshInode(inodeId));
-
-    // cache miss, get s3-inode from metaserver ok, do not need streaming
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(DoAll(SetArgPointee<2>(inode), SetArgPointee<3>(false),
-                        Return(MetaStatusCode::OK)));
-    ASSERT_EQ(CURVEFS_ERROR::OK, iCacheManager_->RefreshInode(inodeId));
-
-    // cache miss, get s3-inode from metaserver, need streaming
-    uint64_t inodeId2 = 200;
-    Inode inode2 = inode;
-    inode2.set_inodeid(inodeId2);
+    // enable cto and not opened and not dirty
+    curvefs::client::common::FLAGS_enableCto = true;
     EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId2, _, _))
         .WillOnce(DoAll(SetArgPointee<2>(inode2), SetArgPointee<3>(true),
                         Return(MetaStatusCode::OK)));
     EXPECT_CALL(*metaClient_,
                 GetOrModifyS3ChunkInfo(fsId_, inodeId2, _, true, _, _))
         .WillOnce(Return(MetaStatusCode::OK));
-    ASSERT_EQ(CURVEFS_ERROR::OK, iCacheManager_->RefreshInode(inodeId2));
-
-    // cache hit, refresh s3-inode from metaserver, do not need streaming
-    Inode inodenew = inode;
-    inodenew.set_length(fileLength * 3);
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(DoAll(SetArgPointee<2>(inodenew), SetArgPointee<3>(false),
-                        Return(MetaStatusCode::OK)));
-    EXPECT_CALL(*metaClient_,
-                GetOrModifyS3ChunkInfo(fsId_, inodeId, _, true, _, _))
-        .WillOnce(Return(MetaStatusCode::OK));
-    ASSERT_EQ(CURVEFS_ERROR::OK, iCacheManager_->RefreshInode(inodeId));
-    std::shared_ptr<InodeWrapper> inodeWrapper;
     ASSERT_EQ(CURVEFS_ERROR::OK,
-              iCacheManager_->GetInode(inodeId, inodeWrapper));
-    ASSERT_EQ(inodenew.length(), inodeWrapper->GetLength());
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
 
-    // cache hit, refresh s3-inode from metaserver, need streaming
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodeId, _, _))
-        .WillOnce(DoAll(SetArgPointee<2>(inodenew), SetArgPointee<3>(true),
-                        Return(MetaStatusCode::OK)));
-    EXPECT_CALL(*metaClient_,
-                GetOrModifyS3ChunkInfo(fsId_, inodeId, _, true, _, _))
-        .WillOnce(Return(MetaStatusCode::OK));
-    ASSERT_EQ(CURVEFS_ERROR::OK, iCacheManager_->RefreshInode(inodeId));
+    // enable cto and opened will hit cache
+    iCacheManager_->AddOpenedInode(inodeId2);
+    ASSERT_EQ(CURVEFS_ERROR::OK,
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
 
-    // cache miss, get file-inode from metaserver
-    Inode inodefile;
-    uint64_t inodefileid = 300;
-    inodefile.set_inodeid(inodefileid);
-    inodefile.set_fsid(fsId_);
-    inodefile.set_type(FsFileType::TYPE_FILE);
-    inodefile.set_length(1);
-    EXPECT_CALL(*metaClient_, GetInode(fsId_, inodefileid, _, _))
-        .WillOnce(DoAll(SetArgPointee<2>(inodefile), SetArgPointee<3>(false),
-                        Return(MetaStatusCode::OK)));
-    EXPECT_CALL(*metaClient_, GetVolumeExtent(fsId_, inodefileid, _, _))
-        .WillOnce(Return(MetaStatusCode::OK));
-    ASSERT_EQ(CURVEFS_ERROR::OK, iCacheManager_->RefreshInode(inodefileid));
+    // enable cto and not opened and inode dirty will hit cache
+    iCacheManager_->RemoveOpenedInode(inodeId2);
+    inodeWrapper->MarkDirty();
+    ASSERT_EQ(CURVEFS_ERROR::OK,
+              iCacheManager_->GetInode(inodeId2, inodeWrapper));
 }
 
 TEST_F(TestInodeCacheManager, GetInodeAttr) {
@@ -282,6 +222,35 @@ TEST_F(TestInodeCacheManager, GetInodeAttr) {
     ASSERT_EQ(inodeId + 1, out.inodeid());
     ASSERT_EQ(fsId_ + 1, out.fsid());
     ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
+
+    // enable cto will get from metaserver
+    curvefs::client::common::FLAGS_enableCto = true;
+    EXPECT_CALL(*metaClient_, BatchGetInodeAttr(fsId_, _, _))
+        .WillOnce(DoAll(SetArgPointee<2>(attrs), Return(MetaStatusCode::OK)));
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(inodeId, out.inodeid());
+    ASSERT_EQ(fsId_, out.fsid());
+    ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
+
+    // set this inode open
+    curvefs::client::common::FLAGS_enableCto = false;
+    iCacheManager_->AddOpenedInode(inodeId + 1);
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(inodeId + 1, out.inodeid());
+    ASSERT_EQ(fsId_ + 1, out.fsid());
+    ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
+
+    // set this inode dirty
+    curvefs::client::common::FLAGS_enableCto = false;
+    iCacheManager_->RemoveOpenedInode(inodeId + 1);
+    inodeWrapper->MarkDirty();
+    ret = iCacheManager_->GetInodeAttr(inodeId + 1, &out);
+    ASSERT_EQ(CURVEFS_ERROR::OK, ret);
+    ASSERT_EQ(inodeId + 1, out.inodeid());
+    ASSERT_EQ(fsId_ + 1, out.fsid());
+    ASSERT_EQ(FsFileType::TYPE_FILE, out.type());
 }
 
 TEST_F(TestInodeCacheManager, CreateAndGetInode) {

curvefs/test/metaserver/copyset/conf_epoch_file_test.cpp

@@ -70,7 +70,7 @@ TEST_F(ConfEpochFileTest, LoadTest_ReadFailed) {
         .WillOnce(Return(0));
     EXPECT_CALL(*mockfs_, Close(_))
         .Times(1);
-    EXPECT_CALL(*mockfs_, Read(_, _, _, _))
+    EXPECT_CALL(*mockfs_, Read(_, Matcher<char*>(_), _, _))
         .WillOnce(Return(0));
 
     EXPECT_NE(0,
@@ -85,7 +85,7 @@ TEST_F(ConfEpochFileTest, LoadTest_DecodeFailed) {
 
     const char* data = "{\"hello\", \"world\"}";
     size_t datasize = strlen(data);
-    EXPECT_CALL(*mockfs_, Read(_, _, _, _))
+    EXPECT_CALL(*mockfs_, Read(_, Matcher<char*>(_), _, _))
         .WillOnce(DoAll(SetArrayArgument<1>(data, data + datasize),
                         Return(datasize)));
 
@@ -102,7 +102,7 @@ TEST_F(ConfEpochFileTest, LoadTest_CrcFailed) {
     const char* data =
         "{\"poolId\": 123, \"copysetId\": 431, \"epoch\": 3, \"checksum\": 12}";
     size_t datasize = strlen(data);
-    EXPECT_CALL(*mockfs_, Read(_, _, _, _))
+    EXPECT_CALL(*mockfs_, Read(_, Matcher<char*>(_), _, _))
         .WillOnce(DoAll(SetArrayArgument<1>(data, data + datasize),
                         Return(datasize)));
 
@@ -119,7 +119,7 @@ TEST_F(ConfEpochFileTest, LoadTest_Success) {
     const char* data =
         "{\"poolId\": 123, \"copysetId\": 431, \"epoch\": 3, \"checksum\": 1347065312}";  // NOLINT
     size_t datasize = strlen(data);
-    EXPECT_CALL(*mockfs_, Read(_, _, _, _))
+    EXPECT_CALL(*mockfs_, Read(_, Matcher<char*>(_), _, _))
         .WillOnce(DoAll(SetArrayArgument<1>(data, data + datasize),
                         Return(datasize)));
 

curvefs/test/metaserver/copyset/copyset_node_manager_test.cpp

@@ -260,7 +260,7 @@ TEST_F(CopysetNodeManagerTest,
 
     EXPECT_CALL(mockfs, DirExists(_))
         .WillOnce(Return(false));
-    EXPECT_CALL(mockfs, Mkdir(_))
+    EXPECT_CALL(mockfs, Mkdir(_, _))
         .WillOnce(Return(-1));
 
     EXPECT_FALSE(nodeManager_->PurgeCopysetNode(kPoolId, kCopysetId));

curvefs/test/metaserver/copyset/copyset_node_snapshot_test.cpp

@@ -132,7 +132,7 @@ class CopysetNodeRaftSnapshotTest : public testing::Test {
 
         ASSERT_EQ(0, server_.Start(listenAddr, nullptr));
 
-        EXPECT_CALL(*mockfs_, Mkdir(_))
+        EXPECT_CALL(*mockfs_, Mkdir(_, _))
             .WillRepeatedly(Return(0));
     }
 

curvefs/test/metaserver/copyset/copyset_node_test.cpp

@@ -46,6 +46,7 @@ using ::testing::Return;
 using ::testing::SetArgPointee;
 using ::testing::SetArrayArgument;
 using ::testing::AtLeast;
+using ::testing::Matcher;
 
 using ::curve::fs::MockLocalFileSystem;
 
@@ -65,7 +66,7 @@ class CopysetNodeTest : public testing::Test {
         options_.localFileSystem = &mockfs_;
         options_.storageOptions.type = "memory";
 
-        EXPECT_CALL(mockfs_, Mkdir(_))
+        EXPECT_CALL(mockfs_, Mkdir(_, _))
             .WillRepeatedly(Return(0));
     }
 
@@ -181,7 +182,7 @@ TEST_F(CopysetNodeTest, LoadConEpochFailed_PoolIdIsNotIdentical) {
         "{\"poolId\": 123, \"copysetId\": 1, \"epoch\": 3, \"checksum\": 3480649501}";  // NOLINT
 
     size_t datasize = strlen(data);
-    EXPECT_CALL(mockfs_, Read(_, _, _, _))
+    EXPECT_CALL(mockfs_, Read(_, Matcher<char*>(_), _, _))
         .WillOnce(DoAll(SetArrayArgument<1>(data, data + datasize),
                         Return(datasize)));
 
@@ -204,7 +205,7 @@ TEST_F(CopysetNodeTest, LoadConEpochFailed_CopysetIdIsNotIdentical) {
         "{\"poolId\": 1, \"copysetId\": 431, \"epoch\": 3, \"checksum\": 2004834618}";  // NOLINT
 
     size_t datasize = strlen(data);
-    EXPECT_CALL(mockfs_, Read(_, _, _, _))
+    EXPECT_CALL(mockfs_, Read(_, Matcher<char*>(_), _, _))
         .WillOnce(DoAll(SetArrayArgument<1>(data, data + datasize),
                         Return(datasize)));
 
@@ -228,7 +229,7 @@ TEST_F(CopysetNodeTest, LoadConEpoch_Success) {
         "{\"poolId\": 1, \"copysetId\": 1, \"epoch\": 3, \"checksum\": 3896751047}";  // NOLINT
 
     size_t datasize = strlen(data);
-    EXPECT_CALL(mockfs_, Read(_, _, _, _))
+    EXPECT_CALL(mockfs_, Read(_, Matcher<char*>(_), _, _))
         .WillOnce(DoAll(SetArrayArgument<1>(data, data + datasize),
                         Return(datasize)));
 

curvefs/test/metaserver/copyset/meta_operator_test.cpp

@@ -419,7 +419,7 @@ TEST_F(MetaOperatorTest, PropostTest_RequestCanBypassProcess) {
     options.localFileSystem = &localFs;
     options.storageOptions.type = "memory";
 
-    EXPECT_CALL(localFs, Mkdir(_))
+    EXPECT_CALL(localFs, Mkdir(_, _))
         .WillOnce(Return(0));
 
     EXPECT_TRUE(node.Init(options));

curvefs/test/metaserver/copyset/trash_test.cpp

@@ -66,7 +66,7 @@ TEST_F(TrashTest, RecycleCopysetTest) {
         EXPECT_CALL(*mocklfs_, DirExists(_))
             .WillOnce(Return(false));
 
-        EXPECT_CALL(*mocklfs_, Mkdir(_))
+        EXPECT_CALL(*mocklfs_, Mkdir(_, _))
             .WillOnce(Return(-1));
 
         EXPECT_FALSE(trash_->RecycleCopyset("/mnt/data/copysets/123456789"));

docs/README.md

@@ -122,6 +122,9 @@ Curve支持部署在私有云和公有云环境，也可以以混合云方式使
 ### 测试用例编译及运行
 [测试用例编译及运行](cn/build_and_run.md#%E6%B5%8B%E8%AF%95%E7%94%A8%E4%BE%8B%E7%BC%96%E8%AF%91%E5%8F%8A%E6%89%A7%E8%A1%8C)
 
+### 最佳实践
+- [CurveBS+NFS搭建共享存储](practical/curvebs_nfs.md)
+
 ## 联系社区
 - [Github Issues](https://github.com/openCURVE/CURVE/issues)：欢迎提交BUG、建议，使用中如遇到问题可参考FAQ或加入我们的User group进行咨询
 - [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：主要根据User group中常见问题整理，还在逐步完善中

docs/practical/curvebs_nfs.md

@@ -0,0 +1,366 @@
+[TOC]
+## CurveBS系统部署
+
+### 环境信息
+
+主机名|系统|IP|用户名密码
+-|-|-|-
+主控机|centos7|172.20.0.62|root/XX....
+server-node1|centos7|172.20.0.38|root/vT2dALVE
+server-node2|centos7|172.20.0.39|root/W0lFAv1h
+server-node3|centos7|172.20.0.58|root/YVcR8Iwl
+client-node1|centos7|172.20.0.61|root/KR0c61Vs
+
+
+### curveadm 部署
+主控机器安装curveadm
+```shell
+# docker 安装，主控节点，工作节点均需要安装
+yum install docker
+
+# curveadm 安装
+bash -c "$(curl -fsSL https://curveadm.nos-eastchina1.126.net/script/install.sh)"
+```
+
+
+### 准备配置文件
+
+```yaml
+hosts.yaml
+
+global:
+  user: root
+  ssh_port: 22
+  private_key_file: /root/.ssh/id_rsa
+
+hosts:
+  - host: server-node1
+    hostname: 172.20.0.38
+  - host: server-node2
+    hostname: 172.20.0.39
+  - host: server-node3
+    hostname: 172.20.0.58
+  - host: client-node1
+    hostname: 172.20.0.61
+```
+
+### 准备主机列表
+```shell
+#添加主机列表
+curveadm hosts commit hosts.yaml
+```
+
+```yaml
+format.yaml
+
+host:
+  - server-node1
+  - server-node2
+  - server-node3
+disk:
+  - /dev/vdc:/data/chunkserver0:90  # device:mount_path:format_percent
+```
+
+### 首先需要主动拉取镜像，加速后续的操作
+```shell
+#nohup  docker pull opencurvedocker/curvebs:v1.2  >log 2>&1 &
+docker pull opencurvedocker/curvebs:v1.2 
+```
+
+
+### 格式化磁盘
+```shell
+#格式化磁盘
+curveadm format -f format.yaml
+
+#查看格式化进度
+curveadm format --status
+```
+
+```yaml
+topology.yaml
+
+kind: curvebs
+global:
+  container_image: opencurvedocker/curvebs:v1.2
+  log_dir: ${home}/logs/${service_role}${service_host_sequence}
+  data_dir: ${home}/data/${service_role}${service_host_sequence}
+  s3.nos_address: <>
+  s3.snapshot_bucket_name: <>
+  s3.ak: <>
+  s3.sk: <>
+  report_usage: false
+  variable:
+    home: /root/curvebs
+    node1: server-node1
+    node2: server-node2
+    node3: server-node3
+
+etcd_services:
+  config:
+    listen.ip: ${service_host}
+    listen.port: 2380
+    listen.client_port: 2379
+  deploy:
+    - host: ${node1}
+    - host: ${node2}
+    - host: ${node3}
+
+mds_services:
+  config:
+    listen.ip: ${service_host}
+    listen.port: 6700
+    listen.dummy_port: 7700
+  deploy:
+    - host: ${node1}
+    - host: ${node2}
+    - host: ${node3}
+
+chunkserver_services:
+  config:
+    listen.ip: ${service_host}
+    listen.port: 8200 # 8200,8201,8202
+    data_dir: /data/chunkserver0  # /data/chunkserver0, /data/chunksever1
+    copysets: 100
+    fs.enable_renameat2: false #3.10 kernel当前内核版本不支持 renameat() 接口 用户可将 chunkserver 中的 fs.enable_renameat2 配置项设置成 false
+  deploy:
+    - host: ${node1}
+    - host: ${node2}
+    - host: ${node3}
+
+snapshotclone_services:
+  config:
+    listen.ip: ${service_host}
+    listen.port: 5555
+    listen.dummy_port: 8081
+    listen.proxy_port: 8080
+  deploy:
+    - host: ${node1}
+    - host: ${node2}
+    - host: ${node3}
+```
+
+### 添加集群
+```shell
+#添加集群
+curveadm cluster add my-cluster -f topology.yaml
+#删除集群
+curveadm cluster rm my-cluster
+#切换集群
+curveadm cluster checkout my-cluster
+
+#执行预检查
+curveadm precheck
+curveadm precheck --skip <item> #topology ssh permission kernel network date service
+
+```
+
+### 部署集群
+```shell
+#部署集群
+curveadm deploy --skip snapshotclone
+
+#查看集群运行情况
+curveadm status
+
+#进入服务容器
+curveadm enter <Id>
+#检查健康情况
+curve_ops_tool status
+```
+
+
+### 添加客户端配置
+```yaml
+client.yaml
+
+kind: curvebs
+container_image: opencurvedocker/curvebs:v1.2
+mds.listen.addr: 172.20.0.38:6700,172.20.0.39:6700,172.20.0.58:6700
+log_dir: /root/curve/curvebs/logs/client
+
+```
+
+### 添加客户端主机
+```shell
+curveadm hosts commit client.yaml
+```
+
+
+centos client ,server 安装nbd模块
+```shell
+
+yum install net-tools
+yum install telnet
+yum install wget
+yum install rpm-build
+yum install vim
+yum install kernel kernel-devel kernel-headers -y
+wget https://vault.centos.org/7.9.2009/os/Source/SPackages/kernel-3.10.0-1160.el7.src.rpm
+useradd mockbuild
+rpm -ihv kernel-3.10.0-1160.el7.src.rpm
+cd ~/rpmbuild/SOURCES/
+tar Jxvf linux-3.10.0-1160.el7.tar.xz -C /usr/src/kernels/
+cd /usr/src/kernels/linux-3.10.0-1160.el7/
+yum install gcc
+yum install elfutils-libelf-devel
+make mrproper
+cp ../3.10.0-1160.76.1.el7.x86_64/Module.symvers .
+cp /boot/config-3.10.0-1160.76.1.el7.x86_64 .config
+make oldconfig
+make prepare
+make scripts
+make CONFIG_BLK_DEV_NBD=m M=drivers/block
+
+vim drivers/block/nbd.c
+// sreq.cmd_type = REQ_TYPE_SPECIAL;
+sreq.cmd_type = 7;
+
+make CONFIG_BLK_DEV_NBD=m M=drivers/block
+modinfo drivers/block/nbd.ko
+cp drivers/block/nbd.ko /lib/modules/$(uname -r)/kernel/drivers/block/
+depmod -a
+modprobe nbd
+lsmod | grep nbd
+
+#设置开机自动加载nbd模块
+cat > /etc/modules-load.d/nbd.conf <<EOF
+# Load nbd.ko at boot
+nbd
+EOF
+``` 
+
+
+### 创建磁盘挂载
+```shell
+#挂载
+curveadm map curvebsuser1:/vdq --host client-node1 -c client.yaml --create --size 10GB
+#非创建挂载
+curveadm map curvebsuser1:/vdq --host client-node1 -c client.yaml --size 10GB
+#解除挂载
+curveadm unmap curvebsuser1:/vdq --host client-node1
+```
+<strong>
+注意: curvebsuser1:/vdq 如下信息只是记录在curve系统 不需要在client系统上创建, 注意次用户名不能用下划线
+map nbd设备的时候，下划线会用来作为卷和用户的分隔符的
+</strong>
+
+### client 查看集群空间情况
+```shell
+curve_ops_tool space
+```
+
+### curveadm 查看client挂载的设备情况
+```shell
+curveadm client status 
+Get Client Status: [OK]  
+
+Id            Kind     Host          Container Id  Status                           Aux Info                                          
+--            ----     ----          ------------  ------                           --------                                          
+c7b6d6ce8a77  curvebs  client-node1  7a35f31c5648  Exited (143) About an hour ago   {"user":"curvebs_user","volume":"/dev/curvebs_bs"}
+f1225dfb6b94  curvebs  client-node1  4a81d6b2ac6f  Exited (143) About a minute ago  {"user":"curvebs_user","volume":"/vdq"}           
+4d01b8991a8a  curvebs  client-node1  5008c7362d66  Exited (143) 25 minutes ago      {"user":"curvebsuser1","volume":"/vdq"}           
+f846805e752e  curvebs  client-node1  c0c0ceb45459  Exited (143) 25 minutes ago      {"user":"curvebs_user","volume":"/vdw"}           
+ce0dc05be2ab  curvebs  client-node1  cc33ba0b4806  Up 55 seconds                    {"user":"curvebsuser1","volume":"/vde"}     
+```
+查看到对应的client挂载情况，可以通过container id 重启client docker容器
+
+之后可以进入容器，然后执行相应的手动挂载
+```shell
+curve-nbd map cbd:pool//vde_curvebsuser1_
+```
+
+### 挂载块设备
+```shell 
+#查看块设备
+fdisk -l
+
+Disk /dev/nbd1: 10.7 GB, 10737418240 bytes, 20971520 sectors
+Units = sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+# 制作文件系统
+mkfs.ext4 /dev/nbd1
+# 挂载块设备
+mkdir /opt/nbd1
+mount /dev/nbd1 /opt/nbd1
+```
+
+
+### nfs-server 安装
+```shell
+yum install nfs-utils
+# 设置开机自启动
+systemctl enable nfs
+# 启动nfs-server
+systemctl start nfs
+# 编辑exportfs
+vim /etc/exports
+
+# 加入导出的目录    
+# no_all_squash: 可以使用普通用户授权
+# no_root_squash: 可以使用 root 授权
+# sync: 同步共享目录
+# rw: 权限设置，可读可写
+# 192.168.0.0/24: 客户端 IP 范围，* 代表所有，即没有限制
+# /data: 共享目录位置
+/opt/nbd0     *(rw,sync,no_root_squash,no_all_squash)
+/opt/nbd1     *(rw,sync,no_root_squash,no_all_squash)
+
+#执行导出,注意必须执行此步骤
+exportfs -a
+# 重启nfs
+systemctl restart nfs
+# 查看导出情况
+showmount -e
+```
+
+
+### nfs-client 安装
+
+```shell
+# 创建目录
+mkdir /opt/nbd1
+# 挂载目录
+mount -o rw -t nfs 172.20.0.61:/opt/nbd1 /opt/nbd1
+# 取消挂载目录
+umount /root/data/
+```
+
+
+
+
+### FAQ
+1.client 端不需要创建user 和 /dev 块设备
+2.可以在client端进入容器执行创建卷：
+```shell
+/bin/bash /curvebs/nebd/sbin/create.sh curvebsuser1 /vde 10
+```
+3. 可以进入leader mds查看相应的mds日志
+```shell
+curveadm status
+curveadm enter <mds id>
+tailf logs
+```
+4. 也可以去client端手动启
+```shell
+docker exec  curvebs-volume-0673e7a472e0dfa4121595b0b457b8f7 /bin/bash /curvebs/nebd/sbin/create.sh curvebsuser1 /vde 10
+```
+5. 注意，也可以手动使用curve_ops_tool 工具检查, 其中curve,curve_ops_tool,curve-nbd，curve_snaptool几个工具，ops 三个工具有重叠，目前基本上找不到curve工具，可以用curve_ops_tool代替
+```shell
+curve_ops_tool list --fileName=/
+```
+6. lsblk可以看到nbd设备,或者fdisk 可以检查挂载的磁盘
+7. 检查空间情况 curve_ops_tool space
+8. 手动挡的可以参考自动挂载命令，或者重新挂载命令：
+```code 
+https://github.com/opencurve/curve/blob/master/k8s/nbd/nbd-package/usr/bin/mount_curve_clouddisk.sh
+```
+9. client自动挂载思路
+```shell
+1. systemd enable 设置docker自启动
+2. curveadm 挂载启动容器以restart always启动
+3. curveadm 启动后需要固化数据，把user,/vde 这类信息固定下来
+4. docker 启动后自动init执行相应的 map挂载
+5. 配置nbd的自动加载
+```

mk-deb.sh

@@ -188,7 +188,7 @@ then
 bazel build ... --copt -DHAVE_ZLIB=1 --compilation_mode=dbg -s --define=with_glog=true \
 --define=libunwind=true --copt -DGFLAGS_NS=google --copt \
 -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} \
---linkopt -L/usr/local/lib ${bazelflags}
+--linkopt -L/usr/local/lib ${bazelflags} #--define=with_spdk=true
 if [ $? -ne 0 ]
 then
     echo "build phase1 failed"
@@ -215,7 +215,7 @@ else
 bazel build ... --copt -DHAVE_ZLIB=1 --copt -O2 -s --define=with_glog=true \
 --define=libunwind=true --copt -DGFLAGS_NS=google --copt \
 -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} \
---linkopt -L/usr/local/lib ${bazelflags}
+--linkopt -L/usr/local/lib ${bazelflags} #--define=with_spdk=true
 if [ $? -ne 0 ]
 then
     echo "build phase1 failed"
@@ -349,6 +349,11 @@ if [ $? -ne 0 ]
 then
     exit
 fi
+mkdir -p build/curve-chunkserver/usr/lib
+if [ $? -ne 0 ]
+then
+    exit
+fi
 mkdir -p build/curve-chunkserver/etc/curve
 if [ $? -ne 0 ]
 then
@@ -360,6 +365,18 @@ if [ $? -ne 0 ]
 then
     exit
 fi
+cp thirdparties/aws/aws-sdk-cpp/build/aws-cpp-sdk-core/libaws-cpp-sdk-core.so \
+build/curve-chunkserver/usr/lib/libaws-cpp-sdk-core.so
+if [ $? -ne 0 ]
+then
+    exit
+fi
+cp thirdparties/aws/aws-sdk-cpp/build/aws-cpp-sdk-s3-crt/libaws-cpp-sdk-s3-crt.so \
+build/curve-chunkserver/usr/lib/libaws-cpp-sdk-s3-crt.so
+if [ $? -ne 0 ]
+then
+    exit
+fi
 cp ./bazel-bin/src/tools/curve_chunkserver_tool \
 build/curve-chunkserver/usr/bin/curve_chunkserver_tool
 if [ $? -ne 0 ]
@@ -447,7 +464,6 @@ if [ $? -ne 0 ]
 then
     exit
 fi
-cp ./bazel-bin/src/client/libcurve.so build/curve-sdk/usr/lib
 cp include/client/libcurve.h build/curve-sdk/usr/include
 cp include/client/libcbd.h build/curve-sdk/usr/include
 cp include/client/libcurve_define.h build/curve-sdk/usr/include
@@ -487,35 +503,11 @@ if [ $? -ne 0 ]
 then
     exit
 fi
-cp -r ./curve-snapshotcloneserver-nginx/app/lib \
-build/curve-nginx/etc/curve/nginx/app
-if [ $? -ne 0 ]
-then
-    exit
-fi
-cp -r ./curve-snapshotcloneserver-nginx/app/src \
-build/curve-nginx/etc/curve/nginx/app
-if [ $? -ne 0 ]
-then
-    exit
-fi
 mkdir -p build/curve-nginx/etc/curve/nginx/conf
 if [ $? -ne 0 ]
 then
     exit
 fi
-cp ./curve-snapshotcloneserver-nginx/conf/mime.types \
-build/curve-nginx/etc/curve/nginx/conf/
-if [ $? -ne 0 ]
-then
-    exit
-fi
-cp -r ./curve-snapshotcloneserver-nginx/docker \
-build/curve-nginx/etc/curve/nginx/
-if [ $? -ne 0 ]
-then
-    exit
-fi
 # step 4.1 prepare for nebd-package
 cp -r nebd/nebd-package build/
 mkdir -p build/nebd-package/usr/include/nebd

proto/heartbeat.proto

@@ -87,6 +87,8 @@ message ChunkServerStatisticInfo {
     required uint64 chunkSizeLeftBytes = 6;
     // 回收站中chunk占用的磁盘空间
     required uint64 chunkSizeTrashedBytes = 7;
+    // chunkfilepool的大小
+    optional uint64 chunkFilepoolSize = 8;
 };
 
 message ChunkServerHeartbeatRequest {

proto/topology.proto

@@ -154,6 +154,9 @@ message ChunkServerRegistRequest {
     // already registed chunkserver should take chunkserverId & token
     optional uint32 chunkServerID = 6;
     optional string token = 7;
+    optional uint64 chunkFilepoolSize = 8;
+    optional bool   useChunkFilePoolAsWalPool = 9;
+    optional uint32 useChunkFilePoolAsWalPoolReserve = 10;
 };
 
 message ChunkServerRegistResponse {

replace-curve-repo.sh

@@ -24,7 +24,7 @@ sed -i "s;https://github.com/gflags/gflags/archive/v2.2.2.tar.gz;https://curve-b
 sed -i "s;https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz;https://curve-build.nos-eastchina1.126.net/leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz;g" WORKSPACE
 
 # brpc
-sed -i "s;https://github.com/apache/incubator-brpc;https://gitee.com/baidu/BRPC;g" WORKSPACE
+sed -i "s;https://github.com/opencurve/incubator-brpc;https://gitee.com/NetEase_Curve/incubator-brpc;g" WORKSPACE
 
 # jsoncpp
 sed -i "s;https://github.com/open-source-parsers/jsoncpp.git;https://gitee.com/mirrors/jsoncpp;g" WORKSPACE

src/chunkserver/BUILD

@@ -14,7 +14,7 @@
 #  limitations under the License.
 #
 
-load("//:copts.bzl", "CURVE_DEFAULT_COPTS", "CURVE_TEST_COPTS")
+load("//:copts.bzl", "CURVE_DEFAULT_COPTS", "CURVE_TEST_COPTS", "SPDK_COPTS", "SPDK_LINK_OPTS", "SPDK_DEPS")
 
 cc_library(
     name = "chunkserver-lib",
@@ -22,12 +22,39 @@ cc_library(
         ["*.cpp"],
         exclude = [
             "chunkserver_main.cpp",
+            "spdk_hook.cpp",
         ],
-    ),
-    hdrs = glob([
-        "*.h",
-    ]),
-    copts = CURVE_DEFAULT_COPTS,
+    ) + select({
+        "//:with_spdk": ["spdk_hook.cpp"],
+        "//conditions:default": [],
+    }),
+    hdrs = glob(
+        ["*.h"],
+        exclude = [
+            "spdk_hook.h",
+        ],
+    ) + select({
+        "//:with_spdk": ["spdk_hook.h"],
+        "//conditions:default": [],
+    }),
+    copts = CURVE_DEFAULT_COPTS + select({
+        "//:with_spdk": SPDK_COPTS,
+        "//conditions:default": [],
+    }),
+
+    linkopts = select({
+        "//:with_spdk": SPDK_LINK_OPTS,
+        "//conditions:default": [],
+    }) + [
+        "-std=c++11",
+        "-luuid",
+        "-lrt",
+        "-lssl",
+        "-lcrypto",
+        "-ldl",
+        "-lz",
+        "-lpthread",
+    ],
     visibility = ["//visibility:public"],
     deps = [
         "//external:braft",
@@ -48,9 +75,12 @@ cc_library(
         "//src/common:curve_s3_adapter",
         "//src/fs:lfs",
         "//src/client:curve_client",
-	"//include/client:include_client",
-	"//proto:scan_cc_proto",
-    ],
+	    "//proto:scan_cc_proto",
+	    "//include/client:include_client",
+    ] + select({
+        "//:with_spdk": SPDK_DEPS,
+        "//conditions:default": [],
+    }),
 )
 
 cc_library(
@@ -59,14 +89,40 @@ cc_library(
         ["*.cpp"],
         exclude = [
             "chunkserver_main.cpp",
+            "spdk_hook.cpp",
         ],
-    ),
-    hdrs = glob([
-        "*.h",
-    ]),
-    copts = CURVE_TEST_COPTS,
+    ) + select({
+        "//:with_spdk": ["spdk_hook.cpp"],
+        "//conditions:default": [],
+    }),
+    hdrs = glob(
+        ["*.h"],
+        exclude = [
+            "spdk_hook.h",
+        ],
+    ) + select({
+        "//:with_spdk": ["spdk_hook.h"],
+        "//conditions:default": [],
+    }),
+    copts = CURVE_DEFAULT_COPTS + select({
+        "//:with_spdk": SPDK_COPTS,
+        "//conditions:default": [],
+    }),
     defines = ["UNIT_TEST"],
     visibility = ["//visibility:public"],
+    linkopts = select({
+        "//:with_spdk": SPDK_LINK_OPTS,
+        "//conditions:default": [],
+    }) + [
+        "-std=c++11",
+        "-luuid",
+        "-lrt",
+        "-lssl",
+        "-lcrypto",
+        "-ldl",
+        "-lz",
+        "-lpthread",
+    ],
     deps = [
         "//external:braft",
         "//external:brpc",
@@ -87,7 +143,10 @@ cc_library(
         "//src/fs:lfs",
         "//src/client:curve_client",
         "//proto:scan_cc_proto",
-    ],
+    ] + select({
+        "//:with_spdk": SPDK_DEPS,
+        "//conditions:default": [],
+    }),
 )
 
 # chunkserver exec

src/chunkserver/chunkserver.cpp

@@ -43,6 +43,10 @@
 #include "src/chunkserver/raftsnapshot/curve_snapshot_storage.h"
 #include "src/chunkserver/raftlog/curve_segment_log_storage.h"
 #include "src/common/curve_version.h"
+#ifdef WITH_SPDK
+#include "src/fs/pfs_filesystem_impl.h"
+#include "src/chunkserver/spdk_hook.h"
+#endif
 
 using ::curve::fs::LocalFileSystem;
 using ::curve::fs::LocalFileSystemOption;
@@ -51,6 +55,10 @@ using ::curve::fs::FileSystemType;
 using ::curve::chunkserver::concurrent::ConcurrentApplyModule;
 using ::curve::common::UriParser;
 
+#ifdef WITH_SPDK
+using ::curve::fs::InitPfsOption;
+#endif
+
 DEFINE_string(conf, "ChunkServer.conf", "Path of configuration file");
 DEFINE_string(chunkServerIp, "127.0.0.1", "chunkserver ip");
 DEFINE_bool(enableExternalServer, false, "start external server or not");
@@ -75,11 +83,28 @@ DEFINE_string(walFilePoolDir, "./0/", "WAL filepool location");
 DEFINE_string(walFilePoolMetaPath, "./walfilepool.meta",
                                     "WAL filepool meta path");
 
+#ifdef WITH_SPDK
+DEFINE_string(pfs_cluster,
+    "",
+    "pfs cluster name, only for pfs");
+
+DEFINE_string(pfs_pbd_name,
+    "",
+    "pfs pbd name, only for pfs");
+DEFINE_int32(pfs_host_id,
+    -1,
+    "pfs host id, only for pfs");
+DEFINE_string(spdk_nvme_controller,
+    "",
+    "spdk nvme controller, only for pfs");
+#endif
+
 const char* kProtocalCurve = "curve";
 
 namespace curve {
 namespace chunkserver {
 
+
 int ChunkServer::Run(int argc, char** argv) {
     gflags::ParseCommandLineFlags(&argc, &argv, true);
 
@@ -100,6 +125,23 @@ int ChunkServer::Run(int argc, char** argv) {
     // 初始化日志模块
     google::InitGoogleLogging(argv[0]);
 
+#ifdef WITH_SPDK
+    FILE *dpdk_log_stream = NULL;
+    cookie_io_functions_t io_funcs;
+    memset(&io_funcs, 0, sizeof(io_funcs));
+
+    io_funcs.write = glog_dpdk_log_func;
+    dpdk_log_stream = fopencookie(NULL, "w", io_funcs);
+    if (dpdk_log_stream == NULL) {
+        LOG(FATAL) << "fopencookie failed";
+    }
+    rte_openlog_stream(dpdk_log_stream);
+    spdk_log_open(glog_spdk_func);
+    pfs_set_trace_func(glog_pfs_func);
+
+    curve::fs::HookIOBufIOFuncs();
+#endif
+
     // 打印参数
     conf.PrintConfig();
     curve::common::ExposeCurveVersion();
@@ -128,11 +170,46 @@ int ChunkServer::Run(int argc, char** argv) {
         << "Failed to initialize concurrentapply module!";
 
     // 初始化本地文件系统
-    std::shared_ptr<LocalFileSystem> fs(
-        LocalFsFactory::CreateFs(FileSystemType::EXT4, ""));
     LocalFileSystemOption lfsOption;
+    std::string fsTypeStr;
+    LOG_IF(FATAL, !conf.GetStringValue(
+        "fs.type", &fsTypeStr));
+    lfsOption.type = fs::StringToFileSystemType(fsTypeStr);
+#ifdef WITH_SPDK
+    if (fs::FileSystemType::PFS  == lfsOption.type) {
+        LOG_IF(FATAL, !conf.GetStringValue(
+            "fs.pfs_cluster", &lfsOption.pfs_cluster));
+        LOG_IF(FATAL, !conf.GetStringValue(
+            "fs.pfs_pbd_name", &lfsOption.pfs_pbd_name));
+        LOG_IF(FATAL, !conf.GetIntValue(
+            "fs.pfs_host_id", &lfsOption.pfs_host_id));
+        LOG_IF(FATAL, !conf.GetStringValue(
+            "fs.spdk_nvme_controller", &lfsOption.spdk_nvme_controller));
+
+        LOG_IF(FATAL, 0 != InitPfsOption(lfsOption))
+            << "InitPfsOption failed";
+
+        // setup spdk
+        LOG_IF(FATAL, 0 != pfs_spdk_setup())
+            << "setup spdk failed";
+        // make iobuf use dpdk malloc & free
+
+        butil::iobuf::set_blockmem_allocate_and_deallocate(
+            dpdk_mem_allocate, dpdk_mem_free);
+        // start pfsdaemon
+        LOG_IF(FATAL, 0 != pfsd_start(true))
+            << "start pfsd failed";
+    } else {
+        LOG(FATAL) <<"PFS filesystem must be used when using spdk! "
+                   << "Current filesystem type: " << fsTypeStr;
+    }
+#endif
+    std::shared_ptr<LocalFileSystem> fs(
+        LocalFsFactory::CreateFs(lfsOption.type, ""));
+
     LOG_IF(FATAL, !conf.GetBoolValue(
         "fs.enable_renameat2", &lfsOption.enableRenameat2));
+
     LOG_IF(FATAL, 0 != fs->Init(lfsOption))
         << "Failed to initialize local filesystem module!";
 
@@ -150,6 +227,7 @@ int ChunkServer::Run(int argc, char** argv) {
     std::string raftLogProtocol = UriParser::GetProtocolFromUri(raftLogUri);
     std::shared_ptr<FilePool> walFilePool = nullptr;
     bool useChunkFilePoolAsWalPool = true;
+    uint32_t useChunkFilePoolAsWalPoolReserve = 15;
     if (raftLogProtocol == kProtocalCurve) {
         LOG_IF(FATAL, !conf.GetBoolValue(
             "walfilepool.use_chunk_file_pool",
@@ -164,6 +242,9 @@ int ChunkServer::Run(int argc, char** argv) {
             LOG(INFO) << "initialize walpool success.";
         } else {
             walFilePool = chunkfilePool;
+            LOG_IF(FATAL, !conf.GetUInt32Value(
+            "walfilepool.use_chunk_file_pool_reserve",
+            &useChunkFilePoolAsWalPoolReserve));
             LOG(INFO) << "initialize to use chunkfilePool as walpool success.";
         }
     }
@@ -190,7 +271,11 @@ int ChunkServer::Run(int argc, char** argv) {
     // 初始化注册模块
     RegisterOptions registerOptions;
     InitRegisterOptions(&conf, &registerOptions);
+    registerOptions.useChunkFilePoolAsWalPoolReserve =
+            useChunkFilePoolAsWalPoolReserve;
+    registerOptions.useChunkFilePoolAsWalPool = useChunkFilePoolAsWalPool;
     registerOptions.fs = fs;
+    registerOptions.chunkFilepool = chunkfilePool;
     Register registerMDS(registerOptions);
     ChunkServerMetadata metadata;
     ChunkServerMetadata localMetadata;
@@ -286,6 +371,7 @@ int ChunkServer::Run(int argc, char** argv) {
     InitHeartbeatOptions(&conf, &heartbeatOptions);
     heartbeatOptions.copysetNodeManager = copysetNodeManager_;
     heartbeatOptions.fs = fs;
+    heartbeatOptions.chunkFilePool = chunkfilePool;
     heartbeatOptions.chunkserverId = metadata.id();
     heartbeatOptions.chunkserverToken = metadata.token();
     heartbeatOptions.scanManager = &scanManager_;
@@ -455,6 +541,11 @@ int ChunkServer::Run(int argc, char** argv) {
         << "Failed to shutdown file pool clean worker.";
     concurrentapply.Stop();
 
+#ifdef WITH_SPDK
+    LOG_IF(ERROR, pfsd_stop() != 0)
+        << "Failed to stop pfsd.";
+#endif
+
     google::ShutdownGoogleLogging();
     return 0;
 }
@@ -857,6 +948,23 @@ void ChunkServer::LoadConfigFromCmdline(common::Configuration *conf) {
     if (GetCommandLineFlagInfo("minIoAlignment", &info) && !info.is_default) {
         conf->SetUInt32Value("global.min_io_alignment", FLAGS_minIoAlignment);
     }
+
+#ifdef WITH_SPDK
+    if (GetCommandLineFlagInfo("pfs_cluster", &info) && !info.is_default) {
+        conf->SetStringValue("fs.pfs_cluster", FLAGS_pfs_cluster);
+    }
+    if (GetCommandLineFlagInfo("pfs_pbd_name", &info) && !info.is_default) {
+        conf->SetStringValue("fs.pfs_pbd_name", FLAGS_pfs_pbd_name);
+    }
+    if (GetCommandLineFlagInfo("pfs_host_id", &info) && !info.is_default) {
+        conf->SetIntValue("fs.pfs_host_id", FLAGS_pfs_host_id);
+    }
+    if (GetCommandLineFlagInfo("spdk_nvme_controller", &info) &&
+        !info.is_default) {
+        conf->SetStringValue("fs.spdk_nvme_controller",
+            FLAGS_spdk_nvme_controller);
+    }
+#endif
 }
 
 int ChunkServer::GetChunkServerMetaFromLocal(

src/chunkserver/copyset_node.cpp

@@ -35,8 +35,9 @@
 #include <future>
 #include <deque>
 #include <set>
+#include <list>
 
-#include "src/chunkserver/raftsnapshot/curve_filesystem_adaptor.h"
+#include "src/chunkserver/filesystem_adaptor/curve_filesystem_adaptor.h"
 #include "src/chunkserver/chunk_closure.h"
 #include "src/chunkserver/op_request.h"
 #include "src/fs/fs_common.h"
@@ -174,11 +175,15 @@ int CopysetNode::Init(const CopysetNodeOptions &options) {
         metric_->MonitorCurveSegmentLogStorage(logStorage);
     };
 
-    LogStorageOptions lsOptions(options.walFilePool, monitorMetricCb);
+
+    scoped_refptr<CurveFilesystemAdaptor> curveFileSystemAdaptor =
+        new CurveFilesystemAdaptor(
+            options.walFilePool, options.localFileSystem);
+    LogStorageOptions lsOptions(curveFileSystemAdaptor, monitorMetricCb);
 
     // In order to get more copysetNode's information in CurveSegmentLogStorage
     // without using global variables.
-    StoreOptForCurveSegmentLogStorage(lsOptions);
+    StoreOptForCurveSegmentLogStorage(&lsOptions);
 
     syncTimerIntervalMs_ = options.syncTimerIntervalMs;
     checkSyncingIntervalMs_ = options.checkSyncingIntervalMs;
@@ -250,12 +255,12 @@ void CopysetNode::InitRaftNodeOptions(const CopysetNodeOptions &options) {
     CurveFilesystemAdaptor* cfa =
         new CurveFilesystemAdaptor(options.chunkFilePool,
                                    options.localFileSystem);
-    std::vector<std::string> filterList;
+    std::list<std::string> filterList;
     std::string snapshotMeta(BRAFT_SNAPSHOT_META_FILE);
     filterList.push_back(kCurveConfEpochFilename);
     filterList.push_back(snapshotMeta);
     filterList.push_back(snapshotMeta.append(BRAFT_PROTOBUF_FILE_TEMP));
-    cfa->SetFilterList(filterList);
+    cfa->AddToFilterList(&filterList);
 
     nodeOptions_.snapshot_file_system_adaptor =
         new scoped_refptr<braft::FileSystemAdaptor>(cfa);

src/chunkserver/datastore/BUILD

@@ -27,6 +27,7 @@ cc_library(
         "//external:braft",
         "//external:gflags",
         "//external:glog",
+        "//external:butil",
         "//include:include-common",
         "//include/chunkserver:include-chunkserver",
         "//src/fs:lfs",

src/chunkserver/datastore/chunkserver_chunkfile.cpp

@@ -488,7 +488,8 @@ CSErrorCode CSChunkFile::Paste(const char * buf, off_t offset, size_t length) {
     return CSErrorCode::Success;
 }
 
-CSErrorCode CSChunkFile::Read(char * buf, off_t offset, size_t length) {
+CSErrorCode CSChunkFile::Read(
+    butil::IOPortal * buf, off_t offset, size_t length) {
     ReadLockGuard readGuard(rwLock_);
     if (!CheckOffsetAndLength(
             offset, length, isCloneChunk_ ? pageSize_ : FLAGS_minIoAlignment)) {

src/chunkserver/datastore/chunkserver_chunkfile.h

@@ -182,6 +182,7 @@ class CSChunkFile {
      * @return: return error code
      */
     CSErrorCode Paste(const char * buf, off_t offset, size_t length);
+
     /**
      * Read chunk files
      * There may be concurrency, add read lock
@@ -190,7 +191,7 @@ class CSChunkFile {
      * @param length: The length of the data requested to be read
      * @return: return error code
      */
-    CSErrorCode Read(char * buf, off_t offset, size_t length);
+    CSErrorCode Read(butil::IOPortal * buf, off_t offset, size_t length);
 
     /**
      * Read chunk meta data
@@ -323,6 +324,10 @@ class CSChunkFile {
         return lfs_->Read(fd_, buf, offset + pageSize_, length);
     }
 
+    inline int readData(butil::IOPortal* buf, off_t offset, size_t length) {
+        return lfs_->Read(fd_, buf, offset + pageSize_, length);
+    }
+
     inline int writeData(const char* buf, off_t offset, size_t length) {
         int rc = lfs_->Write(fd_, buf, offset + pageSize_, length);
         if (rc < 0) {
@@ -364,7 +369,7 @@ class CSChunkFile {
     }
 
     inline int SyncData() {
-        return lfs_->Sync(fd_);
+        return lfs_->Fdatasync(fd_);
     }
 
     inline bool CheckOffsetAndLength(off_t offset, size_t len, size_t align) {

src/chunkserver/datastore/chunkserver_datastore.cpp

@@ -144,7 +144,7 @@ CSErrorCode CSDataStore::DeleteSnapshotChunkOrCorrectSn(
 
 CSErrorCode CSDataStore::ReadChunk(ChunkID id,
                                    SequenceNum sn,
-                                   char * buf,
+                                   butil::IOPortal* buf,
                                    off_t offset,
                                    size_t length) {
     auto chunkFile = metaCache_.Get(id);

src/chunkserver/datastore/chunkserver_datastore.h

@@ -173,6 +173,7 @@ class CSDataStore {
      */
     virtual CSErrorCode DeleteSnapshotChunkOrCorrectSn(
         ChunkID id, SequenceNum correctedSn);
+
     /**
      * Read the contents of the current chunk
      * @param id: the chunk id to be read
@@ -187,6 +188,27 @@ class CSDataStore {
                                   SequenceNum sn,
                                   char * buf,
                                   off_t offset,
+                                  size_t length) {
+        butil::IOPortal iop;
+        CSErrorCode rc = ReadChunk(id, sn, &iop, offset, length);
+        iop.copy_to(buf, length, 0);
+        return rc;
+    }
+
+    /**
+     * Read the contents of the current chunk
+     * @param id: the chunk id to be read
+     * @param sn: used to record trace, not used in actual logic processing,
+     *             indicating the sequence number of the current user file
+     * @param buf: the content of the data read
+     * @param offset: the logical offset of the data requested to be read in the chunk
+     * @param length: the length of the data requested to be read
+     * @return: return error code
+     */
+    virtual CSErrorCode ReadChunk(ChunkID id,
+                                  SequenceNum sn,
+                                  butil::IOPortal* buf,
+                                  off_t offset,
                                   size_t length);
 
     /**

src/chunkserver/datastore/file_pool.cpp

@@ -279,7 +279,7 @@ bool FilePool::CleanChunk(uint64_t chunkid, bool onlyMarked) {
         char* buffer = writeBuffer_.get();
 
         while (nwrite < ntotal) {
-            nbytes = fsptr_->Write(fd, buffer, nwrite,
+            nbytes = fsptr_->WriteZeroIfSupport(fd, buffer, nwrite,
                 std::min(ntotal - nwrite, (uint64_t)bytesPerWrite));
             if (nbytes < 0) {
                 LOG(ERROR) << "Write file failed: " << chunkpath;

src/chunkserver/filesystem_adaptor/BUILD

@@ -0,0 +1,37 @@
+#
+#  Copyright (c) 2022 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+load("//:copts.bzl", "CURVE_DEFAULT_COPTS")
+
+cc_library(
+    name = "filesystem_adaptor",
+    srcs = glob(
+        ["*.cpp"],
+    ),
+    hdrs = glob([
+        "*.h",
+    ]),
+    copts = CURVE_DEFAULT_COPTS,
+    visibility = ["//visibility:public"],
+    deps = [
+        "//external:braft",
+        "//external:bthread",
+        "//external:butil",
+        "//external:gflags",
+        "//external:glog",
+        "//src/chunkserver/datastore:chunkserver_datastore",
+    ]
+)

src/chunkserver/filesystem_adaptor/curve_file_adaptor.cpp

@@ -0,0 +1,65 @@
+/*
+ *  Copyright (c) 2022 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Created Date: 2022-08-24
+ * Author: xuchaojie
+ */
+
+#include "src/chunkserver/filesystem_adaptor/curve_file_adaptor.h"
+
+namespace braft {
+DECLARE_bool(raft_use_fsync_rather_than_fdatasync);
+}
+
+namespace curve {
+namespace chunkserver {
+
+ssize_t CurveFileAdaptor::write(const butil::IOBuf& data, off_t offset) {
+    return lfs_->Write(fd_, data, offset, data.size());
+}
+
+ssize_t CurveFileAdaptor::read(
+    butil::IOPortal* portal, off_t offset, size_t size) {
+    return lfs_->Read(fd_, portal, offset, size);
+}
+
+ssize_t CurveFileAdaptor::size() {
+    off_t sz = lfs_->Lseek(fd_, 0, SEEK_END);
+    return ssize_t(sz);
+}
+
+bool CurveFileAdaptor::sync() {
+    if (braft::FLAGS_raft_use_fsync_rather_than_fdatasync) {
+        return lfs_->Fsync(fd_) == 0;
+    } else {
+        return lfs_->Fdatasync(fd_) == 0;
+    }
+}
+
+bool CurveFileAdaptor::close() {
+    if (!sync()) {
+        return false;
+    }
+    if (fd_ < 0) {
+        return true;
+    }
+    return lfs_->Close(fd_) == 0;
+}
+
+}  // namespace chunkserver
+}  // namespace curve

src/chunkserver/filesystem_adaptor/curve_file_adaptor.h

@@ -20,24 +20,36 @@
  * Author: charisu
  */
 
-#ifndef SRC_CHUNKSERVER_RAFTSNAPSHOT_CURVE_FILE_ADAPTOR_H_
-#define SRC_CHUNKSERVER_RAFTSNAPSHOT_CURVE_FILE_ADAPTOR_H_
+#ifndef SRC_CHUNKSERVER_FILESYSTEM_ADAPTOR_CURVE_FILE_ADAPTOR_H_
+#define SRC_CHUNKSERVER_FILESYSTEM_ADAPTOR_CURVE_FILE_ADAPTOR_H_
 
 #include <braft/file_system_adaptor.h>
 
+#include "src/fs/local_filesystem.h"
+
+using curve::fs::LocalFileSystem;
+
 namespace curve {
 namespace chunkserver {
 
-class CurveFileAdaptor : public braft::PosixFileAdaptor {
+class CurveFileAdaptor : public braft::FileAdaptor {
  public:
-    explicit CurveFileAdaptor(int fd) : PosixFileAdaptor(fd) {}
-    // close之前必须先sync，保证数据落盘，其他逻辑不变
-    bool close() override {
-        return sync() && braft::PosixFileAdaptor::close();
-    }
+    explicit CurveFileAdaptor(int fd,
+        const std::shared_ptr<LocalFileSystem> &lfs)
+        : fd_(fd), lfs_(lfs) {}
+
+    ssize_t write(const butil::IOBuf& data, off_t offset) override;
+    ssize_t read(butil::IOPortal* portal, off_t offset, size_t size) override;
+    ssize_t size() override;
+    bool sync() override;
+    bool close() override;
+
+ private:
+    int fd_;
+    std::shared_ptr<LocalFileSystem> lfs_;
 };
 
 }  // namespace chunkserver
 }  // namespace curve
 
-#endif  // SRC_CHUNKSERVER_RAFTSNAPSHOT_CURVE_FILE_ADAPTOR_H_
+#endif  // SRC_CHUNKSERVER_FILESYSTEM_ADAPTOR_CURVE_FILE_ADAPTOR_H_

src/chunkserver/filesystem_adaptor/curve_filesystem_adaptor.cpp

@@ -22,17 +22,18 @@
 
 #include <butil/fd_utility.h>
 #include <vector>
+#include <list>
 
-#include "src/chunkserver/raftsnapshot/curve_filesystem_adaptor.h"
+#include "src/chunkserver/filesystem_adaptor/curve_filesystem_adaptor.h"
 
 namespace curve {
 namespace chunkserver {
 CurveFilesystemAdaptor::CurveFilesystemAdaptor(
-                                std::shared_ptr<FilePool> chunkFilePool,
+                                std::shared_ptr<FilePool> filePool,
                                 std::shared_ptr<LocalFileSystem> lfs) {
     lfs_ = lfs;
-    chunkFilePool_ = chunkFilePool;
-    uint64_t metapageSize = chunkFilePool->GetFilePoolOpt().metaPageSize;
+    filePool_ = filePool;
+    uint64_t metapageSize = filePool->GetFilePoolOpt().metaPageSize;
     tempMetaPageContent = new (std::nothrow) char[metapageSize];
     CHECK(tempMetaPageContent != nullptr);
     memset(tempMetaPageContent, 0, metapageSize);
@@ -47,7 +48,7 @@ CurveFilesystemAdaptor::~CurveFilesystemAdaptor() {
         delete[] tempMetaPageContent;
         tempMetaPageContent = nullptr;
     }
-    LOG(INFO) << "release raftsnapshot filesystem adaptor!";
+    LOG(INFO) << "release CurveFilesystemAdaptor!";
 }
 
 braft::FileAdaptor* CurveFilesystemAdaptor::open(const std::string& path,
@@ -55,20 +56,12 @@ braft::FileAdaptor* CurveFilesystemAdaptor::open(const std::string& path,
                     butil::File::Error* e) {
     (void) file_meta;
 
-    static std::once_flag local_s_check_cloexec_once;
-    static bool local_s_support_cloexec_on_open = false;
-    std::call_once(local_s_check_cloexec_once, [&](){
-        int fd = lfs_->Open("/dev/zero", O_RDONLY | O_CLOEXEC);
-        local_s_support_cloexec_on_open = (fd != -1);
-        if (fd != -1) {
-            lfs_->Close(fd);
-        }
-    });
-
     bool cloexec = (oflag & O_CLOEXEC);
-    if (cloexec && !local_s_support_cloexec_on_open) {
+    if (cloexec) {
         oflag &= (~O_CLOEXEC);
+        LOG(WARNING) << "CurveFilesystemAdaptor not support O_CLOEXEC!";
     }
+
     // Open就使用sync标志是为了避免集中在close一次性sync，对于16MB的chunk文件可能会造成抖动
     oflag |= O_SYNC;
 
@@ -79,7 +72,7 @@ braft::FileAdaptor* CurveFilesystemAdaptor::open(const std::string& path,
         (oflag & O_CREAT) &&
         false == lfs_->FileExists(path)) {
         // 从chunkfile pool中取出chunk返回
-        int rc = chunkFilePool_->GetFile(path, tempMetaPageContent);
+        int rc = filePool_->GetFile(path, tempMetaPageContent);
         // 如果从FilePool中取失败，返回错误。
         if (rc != 0) {
             LOG(ERROR) << "get chunk from chunkfile pool failed!";
@@ -98,20 +91,17 @@ braft::FileAdaptor* CurveFilesystemAdaptor::open(const std::string& path,
 
     if (fd < 0) {
         if (oflag & O_CREAT) {
-            LOG(ERROR) << "snapshot create chunkfile failed, filename = "
+            LOG(ERROR) << "create chunkfile failed, filename = "
                 << path.c_str() << ", errno = " << errno;
         } else {
-            LOG(WARNING) << "snapshot open chunkfile failed,"
+            LOG(WARNING) << "open chunkfile failed,"
                     << "may be deleted by user, filename = "
                     << path.c_str() << ",errno = " << errno;
         }
         return NULL;
     }
-    if (cloexec && !local_s_support_cloexec_on_open) {
-        butil::make_close_on_exec(fd);
-    }
 
-    return new CurveFileAdaptor(fd);
+    return new CurveFileAdaptor(fd, lfs_);
 }
 
 bool CurveFilesystemAdaptor::delete_file(const std::string& path,
@@ -135,7 +125,7 @@ bool CurveFilesystemAdaptor::delete_file(const std::string& path,
                  return lfs_->Delete(path) == 0;
              } else {
                 // chunkfilepool内部会检查path对应文件合法性，如果不符合就直接删除
-                return chunkFilePool_->RecycleFile(path) == 0;
+                return filePool_->RecycleFile(path) == 0;
              }
         }
     }
@@ -160,7 +150,7 @@ bool CurveFilesystemAdaptor::RecycleDirRecursive(
                     break;
                 }
             } else {
-                int ret = chunkFilePool_->RecycleFile(todeletePath);
+                int ret = filePool_->RecycleFile(todeletePath);
                 if (ret < 0) {
                     rc = false;
                     LOG(ERROR) << "recycle " << path + filepath << ", failed!";
@@ -176,14 +166,108 @@ bool CurveFilesystemAdaptor::rename(const std::string& old_path,
                                             const std::string& new_path) {
     if (!NeedFilter(new_path) && lfs_->FileExists(new_path)) {
         // chunkfilepool内部会检查path对应文件合法性，如果不符合就直接删除
-        chunkFilePool_->RecycleFile(new_path);
+        filePool_->RecycleFile(new_path);
     }
     return lfs_->Rename(old_path, new_path) == 0;
 }
 
+bool CurveFilesystemAdaptor::link(const std::string& old_path,
+          const std::string& new_path) {
+    return lfs_->Link(old_path, new_path) == 0;
+}
+
+bool CurveFilesystemAdaptor::CreateDirectoryAndGetError(
+    const FilePath& full_path,
+    File::Error* error,
+    bool create_parents) {
+  if (!create_parents) {
+    if (lfs_->DirExists(full_path.value())) {
+      return true;
+    }
+    if (lfs_->Mkdir(full_path.value(), false) == 0) {
+      return true;
+    }
+    const int saved_errno = errno;
+    if (lfs_->DirExists(full_path.value())) {
+      return true;
+    }
+    if (error) {
+      *error = File::OSErrorToFileError(saved_errno);
+    }
+    return false;
+  }
+  std::vector<FilePath> subpaths;
+
+  // Collect a list of all parent directories.
+  FilePath last_path = full_path;
+  subpaths.push_back(full_path);
+  for (FilePath path = full_path.DirName();
+       path.value() != last_path.value(); path = path.DirName()) {
+    subpaths.push_back(path);
+    last_path = path;
+  }
+
+  // Iterate through the parents and create the missing ones.
+  for (std::vector<FilePath>::reverse_iterator i = subpaths.rbegin();
+       i != subpaths.rend(); ++i) {
+    if (lfs_->DirExists(i->value()))
+      continue;
+    // NOTE(gejun): permission bits of dir are different from file's
+    // -The write bit allows the affected user to create, rename, or delete
+    //  files within the directory, and modify the directory's attributes
+    // -The read bit allows the affected user to list the files within the
+    //  directory
+    // -The execute bit allows the affected user to enter the directory, and
+    //  access files and directories inside
+    // -The sticky bit states that files and directories within that directory
+    //  may only be deleted or renamed by their owner (or root)
+    if (lfs_->Mkdir(i->value(), false) == 0)
+      continue;
+    // Mkdir failed, but it might have failed with EEXIST, or some other error
+    // due to the the directory appearing out of thin air. This can occur if
+    // two processes are trying to create the same file system tree at the same
+    // time. Check to see if it exists and make sure it is a directory.
+    int saved_errno = errno;
+    if (!lfs_->DirExists(i->value())) {
+      if (error)
+        *error = File::OSErrorToFileError(saved_errno);
+      return false;
+    }
+  }
+  return true;
+}
+
+bool CurveFilesystemAdaptor::create_directory(const std::string& path,
+                              butil::File::Error* error,
+                              bool create_parent_directories) {
+    FilePath dir(path);
+    return CreateDirectoryAndGetError(dir, error, create_parent_directories);
+}
+
+bool CurveFilesystemAdaptor::path_exists(const std::string& path) {
+    return lfs_->PathExists(path);
+}
+
+bool CurveFilesystemAdaptor::directory_exists(const std::string& path) {
+    return lfs_->DirExists(path);
+}
+
+braft::DirReader* CurveFilesystemAdaptor::directory_reader(
+    const std::string& path) {
+    return new CurveDirReader(path.c_str(), lfs_);
+}
+
 void CurveFilesystemAdaptor::SetFilterList(
-                                    const std::vector<std::string>& filter) {
-    filterList_.assign(filter.begin(), filter.end());
+                                    const std::list<std::string>& filter) {
+    filterList_ = filter;
+}
+
+void CurveFilesystemAdaptor::AddToFilterList(std::list<std::string> *files) {
+    filterList_.splice(filterList_.end(), *files);
+}
+
+void CurveFilesystemAdaptor::AddToFilterList(const std::string& file) {
+    filterList_.push_back(file);
 }
 
 bool CurveFilesystemAdaptor::NeedFilter(const std::string& filename) {
@@ -198,5 +282,42 @@ bool CurveFilesystemAdaptor::NeedFilter(const std::string& filename) {
     return ret;
 }
 
+CurveDirReader::CurveDirReader(const std::string& path,
+    const std::shared_ptr<LocalFileSystem> &lfs)
+    : path_(path), lfs_(lfs), dir_(nullptr), dirIter_(nullptr) {
+    dir_ = lfs_->OpenDir(path);
+}
+
+CurveDirReader::~CurveDirReader() {
+    if (dir_ != nullptr) {
+        lfs_->CloseDir(dir_);
+    }
+    dir_ = nullptr;
+}
+
+bool CurveDirReader::is_valid() const {
+    return dir_ != nullptr;
+}
+
+bool CurveDirReader::next() {
+    while ((dirIter_ = lfs_->ReadDir(dir_)) != nullptr) {
+        if (strcmp(dirIter_->d_name, ".") == 0
+                || strcmp(dirIter_->d_name, "..") == 0) {
+            continue;
+        } else {
+            return true;
+        }
+    }
+    return false;
+}
+
+const char* CurveDirReader::name() const {
+    if (nullptr == dirIter_) {
+        return nullptr;
+    }
+    return dirIter_->d_name;
+}
+
+
 }  // namespace chunkserver
 }  // namespace curve

src/chunkserver/filesystem_adaptor/curve_filesystem_adaptor.h

@@ -19,8 +19,8 @@
  * File Created: Monday, 10th June 2019 2:20:12 pm
  * Author: tongguangxun
  */
-#ifndef SRC_CHUNKSERVER_RAFTSNAPSHOT_CURVE_FILESYSTEM_ADAPTOR_H_
-#define SRC_CHUNKSERVER_RAFTSNAPSHOT_CURVE_FILESYSTEM_ADAPTOR_H_
+#ifndef SRC_CHUNKSERVER_FILESYSTEM_ADAPTOR_CURVE_FILESYSTEM_ADAPTOR_H_
+#define SRC_CHUNKSERVER_FILESYSTEM_ADAPTOR_CURVE_FILESYSTEM_ADAPTOR_H_
 
 #include <braft/file_system_adaptor.h>
 #include <google/protobuf/message.h>
@@ -28,34 +28,25 @@
 #include <memory>
 #include <string>
 #include <vector>
+#include <list>
 
 #include "src/chunkserver/datastore/file_pool.h"
-#include "src/chunkserver/raftsnapshot/curve_file_adaptor.h"
-
-/**
- * RaftSnapshotFilesystemAdaptor目的是为了接管braft
- * 内部snapshot创建chunk文件的逻辑，目前curve内部
- * 会从chunkfilepool中直接取出已经格式化好的chunk文件
- * 但是braft内部由于install snapshot也会创建chunk文件
- * 这个创建文件不感知chunkfilepool，因此我们希望install
- * snapshot也能从chunkfilepool中直接取出chunk文件，因此
- * 我们对install snapshot流程中的文件系统做了一层hook，在
- * 创建及删除文件操作上直接使用curve提供的文件系统接口即可。
- */
+#include "src/chunkserver/filesystem_adaptor/curve_file_adaptor.h"
 
 using curve::fs::LocalFileSystem;
 using curve::chunkserver::FilePool;
 
+using butil::FilePath;
+using butil::File;
+
 namespace curve {
 namespace chunkserver {
+
 /**
- * CurveFilesystemAdaptor继承raft的PosixFileSystemAdaptor类，在raft
- * 内部其快照使用PosixFileSystemAdaptor类进行文件操作，因为我们只希望在其创建文件
- * 或者删除文件的时候使用chunkfilepool提供的getchunk和recyclechunk接口，所以这里
- * 我们只实现了open和delete_file两个接口。其他接口在调用的时候仍然使用原来raft的内部
- * 的接口。
+ * CurveFilesystemAdaptor inherits braft::FileSystemAdaptor
+ * and is implemented using the interface of filePool and LocalFileSystem.
  */
-class CurveFilesystemAdaptor : public braft::PosixFileSystemAdaptor {
+class CurveFilesystemAdaptor : public braft::FileSystemAdaptor {
  public:
     /**
      * 构造函数
@@ -63,7 +54,7 @@ class CurveFilesystemAdaptor : public braft::PosixFileSystemAdaptor {
      * @param: lfs用于进行一些文件操作，比如打开或者删除目录
      */
     CurveFilesystemAdaptor(std::shared_ptr<FilePool> filePool,
-                                  std::shared_ptr<LocalFileSystem> lfs);
+                           std::shared_ptr<LocalFileSystem> lfs);
     CurveFilesystemAdaptor();
     virtual ~CurveFilesystemAdaptor();
 
@@ -94,16 +85,16 @@ class CurveFilesystemAdaptor : public braft::PosixFileSystemAdaptor {
      *             int _fd;
      *          };
      */
-    virtual braft::FileAdaptor* open(const std::string& path, int oflag,
+    braft::FileAdaptor* open(const std::string& path, int oflag,
                               const ::google::protobuf::Message* file_meta,
-                              butil::File::Error* e);
+                              butil::File::Error* e) override;
     /**
      * 删除path对应的文件或目录
      * @param: path是待删除的文件路径
      * @param: recursive是否递归删除
      * @return: 成功返回true，否则返回false
      */
-    virtual bool delete_file(const std::string& path, bool recursive);
+    bool delete_file(const std::string& path, bool recursive) override;
 
     /**
      * rename到新路径
@@ -118,12 +109,52 @@ class CurveFilesystemAdaptor : public braft::PosixFileSystemAdaptor {
      * @param: old_path旧文件路径
      * @param: new_path新文件路径
      */
-    virtual bool rename(const std::string& old_path,
-                       const std::string& new_path);
+    bool rename(const std::string& old_path,
+                       const std::string& new_path) override;
+
+    bool link(const std::string& old_path,
+              const std::string& new_path) override;
+
+    bool create_directory(const std::string& path,
+                                  butil::File::Error* error,
+                                  bool create_parent_directories) override;
+
+    bool path_exists(const std::string& path) override;
+
+    bool directory_exists(const std::string& path) override;
+
+    braft::DirReader* directory_reader(const std::string& path) override;
+
+    bool CreateDirectoryAndGetError(
+        const FilePath& full_path,
+        File::Error* error,
+        bool create_parents);
 
     // 设置过滤哪些文件，这些文件不从chunkfilepool取
     // 回收的时候也直接删除这些文件，不进入chunkfilepool
-    void SetFilterList(const std::vector<std::string>& filter);
+    void SetFilterList(const std::list<std::string>& filter);
+
+    /**
+     * @brief add files which is not get from chunkfilepool to filterlist
+     *
+     * @param files files to add
+     */
+    void AddToFilterList(std::list<std::string> *files);
+
+    /**
+     * @brief add a file which is not get from chunkfilepool to filterlist
+     *
+     * @param file file to add
+     */
+    void AddToFilterList(const std::string& file);
+
+    std::shared_ptr<FilePool> GetFilePool() const {
+        return filePool_;
+    }
+
+    std::shared_ptr<LocalFileSystem> GetLocalFileSystem() const {
+        return lfs_;
+    }
 
  private:
    /**
@@ -146,12 +177,39 @@ class CurveFilesystemAdaptor : public braft::PosixFileSystemAdaptor {
     std::shared_ptr<LocalFileSystem> lfs_;
     // 操作chunkfilepool的指针，这个FilePool_与copysetnode的
     // chunkfilepool_应该是全局唯一的，保证操作chunkfilepool的原子性
-    std::shared_ptr<FilePool> chunkFilePool_;
+    std::shared_ptr<FilePool> filePool_;
     // 过滤名单，在当前vector中的文件名，都不从chunkfilepool中取文件
     // 回收的时候也直接删除这些文件，不进入chunkfilepool
-    std::vector<std::string> filterList_;
+    std::list<std::string> filterList_;
 };
+
+class CurveDirReader : public braft::DirReader {
+    friend class CurveFilesystemAdaptor;
+ public:
+    virtual ~CurveDirReader();
+
+    // Check if the dir reader is valid
+    virtual bool is_valid() const;
+
+    // Move to next entry in the directory
+    // Return true if a entry can be found, false otherwise
+    virtual bool next();
+
+    // Get the name of current entry
+    virtual const char* name() const;
+
+ protected:
+    CurveDirReader(const std::string& path,
+        const std::shared_ptr<LocalFileSystem> &lfs);
+
+ private:
+    std::string path_;
+    std::shared_ptr<LocalFileSystem> lfs_;
+    DIR *dir_;
+    struct dirent *dirIter_;
+};
+
 }  // namespace chunkserver
 }  // namespace curve
 
-#endif  // SRC_CHUNKSERVER_RAFTSNAPSHOT_CURVE_FILESYSTEM_ADAPTOR_H_
+#endif  // SRC_CHUNKSERVER_FILESYSTEM_ADAPTOR_CURVE_FILESYSTEM_ADAPTOR_H_

src/chunkserver/heartbeat.cpp

@@ -261,10 +261,13 @@ int Heartbeat::BuildRequest(HeartbeatRequest* req) {
                                 * walSegmentFileSize;
     uint64_t trashedChunkSize = metric->GetChunkTrashedCount() * chunkFileSize;
     uint64_t leftChunkSize = metric->GetChunkLeftCount() * chunkFileSize;
+
     // leftWalSegmentSize will be 0 when CHUNK and WAL share file pool
     uint64_t leftWalSegmentSize = metric->GetWalSegmentLeftCount()
                                 * walSegmentFileSize;
-
+    uint64_t chunkPoolSize = options_.chunkFilePool->Size() *
+                options_.chunkFilePool->GetFilePoolOpt().fileSize;
+    stats->set_chunkfilepoolsize(chunkPoolSize);
     stats->set_chunksizeusedbytes(usedChunkSize+usedWalSegmentSize);
     stats->set_chunksizeleftbytes(leftChunkSize+leftWalSegmentSize);
     stats->set_chunksizetrashedbytes(trashedChunkSize);

src/chunkserver/heartbeat.h

@@ -71,6 +71,7 @@ struct HeartbeatOptions {
     ScanManager*            scanManager;
 
     std::shared_ptr<LocalFileSystem> fs;
+    std::shared_ptr<FilePool> chunkFilePool;
 };
 
 /**

src/chunkserver/op_request.cpp

@@ -395,22 +395,16 @@ static void ReadBufferDeleter(void* ptr) {
 }
 
 void ReadChunkRequest::ReadChunk() {
-    char *readBuffer = nullptr;
     size_t size = request_->size();
 
-    readBuffer = new(std::nothrow)char[size];
-    CHECK(nullptr != readBuffer)
-        << "new readBuffer failed " << strerror(errno);
-
+    butil::IOPortal readBuffer;
     auto ret = datastore_->ReadChunk(request_->chunkid(),
                                      request_->sn(),
-                                     readBuffer,
+                                     &readBuffer,
                                      request_->offset(),
                                      size);
-    butil::IOBuf wrapper;
-    wrapper.append_user_data(readBuffer, size, ReadBufferDeleter);
     if (CSErrorCode::Success == ret) {
-        cntl_->response_attachment().append(wrapper);
+        cntl_->response_attachment().append(readBuffer);
         response_->set_status(CHUNK_OP_STATUS::CHUNK_OP_STATUS_SUCCESS);
     } else if (CSErrorCode::ChunkNotExistError == ret) {
         response_->set_status(
@@ -512,7 +506,6 @@ void WriteChunkRequest::OnApplyFromLog(std::shared_ptr<CSDataStore> datastore,
                             request.clonefileoffset());
     }
 
-
     auto ret = datastore->WriteChunk(request.chunkid(),
                                      request.sn(),
                                      data,

src/chunkserver/raftlog/BUILD

@@ -33,6 +33,7 @@ cc_library(
         "//external:gflags",
         "//external:glog",
         "//external:protobuf",
+        "//src/chunkserver/filesystem_adaptor:filesystem_adaptor",
         "//src/chunkserver/datastore:chunkserver_datastore",
     ],
 )

src/chunkserver/raftlog/curve_segment.cpp

@@ -46,6 +46,10 @@
 #include "src/chunkserver/raftlog/curve_segment.h"
 #include "src/chunkserver/raftlog/define.h"
 
+namespace braft {
+DECLARE_bool(raft_use_fsync_rather_than_fdatasync);
+}
+
 namespace curve {
 namespace chunkserver {
 
@@ -71,15 +75,13 @@ int CurveSegment::create() {
         LOG(ERROR) << "Get segment from chunk file pool fail!";
         return -1;
     }
-    _fd = ::open(path.c_str(), O_RDWR|O_NOATIME, 0644);
-    if (_fd >= 0) {
-        butil::make_close_on_exec(_fd);
-    } else {
+    _fd = _lfs->Open(path, O_RDWR|O_NOATIME);
+    if (_fd < 0) {
         LOG(ERROR) << "Open path: " << path << " fail, error: "
                    << strerror(errno);
         return -1;
     }
-    res = ::lseek(_fd, _meta_page_size, SEEK_SET);
+    res = _lfs->Lseek(_fd, _meta_page_size, SEEK_SET);
     if (res != _meta_page_size) {
         LOG(ERROR) << "lseek fail! error: " << strerror(errno);
         return -1;
@@ -87,10 +89,9 @@ int CurveSegment::create() {
     LOG_IF(INFO, _fd >= 0) << "Created new segment `" << path
                            << "' with fd=" << _fd;
     if (FLAGS_enableWalDirectWrite) {
-        _direct_fd = ::open(path.c_str(), O_RDWR|O_NOATIME|O_DIRECT, 0644);
+        _direct_fd = _lfs->Open(path, O_RDWR|O_NOATIME|O_DIRECT);
         LOG_IF(FATAL, _direct_fd < 0) << "failed to open file with O_DIRECT"
                                          ", error: " << strerror(errno);
-        butil::make_close_on_exec(_direct_fd);
     }
     _meta.bytes += _meta_page_size;
     _update_meta_page();
@@ -126,16 +127,14 @@ int CurveSegment::load(braft::ConfigurationManager* configuration_manager) {
         butil::string_appendf(&path, "/" CURVE_SEGMENT_CLOSED_PATTERN,
                              _first_index, _last_index.load());
     }
-    _fd = ::open(path.c_str(), O_RDWR|O_NOATIME);
+    _fd = _lfs->Open(path.c_str(), O_RDWR|O_NOATIME);
     if (_fd < 0) {
         LOG(ERROR) << "Fail to open " << path << ", " << berror();
         return -1;
     }
-    butil::make_close_on_exec(_fd);
     if (FLAGS_enableWalDirectWrite) {
-        _direct_fd = ::open(path.c_str(), O_RDWR|O_NOATIME|O_DIRECT);
+        _direct_fd = _lfs->Open(path.c_str(), O_RDWR|O_NOATIME|O_DIRECT);
         LOG_IF(FATAL, _direct_fd < 0) << "failed to open file with O_DIRECT";
-        butil::make_close_on_exec(_direct_fd);
     }
 
     // load meta page
@@ -220,7 +219,7 @@ int CurveSegment::load(braft::ConfigurationManager* configuration_manager) {
     }
 
     // seek to end, for opening segment
-    ::lseek(_fd, entry_off, SEEK_SET);
+    _lfs->Lseek(_fd, entry_off, SEEK_SET);
 
     _meta.bytes = entry_off;
     return ret;
@@ -228,7 +227,7 @@ int CurveSegment::load(braft::ConfigurationManager* configuration_manager) {
 
 int CurveSegment::_load_meta() {
     char* metaPage = new char[_meta_page_size];
-    int res = ::pread(_fd, metaPage, _meta_page_size, 0);
+    int res = _lfs->Read(_fd, metaPage, 0, _meta_page_size);
     if (res != _meta_page_size) {
         delete metaPage;
         return -1;
@@ -306,7 +305,7 @@ int CurveSegment::_load_entry(off_t offset, EntryHeader* head,
                               butil::IOBuf* data, size_t size_hint) const {
     butil::IOPortal buf;
     size_t to_read = std::max(size_hint, kEntryHeaderSize);
-    const ssize_t n = braft::file_pread(&buf, _fd, offset, to_read);
+    const ssize_t n = _lfs->Read(_fd, &buf, offset, to_read);
     if (n != (ssize_t)to_read) {
         return n < 0 ? -1 : 1;
     }
@@ -346,7 +345,7 @@ int CurveSegment::_load_entry(off_t offset, EntryHeader* head,
         if (buf.length() < kEntryHeaderSize + data_real_len) {
             const size_t to_read = kEntryHeaderSize + data_real_len
                                                     - buf.length();
-            const ssize_t n = braft::file_pread(&buf, _fd,
+            const ssize_t n = _lfs->Read(_fd, &buf,
                                     offset + buf.length(), to_read);
             if (n != (ssize_t)to_read) {
                 return n < 0 ? -1 : 1;
@@ -433,7 +432,7 @@ int CurveSegment::append(const braft::LogEntry* entry) {
                   _checksum_type, write_buf, kEntryHeaderSize - 4));
     if (FLAGS_enableWalDirectWrite) {
         data.copy_to(write_buf + kEntryHeaderSize, real_length);
-        int ret = ::pwrite(_direct_fd, write_buf, to_write, _meta.bytes);
+        int ret = _lfs->Write(_direct_fd, write_buf, _meta.bytes, to_write);
         free(write_buf);
         if (ret != to_write) {
             LOG(ERROR) << "Fail to write directly to fd=" << _direct_fd;
@@ -477,9 +476,9 @@ int CurveSegment::_update_meta_page() {
     memset(metaPage, 0, _meta_page_size);
     memcpy(metaPage, &_meta.bytes, sizeof(_meta.bytes));
     if (FLAGS_enableWalDirectWrite) {
-        ret = ::pwrite(_direct_fd, metaPage, _meta_page_size, 0);
+        ret = _lfs->Write(_direct_fd, metaPage, 0, _meta_page_size);
     } else {
-        ret = ::pwrite(_fd, metaPage, _meta_page_size, 0);
+        ret = _lfs->Write(_fd, metaPage, 0, _meta_page_size);
     }
     free(metaPage);
     if (ret != _meta_page_size) {
@@ -604,12 +603,16 @@ int CurveSegment::close(bool will_sync) {
     if (_last_index > _first_index) {
         if (FLAGS_raftSyncSegments && will_sync &&
                                 !FLAGS_enableWalDirectWrite) {
-            ret = braft::raft_fsync(_fd);
+            if (braft::FLAGS_raft_use_fsync_rather_than_fdatasync) {
+                ret = _lfs->Fsync(_fd);
+            } else {
+                ret = _lfs->Fdatasync(_fd);
+            }
         }
     }
     if (ret == 0) {
         _is_open = false;
-        const int rc = ::rename(old_path.c_str(), new_path.c_str());
+        const int rc = _lfs->Rename(old_path.c_str(), new_path.c_str());
         LOG_IF(INFO, rc == 0) << "Renamed `" << old_path
                               << "' to `" << new_path <<'\'';
         LOG_IF(ERROR, rc != 0) << "Fail to rename `" << old_path
@@ -625,7 +628,11 @@ int CurveSegment::sync(bool will_sync) {
         // CHECK(_is_open);
         if (!FLAGS_enableWalDirectWrite && braft::FLAGS_raft_sync
                                             && will_sync) {
-            return braft::raft_fsync(_fd);
+            if (braft::FLAGS_raft_use_fsync_rather_than_fdatasync) {
+                return _lfs->Fsync(_fd);
+            } else {
+                return _lfs->Fdatasync(_fd);
+            }
         } else {
             return 0;
         }
@@ -676,7 +683,7 @@ int CurveSegment::truncate(const int64_t last_index_kept) {
         std::string new_path(_path);
         butil::string_appendf(&new_path, "/" CURVE_SEGMENT_OPEN_PATTERN,
                               _first_index);
-        int ret = ::rename(old_path.c_str(), new_path.c_str());
+        int ret = _lfs->Rename(old_path.c_str(), new_path.c_str());
         LOG_IF(INFO, ret == 0) << "Renamed `" << old_path << "' to `"
                                << new_path << '\'';
         LOG_IF(ERROR, ret != 0) << "Fail to rename `" << old_path << "' to `"
@@ -694,7 +701,7 @@ int CurveSegment::truncate(const int64_t last_index_kept) {
     }
 
     // seek fd
-    off_t ret_off = ::lseek(_fd, truncate_size, SEEK_SET);
+    off_t ret_off = _lfs->Lseek(_fd, truncate_size, SEEK_SET);
     if (ret_off < 0) {
         PLOG(ERROR) << "Fail to lseek fd=" << _fd << " to size="
                     << truncate_size << " path: " << _path;

src/chunkserver/raftlog/curve_segment.h

@@ -70,22 +70,27 @@ class BAIDU_CACHELINE_ALIGNMENT CurveSegment:
           public Segment {
  public:
     CurveSegment(const std::string& path, const int64_t first_index,
-                 int checksum_type, std::shared_ptr<FilePool> walFilePool)
+                 int checksum_type,
+                 const std::shared_ptr<FilePool> &walFilePool,
+                 const std::shared_ptr<LocalFileSystem> &lfs)
         : _path(path), _meta(CurveSegmentMeta()),
         _fd(-1), _direct_fd(-1), _is_open(true),
         _first_index(first_index), _last_index(first_index - 1),
         _checksum_type(checksum_type),
         _walFilePool(walFilePool),
+        _lfs(lfs),
         _meta_page_size(walFilePool->GetFilePoolOpt().metaPageSize) {
     }
     CurveSegment(const std::string& path, const int64_t first_index,
                  const int64_t last_index, int checksum_type,
-                 std::shared_ptr<FilePool> walFilePool)
+                 const std::shared_ptr<FilePool> &walFilePool,
+                 const std::shared_ptr<LocalFileSystem> &lfs)
         : _path(path), _meta(CurveSegmentMeta()),
         _fd(-1), _direct_fd(-1), _is_open(false),
         _first_index(first_index), _last_index(last_index),
         _checksum_type(checksum_type),
         _walFilePool(walFilePool),
+        _lfs(lfs),
         _meta_page_size(walFilePool->GetFilePoolOpt().metaPageSize) {
     }
     ~CurveSegment() {
@@ -173,6 +178,7 @@ class BAIDU_CACHELINE_ALIGNMENT CurveSegment:
     int _checksum_type;
     std::vector<std::pair<int64_t, int64_t> > _offset_and_term;
     std::shared_ptr<FilePool> _walFilePool;
+    std::shared_ptr<LocalFileSystem> _lfs;
     uint32_t _meta_page_size;
 };
 

src/chunkserver/raftlog/curve_segment_log_storage.cpp

@@ -48,10 +48,10 @@ namespace curve {
 namespace chunkserver {
 
 LogStorageOptions StoreOptForCurveSegmentLogStorage(
-    LogStorageOptions options) {
+    LogStorageOptions *options) {
     static LogStorageOptions options_;
-    if (nullptr != options.walFilePool) {
-        options_ = options;
+    if (nullptr != options) {
+        options_ = *options;
     }
 
     return options_;
@@ -67,7 +67,7 @@ int CurveSegmentLogStorage::init(
                     braft::ConfigurationManager* configuration_manager) {
     butil::FilePath dir_path(_path);
     butil::File::Error e;
-    if (!butil::CreateDirectoryAndGetError(
+    if (!_cfsAdaptor->CreateDirectoryAndGetError(
                 dir_path, &e, braft::FLAGS_raft_create_parent_directories)) {
         LOG(ERROR) << "Fail to create " << dir_path.value() << " : " << e;
         return -1;
@@ -83,6 +83,9 @@ int CurveSegmentLogStorage::init(
                 << "Use murmurhash32 as the checksum type of appending entries";
     }
 
+    std::string logMeta(BRAFT_SEGMENT_META_FILE);
+    _cfsAdaptor->AddToFilterList(logMeta);
+
     int ret = 0;
     bool is_empty = false;
     do {
@@ -120,7 +123,7 @@ int CurveSegmentLogStorage::load_meta() {
     std::string meta_path(_path);
     meta_path.append("/" BRAFT_SEGMENT_META_FILE);
 
-    braft::ProtoBufFile pb_file(meta_path);
+    braft::ProtoBufFile pb_file(meta_path, _cfsAdaptor);
     braft::LogPBMeta meta;
     if (0 != pb_file.load(&meta)) {
         PLOG_IF(ERROR, errno != ENOENT)
@@ -138,25 +141,25 @@ int CurveSegmentLogStorage::load_meta() {
 }
 
 int CurveSegmentLogStorage::list_segments(bool is_empty) {
-    butil::DirReaderPosix dir_reader(_path.c_str());
-    if (!dir_reader.IsValid()) {
+    braft::DirReader* dir_reader = _cfsAdaptor->directory_reader(_path.c_str());
+    if (!dir_reader->is_valid()) {
         LOG(WARNING) << "directory reader failed, maybe NOEXIST or PERMISSION."
                      << " path: " << _path;
         return -1;
     }
 
     // restore segment meta
-    while (dir_reader.Next()) {
+    while (dir_reader->next()) {
         // unlink unneed segments and unfinished unlinked segments
-        if ((is_empty && 0 == strncmp(dir_reader.name(),
+        if ((is_empty && 0 == strncmp(dir_reader->name(),
                                         "log_", strlen("log_"))) ||
-            (0 == strncmp(dir_reader.name() +
-                        (strlen(dir_reader.name()) - strlen(".tmp")),
+            (0 == strncmp(dir_reader->name() +
+                        (strlen(dir_reader->name()) - strlen(".tmp")),
                         ".tmp", strlen(".tmp")))) {
             std::string segment_path(_path);
             segment_path.append("/");
-            segment_path.append(dir_reader.name());
-            ::unlink(segment_path.c_str());
+            segment_path.append(dir_reader->name());
+            _lfs->Delete(segment_path.c_str());
 
             LOG(WARNING) << "unlink unused segment, path: " << segment_path;
 
@@ -167,7 +170,7 @@ int CurveSegmentLogStorage::list_segments(bool is_empty) {
         int match = 0;
         int64_t first_index = 0;
         int64_t last_index = 0;
-        match = sscanf(dir_reader.name(), BRAFT_SEGMENT_CLOSED_PATTERN,
+        match = sscanf(dir_reader->name(), BRAFT_SEGMENT_CLOSED_PATTERN,
                        &first_index, &last_index);
         if (match == 2) {
             LOG(INFO) << "restore closed segment, path: " << _path
@@ -179,7 +182,7 @@ int CurveSegmentLogStorage::list_segments(bool is_empty) {
             continue;
         }
 
-        match = sscanf(dir_reader.name(), BRAFT_SEGMENT_OPEN_PATTERN,
+        match = sscanf(dir_reader->name(), BRAFT_SEGMENT_OPEN_PATTERN,
                        &first_index);
         if (match == 1) {
             BRAFT_VLOG << "restore open segment, path: " << _path
@@ -196,7 +199,7 @@ int CurveSegmentLogStorage::list_segments(bool is_empty) {
         }
 
         // Is curve log pattern
-        match = sscanf(dir_reader.name(), CURVE_SEGMENT_CLOSED_PATTERN,
+        match = sscanf(dir_reader->name(), CURVE_SEGMENT_CLOSED_PATTERN,
                        &first_index, &last_index);
         if (match == 2) {
             LOG(INFO) << "restore closed segment, path: " << _path
@@ -205,12 +208,13 @@ int CurveSegmentLogStorage::list_segments(bool is_empty) {
             CurveSegment* segment = new CurveSegment(_path, first_index,
                                                      last_index,
                                                      _checksum_type,
-                                                     _walFilePool);
+                                                     _walFilePool,
+                                                     _lfs);
             _segments[first_index] = segment;
             continue;
         }
 
-        match = sscanf(dir_reader.name(), CURVE_SEGMENT_OPEN_PATTERN,
+        match = sscanf(dir_reader->name(), CURVE_SEGMENT_OPEN_PATTERN,
                        &first_index);
         if (match == 1) {
             BRAFT_VLOG << "restore open segment, path: " << _path
@@ -218,7 +222,8 @@ int CurveSegmentLogStorage::list_segments(bool is_empty) {
             if (!_open_segment) {
                 _open_segment =
                     new CurveSegment(_path, first_index, _checksum_type,
-                                     _walFilePool);
+                                     _walFilePool,
+                                     _lfs);
                 continue;
             } else {
                 LOG(WARNING) << "open segment conflict, path: " << _path
@@ -488,7 +493,7 @@ int CurveSegmentLogStorage::save_meta(const int64_t log_index) {
 
     braft::LogPBMeta meta;
     meta.set_first_log_index(log_index);
-    braft::ProtoBufFile pb_file(meta_path);
+    braft::ProtoBufFile pb_file(meta_path, _cfsAdaptor);
     int ret = pb_file.save(&meta, braft::raft_sync_meta());
 
     timer.stop();
@@ -678,13 +683,13 @@ void CurveSegmentLogStorage::sync() {
 
 braft::LogStorage* CurveSegmentLogStorage::new_instance(
     const std::string& uri) const {
-    LogStorageOptions options = StoreOptForCurveSegmentLogStorage(
-        LogStorageOptions());
+    LogStorageOptions options = StoreOptForCurveSegmentLogStorage(nullptr);
 
-    CHECK(nullptr != options.walFilePool) << "wal file pool is null";
+    CHECK(nullptr != options.curveFileSystemAdaptor)
+        << "curveFileSystemAdaptor is null";
 
     CurveSegmentLogStorage* logStorage = new CurveSegmentLogStorage(
-        uri, true, options.walFilePool);
+        uri, true, options.curveFileSystemAdaptor);
     options.monitorMetricCb(logStorage);
 
     return logStorage;
@@ -697,7 +702,8 @@ scoped_refptr<Segment> CurveSegmentLogStorage::open_segment(
         BAIDU_SCOPED_LOCK(_mutex);
         if (!_open_segment) {
             _open_segment = new CurveSegment(_path, last_log_index() + 1,
-                                             _checksum_type, _walFilePool);
+                                             _checksum_type,
+                                             _walFilePool, _lfs);
             if (_open_segment->create() != 0) {
                 _open_segment = NULL;
                 return NULL;
@@ -715,7 +721,8 @@ scoped_refptr<Segment> CurveSegmentLogStorage::open_segment(
             if (prev_open_segment->close(_enable_sync) == 0) {
                 BAIDU_SCOPED_LOCK(_mutex);
                 _open_segment = new CurveSegment(_path, last_log_index() + 1,
-                                                 _checksum_type, _walFilePool);
+                                                 _checksum_type,
+                                                 _walFilePool, _lfs);
                 if (_open_segment->create() == 0) {
                     // success
                     break;

src/chunkserver/raftlog/curve_segment_log_storage.h

@@ -56,6 +56,10 @@
 #include "src/chunkserver/raftlog/segment.h"
 #include "src/chunkserver/raftlog/curve_segment.h"
 #include "src/chunkserver/raftlog/braft_segment.h"
+#include "src/chunkserver/filesystem_adaptor/curve_filesystem_adaptor.h"
+
+using butil::FilePath;
+using butil::File;
 
 namespace curve {
 namespace chunkserver {
@@ -63,13 +67,15 @@ namespace chunkserver {
 class CurveSegmentLogStorage;
 
 struct LogStorageOptions {
-    std::shared_ptr<FilePool> walFilePool;
+    scoped_refptr<CurveFilesystemAdaptor> curveFileSystemAdaptor;
     std::function<void(CurveSegmentLogStorage*)> monitorMetricCb;
 
     LogStorageOptions() = default;
-    LogStorageOptions(std::shared_ptr<FilePool> walFilePool,
+    LogStorageOptions(
+        const scoped_refptr<CurveFilesystemAdaptor> &curveFileSystemAdaptor,
         std::function<void(CurveSegmentLogStorage*)> monitorMetricCb)
-        : walFilePool(walFilePool), monitorMetricCb(monitorMetricCb) {
+        : curveFileSystemAdaptor(curveFileSystemAdaptor),
+          monitorMetricCb(monitorMetricCb) {
     }
 };
 
@@ -82,7 +88,7 @@ struct LogStorageStatus {
 };
 
 LogStorageOptions StoreOptForCurveSegmentLogStorage(
-    LogStorageOptions options);
+    LogStorageOptions *options);
 
 void RegisterCurveSegmentLogStorageOrDie();
 
@@ -100,13 +106,30 @@ class CurveSegmentLogStorage : public braft::LogStorage {
 
     explicit CurveSegmentLogStorage(const std::string& path,
         bool enable_sync = true,
-        std::shared_ptr<FilePool> walFilePool = nullptr)
+        const scoped_refptr<CurveFilesystemAdaptor> &cfsAdaptor = nullptr)
         : _path(path)
         , _first_log_index(1)
         , _last_log_index(0)
         , _checksum_type(0)
         , _enable_sync(enable_sync)
+        , _cfsAdaptor(cfsAdaptor)
+        , _walFilePool(cfsAdaptor->GetFilePool())
+        , _lfs(cfsAdaptor->GetLocalFileSystem())
+    {}
+
+    CurveSegmentLogStorage(const std::string& path,
+        bool enable_sync,
+        const scoped_refptr<CurveFilesystemAdaptor> &cfsAdaptor,
+        const std::shared_ptr<FilePool> &walFilePool,
+        const std::shared_ptr<LocalFileSystem> &lfs)
+        : _path(path)
+        , _first_log_index(1)
+        , _last_log_index(0)
+        , _checksum_type(0)
+        , _enable_sync(enable_sync)
+        , _cfsAdaptor(cfsAdaptor)
         , _walFilePool(walFilePool)
+        , _lfs(lfs)
     {}
 
     CurveSegmentLogStorage()
@@ -114,7 +137,9 @@ class CurveSegmentLogStorage : public braft::LogStorage {
         , _last_log_index(0)
         , _checksum_type(0)
         , _enable_sync(true)
+        , _cfsAdaptor(nullptr)
         , _walFilePool(nullptr)
+        , _lfs(nullptr)
     {}
 
     virtual ~CurveSegmentLogStorage() {}
@@ -185,9 +210,12 @@ class CurveSegmentLogStorage : public braft::LogStorage {
     braft::raft_mutex_t _mutex;
     SegmentMap _segments;
     scoped_refptr<Segment> _open_segment;
-    std::shared_ptr<FilePool> _walFilePool;
     int _checksum_type;
     bool _enable_sync;
+
+    scoped_refptr<CurveFilesystemAdaptor> _cfsAdaptor;
+    std::shared_ptr<FilePool> _walFilePool;
+    std::shared_ptr<LocalFileSystem> _lfs;
 };
 
 }  // namespace chunkserver

src/chunkserver/raftsnapshot/BUILD

@@ -38,6 +38,7 @@ cc_library(
         "//external:leveldb",
         "//external:protobuf",
         "//proto:chunkserver-cc-protos",
+        "//src/chunkserver/filesystem_adaptor:filesystem_adaptor",
         "//src/chunkserver/datastore:chunkserver_datastore",
     ],
 )

src/chunkserver/register.cpp

@@ -64,6 +64,14 @@ int Register::RegisterToMDS(const ChunkServerMetadata *localMetadata,
         req.set_externalip(ops_.chunkserverExternalIp);
     }
     req.set_port(ops_.chunkserverPort);
+    uint64_t chunkPoolSize = ops_.chunkFilepool->Size() *
+                ops_.chunkFilepool->GetFilePoolOpt().fileSize;
+    req.set_chunkfilepoolsize(chunkPoolSize);
+    if (ops_.chunkFilepool->GetFilePoolOpt().getFileFromPool) {
+        req.set_usechunkfilepoolaswalpool(ops_.useChunkFilePoolAsWalPool);
+        req.set_usechunkfilepoolaswalpoolreserve(
+            ops_.useChunkFilePoolAsWalPoolReserve);
+    }
 
     if (localMetadata != nullptr) {
         req.set_chunkserverid(localMetadata->id());
@@ -74,7 +82,8 @@ int Register::RegisterToMDS(const ChunkServerMetadata *localMetadata,
               << ". internal ip: " << ops_.chunkserverInternalIp
               << ", port: " << ops_.chunkserverPort
               << ", enable external server: " << ops_.enableExternalServer
-              << ", external ip: " << ops_.chunkserverExternalIp;
+              << ", external ip: " << ops_.chunkserverExternalIp
+              << ", size =" << chunkPoolSize;
 
     int retries = ops_.registerRetries;
     while (retries >= 0) {

src/chunkserver/register.h

@@ -29,6 +29,7 @@
 #include "src/fs/local_filesystem.h"
 #include "proto/chunkserver.pb.h"
 #include "src/chunkserver/epoch_map.h"
+#include "src/chunkserver/datastore/file_pool.h"
 
 using ::curve::fs::LocalFileSystem;
 
@@ -49,6 +50,10 @@ struct RegisterOptions {
     int registerRetries;
     int registerTimeout;
 
+    uint32_t useChunkFilePoolAsWalPoolReserve;
+    bool useChunkFilePoolAsWalPool;
+    std::shared_ptr<FilePool> chunkFilepool;
+
     std::shared_ptr<LocalFileSystem> fs;
 };
 

src/chunkserver/spdk_hook.cpp

@@ -0,0 +1,127 @@
+/*
+ *  Copyright (c) 2020 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Created Date: 20221103
+ * Author: xuchaojie
+ */
+
+#include <glog/logging.h>
+
+#include "src/chunkserver/spdk_hook.h"
+
+void glog_pfs_func(int level, const char *file, const char *func, int line,
+    const char *fmt, va_list ap) {
+    char buf[8192];
+
+    int glevel = google::GLOG_INFO;
+    vsnprintf(buf, sizeof(buf), fmt, ap);
+    switch (level) {
+    case PFS_TRACE_FATAL:
+        glevel = google::GLOG_FATAL;
+        google::LogMessage(file, line, glevel).stream() << buf;
+        abort();
+        break;
+    case PFS_TRACE_ERROR:
+        glevel = google::GLOG_ERROR;
+        break;
+    case PFS_TRACE_WARN:
+        glevel = google::GLOG_WARNING;
+        break;
+    case PFS_TRACE_INFO:
+    case PFS_TRACE_DBG:
+    case PFS_TRACE_VERB:
+    default:
+        glevel = google::GLOG_INFO;
+        break;
+    }
+    google::LogMessage(file, line, glevel).stream() << buf;
+}
+
+void glog_spdk_func(int level, const char *a_file, const int a_line,
+    const char *func, const char *fmt, va_list ap) {
+    char buf[8192];
+    const char *file = a_file;
+    int line = a_line;
+
+    if (file == NULL) {
+        file = "<spdk>";
+        if (a_line <= 0)
+            line = 1;
+    }
+
+    int glevel = google::GLOG_INFO;
+    vsnprintf(buf, sizeof(buf), fmt, ap);
+    switch (level) {
+    case SPDK_LOG_ERROR:
+        glevel = google::GLOG_ERROR;
+        break;
+    case SPDK_LOG_WARN:
+        glevel = google::GLOG_WARNING;
+        break;
+    case SPDK_LOG_NOTICE:
+    case SPDK_LOG_INFO:
+        glevel = google::GLOG_INFO;
+        break;
+#ifndef NDEBUG
+    case SPDK_LOG_DEBUG:
+        glevel = google::GLOG_INFO;
+        break;
+#endif
+    }
+    google::LogMessage(file, line, glevel).stream() << buf;
+}
+
+ssize_t glog_dpdk_log_func(void *cookie, const char *buf, size_t size) {
+    int level = rte_log_cur_msg_loglevel();
+    int glevel = google::GLOG_INFO;
+
+    switch (level) {
+    case RTE_LOG_EMERG:
+    case RTE_LOG_ALERT:
+    case RTE_LOG_CRIT:
+    case RTE_LOG_ERR:
+        glevel = google::GLOG_ERROR;
+        break;
+    case RTE_LOG_WARNING:
+        glevel = google::GLOG_WARNING;
+        break;
+    case RTE_LOG_NOTICE:
+    case RTE_LOG_INFO:
+    default:
+        glevel = google::GLOG_INFO;
+        break;
+#ifndef NDEBUG
+    case RTE_LOG_DEBUG:
+        glevel = google::GLOG_INFO;
+        return 0;
+#endif
+    }
+    google::LogMessage("<dpdk>", 0, glevel).stream().write(buf, size);
+    if (level == RTE_LOG_EMERG)
+        abort();
+    return size;
+}
+
+void* dpdk_mem_allocate(size_t align, size_t sz) {
+    /* rte_malloc seems fast enough, otherwise we need to use mempool */
+    return rte_malloc("iobuf", sz, align);
+}
+
+void dpdk_mem_free(void* p) {
+    rte_free(p);
+}

src/chunkserver/spdk_hook.h

@@ -0,0 +1,48 @@
+/*
+ *  Copyright (c) 2020 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Created Date: 20221103
+ * Author: xuchaojie
+ */
+
+#ifndef SRC_CHUNKSERVER_SPDK_HOOK_H_
+#define SRC_CHUNKSERVER_SPDK_HOOK_H_
+
+#include <spdk/log.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <pfs_api.h>
+#include <pfs_trace_func.h>
+#include <pfsd.h>
+#include <pfsd_sdk.h>
+#include <pfsd_sdk_log.h>
+#include <pfs_spdk_api.h>
+
+void glog_pfs_func(int level, const char *file, const char *func, int line,
+    const char *fmt, va_list ap);
+
+void glog_spdk_func(int level, const char *a_file, const int a_line,
+    const char *func, const char *fmt, va_list ap);
+
+ssize_t glog_dpdk_log_func(void *cookie, const char *buf, size_t size);
+
+void* dpdk_mem_allocate(size_t align, size_t sz);
+
+void dpdk_mem_free(void* p);
+
+#endif  // SRC_CHUNKSERVER_SPDK_HOOK_H_

src/fs/BUILD

@@ -18,18 +18,40 @@ load("//:copts.bzl", "CURVE_DEFAULT_COPTS")
 
 cc_library(
     name = "lfs",
-    srcs = glob([
-                "*.cpp",
-                "ext4_filesystem_impl.h",
-                "ext4_util.h",
-                "wrap_posix.h"
-           ]),
-    hdrs = ["local_filesystem.h","fs_common.h"],
+    srcs = ["ext4_filesystem_impl.cpp",
+            "local_filesystem.cpp",
+            "wrap_posix.cpp",
+    ] + select({
+        "//:with_spdk": ["pfs_filesystem_impl.cpp"],
+        "//conditions:default": [],
+    }),
+    hdrs = ["ext4_filesystem_impl.h",
+            "fs_common.h",
+            "local_filesystem.h",
+            "wrap_posix.h",
+    ] + select({
+        "//:with_spdk": ["pfs_filesystem_impl.h"],
+        "//conditions:default": [],
+    }),
+    copts = CURVE_DEFAULT_COPTS + select({
+        "//:with_spdk": ["-DWITH_SPDK", "-Iexternal/pfs"],
+        "//conditions:default": [],
+    }),
+    linkopts = select({
+        "//:with_spdk": [ "-L/usr/local/curvestore/pfsd/lib", "-lpfsd" ],
+        "//conditions:default": [],
+    }) + [
+        "-std=c++11",
+    ],
     deps = [
-                "//src/common:curve_common",
-                "//external:glog",
-                "//external:butil",
-            ],
+        "//src/common:curve_common",
+        "//external:glog",
+        "//external:butil",
+    ] + select({
+        "//:with_spdk": ["@pfs//:headers"],
+        "//conditions:default": [],
+    }),
     visibility = ["//visibility:public"],
-    copts = CURVE_DEFAULT_COPTS,
 )
+
+

src/fs/ext4_filesystem_impl.cpp

@@ -185,7 +185,7 @@ int Ext4FileSystemImpl::Delete(const string& path) {
     return rc;
 }
 
-int Ext4FileSystemImpl::Mkdir(const string& dirName) {
+int Ext4FileSystemImpl::Mkdir(const string& dirName, bool create_parents) {
     vector<string> names;
     ::curve::common::SplitString(dirName, "/", &names);
 
@@ -193,6 +193,14 @@ int Ext4FileSystemImpl::Mkdir(const string& dirName) {
     if (0 == names.size())
         return 0;
 
+    if (!create_parents) {
+        if (posixWrapper_->mkdir(dirName.c_str(), 0755) < 0) {
+            LOG(WARNING) << "mkdir " << dirName
+                << " failed. "<< strerror(errno);
+            return -errno;
+        }
+    }
+
     string path;
     for (size_t i = 0; i < names.size(); ++i) {
         if (0 == i && dirName[0] != '/')  // 相对路径
@@ -227,6 +235,10 @@ bool Ext4FileSystemImpl::FileExists(const string& filePath) {
         return false;
 }
 
+bool Ext4FileSystemImpl::PathExists(const string& path) {
+    return posixWrapper_->access(path.c_str(), F_OK) == 0;
+}
+
 int Ext4FileSystemImpl::DoRename(const string& oldPath,
                                  const string& newPath,
                                  unsigned int flags) {
@@ -269,6 +281,19 @@ int Ext4FileSystemImpl::List(const string& dirName,
     return -errno;
 }
 
+DIR* Ext4FileSystemImpl::OpenDir(const string& dirPath) {
+    return posixWrapper_->opendir(dirPath.c_str());
+}
+
+struct dirent* Ext4FileSystemImpl::ReadDir(DIR *dir) {
+    return posixWrapper_->readdir(dir);
+}
+
+int Ext4FileSystemImpl::CloseDir(DIR *dir) {
+    posixWrapper_->closedir(dir);
+    return -errno;
+}
+
 int Ext4FileSystemImpl::Read(int fd,
                              char *buf,
                              uint64_t offset,
@@ -303,6 +328,30 @@ int Ext4FileSystemImpl::Read(int fd,
     return length - remainLength;
 }
 
+int Ext4FileSystemImpl::Read(int fd, butil::IOPortal* portal,
+             uint64_t offset, int length) {
+    off_t orig_offset = offset;
+    ssize_t left = length;
+    int retryTimes = 0;
+    while (left > 0) {
+        ssize_t read_len = portal->pappend_from_file_descriptor(
+                fd, offset, static_cast<size_t>(left));
+        if (read_len > 0) {
+            left -= read_len;
+            offset += read_len;
+        } else if (read_len == 0) {
+            break;
+        } else if (errno == EINTR && retryTimes < 3) {
+            ++retryTimes;
+            continue;
+        } else {
+            LOG(ERROR) << "pread failed: " << strerror(errno);
+            return -errno;
+        }
+    }
+    return length - left;
+}
+
 int Ext4FileSystemImpl::Write(int fd,
                               const char *buf,
                               uint64_t offset,
@@ -357,7 +406,12 @@ int Ext4FileSystemImpl::Write(int fd,
     return length;
 }
 
-int Ext4FileSystemImpl::Sync(int fd) {
+int Ext4FileSystemImpl::WriteZero(int fd, uint64_t offset, int length) {
+    LOG(ERROR) << "not supprted!";
+    return -1;
+}
+
+int Ext4FileSystemImpl::Fdatasync(int fd) {
     int rc = posixWrapper_->fdatasync(fd);
     if (rc < 0) {
         LOG(ERROR) << "fdatasync failed: " << strerror(errno);
@@ -403,5 +457,18 @@ int Ext4FileSystemImpl::Fsync(int fd) {
     return 0;
 }
 
+off_t Ext4FileSystemImpl::Lseek(int fd, off_t offset, int whence) {
+    off_t rc = posixWrapper_->lseek(fd, offset, whence);
+    if (rc < 0) {
+        LOG(ERROR) << "lseek failed: " << strerror(errno);
+    }
+    return rc;
+}
+
+int Ext4FileSystemImpl::Link(const std::string &oldPath,
+    const std::string &newPath) {
+    return ::link(oldPath.c_str(), newPath.c_str());
+}
+
 }  // namespace fs
 }  // namespace curve

src/fs/ext4_filesystem_impl.h

@@ -48,19 +48,33 @@ class Ext4FileSystemImpl : public LocalFileSystem {
     int Open(const string& path, int flags) override;
     int Close(int fd) override;
     int Delete(const string& path) override;
-    int Mkdir(const string& dirPath) override;
+    int Mkdir(const string& dirPath, bool create_parents = true) override;
     bool DirExists(const string& dirPath) override;
     bool FileExists(const string& filePath) override;
+    bool PathExists(const string& path) override;
     int List(const string& dirPath, vector<std::string>* names) override;
+    DIR* OpenDir(const string& dirPath) override;
+    struct dirent* ReadDir(DIR *dir) override;
+    int CloseDir(DIR *dir) override;
     int Read(int fd, char* buf, uint64_t offset, int length) override;
+    int Read(int fd, butil::IOPortal* portal,
+             uint64_t offset, int length) override;
     int Write(int fd, const char* buf, uint64_t offset, int length) override;
     int Write(int fd, butil::IOBuf buf, uint64_t offset, int length) override;
-    int Sync(int fd) override;
+    int WriteZero(int fd, uint64_t offset, int length) override;
+    int WriteZeroIfSupport(int fd, const char* buf,
+            uint64_t offset, int length) override {
+        return Write(fd, buf, offset, length);
+    }
+    int Fdatasync(int fd) override;
     int Append(int fd, const char* buf, int length) override;
     int Fallocate(int fd, int op, uint64_t offset,
                   int length) override;
     int Fstat(int fd, struct stat* info) override;
     int Fsync(int fd) override;
+    off_t Lseek(int fd, off_t offset, int whence) override;
+    int Link(const std::string &oldPath,
+        const std::string &newPath) override;
 
  private:
     explicit Ext4FileSystemImpl(std::shared_ptr<PosixWrapper>);

src/fs/fs_common.h

@@ -23,14 +23,31 @@
 #ifndef SRC_FS_FS_COMMON_H_
 #define SRC_FS_FS_COMMON_H_
 
+#include <string>
+
 namespace curve {
 namespace fs {
 
 enum class FileSystemType {
     // SFS,
     EXT4,
+    PFS,
+    UNKOWN,
 };
 
+inline FileSystemType StringToFileSystemType(
+    const std::string &typeStr) {
+    FileSystemType type;
+    if (typeStr == "ext4") {
+        type = FileSystemType::EXT4;
+    } else if (typeStr == "pfs") {
+        type = FileSystemType::PFS;
+    } else {
+        type = FileSystemType::UNKOWN;
+    }
+    return type;
+}
+
 struct FileSystemInfo {
     uint64_t total = 0;         // Total bytes
     uint64_t available = 0;     // Free bytes available for unprivileged users

src/fs/local_filesystem.cpp

@@ -25,6 +25,9 @@
 #include "src/fs/local_filesystem.h"
 #include "src/fs/ext4_filesystem_impl.h"
 #include "src/fs/wrap_posix.h"
+#ifdef WITH_SPDK
+#include "src/fs/pfs_filesystem_impl.h"
+#endif
 
 namespace curve {
 namespace fs {
@@ -33,10 +36,21 @@ std::shared_ptr<LocalFileSystem> LocalFsFactory::CreateFs(
     FileSystemType type,
     const std::string& deviceID) {
     std::shared_ptr<LocalFileSystem> localFs;
-    if (type == FileSystemType::EXT4) {
+    switch (type) {
+    case FileSystemType::EXT4:
         localFs = Ext4FileSystemImpl::getInstance();
-    } else {
-        LOG(ERROR) << "Unknown filesystem type.";
+        break;
+    case FileSystemType::PFS:
+#ifdef WITH_SPDK
+        localFs = PfsFileSystemImpl::getInstance();
+        break;
+#else
+        LOG(ERROR) << "PFS filesystem type is not be complied, "
+            "maybe you need to recompile after adding "
+            "the flag \"--define with_spdk=true\" ";
+#endif
+    default:
+        LOG(ERROR) << "Unknown filesystem type";
         return nullptr;
     }
     return localFs;

src/fs/local_filesystem.h

@@ -23,6 +23,7 @@
 #ifndef SRC_FS_LOCAL_FILESYSTEM_H_
 #define SRC_FS_LOCAL_FILESYSTEM_H_
 
+#include <dirent.h>
 #include <inttypes.h>
 #include <assert.h>
 #include <sys/stat.h>
@@ -44,13 +45,22 @@ namespace curve {
 namespace fs {
 
 struct LocalFileSystemOption {
+    FileSystemType type;
     bool enableRenameat2;
+#ifdef WITH_SPDK
+    std::string pfs_cluster;
+    std::string pfs_pbd_name;
+    int pfs_host_id;
+    std::string spdk_nvme_controller;
+    LocalFileSystemOption() : enableRenameat2(false), pfs_host_id(2) {}
+#else
     LocalFileSystemOption() : enableRenameat2(false) {}
+#endif
 };
 
 class LocalFileSystem {
  public:
-     LocalFileSystem() {}
+    LocalFileSystem() {}
     virtual ~LocalFileSystem() {}
 
     /**
@@ -99,7 +109,7 @@ class LocalFileSystem {
      * @param dirPath: 目录路径
      * @return 成功返回0
      */
-    virtual int Mkdir(const string& dirPath) = 0;
+    virtual int Mkdir(const string& dirPath, bool create_parents = true) = 0;
 
     /**
      * 判断目录是否存在
@@ -115,6 +125,15 @@ class LocalFileSystem {
      */
     virtual bool FileExists(const string& filePath) = 0;
 
+    /**
+     * @brief Check if the path exists
+     *
+     * @param path: path
+     *
+     * @return true for exists, false for not
+     */
+    virtual bool PathExists(const string& path) = 0;
+
     /**
      * 重命名文件/目录
      * 将文件或目录重命名或者移到其他路径,不会覆盖已存在的文件
@@ -140,6 +159,33 @@ class LocalFileSystem {
      */
     virtual int List(const string& dirPath, vector<std::string>* names) = 0;
 
+    /**
+     * @brief  open the dir
+     *
+     * @param dirPath: dir path
+     *
+     * @return DIR
+     */
+    virtual DIR* OpenDir(const string& dirPath) = 0;
+
+    /**
+     * @brief read dir
+     *
+     * @param dir: DIR
+     *
+     * @return dirent
+     */
+    virtual struct dirent* ReadDir(DIR *dir) = 0;
+
+    /**
+     * @brief close the dir
+     *
+     * @param dir: DIR
+     *
+     * @return 0 for success, -1 for failed
+     */
+    virtual int CloseDir(DIR *dir) = 0;
+
     /**
      * 从文件指定区域读取数据
      * @param fd：文件句柄id，通过Open接口获取
@@ -150,6 +196,20 @@ class LocalFileSystem {
      */
     virtual int Read(int fd, char* buf, uint64_t offset, int length) = 0;
 
+
+    /**
+     * @brief  read for [offset, length]
+     *
+     * @param fd: fd
+     * @param portal: data recevied
+     * @param offset: read start offset
+     * @param length: read length
+     *
+     * @return return read length when success, return -errno when failed.
+     */
+    virtual int Read(int fd, butil::IOPortal* portal,
+             uint64_t offset, int length) = 0;
+
     /**
      * 向文件指定区域写入数据
      * @param fd：文件句柄id，通过Open接口获取
@@ -171,14 +231,40 @@ class LocalFileSystem {
     virtual int Write(int fd, butil::IOBuf buf, uint64_t offset,
                       int length) = 0;
 
+    /**
+     * @brief write zero to [offset, length]
+     *
+     * @param fd:  fd
+     * @param offset: start offset
+     * @param length: write length
+     *
+     * @return return actutal write length when success,
+     *  return -errno when failed.
+     */
+    virtual int WriteZero(int fd, uint64_t offset, int length) = 0;
+
+    /**
+     * @brief WriteZero if support, otherwise fallback to Write
+     *
+     * @param fd:  fd
+     * @param buf: the buffer filled with zero
+     * @param offset: start offset
+     * @param length: write length
+     *
+     * @return return actutal write length when success,
+     *  return -errno when failed.
+     */
+    virtual int WriteZeroIfSupport(int fd, const char* buf,
+            uint64_t offset, int length) = 0;
+
     /**
      * @brief sync one fd
      *
      * @param fd : file descriptor
      *
-     * @return succcess return 0, otherwsie reutrn -1
+     * @return succcess return 0, otherwise return -1
      */
-    virtual int Sync(int fd) = 0;
+    virtual int Fdatasync(int fd) = 0;
 
     /**
      * 向文件末尾追加数据
@@ -215,6 +301,28 @@ class LocalFileSystem {
      */
     virtual int Fsync(int fd) = 0;
 
+    /**
+     * @brief lseek
+     *
+     * @param fd: fd
+     * @param offset: lseek offset
+     * @param whence: lseek hence
+     *
+     * @return offset of file pointer
+     */
+    virtual off_t Lseek(int fd, off_t offset, int whence) = 0;
+
+    /**
+     * @brief link
+     *
+     * @param oldPath: link source
+     * @param newPath: link dest
+     *
+     * @return 0 for success, -1 for failed.
+     */
+    virtual int Link(const std::string &oldPath,
+        const std::string &newPath) = 0;
+
  private:
     virtual int DoRename(const string& /* oldPath */,
                          const string& /* newPath */,

src/fs/pfs_filesystem_impl.cpp

@@ -0,0 +1,566 @@
+/*
+ *  Copyright (c) 2022 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * File Created: 2022-06-01
+ * Author: xuchaojie
+ */
+
+#include "src/fs/pfs_filesystem_impl.h"
+
+#include <glog/logging.h>
+#include <sys/vfs.h>
+#include <sys/utsname.h>
+#include <linux/version.h>
+#include <dirent.h>
+#include <brpc/server.h>
+#include <butil/iobuf.h>
+
+#include <pfs_api.h>
+#include <pfs_option_api.h>
+#include <pfs_spdk_api.h>
+#include <sys/uio.h>
+
+#include <string>
+#include <vector>
+
+#include "src/common/string_util.h"
+
+
+#define NVME_ALIGN 8
+#define SECTOR_SIZE 512
+
+namespace curve {
+namespace fs {
+
+std::string DumpIOVec(const struct iovec *vec, size_t nvec) {
+    std::stringstream ss;
+    ss << "{";
+    for (size_t i = 0; i < nvec; ++i) {
+        ss << "[" << uintptr_t(vec[i].iov_base) << ","
+           << vec[i].iov_len << "]";
+    }
+    ss << "}";
+    return ss.str();
+}
+
+/**
+ * @brief The hook function used by iobuf to dispatch preadv using dma or not
+ *
+ * @param fd: fd
+ * @param vector: io vector
+ * @param nvec: size of io vector
+ * @param offset: io offset
+ *
+ * @return return size when success, -1 for failed
+ */
+ssize_t preadv_dispatch(int fd, const struct iovec *vector,
+                        int nvec, off_t offset) {
+    if (PFS_FD_ISVALID(fd)) {
+        int ret = pfs_preadv_dma(fd, vector, nvec, offset);
+        if (ret < 0) {
+            LOG(ERROR) << "preadv_dispatch failed, ret: " << ret
+                       << ", fd: " << fd
+                       << ", nvec: " << nvec
+                       << ", offset: " << offset
+                       << ", iovec: " << DumpIOVec(vector, nvec);
+        }
+        return ret;
+    }
+    return preadv(fd, vector, nvec, offset);
+}
+
+/**
+ * @brief The hook function used by iobuf to dispatch pwritev using dma or not
+ *
+ * @param fd: fd
+ * @param vector: io vector
+ * @param nvec: size of io vector
+ * @param offset io offset
+ *
+ * @return return size when success, -1 for failed
+ */
+ssize_t pwritev_dispatch(int fd, const struct iovec *vector,
+                         int nvec, off_t offset) {
+    if (PFS_FD_ISVALID(fd)) {
+        // TODO(xuchaojie): use pfs_pwritev_dma in future
+        int ret = pfs_pwritev(fd, vector, nvec, offset);
+        if (ret < 0) {
+            LOG(ERROR) << "pwritev_dispatch failed, ret: " << ret
+                       << ", fd: " << fd
+                       << ", nvec: " << nvec
+                       << ", offset: " << offset
+                       << ", iovec: " << DumpIOVec(vector, nvec);
+        }
+        return ret;
+    }
+    return pwritev(fd, vector, nvec, offset);
+}
+
+/**
+ * @brief The hook function used by iobuf to dispatch readv using dma or not
+ *
+ * @param fd: fd
+ * @param vector: io vector
+ * @param nvec: size of io vector
+ *
+ * @return return size when success, -1 for failed
+ */
+ssize_t readv_dispatch(int fd, const struct iovec *vector, int nvec) {
+    if (PFS_FD_ISVALID(fd)) {
+        int ret = pfs_readv_dma(fd, vector, nvec);
+        if (ret < 0) {
+            LOG(ERROR) << "readv_dispatch failed, ret: " << ret
+                       << ", fd: " << fd
+                       << ", nvec: " << nvec
+                       << ", iovec: " << DumpIOVec(vector, nvec);
+        }
+        return ret;
+    }
+    return readv(fd, vector, nvec);
+}
+
+/**
+ * @brief The hook function used by iobuf to dispatch writev using dma or not
+ *
+ * @param fd: fd
+ * @param vector: io vector
+ * @param nvec: size of io vector
+ *
+ * @return return size when success, -1 for failed
+ */
+ssize_t writev_dispatch(int fd, const struct iovec *vector, int nvec) {
+    if (PFS_FD_ISVALID(fd)) {
+        // TODO(xuchaojie): use pfs_writev_dma in future
+        int ret = pfs_writev(fd, vector, nvec);
+        if (ret < 0) {
+            LOG(ERROR) << "pwritev_dispatch failed, ret: " << ret
+                       << ", fd: " << fd
+                       << ", nvec: " << nvec
+                       << ", iovec: " << DumpIOVec(vector, nvec);
+        }
+        return ret;
+    }
+    return writev(fd, vector, nvec);
+}
+
+void HookIOBufIOFuncs() {
+    butil::iobuf::iobuf_io_funcs iofuncs;
+    iofuncs.iof_preadv = preadv_dispatch;
+    iofuncs.iof_pwritev = pwritev_dispatch;
+    iofuncs.iof_readv = readv_dispatch;
+    iofuncs.iof_writev = writev_dispatch;
+    butil::iobuf::set_external_io_funcs(iofuncs);
+}
+
+std::shared_ptr<PfsFileSystemImpl> PfsFileSystemImpl::self_ = nullptr;
+std::mutex PfsFileSystemImpl::mutex_;
+
+std::shared_ptr<PfsFileSystemImpl> PfsFileSystemImpl::getInstance() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    if (self_ == nullptr) {
+        self_ = std::shared_ptr<PfsFileSystemImpl>(
+                new(std::nothrow) PfsFileSystemImpl());
+        CHECK(self_ != nullptr) << "Failed to new pfs.";
+    }
+    return self_;
+}
+
+int PfsFileSystemImpl::Init(const LocalFileSystemOption& option) {
+    int rc = pfs_mount_acquire(option.pfs_cluster.c_str(),
+        option.pfs_pbd_name.c_str(),
+        option.pfs_host_id, PFS_RDWR|MNTFLG_PFSD_INTERNAL);
+    if (rc) {
+        LOG(ERROR) << "can not mount pfs"
+                   << ", cluster: " << option.pfs_cluster
+                   << ", host_id: " << option.pfs_host_id
+                   << ", pbd name: " << option.pfs_pbd_name;
+        return rc;
+    }
+    return 0;
+}
+
+int PfsFileSystemImpl::Statfs(const string& path, struct FileSystemInfo* info) {
+    struct statfs diskInfo;
+    int rc = pfs_statfs(path.c_str(), &diskInfo);
+    if (rc < 0) {
+        LOG(WARNING) << "pfs_statfs failed: " << strerror(errno)
+                   << ", past: " << path;
+        return -errno;
+    }
+    info->total = diskInfo.f_blocks * diskInfo.f_bsize;
+    info->available = diskInfo.f_bavail * diskInfo.f_bsize;
+    info->stored = info->total - diskInfo.f_bfree * diskInfo.f_bsize;
+    info->allocated = info->stored;
+    return 0;
+}
+
+int PfsFileSystemImpl::Open(const string& path, int flags) {
+    int fd = pfs_open(path.c_str(), flags, 0644);
+    if (fd < 0) {
+        LOG(WARNING) << "pfs_open failed: " << strerror(errno)
+                     << ", path: " << path;
+        return -errno;
+    }
+    return fd;
+}
+
+int PfsFileSystemImpl::Close(int fd) {
+    int rc = pfs_close(fd);
+    if (rc < 0) {
+        LOG(ERROR) << "close failed: " << strerror(errno)
+                   << ", fd: " << fd;
+        return -errno;
+    }
+    return rc;
+}
+
+int PfsFileSystemImpl::Delete(const string& path) {
+    int rc = 0;
+    if (DirExists(path)) {
+        std::vector<std::string> names;
+        rc = List(path, &names);
+        if (rc < 0) {
+            LOG(WARNING) << "List " << path << " failed.";
+            return rc;
+        }
+        for (auto &name : names) {
+            string subPath = path + "/" + name;
+            rc = Delete(subPath);
+            if (rc < 0) {
+                LOG(WARNING) << "Delete " << subPath << " failed.";
+                return rc;
+            }
+        }
+    }
+    rc = pfs_unlink(path.c_str());
+    if (rc < 0) {
+        LOG(WARNING) << "pfs_unlink failed: " << strerror(errno)
+                     << ", path: " << path;
+        return -errno;
+    }
+    return rc;
+}
+
+int PfsFileSystemImpl::Mkdir(const string& dirPath, bool create_parents) {
+    std::vector<std::string> names;
+    ::curve::common::SplitString(dirPath, "/", &names);
+
+    // root dir must exists
+    if (0 == names.size())
+        return 0;
+
+    if (!create_parents) {
+        if (pfs_mkdir(dirPath.c_str(), 0755) < 0) {
+            LOG(WARNING) << "pfs_mkdir " << dirPath
+                         << " failed: "<< strerror(errno);
+            return -errno;
+        }
+    }
+
+    std::string path;
+    for (size_t i = 0; i < names.size(); ++i) {
+        if (0 == i && dirPath[0] != '/') {
+            path = path + names[i];
+        } else {
+            path = path + "/" + names[i];
+        }
+        if (DirExists(path))
+            continue;
+        if (pfs_mkdir(path.c_str(), 0755) < 0) {
+            LOG(WARNING) << "pfs_mkdir " << path
+                         << " failed: "<< strerror(errno);
+            return -errno;
+        }
+    }
+    return 0;
+}
+
+bool PfsFileSystemImpl::DirExists(const string& dirPath) {
+    struct stat path_stat;
+    if (0 == pfs_stat(dirPath.c_str(), &path_stat))
+        return S_ISDIR(path_stat.st_mode);
+    else
+        return false;
+}
+
+bool PfsFileSystemImpl::FileExists(const string& filePath) {
+    struct stat path_stat;
+    if (0 == pfs_stat(filePath.c_str(), &path_stat))
+        return S_ISREG(path_stat.st_mode);
+    else
+        return false;
+}
+
+bool PfsFileSystemImpl::PathExists(const string& path) {
+    return pfs_access(path.c_str(), F_OK) == 0;
+}
+
+int PfsFileSystemImpl::DoRename(const string& oldPath,
+                   const string& newPath,
+                   unsigned int flags) {
+    int rc = pfs_rename2(oldPath.c_str(), newPath.c_str(), flags);
+    if (rc < 0) {
+        LOG(WARNING) << "rename failed: " << strerror(errno)
+                     << ". old path: " << oldPath
+                     << ", new path: " << newPath
+                     << ", flag: " << flags;
+        return -errno;
+    }
+    return 0;
+}
+
+int PfsFileSystemImpl::List(const string& dirPath, vector<std::string>* names) {
+    DIR *dir = pfs_opendir(dirPath.c_str());
+    if (nullptr == dir) {
+        LOG(WARNING) << "pfs_opendir:" << dirPath
+                     << " failed:" << strerror(errno);
+        return -errno;
+    }
+    struct dirent *dirIter;
+    errno = 0;
+    while ((dirIter = pfs_readdir(dir)) != nullptr) {
+        if (strcmp(dirIter->d_name, ".") == 0
+                || strcmp(dirIter->d_name, "..") == 0)
+            continue;
+        names->push_back(dirIter->d_name);
+    }
+    if (errno != 0) {
+        LOG(WARNING) << "readdir failed: " << strerror(errno);
+    }
+    pfs_closedir(dir);
+    return -errno;
+}
+
+DIR* PfsFileSystemImpl::OpenDir(const string& dirPath) {
+    return pfs_opendir(dirPath.c_str());
+}
+
+struct dirent* PfsFileSystemImpl::ReadDir(DIR *dir) {
+    return pfs_readdir(dir);
+}
+
+int PfsFileSystemImpl::CloseDir(DIR *dir) {
+    pfs_closedir(dir);
+    return -errno;
+}
+
+int PfsFileSystemImpl::Read(int fd, char* buf, uint64_t offset, int length) {
+    int remainLength = length;
+    int relativeOffset = 0;
+    int retryTimes = 0;
+    while (remainLength > 0) {
+        int ret = pfs_pread(fd,
+                            buf + relativeOffset,
+                            remainLength,
+                            offset);
+        if (ret == 0) {
+            LOG(WARNING) << "pread returns zero."
+                         << "offset: " << offset
+                         << ", length: " << remainLength;
+            break;
+        }
+        if (ret < 0) {
+            if (errno == EINTR && retryTimes < 3) {
+                ++retryTimes;
+                continue;
+            }
+            LOG(ERROR) << "pread failed: " << strerror(errno);
+            return -errno;
+        }
+        remainLength -= ret;
+        offset += ret;
+        relativeOffset += ret;
+    }
+    return length - remainLength;
+}
+
+int PfsFileSystemImpl::Read(int fd, butil::IOPortal* portal,
+             uint64_t offset, int length) {
+    off_t orig_offset = offset;
+    ssize_t left = length;
+    int retryTimes = 0;
+    while (left > 0) {
+        ssize_t read_len = portal->pappend_from_dev_descriptor(
+                fd, offset, static_cast<size_t>(left));
+        if (read_len > 0) {
+            left -= read_len;
+            offset += read_len;
+        } else if (read_len == 0) {
+            break;
+        } else if (errno == EINTR && retryTimes < 3) {
+            ++retryTimes;
+            continue;
+        } else {
+            LOG(ERROR) << "pread failed: " << strerror(errno);
+            return -errno;
+        }
+    }
+    return length - left;
+}
+
+int PfsFileSystemImpl::Write(int fd, const char* buf,
+                             uint64_t offset, int length) {
+    int remainLength = length;
+    int relativeOffset = 0;
+    int retryTimes = 0;
+    while (remainLength > 0) {
+        int ret = pfs_pwrite(fd,
+                             buf + relativeOffset,
+                             remainLength,
+                             offset);
+        if (ret < 0) {
+            if (errno == EINTR && retryTimes < 3) {
+                ++retryTimes;
+                continue;
+            }
+            LOG(ERROR) << "pwrite failed: " << strerror(errno);
+            return -errno;
+        }
+        remainLength -= ret;
+        offset += ret;
+        relativeOffset += ret;
+    }
+    return length;
+}
+
+int PfsFileSystemImpl::Write(int fd, butil::IOBuf buf, uint64_t offset,
+    int length) {
+    int remainLength = length;
+    int relativeOffset = 0;
+    int retryTimes = 0;
+
+    while (remainLength > 0) {
+        ssize_t ret = buf.pcut_into_file_descriptor(fd, offset, remainLength);
+        if (ret < 0) {
+            if (errno == EINTR || retryTimes < 3) {
+                ++retryTimes;
+                continue;
+            }
+            LOG(ERROR) << "IOBuf::pcut_into_file_descriptor failed: "
+                       << strerror(errno)
+                       << ", fd: " << fd
+                       << ", offset: " << offset
+                       << ", length: " << length;
+            return -errno;
+        }
+
+        remainLength -= ret;
+        offset += ret;
+    }
+
+    return length;
+}
+
+int PfsFileSystemImpl::WriteZero(int fd, uint64_t offset, int length) {
+    int remainLength = length;
+    int relativeOffset = 0;
+    int retryTimes = 0;
+    while (remainLength > 0) {
+        int ret = pfs_pwrite_zero(fd, remainLength, offset);
+        if (ret < 0) {
+            if (errno == EINTR && retryTimes < 3) {
+                ++retryTimes;
+                continue;
+            }
+            LOG(ERROR) << "pwrite zero failed: " << strerror(errno);
+            return -errno;
+        }
+        remainLength -= ret;
+        offset += ret;
+        relativeOffset += ret;
+    }
+    return length;
+}
+
+int PfsFileSystemImpl::Fdatasync(int fd) {
+    int rc = pfs_fsync(fd);
+    if (rc < 0) {
+        LOG(ERROR) << "fsync failed: " << strerror(errno);
+        return -errno;
+    }
+    return 0;
+}
+
+int PfsFileSystemImpl::Append(int fd, const char* buf, int length) {
+    // not implemented
+    return 0;
+}
+
+int PfsFileSystemImpl::Fallocate(int fd, int op, uint64_t offset, int length) {
+    int rc = pfs_fallocate(fd, op, offset, length);
+    if (rc < 0) {
+        LOG(ERROR) << "fallocate failed: " << strerror(errno);
+        return -errno;
+    }
+    return 0;
+}
+
+int PfsFileSystemImpl::Fstat(int fd, struct stat* info) {
+    int rc = pfs_fstat(fd, info);
+    if (rc < 0) {
+        LOG(ERROR) << "fstat failed: " << strerror(errno);
+        return -errno;
+    }
+    return 0;
+}
+
+int PfsFileSystemImpl::Fsync(int fd) {
+    int rc = pfs_fsync(fd);
+    if (rc < 0) {
+        LOG(ERROR) << "fsync failed: " << strerror(errno);
+        return -errno;
+    }
+    return 0;
+}
+
+off_t PfsFileSystemImpl::Lseek(int fd, off_t offset, int whence) {
+    off_t rc = pfs_lseek(fd, offset, whence);
+    if (rc < 0) {
+        LOG(ERROR) << "lseek failed: " << strerror(errno);
+    }
+    return rc;
+}
+
+int PfsFileSystemImpl::Link(const std::string &oldPath,
+    const std::string &newPath) {
+    LOG(ERROR) << "PfsFileSystem do not support link!";
+    return -1;
+}
+
+int InitPfsOption(const LocalFileSystemOption& option) {
+    int rc = pfs_option_set("spdk_nvme_controller",
+        option.spdk_nvme_controller.c_str());
+    if (rc) {
+        LOG(ERROR) << "pfs_option_set spdk_nvme_controller failed, rc: " << rc;
+        return rc;
+    }
+    rc = pfs_option_set("spdk_env_context", "--in-memory");
+    if (rc) {
+        LOG(ERROR) << "pfs_option_set spdk_env_context failed, rc: " << rc;
+        return rc;
+    }
+    rc = pfs_option_set("pbd_name", option.pfs_pbd_name.c_str());
+    if (rc) {
+        LOG(ERROR) << "pfs_option_set pbd_name failed, rc: " << rc;
+        return rc;
+    }
+    return 0;
+}
+
+}  // namespace fs
+}  // namespace curve

src/fs/pfs_filesystem_impl.h

@@ -0,0 +1,121 @@
+/*
+ *  Copyright (c) 2022 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * File Created: 2022-06-01
+ * Author: xuchaojie
+ */
+
+#ifndef SRC_FS_PFS_FILESYSTEM_IMPL_H_
+#define SRC_FS_PFS_FILESYSTEM_IMPL_H_
+
+#include <err.h>
+#include <pfs_api.h>
+
+#include <vector>
+#include <string>
+
+#include "src/fs/local_filesystem.h"
+
+namespace curve {
+namespace fs {
+
+void HookIOBufIOFuncs();
+
+class PfsFileSystemImpl : public LocalFileSystem {
+ public:
+    virtual ~PfsFileSystemImpl() {}
+
+    static std::shared_ptr<PfsFileSystemImpl> getInstance();
+
+    int Init(const LocalFileSystemOption& option) override;
+
+    int Statfs(const string& path, struct FileSystemInfo* info) override;
+
+    int Open(const string& path, int flags) override;
+
+    int Close(int fd) override;
+
+    int Delete(const string& path) override;
+
+    int Mkdir(const string& dirPath, bool create_parents = true) override;
+
+    bool DirExists(const string& dirPath) override;
+
+    bool FileExists(const string& filePath) override;
+
+    bool PathExists(const string& path) override;
+
+    int List(const string& dirPath, vector<std::string>* names) override;
+
+    DIR* OpenDir(const string& dirPath) override;
+
+    struct dirent* ReadDir(DIR *dir) override;
+
+    int CloseDir(DIR *dir) override;
+
+    int Read(int fd, char* buf, uint64_t offset, int length) override;
+
+    int Read(int fd, butil::IOPortal* portal,
+             uint64_t offset, int length) override;
+
+    int Write(int fd, const char* buf, uint64_t offset, int length) override;
+
+    int Write(int fd, butil::IOBuf buf, uint64_t offset, int length) override;
+
+    int WriteZero(int fd, uint64_t offset, int length) override;
+
+    int WriteZeroIfSupport(int fd, const char* buf,
+            uint64_t offset, int length) override {
+        return WriteZero(fd, offset, length);
+    }
+
+    int Fdatasync(int fd) override;
+
+    int Append(int fd, const char* buf, int length) override;
+
+    int Fallocate(int fd, int op, uint64_t offset, int length) override;
+
+    int Fstat(int fd, struct stat* info) override;
+
+    int Fsync(int fd) override;
+
+    off_t Lseek(int fd, off_t offset, int whence) override;
+
+    int Link(const std::string &oldPath,
+        const std::string &newPath) override;
+
+ private:
+    PfsFileSystemImpl() {}
+
+    int DoRename(const string& oldPath,
+                 const string& newPath,
+                 unsigned int flags) override;
+
+ private:
+    static std::shared_ptr<PfsFileSystemImpl> self_;
+    static std::mutex mutex_;
+};
+
+int InitPfsOption(const LocalFileSystemOption& option);
+
+}  // namespace fs
+}  // namespace curve
+
+
+
+#endif  // SRC_FS_PFS_FILESYSTEM_IMPL_H_

src/fs/wrap_posix.cpp

@@ -129,5 +129,13 @@ int PosixWrapper::uname(struct utsname *buf) {
     return ::uname(buf);
 }
 
+int PosixWrapper::access(const char *path, int amode) {
+    return ::access(path, amode);
+}
+
+off_t PosixWrapper::lseek(int fd, off_t offset, int whence) {
+    return ::lseek(fd, offset, whence);
+}
+
 }  // namespace fs
 }  // namespace curve

src/fs/wrap_posix.h

@@ -66,6 +66,8 @@ class PosixWrapper {
     virtual int fsync(int fd);
     virtual int statfs(const char *path, struct statfs *buf);
     virtual int uname(struct utsname *buf);
+    virtual int access(const char *path, int amode);
+    virtual off_t lseek(int fd, off_t offset, int whence);
 };
 
 }  // namespace fs

src/mds/heartbeat/heartbeat_manager.cpp

@@ -133,6 +133,9 @@ void HeartbeatManager::UpdateChunkServerStatistics(
         stat.chunkSizeUsedBytes = request.stats().chunksizeusedbytes();
         stat.chunkSizeLeftBytes = request.stats().chunksizeleftbytes();
         stat.chunkSizeTrashedBytes = request.stats().chunksizetrashedbytes();
+        if (request.stats().has_chunkfilepoolsize()) {
+            stat.chunkFilepoolSize = request.stats().chunkfilepoolsize();
+        }
 
         for (int i = 0; i < request.copysetinfos_size(); i++) {
             CopysetStat cstat;

src/mds/server/mds.cpp

@@ -367,7 +367,10 @@ void MDS::InitTopologyServiceManager(const TopologyOption& option) {
         std::make_shared<ChunkServerRegistInfoBuilderImpl>(&kCurveFS);
     topologyServiceManager_ =
         std::make_shared<TopologyServiceManager>(topology_,
-        copysetManager, registerInfoBuilder);
+        topologyStat_,
+        topologyChunkAllocator_,
+        copysetManager,
+        registerInfoBuilder);
     topologyServiceManager_->Init(option);
     LOG(INFO) << "init topologyServiceManager success.";
 }
@@ -389,9 +392,12 @@ void MDS::InitCopysetOption(CopysetOption *copysetOption) {
 
 
 void MDS::InitTopologyChunkAllocator(const TopologyOption& option) {
+    std::shared_ptr<ChunkFilePoolAllocHelp> chunkFilePoolAllocHelp_ =
+          std::make_shared<ChunkFilePoolAllocHelp>();
     topologyChunkAllocator_ =
           std::make_shared<TopologyChunkAllocatorImpl>(topology_,
-               segmentAllocStatistic_, option);
+               segmentAllocStatistic_, topologyStat_,
+                chunkFilePoolAllocHelp_, option);
     LOG(INFO) << "init topologyChunkAllocator success.";
 }
 

src/mds/server/mds.h

@@ -65,6 +65,7 @@ using ::curve::mds::topology::DefaultTokenGenerator;
 using ::curve::mds::topology::TopologyImpl;
 using ::curve::mds::topology::TopologyOption;
 using ::curve::mds::topology::TopologyStatImpl;
+using ::curve::mds::topology::ChunkFilePoolAllocHelp;
 using ::curve::mds::topology::TopologyMetricService;
 using ::curve::mds::topology::TopologyServiceManager;
 using ::curve::mds::copyset::CopysetManager;

src/mds/topology/topology.h

@@ -239,6 +239,8 @@ class Topology {
                                 uint32_t port,
                                 ChunkServer *out) const = 0;
 
+    virtual int GetBelongPhysicalPoolId(ChunkServerIdType csId,
+        PoolIdType *physicalPoolIdOut)  = 0;
 
     virtual std::vector<ChunkServerIdType> GetChunkServerInCluster(
         ChunkServerFilter filter = [](const ChunkServer&) {

src/mds/topology/topology_chunk_allocator.cpp

@@ -198,9 +198,19 @@ void TopologyChunkAllocatorImpl::GetRemainingSpaceInLogicalPool(
         if (!topology_->GetPhysicalPool(lPool.GetPhysicalPoolId(), &pPool)) {
             continue;
         }
-        uint64_t diskCapacity = pPool.GetDiskCapacity();
-        // calculate actual capacity available
-        diskCapacity = diskCapacity * poolUsagePercentLimit_ / 100;
+        uint64_t diskCapacity = 0;
+        double available = available_;
+        if (chunkFilePoolAllocHelp_->GetUseChunkFilepool()) {
+            topoStat_->GetChunkPoolSize(lPool.GetPhysicalPoolId(),
+                        &diskCapacity);
+            available = available *
+                chunkFilePoolAllocHelp_->GetAvailable() / 100;
+            diskCapacity = diskCapacity * available / 100;
+        } else {
+            diskCapacity = pPool.GetDiskCapacity();
+            // calculate actual capacity available
+            diskCapacity = diskCapacity * available / 100;
+        }
 
         // TODO(xuchaojie): if create more than one logical pools is supported,
         //                  the logic here need to be fixed

src/mds/topology/topology_chunk_allocator.h

@@ -32,6 +32,7 @@
 #include "proto/nameserver2.pb.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/mds/topology/topology_item.h"
+#include "src/mds/topology/topology_stat.h"
 #include "src/mds/nameserver2/allocstatistic/alloc_statistic.h"
 
 namespace curve {
@@ -45,6 +46,43 @@ enum class ChoosePoolPolicy {
     kWeight,
 };
 
+class ChunkFilePoolAllocHelp {
+ public:
+    ChunkFilePoolAllocHelp()
+    : ChunkFilePoolPoolWalReserve(0),
+      useChunkFilepool(false),
+      useChunkFilePoolAsWalPool(false) {}
+    ~ChunkFilePoolAllocHelp() {}
+    void UpdateChunkFilePoolAllocConfig(bool useChunkFilepool_,
+        bool useChunkFilePoolAsWalPool_,
+        uint32_t useChunkFilePoolAsWalPoolReserve_)  {
+        useChunkFilepool.store(useChunkFilepool_, std::memory_order_release);
+        useChunkFilePoolAsWalPool.store(useChunkFilePoolAsWalPool_,
+            std::memory_order_release);
+        ChunkFilePoolPoolWalReserve.store(useChunkFilePoolAsWalPoolReserve_,
+            std::memory_order_release);
+    }
+    bool GetUseChunkFilepool() {
+        return useChunkFilepool.load(std::memory_order_acquire);
+    }
+    // After removing the reserved space, the remaining percentage
+    uint32_t GetAvailable() {
+        if (useChunkFilePoolAsWalPool.load(std::memory_order_acquire)) {
+            return 100 - ChunkFilePoolPoolWalReserve.load(
+                    std::memory_order_acquire);
+        } else {
+            return 100;
+        }
+    }
+
+ private:
+    // use chunkfile as allocation condition
+    std::atomic<bool> useChunkFilepool;
+    std::atomic<bool> useChunkFilePoolAsWalPool;
+    // Reserve extra space for walpool
+    std::atomic<uint32_t>  ChunkFilePoolPoolWalReserve;
+};
+
 class TopologyChunkAllocator {
  public:
     TopologyChunkAllocator() {}
@@ -62,16 +100,23 @@ class TopologyChunkAllocator {
     virtual void GetRemainingSpaceInLogicalPool(
         const std::vector<PoolIdType>& logicalPools,
         std::map<PoolIdType, double>* remianingSpace) = 0;
+    virtual void UpdateChunkFilePoolAllocConfig(
+        bool useChunkFilepool_, bool useChunkFilePoolAsWalPool_,
+        uint32_t useChunkFilePoolAsWalPoolReserve_) = 0;
 };
 
 class TopologyChunkAllocatorImpl : public TopologyChunkAllocator {
  public:
     TopologyChunkAllocatorImpl(std::shared_ptr<Topology> topology,
         std::shared_ptr<AllocStatistic> allocStatistic,
+        std::shared_ptr<TopologyStat> topologyStat,
+        std::shared_ptr<ChunkFilePoolAllocHelp> ChunkFilePoolAllocHelp,
         const TopologyOption &option)
         : topology_(topology),
         allocStatistic_(allocStatistic),
-        poolUsagePercentLimit_(option.PoolUsagePercentLimit),
+        topoStat_(topologyStat),
+        chunkFilePoolAllocHelp_(ChunkFilePoolAllocHelp),
+        available_(option.PoolUsagePercentLimit),
         policy_(static_cast<ChoosePoolPolicy>(option.choosePoolPolicy)),
         enableLogicalPoolStatus_(option.enableLogicalPoolStatus) {
         std::srand(std::time(nullptr));
@@ -115,6 +160,13 @@ class TopologyChunkAllocatorImpl : public TopologyChunkAllocator {
     void GetRemainingSpaceInLogicalPool(
         const std::vector<PoolIdType>& logicalPools,
         std::map<PoolIdType, double>* remianingSpace) override;
+    void UpdateChunkFilePoolAllocConfig(bool useChunkFilepool_,
+            bool useChunkFilePoolAsWalPool_,
+            uint32_t useChunkFilePoolAsWalPoolReserve_) override {
+              chunkFilePoolAllocHelp_->UpdateChunkFilePoolAllocConfig(
+                useChunkFilepool_, useChunkFilePoolAsWalPool_,
+                useChunkFilePoolAsWalPoolReserve_);
+        }
 
  private:
     /**
@@ -135,9 +187,17 @@ class TopologyChunkAllocatorImpl : public TopologyChunkAllocator {
     // allocation statistic module
     std::shared_ptr<AllocStatistic> allocStatistic_;
 
-    // usage limit of pool
-    uint32_t poolUsagePercentLimit_;
+    // usage limit percentage of pool
+    uint32_t available_;
 
+    /**
+     * @brief topology statistic module
+     */
+    std::shared_ptr<TopologyStat> topoStat_;
+    /**
+     * @brief chunkfilepool help with capacity allocation
+     */
+    std::shared_ptr<ChunkFilePoolAllocHelp> chunkFilePoolAllocHelp_;
     /**
      * @brief starting point of round robin for (copysets of) every logical pool
      */

src/mds/topology/topology_config.h

@@ -53,7 +53,6 @@ struct TopologyOption {
           CreateCopysetRpcRetryTimes(3),
           CreateCopysetRpcRetrySleepTimeMs(500),
           UpdateMetricIntervalSec(0),
-          PoolUsagePercentLimit(100),
           choosePoolPolicy(0),
           enableLogicalPoolStatus(false) {}
 };

src/mds/topology/topology_service_manager.cpp

@@ -61,6 +61,29 @@ void TopologyServiceManager::RegistChunkServer(
     ChunkServerRegistResponse *response) {
     std::string hostIp = request->hostip();
     uint32_t port = request->port();
+    ChunkServerStat stat;
+    bool useChunkFilePoolAsWalPool = false;
+    uint32_t useChunkFilePoolAsWalPoolReserve;
+    bool useChunkFilepool = false;
+    if (request->has_chunkfilepoolsize()) {
+        stat.chunkFilepoolSize = request->chunkfilepoolsize();
+    }
+    if (request->has_usechunkfilepoolaswalpool()) {
+        useChunkFilepool = true;
+        useChunkFilePoolAsWalPool = request->usechunkfilepoolaswalpool();
+    }
+    if (request->has_usechunkfilepoolaswalpoolreserve()) {
+        useChunkFilePoolAsWalPoolReserve =
+            request->usechunkfilepoolaswalpoolreserve();
+    } else if (useChunkFilePoolAsWalPool) {
+        // Error occurs if usechunkfilepoolaswalpool is used,
+        // but no reserve percentage is set
+        LOG(WARNING) << "Received RegistChunkServer request from "
+                             << "parameter setting error, "
+                             << "chunkserverid: " << request->chunkserverid();
+        response->set_statuscode(kTopoErrCodeInvalidParam);
+        return;
+    }
     ::curve::common::NameLockGuard lock(registCsMutex,
         hostIp + ":" + std::to_string(port));
 
@@ -91,6 +114,10 @@ void TopologyServiceManager::RegistChunkServer(
             response->set_statuscode(kTopoErrCodeSuccess);
             response->set_chunkserverid(cs.GetId());
             response->set_token(cs.GetToken());
+            topoStat_->UpdateChunkServerStat(cs.GetId(), stat);
+            topologyChunkAllocator_->UpdateChunkFilePoolAllocConfig(
+                useChunkFilepool, useChunkFilePoolAsWalPool,
+                useChunkFilePoolAsWalPoolReserve);
             ::google::protobuf::Map<::google::protobuf::uint64,
                 ::google::protobuf::uint64> epochMap;
             int err = registInfoBuilder_->BuildEpochMap(&epochMap);
@@ -124,6 +151,11 @@ void TopologyServiceManager::RegistChunkServer(
         response->set_statuscode(kTopoErrCodeSuccess);
         response->set_chunkserverid(cs.GetId());
         response->set_token(cs.GetToken());
+        topoStat_->UpdateChunkServerStat(cs.GetId(),
+                    stat);
+        topologyChunkAllocator_->UpdateChunkFilePoolAllocConfig(
+            useChunkFilepool, useChunkFilePoolAsWalPool,
+            useChunkFilePoolAsWalPoolReserve);
         ::google::protobuf::Map<::google::protobuf::uint64,
             ::google::protobuf::uint64> epochMap;
         int err = registInfoBuilder_->BuildEpochMap(&epochMap);
@@ -198,6 +230,12 @@ void TopologyServiceManager::RegistChunkServer(
         response->set_statuscode(kTopoErrCodeSuccess);
         response->set_chunkserverid(chunkserver.GetId());
         response->set_token(chunkserver.GetToken());
+        ChunkServerStat stat;
+        stat.chunkFilepoolSize = request->chunkfilepoolsize();
+        topoStat_->UpdateChunkServerStat(chunkserver.GetId(), stat);
+        topologyChunkAllocator_->UpdateChunkFilePoolAllocConfig(
+                useChunkFilepool, useChunkFilePoolAsWalPool,
+                useChunkFilePoolAsWalPoolReserve);
         ::google::protobuf::Map<::google::protobuf::uint64,
             ::google::protobuf::uint64> epochMap;
         int err = registInfoBuilder_->BuildEpochMap(&epochMap);

src/mds/topology/topology_service_manager.h

@@ -32,7 +32,8 @@
 #include "src/mds/copyset/copyset_policy.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/concurrent/name_lock.h"
-
+#include "src/mds/topology/topology_stat.h"
+#include "src/mds/topology/topology_chunk_allocator.h"
 
 using curve::mds::copyset::CopysetManager;
 
@@ -52,9 +53,13 @@ class TopologyServiceManager {
  public:
     TopologyServiceManager(
         const std::shared_ptr<Topology> &topology,
+        const std::shared_ptr<TopologyStat> &topologyStat,
+        const std::shared_ptr<TopologyChunkAllocator> &topologychunkallocator,
         const std::shared_ptr<CopysetManager> &copysetManager,
         const std::shared_ptr<ChunkServerRegistInfoBuilder> &registInfoBuilder)
         : topology_(topology),
+          topoStat_(topologyStat),
+          topologyChunkAllocator_(topologychunkallocator),
           copysetManager_(copysetManager),
           registInfoBuilder_(registInfoBuilder) {}
 
@@ -240,6 +245,16 @@ class TopologyServiceManager {
      */
     std::shared_ptr<Topology> topology_;
 
+    /**
+     * @brief topology statistic module
+     */
+    std::shared_ptr<TopologyStat> topoStat_;
+
+    /**
+     * @brief topology allocation module
+     */
+    std::shared_ptr<TopologyChunkAllocator> topologyChunkAllocator_;
+
     /**
      * @brief copyset manager module
      */

src/mds/topology/topology_stat.cpp

@@ -27,6 +27,7 @@
 #include <utility>
 
 #include "src/common/concurrent/concurrent.h"
+#include "src/mds/common/mds_define.h"
 
 using ::curve::common::WriteLockGuard;
 using ::curve::common::ReadLockGuard;
@@ -38,11 +39,23 @@ namespace topology {
 void TopologyStatImpl::UpdateChunkServerStat(ChunkServerIdType csId,
     const ChunkServerStat &stat) {
     WriteLockGuard wLock(statsLock_);
+
+    PoolIdType belongPhysicalPoolId = UNINTIALIZE_ID;
+    int ret = topo_-> GetBelongPhysicalPoolId(csId, &belongPhysicalPoolId);
+    if (ret != kTopoErrCodeSuccess) {
+        LOG(ERROR) << "UpdateChunkServerStat, GetBelongPhysicalPoolId fail,"
+                   << " chunkserverId = "
+                   << csId;
+        return;
+    }
     auto it = chunkServerStats_.find(csId);
     if (it != chunkServerStats_.end()) {
+        int64_t diff = stat.chunkFilepoolSize - it->second.chunkFilepoolSize;
+        ChunkPoolSize_[belongPhysicalPoolId] += diff;
         it->second = stat;
     } else {
         chunkServerStats_.emplace(csId, stat);
+        ChunkPoolSize_[belongPhysicalPoolId] += stat.chunkFilepoolSize;
     }
     return;
 }
@@ -58,6 +71,18 @@ bool TopologyStatImpl::GetChunkServerStat(ChunkServerIdType csId,
     return false;
 }
 
+bool TopologyStatImpl::GetChunkPoolSize(PoolIdType pId,
+    uint64_t *chunkPoolSize) {
+    ReadLockGuard rLock(statsLock_);
+    auto it = ChunkPoolSize_.find(pId);
+    if (it != ChunkPoolSize_.end()) {
+        *chunkPoolSize = it->second;
+        return true;
+    }
+    chunkPoolSize = 0;
+    return false;
+}
+
 int TopologyStatImpl::Init() {
     return kTopoErrCodeSuccess;
 }

src/mds/topology/topology_stat.h

@@ -83,6 +83,8 @@ struct ChunkServerStat {
     uint64_t chunkSizeLeftBytes;
     // Size of chunks in recycle bin
     uint64_t chunkSizeTrashedBytes;
+    // Size of chunkfilepool
+    uint64_t chunkFilepoolSize;
 
     // Copyset statistic
     std::vector<CopysetStat> copysetStats;
@@ -102,7 +104,7 @@ struct ChunkServerStat {
 class TopologyStat {
  public:
     TopologyStat() {}
-    ~TopologyStat() {}
+    virtual ~TopologyStat() {}
 
     /**
      * @brief Update the statistic of the chunkservers that sent heartbeat
@@ -123,6 +125,14 @@ class TopologyStat {
      */
     virtual bool GetChunkServerStat(ChunkServerIdType csId,
         ChunkServerStat *stat) = 0;
+    /**
+     * @brief fetch the statistic information of chunkPool size that sent by heartbeat
+     *
+     * @param pId physicalId
+     * @param chunkpoolsize the size of chunkpool
+     */    
+    virtual bool GetChunkPoolSize(PoolIdType pId,
+    uint64_t *chunkPoolSize) = 0;
 };
 
 class TopologyStatImpl : public TopologyStat {
@@ -136,12 +146,18 @@ class TopologyStatImpl : public TopologyStat {
         const ChunkServerStat &stat) override;
     bool GetChunkServerStat(ChunkServerIdType csId,
         ChunkServerStat *stat) override;
+    bool GetChunkPoolSize(PoolIdType pId,
+    uint64_t *chunkPoolSize) override;
 
  private:
     /**
      * @brief chunkserver statistic
      */
     std::map<ChunkServerIdType, ChunkServerStat>  chunkServerStats_;
+      /**
+     * @brief Count the size of chunkFilePool
+     */ 
+    std::map<PoolIdType, uint64_t> ChunkPoolSize_;
     /**
      * @brief the lock for protecting concurrent visit of chunkServerStats_
      */

src/tools/BUILD

@@ -14,14 +14,17 @@
 #  limitations under the License.
 #
 
-load("//:copts.bzl", "CURVE_DEFAULT_COPTS")
+load("//:copts.bzl", "CURVE_DEFAULT_COPTS", "CURVE_TEST_COPTS", "SPDK_COPTS", "SPDK_LINK_OPTS", "SPDK_DEPS")
 
 cc_binary(
     name = "curve_format",
     srcs = [
         "curve_format_main.cpp",
     ],
-    copts = CURVE_DEFAULT_COPTS,
+    copts = CURVE_DEFAULT_COPTS + select({
+        "//:with_spdk": SPDK_COPTS,
+        "//conditions:default": [],
+    }),
     deps = [
         "//external:braft",
         "//external:brpc",

src/tools/createtool.cpp

@@ -1,80 +0,0 @@
-/*
- *  Copyright (c) 2020 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: curve
- * File Created: Wednesday, 3rd October 2018 5:08:08 pm
- * Author: tongguangxun
- */
-
-#include <gflags/gflags.h>
-#include <glog/logging.h>
-#include <brpc/channel.h>
-
-#include "proto/nameserver2.pb.h"
-#include "proto/topology.pb.h"
-
-#include "include/client/libcurve.h"
-#include "src/client/client_common.h"
-
-DEFINE_string(mds_addr, "127.0.0.1:9000", "mds addr");
-DEFINE_string(file_name, "vdisk_001", "file name");
-DEFINE_uint64(file_size, 1 * 1024 * 1024 * 1024, "file size");
-
-using curve::mds::CurveFSService;
-using curve::mds::topology::TopologyService;
-
-int main(int argc, char** argv) {
-    google::InitGoogleLogging(argv[0]);
-    google::ParseCommandLineFlags(&argc, &argv, false);
-
-    brpc::Channel channel;
-    if (channel.Init(FLAGS_mds_addr.c_str(), nullptr) != 0) {
-        LOG(FATAL) << "Init channel  failed!";
-        return LIBCURVE_ERROR::FAILED;
-    }
-    curve::mds::CurveFSService_Stub stub(&channel);
-
-    // CreateFile
-    curve::mds::CreateFileRequest request;
-    curve::mds::CreateFileResponse response;
-    brpc::Controller cntl;
-
-    request.set_filename(FLAGS_file_name);
-    request.set_filetype(curve::mds::FileType::INODE_PAGEFILE);
-    request.set_filelength(FLAGS_file_size);
-
-    cntl.set_log_id(1);  // TODO(tongguangxun) : specify the log id usage
-    stub.CreateFile(&cntl, &request, &response, NULL);
-
-    if (cntl.Failed()) {
-        LOG(ERROR) << "Create file failed, errcorde = "
-                    << response.statuscode()
-                    << ", error content:"
-                    << cntl.ErrorText();
-    }
-    if (response.has_statuscode()) {
-        if (response.statuscode() == curve::mds::StatusCode::kFileExists) {
-            LOG(INFO) << "file already exists!";
-        } else if (response.statuscode() == curve::mds::StatusCode::kOK) {
-            LOG(INFO) << "Create file success!";
-        } else {
-            LOG(INFO) << "Create file failed, "
-                      << cntl.ErrorText();
-        }
-    }
-    return 0;
-}

src/tools/curve_format_main.cpp

@@ -37,6 +37,19 @@
 #include "src/common/crc32.h"
 #include "src/common/curve_define.h"
 #include "src/chunkserver/datastore/file_pool.h"
+#ifdef WITH_SPDK
+#include "src/fs/pfs_filesystem_impl.h"
+#include "src/chunkserver/spdk_hook.h"
+#endif
+
+using curve::fs::FileSystemType;
+using curve::fs::LocalFsFactory;
+using curve::fs::FileSystemInfo;
+using curve::fs::LocalFileSystem;
+using curve::common::kFilePoolMaigic;
+#ifdef WITH_SPDK
+using ::curve::fs::InitPfsOption;
+#endif
 
 /**
  * chunkfile pool预分配工具，提供两种分配方式
@@ -86,11 +99,26 @@ DEFINE_bool(needWriteZero,
         true,
         "not write zero for test.");
 
-using curve::fs::FileSystemType;
-using curve::fs::LocalFsFactory;
-using curve::fs::FileSystemInfo;
-using curve::fs::LocalFileSystem;
-using curve::common::kFilePoolMaigic;
+DEFINE_string(fileSystemType,
+    "ext4",
+    "file system type");
+
+#ifdef WITH_SPDK
+DEFINE_string(pfs_cluster,
+    "spdk",
+    "pfs cluster name, only for pfs");
+
+DEFINE_string(pfs_pbd_name,
+    "0000:c2:00.0n1",
+    "pfs pbd name, only for pfs");
+
+DEFINE_int32(pfs_host_id,
+    2,
+    "pfs host id, only for pfs");
+DEFINE_string(spdk_nvme_controller,
+    "",
+    "spdk nvme controller, only for pfs");
+#endif
 
 class CompareInternal {
  public:
@@ -145,7 +173,7 @@ int AllocateFiles(AllocateStruct* allocatestruct) {
         }
 
         if (FLAGS_needWriteZero) {
-            ret = allocatestruct->fsptr->Write(fd, data, 0,
+            ret = allocatestruct->fsptr->WriteZeroIfSupport(fd, data, 0,
                 FLAGS_fileSize + FLAGS_metaPagSize);
             if (ret < 0) {
                 allocatestruct->fsptr->Close(fd);
@@ -177,12 +205,60 @@ int AllocateFiles(AllocateStruct* allocatestruct) {
 
 // TODO(tongguangxun) :添加单元测试
 int main(int argc, char** argv) {
+#ifdef WITH_SPDK
+    FILE *dpdk_log_stream = NULL;
+    cookie_io_functions_t io_funcs;
+    memset(&io_funcs, 0, sizeof(io_funcs));
+#endif
+
     google::ParseCommandLineFlags(&argc, &argv, false);
     google::InitGoogleLogging(argv[0]);
 
+#ifdef WITH_SPDK
+    io_funcs.write = glog_dpdk_log_func;
+    dpdk_log_stream = fopencookie(NULL, "w", io_funcs);
+    if (dpdk_log_stream == NULL) {
+        LOG(FATAL) << "fopencookie failed";
+    }
+    rte_openlog_stream(dpdk_log_stream);
+    spdk_log_open(glog_spdk_func);
+    pfs_set_trace_func(glog_pfs_func);
+
+    curve::fs::HookIOBufIOFuncs();
+#endif
+
+    ::curve::fs::LocalFileSystemOption lfsOption;
+    lfsOption.type =
+        ::curve::fs::StringToFileSystemType(FLAGS_fileSystemType);
+#ifdef WITH_SPDK
+    if (::curve::fs::FileSystemType::PFS  == lfsOption.type) {
+        lfsOption.pfs_cluster = FLAGS_pfs_cluster;
+        lfsOption.pfs_pbd_name = FLAGS_pfs_pbd_name;
+        lfsOption.pfs_host_id = FLAGS_pfs_host_id;
+        lfsOption.spdk_nvme_controller = FLAGS_spdk_nvme_controller;
+
+        LOG_IF(FATAL, 0 != InitPfsOption(lfsOption))
+            << "InitPfsOption failed";
+
+        // setup spdk
+        LOG_IF(FATAL, 0 != pfs_spdk_setup())
+            << "setup spdk failed";
+        // start pfsdaemon
+        LOG_IF(FATAL, 0 != pfsd_start(true))
+            << "start pfsd failed";
+    } else {
+        LOG(FATAL) <<"PFS filesystem must be used when using spdk! "
+                   << "Current filesystem type: " << FLAGS_fileSystemType;
+    }
+#endif
     // load current chunkfile pool
     std::mutex mtx;
-    std::shared_ptr<LocalFileSystem> fsptr = LocalFsFactory::CreateFs(FileSystemType::EXT4, "");   // NOLINT
+    std::shared_ptr<LocalFileSystem> fsptr =
+        LocalFsFactory::CreateFs(lfsOption.type, "");
+
+    LOG_IF(FATAL, 0 != fsptr->Init(lfsOption))
+        << "Failed to initialize local filesystem module!";
+
     std::set<std::string, CompareInternal> tmpChunkSet_;
     std::atomic<uint64_t> allocateChunknum_(0);
     std::vector<std::string> tmpvec;

src/tools/curve_tool_define.h

@@ -58,12 +58,13 @@ const char kSnapshotCloneStatusCmd[] = "snapshot-clone-status";
 const char kClusterStatusCmd[] = "cluster-status";
 const char kScanStatusCmd[] = "scan-status";
 
-// NamesPaceTool相关命令
+// NameSpaceTool相关命令
 const char kGetCmd[] = "get";
 const char kListCmd[] = "list";
 const char kSegInfoCmd[] = "seginfo";
 const char kDeleteCmd[] = "delete";
 const char kCreateCmd[] = "create";
+const char kExtendCmd[] = "extend";
 const char kCleanRecycleCmd[] = "clean-recycle";
 const char kChunkLocatitonCmd[] = "chunk-location";
 const char kUpdateThrottle[] = "update-throttle";

src/tools/curve_tool_main.cpp

@@ -45,6 +45,7 @@ const char* kHelpStr = "Usage: curve_ops_tool [Command] [OPTIONS...]\n"
         "delete : delete the file, to force delete, should specify the --forcedelete=true\n"  //NOLINT
         "clean-recycle : clean the RecycleBin\n"
         "create : create file, file length unit is GB\n"
+        "extend : extend volume of file\n"
         "chunk-location : query the location of the chunk corresponding to the offset\n"  //NOLINT
         "check-consistency : check the consistency of three copies\n"
         "remove-peer : remove the peer from the copyset\n"

src/tools/mds_client.cpp

@@ -271,12 +271,15 @@ int MDSClient::DeleteFile(const std::string& fileName, bool forcedelete) {
     return -1;
 }
 
-int MDSClient::CreateFile(const std::string& fileName, uint64_t length) {
+int MDSClient::CreateFile(const std::string& fileName, uint64_t length,
+                   uint64_t stripeUnit, uint64_t stripeCount) {
     curve::mds::CreateFileRequest request;
     curve::mds::CreateFileResponse response;
     request.set_filename(fileName);
     request.set_filetype(curve::mds::FileType::INODE_PAGEFILE);
     request.set_filelength(length);
+    request.set_stripeunit(stripeUnit);
+    request.set_stripecount(stripeCount);
     FillUserInfo(&request);
     curve::mds::CurveFSService_Stub stub(&channel_);
 
@@ -295,6 +298,29 @@ int MDSClient::CreateFile(const std::string& fileName, uint64_t length) {
     return -1;
 }
 
+int MDSClient::ExtendVolume(const std::string& fileName, uint64_t newSize) {
+    curve::mds::ExtendFileRequest request;
+    curve::mds::ExtendFileResponse response;
+    request.set_filename(fileName);
+    request.set_newsize(newSize);
+    FillUserInfo(&request);
+    curve::mds::CurveFSService_Stub stub(&channel_);
+    auto fp = &curve::mds::CurveFSService_Stub::ExtendFile;
+    if (SendRpcToMds(&request, &response, &stub, fp) != 0) {
+        std::cout << "extendFile from all mds fail!" << std::endl;
+        return -1;
+    }
+
+    if (response.has_statuscode() &&
+                response.statuscode() == StatusCode::kOK) {
+                std::cout << "extendFile success!" << std::endl;
+        return 0;
+    }
+    std::cout << "extendFile fail with errCode: "
+              << response.statuscode() << std::endl;
+    return -1;
+}
+
 int MDSClient::ListVolumesOnCopyset(
                         const std::vector<common::CopysetInfo>& copysets,
                         std::vector<std::string>* fileNames) {

src/tools/mds_client.h

@@ -170,9 +170,12 @@ class MDSClient {
      *  @brief 创建pageFile文件
      *  @param fileName 文件名
      *  @param length 文件长度
+     *  @param stripeUnit stripe unit size
+     *  @param stripeCount the amount of stripes
      *  @return 成功返回0，失败返回-1
      */
-    virtual int CreateFile(const std::string& fileName, uint64_t length);
+    virtual int CreateFile(const std::string& fileName, uint64_t length,
+                           uint64_t stripeUnit, uint64_t stripeCount);
 
     /**
      *  @brief List all volumes on copysets
@@ -184,6 +187,14 @@ class MDSClient {
                         const std::vector<common::CopysetInfo>& copysets,
                         std::vector<std::string>* fileNames);
 
+    /**
+     *  @brief 扩容卷
+     *  @param fileName 文件名
+     *  @param newSize 扩容后的卷大小
+     *  @return 成功返回0，失败返回-1
+     */
+    virtual int ExtendVolume(const std::string& fileName, uint64_t newSize);
+
     /**
      *  @brief 列出client的dummyserver的地址
      *  @param[out] clientAddrs client地址列表，返回0时有效

src/tools/namespace_tool.cpp

@@ -27,6 +27,7 @@ DEFINE_string(expireTime, "7d", "Time for file in recyclebin exceed expire time
                                 "will be deleted (default: 7d)");
 DEFINE_bool(forcedelete, false, "force delete file or not");
 DEFINE_uint64(fileLength, 20, "file length (GB)");
+DEFINE_uint64(newSize, 30, "the new size of expanded volume(GB)");
 DEFINE_bool(isTest, false, "is unit test or not");
 DEFINE_uint64(offset, 0, "offset to query chunk location");
 DEFINE_uint64(rpc_timeout, 3000, "millisecond for rpc timeout");
@@ -40,6 +41,8 @@ DEFINE_string(throttleType, "", "throttle type");
 DEFINE_uint64(limit, 0, "throttle limit");
 DEFINE_int64(burst, -1, "throttle burst");
 DEFINE_int64(burstLength, -1, "throttle burst length");
+DEFINE_uint64(stripeUnit, 0, "stripe unit size");
+DEFINE_uint64(stripeCount, 0, "strip count");
 
 namespace curve {
 namespace tool {
@@ -61,6 +64,7 @@ bool NameSpaceTool::SupportCommand(const std::string& command) {
                                || command == kSegInfoCmd
                                || command == kDeleteCmd
                                || command == kCreateCmd
+                               || command == kExtendCmd
                                || command == kCleanRecycleCmd
                                || command == kChunkLocatitonCmd
                                || command == kUpdateThrottle);
@@ -118,7 +122,10 @@ int NameSpaceTool::RunCommand(const std::string &cmd) {
             return 0;
         }
     } else if (cmd == kCreateCmd) {
-        return core_->CreateFile(fileName, FLAGS_fileLength * mds::kGB);
+        return core_->CreateFile(fileName, FLAGS_fileLength * mds::kGB,
+                             FLAGS_stripeUnit, FLAGS_stripeCount);
+    } else if (cmd == kExtendCmd) {
+        return core_->ExtendVolume(fileName, FLAGS_newSize * mds::kGB);
     } else if (cmd == kChunkLocatitonCmd) {
         return PrintChunkLocation(fileName, FLAGS_offset);
     } else if (cmd == kUpdateThrottle) {
@@ -143,7 +150,9 @@ void NameSpaceTool::PrintHelp(const std::string &cmd) {
         std::cout << "If -fileName is specified, delete the files in recyclebin that the original directory is fileName" << std::endl;  // NOLINT
         std::cout << "expireTime: s=second, m=minute, h=hour, d=day, M=month, y=year" << std::endl;  // NOLINT
     } else if (cmd == kCreateCmd) {
-        std::cout << "curve_ops_tool " << cmd << " -fileName=/test -userName=test -password=123 -fileLength=20‬  [-mdsAddr=127.0.0.1:6666] [-confPath=/etc/curve/tools.conf]" << std::endl;  // NOLINT
+        std::cout << "curve_ops_tool " << cmd << " -fileName=/test -userName=test -password=123 -filelength=20 -stripeUnit=32768 -stripeCount=32  [-mdsAddr=127.0.0.1:6666] [-confPath=/etc/curve/tools.conf]" << std::endl;  // NOLINT
+    } else if (cmd == kExtendCmd) {
+        std::cout << "curve_ops_tool " << cmd << " -fileName=/test -userName=test -password=123 -newSize=30  [-mdsAddr=127.0.0.1:6666] [-confPath=/etc/curve/tools.conf]" << std::endl;  // NOLINT
     } else if (cmd == kDeleteCmd) {
         std::cout << "curve_ops_tool " << cmd << " -fileName=/test -userName=test -password=123 -forcedelete=true  [-mdsAddr=127.0.0.1:6666] [-confPath=/etc/curve/tools.conf]" << std::endl;  // NOLINT
     } else if (cmd == kChunkLocatitonCmd) {

src/tools/namespace_tool_core.cpp

@@ -61,10 +61,16 @@ int NameSpaceToolCore::DeleteFile(const std::string& fileName,
 }
 
 int NameSpaceToolCore::CreateFile(const std::string& fileName,
-                                  uint64_t length) {
-    return client_->CreateFile(fileName, length);
+                                  uint64_t length,
+                                  uint64_t stripeUnit,
+                                  uint64_t stripeCount) {
+    return client_->CreateFile(fileName, length,
+                               stripeUnit, stripeCount);
+}
+int NameSpaceToolCore::ExtendVolume(const std::string& fileName,
+                                     uint64_t newSize) {
+    return client_->ExtendVolume(fileName, newSize);
 }
-
 int NameSpaceToolCore::GetAllocatedSize(const std::string& fileName,
                                         uint64_t* allocSize,
                                         AllocMap* allocMap) {

src/tools/namespace_tool_core.h

@@ -105,9 +105,20 @@ class NameSpaceToolCore {
      *  @brief 创建pageFile文件
      *  @param fileName 文件名
      *  @param length 文件长度
+     *  @param stripeUnit stripe unit size
+     *  @param stripeCount the amount of stripes
      *  @return 成功返回0，失败返回-1
      */
-    virtual int CreateFile(const std::string& fileName, uint64_t length);
+    virtual int CreateFile(const std::string& fileName, uint64_t length,
+                          uint64_t stripeUnit, uint64_t stripeCount);
+
+   /**
+     *  @brief 扩容卷
+     *  @param fileName 文件名
+     *  @param newSize 扩容后的文件长度
+     *  @return 成功返回0，失败返回-1
+     */
+    virtual int ExtendVolume(const std::string& fileName, uint64_t newSize);
 
     /**
      *  @brief 计算文件或目录实际分配的空间

test/chunkserver/chunkserver_test_util.cpp

@@ -153,6 +153,7 @@ int StartChunkserver(const char *ip,
     if (nullptr == copysetNodeOptions.chunkFilePool) {
         LOG(FATAL) << "new chunfilepool failed";
     }
+    copysetNodeOptions.walFilePool = copysetNodeOptions.chunkFilePool;
     FilePoolOptions cfop;
     if (false == copysetNodeOptions.chunkFilePool->Initialize(cfop)) {
         LOG(FATAL) << "chunfilepool init failed";

test/chunkserver/clone/clone_core_test.cpp

@@ -40,6 +40,7 @@ namespace chunkserver {
 using curve::chunkserver::CHUNK_OP_TYPE;
 using curve::fs::LocalFsFactory;
 using curve::fs::FileSystemType;
+using ::testing::Matcher;
 
 ACTION_TEMPLATE(SaveBraftTask,
                 HAS_1_TEMPLATE_PARAMS(int, k),
@@ -158,7 +159,8 @@ TEST_F(CloneCoreTest, ReadChunkTest1) {
         .WillOnce(DoAll(SetArgPointee<1>(info),
                         Return(CSErrorCode::Success)));
     // 读chunk文件
-    EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+    EXPECT_CALL(*datastore_, ReadChunk(_, _,
+        Matcher<char*>(_), offset, length))
         .Times(1);
     // 更新 applied index
     EXPECT_CALL(*node_, UpdateAppliedIndex(_))
@@ -212,7 +214,8 @@ TEST_F(CloneCoreTest, ReadChunkTest2) {
         // 读chunk文件
         char chunkData[length];  // NOLINT
         memset(chunkData, 'a', length);
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), offset, length))
             .WillOnce(DoAll(SetArrayArgument<2>(chunkData,
                                                 chunkData + length),
                             Return(CSErrorCode::Success)));
@@ -254,7 +257,8 @@ TEST_F(CloneCoreTest, ReadChunkTest2) {
             .WillRepeatedly(DoAll(SetArgPointee<1>(info),
                                   Return(CSErrorCode::Success)));
         // 读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), offset, length))
             .Times(0);
         // 更新 applied index
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
@@ -307,7 +311,8 @@ TEST_F(CloneCoreTest, ReadChunkTest2) {
         // 读chunk文件
         char chunkData[3 * PAGE_SIZE];
         memset(chunkData, 'a', 3 * PAGE_SIZE);
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, 0, 3 * PAGE_SIZE))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), 0, 3 * PAGE_SIZE))
             .WillOnce(DoAll(SetArrayArgument<2>(chunkData,
                                                 chunkData + 3 * PAGE_SIZE),
                             Return(CSErrorCode::Success)));
@@ -358,7 +363,8 @@ TEST_F(CloneCoreTest, ReadChunkTest2) {
         EXPECT_CALL(*copyer_, DownloadAsync(_))
             .Times(0);
         // 读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), _, _))
             .Times(0);
         // 更新 applied index
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
@@ -413,7 +419,8 @@ TEST_F(CloneCoreTest, ReadChunkTest3) {
             .Times(2)
             .WillRepeatedly(Return(CSErrorCode::ChunkNotExistError));
         // 读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), offset, length))
             .Times(0);
         // 更新 applied index
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
@@ -475,7 +482,8 @@ TEST_F(CloneCoreTest, ReadChunkErrorTest) {
             .WillOnce(Return(CSErrorCode::InternalError));
         EXPECT_CALL(*copyer_, DownloadAsync(_))
             .Times(0);
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), _, _))
             .Times(0);
         EXPECT_CALL(*node_, Propose(_))
             .Times(0);
@@ -504,7 +512,8 @@ TEST_F(CloneCoreTest, ReadChunkErrorTest) {
                 brpc::ClosureGuard guard(closure);
                 closure->SetFailed();
             }));
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), _, _))
             .Times(0);
 
         ASSERT_EQ(0, core->HandleReadRequest(readRequest,
@@ -533,7 +542,8 @@ TEST_F(CloneCoreTest, ReadChunkErrorTest) {
                 AsyncDownloadContext* context = closure->GetDownloadContext();
                 memcpy(context->buf, cloneData, length);
             }));
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), _, _))
             .WillOnce(Return(CSErrorCode::InternalError));
          // 产生PasteChunkRequest
         braft::Task task;
@@ -581,7 +591,8 @@ TEST_F(CloneCoreTest, RecoverChunkTest1) {
         .WillOnce(DoAll(SetArgPointee<1>(info),
                         Return(CSErrorCode::Success)));
     // 读chunk文件
-    EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+    EXPECT_CALL(*datastore_, ReadChunk(_, _,
+        Matcher<char*>(_), _, _))
         .Times(0);
     // 不会产生PasteChunkRequest
     EXPECT_CALL(*node_, Propose(_))
@@ -626,7 +637,8 @@ TEST_F(CloneCoreTest, RecoverChunkTest2) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 不会读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), _, _))
             .Times(0);
         // 不会产生PasteChunkRequest
         EXPECT_CALL(*node_, Propose(_))
@@ -659,7 +671,8 @@ TEST_F(CloneCoreTest, RecoverChunkTest2) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 不会读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), offset, length))
             .Times(0);
         // 产生PasteChunkRequest
         braft::Task task;
@@ -719,7 +732,8 @@ TEST_F(CloneCoreTest, DisablePasteTest) {
             .WillRepeatedly(DoAll(SetArgPointee<1>(info),
                                   Return(CSErrorCode::Success)));
         // 读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), offset, length))
             .Times(0);
         // 更新 applied index
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
@@ -757,7 +771,8 @@ TEST_F(CloneCoreTest, DisablePasteTest) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 不会读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<char*>(_), offset, length))
             .Times(0);
         // 产生PasteChunkRequest
         braft::Task task;

test/chunkserver/clone/op_request_test.cpp

@@ -36,6 +36,7 @@ namespace chunkserver {
 
 using curve::chunkserver::CHUNK_OP_TYPE;
 using curve::chunkserver::concurrent::ConcurrentApplyOption;
+using ::testing::Matcher;
 
 const char PEER_STRING[] = "127.0.0.1:8200:0";
 
@@ -670,8 +671,11 @@ TEST_F(OpRequestTest, ReadChunkTest) {
 
         char chunkData[length];  // NOLINT
         memset(chunkData, 'a', length);
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
-            .WillOnce(DoAll(SetArrayArgument<2>(chunkData, chunkData + length),
+        butil::IOPortal buf;
+        buf.append(chunkData, length);
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
+            .WillOnce(DoAll(SetArgPointee<2>(buf),
                             Return(CSErrorCode::Success)));
         EXPECT_CALL(*node_, UpdateAppliedIndex(_)).Times(1);
 
@@ -706,8 +710,11 @@ TEST_F(OpRequestTest, ReadChunkTest) {
         // 读chunk文件
         char chunkData[length];  // NOLINT
         memset(chunkData, 'a', length);
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
-            .WillOnce(DoAll(SetArrayArgument<2>(chunkData, chunkData + length),
+        butil::IOPortal buf;
+        buf.append(chunkData, length);
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
+            .WillOnce(DoAll(SetArgPointee<2>(buf),
                             Return(CSErrorCode::Success)));
         EXPECT_CALL(*node_, UpdateAppliedIndex(_)).Times(1);
 
@@ -743,8 +750,11 @@ TEST_F(OpRequestTest, ReadChunkTest) {
         // 读chunk文件
         char chunkData[length];  // NOLINT
         memset(chunkData, 'a', length);
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
-            .WillOnce(DoAll(SetArrayArgument<2>(chunkData, chunkData + length),
+        butil::IOPortal buf;
+        buf.append(chunkData, length);
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
+            .WillOnce(DoAll(testing::SetArgPointee<2>(buf),
                             Return(CSErrorCode::Success)));
         EXPECT_CALL(*node_, UpdateAppliedIndex(_)).Times(1);
 
@@ -780,7 +790,8 @@ TEST_F(OpRequestTest, ReadChunkTest) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), _, _))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);
@@ -812,7 +823,8 @@ TEST_F(OpRequestTest, ReadChunkTest) {
         EXPECT_CALL(*datastore_, GetChunkInfo(_, _))
             .WillOnce(Return(CSErrorCode::ChunkNotExistError));
         // 不会读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);
@@ -841,7 +853,8 @@ TEST_F(OpRequestTest, ReadChunkTest) {
         EXPECT_CALL(*datastore_, GetChunkInfo(_, _))
             .WillOnce(Return(CSErrorCode::ChunkNotExistError));
         // 读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), _, _))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);
@@ -881,9 +894,11 @@ TEST_F(OpRequestTest, ReadChunkTest) {
         // 读chunk文件
         char chunkData[length];  // NOLINT
         memset(chunkData, 'a', length);
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
-            .WillOnce(DoAll(SetArrayArgument<2>(chunkData,
-                                                chunkData + length),
+        butil::IOPortal buf;
+        buf.append(chunkData, length);
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
+            .WillOnce(DoAll(testing::SetArgPointee<2>(buf),
                             Return(CSErrorCode::Success)));
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(1);
@@ -913,7 +928,8 @@ TEST_F(OpRequestTest, ReadChunkTest) {
         EXPECT_CALL(*datastore_, GetChunkInfo(_, _))
             .WillOnce(Return(CSErrorCode::InternalError));
         // 不会读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);
@@ -941,7 +957,8 @@ TEST_F(OpRequestTest, ReadChunkTest) {
             .WillRepeatedly(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 读chunk文件失败
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
             .WillRepeatedly(Return(CSErrorCode::InternalError));
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);
@@ -965,7 +982,8 @@ TEST_F(OpRequestTest, ReadChunkTest) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), _, _))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);
@@ -1092,7 +1110,8 @@ TEST_F(OpRequestTest, RecoverChunkTest) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 不读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(1);
@@ -1134,7 +1153,8 @@ TEST_F(OpRequestTest, RecoverChunkTest) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 不读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(1);
@@ -1174,7 +1194,8 @@ TEST_F(OpRequestTest, RecoverChunkTest) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 不读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(1);
@@ -1204,7 +1225,8 @@ TEST_F(OpRequestTest, RecoverChunkTest) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 不读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(1);
@@ -1234,7 +1256,8 @@ TEST_F(OpRequestTest, RecoverChunkTest) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), _, _))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);
@@ -1266,7 +1289,8 @@ TEST_F(OpRequestTest, RecoverChunkTest) {
         EXPECT_CALL(*datastore_, GetChunkInfo(_, _))
             .WillOnce(Return(CSErrorCode::ChunkNotExistError));
         // 不会读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);
@@ -1292,7 +1316,8 @@ TEST_F(OpRequestTest, RecoverChunkTest) {
         EXPECT_CALL(*datastore_, GetChunkInfo(_, _))
             .WillOnce(Return(CSErrorCode::InternalError));
         // 不会读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, offset, length))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), offset, length))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);
@@ -1322,7 +1347,8 @@ TEST_F(OpRequestTest, RecoverChunkTest) {
             .WillOnce(DoAll(SetArgPointee<1>(info),
                             Return(CSErrorCode::Success)));
         // 读chunk文件
-        EXPECT_CALL(*datastore_, ReadChunk(_, _, _, _, _))
+        EXPECT_CALL(*datastore_, ReadChunk(_, _,
+            Matcher<butil::IOPortal*>(_), _, _))
             .Times(0);
         EXPECT_CALL(*node_, UpdateAppliedIndex(_))
             .Times(0);

test/chunkserver/conf_epoch_file_test.cpp

@@ -99,7 +99,8 @@ TEST(ConfEpochFileTest, load_save) {
         CopysetID loadCopysetID;
         uint64_t loadEpoch;
         EXPECT_CALL(*fs, Open(_, _)).Times(1).WillOnce(Return(10));
-        EXPECT_CALL(*fs, Read(_, _, _, _)).Times(1).WillOnce(Return(-1));
+        EXPECT_CALL(*fs, Read(_, Matcher<char*>(_), _, _))
+            .Times(1).WillOnce(Return(-1));
         EXPECT_CALL(*fs, Close(_)).Times(1).WillOnce(Return(0));
         ASSERT_EQ(-1, confEpochFile.Load(path,
                                          &loadLogicPoolID,
@@ -117,7 +118,7 @@ TEST(ConfEpochFileTest, load_save) {
         CopysetID loadCopysetID;
         uint64_t loadEpoch;
         EXPECT_CALL(*fs, Open(_, _)).Times(1).WillOnce(Return(10));
-        EXPECT_CALL(*fs, Read(_, _, _, _)).Times(1)
+        EXPECT_CALL(*fs, Read(_, Matcher<char*>(_), _, _)).Times(1)
             .WillOnce(DoAll(SetArrayArgument<1>(json, json + jsonStr.size()),
                             Return(jsonStr.size())));
         EXPECT_CALL(*fs, Close(_)).Times(1).WillOnce(Return(0));
@@ -137,7 +138,7 @@ TEST(ConfEpochFileTest, load_save) {
         CopysetID loadCopysetID;
         uint64_t loadEpoch;
         EXPECT_CALL(*fs, Open(_, _)).Times(1).WillOnce(Return(10));
-        EXPECT_CALL(*fs, Read(_, _, _, _)).Times(1)
+        EXPECT_CALL(*fs, Read(_, Matcher<char*>(_), _, _)).Times(1)
             .WillOnce(DoAll(SetArrayArgument<1>(json, json + jsonStr.size()),
                             Return(jsonStr.size())));
         EXPECT_CALL(*fs, Close(_)).Times(1).WillOnce(Return(0));
@@ -157,7 +158,7 @@ TEST(ConfEpochFileTest, load_save) {
         CopysetID loadCopysetID;
         uint64_t loadEpoch;
         EXPECT_CALL(*fs, Open(_, _)).Times(1).WillOnce(Return(10));
-        EXPECT_CALL(*fs, Read(_, _, _, _)).Times(1)
+        EXPECT_CALL(*fs, Read(_, Matcher<char*>(_), _, _)).Times(1)
             .WillOnce(DoAll(SetArrayArgument<1>(json, json + jsonStr.size()),
                             Return(jsonStr.size())));
         EXPECT_CALL(*fs, Close(_)).Times(1).WillOnce(Return(0));

test/chunkserver/copyset_node_test.cpp

@@ -143,6 +143,7 @@ class CopysetNodeTest : public ::testing::Test {
         defaultOptions_.localFileSystem = fs;
         defaultOptions_.chunkFilePool =
             std::make_shared<FilePool>(fs);
+        defaultOptions_.walFilePool = defaultOptions_.chunkFilePool;
         defaultOptions_.trash = std::make_shared<Trash>();
         defaultOptions_.enableOdsyncWhenOpenChunkFile = true;
     }
@@ -959,7 +960,7 @@ TEST_F(CopysetNodeTest, get_hash) {
             .WillOnce(Return(3));
         EXPECT_CALL(*mockfs, Fstat(_, _)).Times(1)
             .WillOnce(DoAll(SetArgPointee<1>(fileInfo), Return(0)));
-        EXPECT_CALL(*mockfs, Read(_, _, _, _)).Times(1)
+        EXPECT_CALL(*mockfs, Read(_, Matcher<char*>(_), _, _)).Times(1)
             .WillOnce(Return(-1));
 
         ASSERT_EQ(-1, copysetNode.GetHash(&hash));
@@ -987,7 +988,7 @@ TEST_F(CopysetNodeTest, get_hash) {
             .WillOnce(Return(3));
         EXPECT_CALL(*mockfs, Fstat(_, _)).Times(1)
             .WillOnce(DoAll(SetArgPointee<1>(fileInfo), Return(0)));
-        EXPECT_CALL(*mockfs, Read(_, _, _, _)).Times(1)
+        EXPECT_CALL(*mockfs, Read(_, Matcher<char*>(_), _, _)).Times(1)
             .WillOnce(DoAll(SetArgPointee<1>(*buff), Return(1024)));
 
         ASSERT_EQ(0, copysetNode.GetHash(&hash));

test/chunkserver/copyset_service_test.cpp

@@ -117,6 +117,7 @@ TEST_F(CopysetServiceTest, basic) {
     copysetNodeOptions.localFileSystem = fs;
     copysetNodeOptions.chunkFilePool =
         std::make_shared<FilePool>(fs);
+    copysetNodeOptions.walFilePool = copysetNodeOptions.chunkFilePool;
     copysetNodeOptions.trash = trash_;
     copysetNodeOptions.enableOdsyncWhenOpenChunkFile = true;
     ASSERT_EQ(0, copysetNodeManager->Init(copysetNodeOptions));
@@ -259,6 +260,7 @@ TEST_F(CopysetServiceTest, basic2) {
     copysetNodeOptions.localFileSystem = fs;
     copysetNodeOptions.chunkFilePool =
         std::make_shared<FilePool>(fs);
+    copysetNodeOptions.walFilePool = copysetNodeOptions.chunkFilePool;
     copysetNodeOptions.enableOdsyncWhenOpenChunkFile = true;
     ASSERT_EQ(0, copysetNodeManager->Init(copysetNodeOptions));
     ASSERT_EQ(0, copysetNodeManager->Run());

test/chunkserver/datastore/BUILD

@@ -14,7 +14,7 @@
 #  limitations under the License.
 #
 
-load("//:copts.bzl", "CURVE_TEST_COPTS")
+load("//:copts.bzl", "CURVE_DEFAULT_COPTS", "CURVE_TEST_COPTS", "SPDK_COPTS", "SPDK_LINK_OPTS", "SPDK_DEPS")
 
 cc_library(
     name = "datastore_mock",
@@ -52,7 +52,14 @@ cc_test(
         "datastore_unittest_main.cpp",
         "file_helper_unittest.cpp",
     ],
-    copts = CURVE_TEST_COPTS,
+    copts = CURVE_TEST_COPTS + select({
+        "//:with_spdk": SPDK_COPTS,
+        "//conditions:default": [],
+    }),
+    linkopts = select({
+        "//:with_spdk": SPDK_LINK_OPTS,
+        "//conditions:default": [],
+    }),
     deps = [
         "@com_google_googletest//:gtest",
         "//external:gflags",
@@ -64,5 +71,8 @@ cc_test(
         "//test/chunkserver/datastore:datastore_mock",
         "//external:json",
         "//test/chunkserver/datastore:filepool_helper",
-    ],
+    ] + select({
+        "//:with_spdk": SPDK_DEPS,
+        "//conditions:default": [],
+    }),
 )

test/chunkserver/datastore/datastore_mock_unittest.cpp

@@ -207,7 +207,7 @@ class CSDataStore_test : public testing::Test {
                 .WillRepeatedly(DoAll(SetArgPointee<1>(fileInfo),
                                 Return(0)));
             // fake Read
-            ON_CALL(*lfs_, Read(Ge(1), NotNull(), Ge(0), Gt(0)))
+            ON_CALL(*lfs_, Read(Ge(1), Matcher<char*>(NotNull()), Ge(0), Gt(0)))
                 .WillByDefault(ReturnArg<3>());
             // fake Write
             ON_CALL(*lfs_,
@@ -217,21 +217,21 @@ class CSDataStore_test : public testing::Test {
                 .WillByDefault(ReturnArg<3>());
             // fake read chunk1 metapage
             FakeEncodeChunk(chunk1MetaPage, 0, 2);
-            EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+            EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
                 .WillRepeatedly(DoAll(
                                 SetArrayArgument<1>(chunk1MetaPage,
                                 chunk1MetaPage + PAGE_SIZE),
                                 Return(PAGE_SIZE)));
             // fake read chunk1's snapshot1 metapage
             FakeEncodeSnapshot(chunk1SnapMetaPage, 1);
-            EXPECT_CALL(*lfs_, Read(2, NotNull(), 0, PAGE_SIZE))
+            EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
                 .WillRepeatedly(DoAll(
                                 SetArrayArgument<1>(chunk1SnapMetaPage,
                                 chunk1SnapMetaPage + PAGE_SIZE),
                                 Return(PAGE_SIZE)));
             // fake read chunk2 metapage
             FakeEncodeChunk(chunk2MetaPage, 0, 2);
-            EXPECT_CALL(*lfs_, Read(3, NotNull(), 0, PAGE_SIZE))
+            EXPECT_CALL(*lfs_, Read(3, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
                 .WillRepeatedly(DoAll(
                                 SetArrayArgument<1>(chunk2MetaPage,
                                 chunk2MetaPage + PAGE_SIZE),
@@ -283,7 +283,7 @@ TEST_F(CSDataStore_test, InitializeTest1) {
     EXPECT_CALL(*lfs_, DirExists(baseDir))
         .Times(1)
         .WillOnce(Return(true));
-    EXPECT_CALL(*lfs_, Mkdir(baseDir))
+    EXPECT_CALL(*lfs_, Mkdir(baseDir, _))
         .Times(0);
     vector<string> fileNames;
     fileNames.push_back(temp1);
@@ -391,7 +391,7 @@ TEST_F(CSDataStore_test, InitializeErrorTest1) {
     EXPECT_CALL(*lfs_, DirExists(baseDir))
         .Times(1)
         .WillOnce(Return(false));
-    EXPECT_CALL(*lfs_, Mkdir(baseDir))
+    EXPECT_CALL(*lfs_, Mkdir(baseDir, _))
         .Times(1)
         .WillOnce(Return(-UT_ERRNO));
     // List should not be called
@@ -410,7 +410,7 @@ TEST_F(CSDataStore_test, InitializeErrorTest2) {
     EXPECT_CALL(*lfs_, DirExists(baseDir))
         .Times(1)
         .WillOnce(Return(false));
-    EXPECT_CALL(*lfs_, Mkdir(baseDir))
+    EXPECT_CALL(*lfs_, Mkdir(baseDir, _))
         .Times(1)
         .WillOnce(Return(0));
     // List failed
@@ -470,7 +470,7 @@ TEST_F(CSDataStore_test, InitializeErrorTest3) {
         .WillOnce(DoAll(SetArgPointee<1>(fileInfo),
                         Return(0)));
     // read metapage failed
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(Return(-UT_ERRNO));
     EXPECT_FALSE(dataStore->Initialize());
 
@@ -488,7 +488,7 @@ TEST_F(CSDataStore_test, InitializeErrorTest3) {
     // read metapage success, but version incompatible
     uint8_t version = FORMAT_VERSION + 1;
     memcpy(chunk1MetaPage, &version, sizeof(uint8_t));
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
                 .WillOnce(DoAll(SetArrayArgument<1>(chunk1MetaPage,
                                 chunk1MetaPage + PAGE_SIZE),
                                 Return(PAGE_SIZE)));
@@ -509,7 +509,7 @@ TEST_F(CSDataStore_test, InitializeErrorTest3) {
     version = FORMAT_VERSION;
     chunk1MetaPage[1] += 1;  // change the page data
     memcpy(chunk1MetaPage, &version, sizeof(uint8_t));
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
                 .WillOnce(DoAll(SetArrayArgument<1>(chunk1MetaPage,
                                 chunk1MetaPage + PAGE_SIZE),
                                 Return(PAGE_SIZE)));
@@ -575,7 +575,7 @@ TEST_F(CSDataStore_test, InitializeErrorTest4) {
         .WillOnce(DoAll(SetArgPointee<1>(fileInfo),
                         Return(0)));
     // read metapage failed
-    EXPECT_CALL(*lfs_, Read(2, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(Return(-UT_ERRNO));
     EXPECT_FALSE(dataStore->Initialize());
 
@@ -596,7 +596,7 @@ TEST_F(CSDataStore_test, InitializeErrorTest4) {
     // read metapage success, but version incompatible
     uint8_t version = FORMAT_VERSION + 1;
     memcpy(chunk1SnapMetaPage, &version, sizeof(uint8_t));
-    EXPECT_CALL(*lfs_, Read(2, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
                 .WillOnce(DoAll(SetArrayArgument<1>(chunk1SnapMetaPage,
                                 chunk1SnapMetaPage + PAGE_SIZE),
                                 Return(PAGE_SIZE)));
@@ -620,7 +620,7 @@ TEST_F(CSDataStore_test, InitializeErrorTest4) {
     version = FORMAT_VERSION;
     chunk1SnapMetaPage[1] += 1;  // change the page data
     memcpy(chunk1SnapMetaPage, &version, sizeof(uint8_t));
-    EXPECT_CALL(*lfs_, Read(2, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
                 .WillOnce(DoAll(SetArrayArgument<1>(chunk1SnapMetaPage,
                                 chunk1SnapMetaPage + PAGE_SIZE),
                                 Return(PAGE_SIZE)));
@@ -694,7 +694,7 @@ TEST_F(CSDataStore_test, WriteChunkTest1) {
     char chunk3MetaPage[PAGE_SIZE];
     memset(chunk3MetaPage, 0, sizeof(chunk3MetaPage));
     FakeEncodeChunk(chunk3MetaPage, 0, 1);
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                         chunk3MetaPage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
@@ -710,7 +710,7 @@ TEST_F(CSDataStore_test, WriteChunkTest1) {
                                                           length,
                                                           nullptr));
 
-    EXPECT_CALL(*lfs_, Sync(4))
+    EXPECT_CALL(*lfs_, Fdatasync(4))
         .WillOnce(Return(0))
         .WillOnce(Return(-1));
 
@@ -801,7 +801,7 @@ TEST_F(CSDataStore_test, WriteChunkTest3) {
     FakeEnv();
     // set chunk2's correctedSn as 3
     FakeEncodeChunk(chunk2MetaPage, 4, 2);
-    EXPECT_CALL(*lfs_, Read(3, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(3, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk2MetaPage,
                         chunk2MetaPage + PAGE_SIZE),
@@ -943,7 +943,7 @@ TEST_F(CSDataStore_test, WriteChunkTest6) {
     FakeEnv();
     // set chunk2's correctedSn as 3
     FakeEncodeChunk(chunk2MetaPage, 3, 2);
-    EXPECT_CALL(*lfs_, Read(3, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(3, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk2MetaPage,
                         chunk2MetaPage + PAGE_SIZE),
@@ -1017,7 +1017,7 @@ TEST_F(CSDataStore_test, WriteChunkTest7) {
     char metapage[PAGE_SIZE];
     memset(metapage, 0, sizeof(metapage));
     FakeEncodeSnapshot(metapage, 2);
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(DoAll(SetArrayArgument<1>(metapage,
                         metapage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
@@ -1025,7 +1025,8 @@ TEST_F(CSDataStore_test, WriteChunkTest7) {
     EXPECT_CALL(*lfs_, Write(3, Matcher<const char*>(NotNull()), 0, PAGE_SIZE))
         .Times(1);
     // will copy on write
-    EXPECT_CALL(*lfs_, Read(3, NotNull(), PAGE_SIZE + offset, length))
+    EXPECT_CALL(*lfs_, Read(3, Matcher<char*>(NotNull()),
+        PAGE_SIZE + offset, length))
         .Times(1);
     EXPECT_CALL(*lfs_, Write(4, Matcher<const char*>(NotNull()),
                              PAGE_SIZE + offset, length))
@@ -1100,7 +1101,8 @@ TEST_F(CSDataStore_test, WriteChunkTest9) {
     memset(buf, 0, sizeof(buf));
     // will not create snapshot
     // will copy on write
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), PAGE_SIZE + offset, length))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()),
+        PAGE_SIZE + offset, length))
         .Times(1);
     EXPECT_CALL(*lfs_, Write(2, Matcher<const char*>(NotNull()),
                              PAGE_SIZE + offset, length))
@@ -1144,7 +1146,7 @@ TEST_F(CSDataStore_test, WriteChunkTest10) {
     FakeEnv();
     // set chunk1's correctedSn as 3
     FakeEncodeChunk(chunk1MetaPage, 3, 2);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1MetaPage,
                         chunk1MetaPage + PAGE_SIZE),
@@ -1197,7 +1199,7 @@ TEST_F(CSDataStore_test, WriteChunkTest11) {
     FakeEnv();
     // set chunk1's correctedSn as 3
     FakeEncodeChunk(chunk1MetaPage, 3, 2);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1MetaPage,
                         chunk1MetaPage + PAGE_SIZE),
@@ -1278,7 +1280,7 @@ TEST_F(CSDataStore_test, WriteChunkTest13) {
             .Times(1)
             .WillOnce(Return(4));
         // will read metapage
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
             .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                             chunk3MetaPage + PAGE_SIZE),
                             Return(PAGE_SIZE)));
@@ -1463,7 +1465,7 @@ TEST_F(CSDataStore_test, WriteChunkTest14) {
             .Times(1)
             .WillOnce(Return(4));
         // will read metapage
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
             .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                             chunk3MetaPage + PAGE_SIZE),
                             Return(PAGE_SIZE)));
@@ -1628,14 +1630,14 @@ TEST_F(CSDataStore_test, WriteChunkTest15) {
     FakeEnv();
     // fake read chunk1 metapage
     FakeEncodeChunk(chunk1MetaPage, 0, 2);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1MetaPage,
                         chunk1MetaPage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
     // fake read chunk1's snapshot1 metapage,chunk.sn<snap.sn
     FakeEncodeSnapshot(chunk1SnapMetaPage, 3);
-    EXPECT_CALL(*lfs_, Read(2, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1SnapMetaPage,
                         chunk1SnapMetaPage + PAGE_SIZE),
@@ -1687,14 +1689,14 @@ TEST_F(CSDataStore_test, WriteChunkTest16) {
     FakeEnv();
     // fake read chunk1 metapage
     FakeEncodeChunk(chunk1MetaPage, 0, 2);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1MetaPage,
                         chunk1MetaPage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
     // fake read chunk1's snapshot1 metapage
     FakeEncodeSnapshot(chunk1SnapMetaPage, 3);
-    EXPECT_CALL(*lfs_, Read(2, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1SnapMetaPage,
                         chunk1SnapMetaPage + PAGE_SIZE),
@@ -1796,7 +1798,7 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest1) {
         .WillOnce(Return(true));
     EXPECT_CALL(*lfs_, Open(snapPath, _))
         .WillOnce(Return(4));
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(Return(-UT_ERRNO));
     EXPECT_CALL(*lfs_, Close(4))
         .Times(1);
@@ -1850,7 +1852,7 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest2) {
     char metapage[PAGE_SIZE];
     memset(metapage, 0, sizeof(metapage));
     FakeEncodeSnapshot(metapage, 2);
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(DoAll(SetArrayArgument<1>(metapage,
                         metapage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
@@ -1912,7 +1914,7 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest3) {
     char metapage[PAGE_SIZE];
     memset(metapage, 0, sizeof(metapage));
     FakeEncodeSnapshot(metapage, 2);
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(DoAll(SetArrayArgument<1>(metapage,
                         metapage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
@@ -1922,7 +1924,8 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest3) {
 
     LOG(INFO) << "case 1";
     // copy data failed
-    EXPECT_CALL(*lfs_, Read(3, NotNull(), PAGE_SIZE + offset, length))
+    EXPECT_CALL(*lfs_, Read(3, Matcher<char*>(NotNull()),
+        PAGE_SIZE + offset, length))
         .WillOnce(Return(-UT_ERRNO));
 
     EXPECT_EQ(CSErrorCode::InternalError,
@@ -1939,7 +1942,8 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest3) {
 
     LOG(INFO) << "case 2";
     // copy data success
-    EXPECT_CALL(*lfs_, Read(3, NotNull(), PAGE_SIZE + offset, length))
+    EXPECT_CALL(*lfs_, Read(3, Matcher<char*>(NotNull()),
+        PAGE_SIZE + offset, length))
         .Times(1);
     // write data to snapshot failed
     EXPECT_CALL(*lfs_, Write(4, Matcher<const char*>(NotNull()),
@@ -1958,7 +1962,8 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest3) {
 
     LOG(INFO) << "case 3";
     // copy data success
-    EXPECT_CALL(*lfs_, Read(3, NotNull(), PAGE_SIZE + offset, length))
+    EXPECT_CALL(*lfs_, Read(3, Matcher<char*>(NotNull()),
+        PAGE_SIZE + offset, length))
         .Times(1);
     // write data to snapshot success
     EXPECT_CALL(*lfs_,
@@ -1981,7 +1986,8 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest3) {
     // 再次写入仍会cow
     // will copy on write
     LOG(INFO) << "case 4";
-    EXPECT_CALL(*lfs_, Read(3, NotNull(), PAGE_SIZE + offset, length))
+    EXPECT_CALL(*lfs_, Read(3, Matcher<char*>(NotNull()),
+        PAGE_SIZE + offset, length))
         .Times(1);
     EXPECT_CALL(*lfs_, Write(4, Matcher<const char*>(NotNull()),
                              PAGE_SIZE + offset, length))
@@ -2044,7 +2050,7 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest4) {
     char metapage[PAGE_SIZE];
     memset(metapage, 0, sizeof(metapage));
     FakeEncodeSnapshot(metapage, 2);
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(DoAll(SetArrayArgument<1>(metapage,
                         metapage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
@@ -2052,7 +2058,8 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest4) {
     EXPECT_CALL(*lfs_, Write(3, Matcher<const char*>(NotNull()), 0, PAGE_SIZE))
         .Times(1);
     // will copy on write
-    EXPECT_CALL(*lfs_, Read(3, NotNull(), PAGE_SIZE + offset, length))
+    EXPECT_CALL(*lfs_, Read(3, Matcher<char*>(NotNull()),
+        PAGE_SIZE + offset, length))
         .Times(1);
     EXPECT_CALL(*lfs_, Write(4, Matcher<const char*>(NotNull()),
                              PAGE_SIZE + offset, length))
@@ -2196,7 +2203,7 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest5) {
         .WillOnce(DoAll(SetArgPointee<1>(fileInfo),
                         Return(0)));
     // read metapage failed
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(Return(-UT_ERRNO));
     EXPECT_EQ(CSErrorCode::InternalError, dataStore->WriteChunk(id,
                                                                 sn,
@@ -2265,7 +2272,7 @@ TEST_F(CSDataStore_test, WriteChunkErrorTest6) {
             .Times(1)
             .WillOnce(Return(4));
         // will read metapage
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
             .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                             chunk3MetaPage + PAGE_SIZE),
                             Return(PAGE_SIZE)));
@@ -2434,7 +2441,8 @@ TEST_F(CSDataStore_test, ReadChunkTest3) {
     char buf[length];  // NOLINT
     memset(buf, 0, sizeof(buf));
     // test chunk exists
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), offset + PAGE_SIZE, length))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<butil::IOPortal*>(NotNull()),
+        offset + PAGE_SIZE, length))
         .Times(1);
     EXPECT_EQ(CSErrorCode::Success,
               dataStore->ReadChunk(id,
@@ -2488,7 +2496,7 @@ TEST_F(CSDataStore_test, ReadChunkTest4) {
         .Times(1)
         .WillOnce(Return(4));
     // will read metapage
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                         chunk3MetaPage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
@@ -2504,7 +2512,7 @@ TEST_F(CSDataStore_test, ReadChunkTest4) {
     size_t length = PAGE_SIZE;
     char buf[2 * length];  // NOLINT
     memset(buf, 0, sizeof(buf));
-    EXPECT_CALL(*lfs_, Read(_, _, _, _))
+    EXPECT_CALL(*lfs_, Read(_, Matcher<char*>(_), _, _))
         .Times(0);
     EXPECT_EQ(CSErrorCode::PageNerverWrittenError,
               dataStore->ReadChunk(id,
@@ -2516,7 +2524,7 @@ TEST_F(CSDataStore_test, ReadChunkTest4) {
     // case2: 读取区域部分被写过
     offset = 0;
     length = 2 * PAGE_SIZE;
-    EXPECT_CALL(*lfs_, Read(_, _, _, _))
+    EXPECT_CALL(*lfs_, Read(_, Matcher<char*>(_), _, _))
         .Times(0);
     EXPECT_EQ(CSErrorCode::PageNerverWrittenError,
               dataStore->ReadChunk(id,
@@ -2528,7 +2536,8 @@ TEST_F(CSDataStore_test, ReadChunkTest4) {
     // case3: 读取区域已写过
     offset = 0;
     length = PAGE_SIZE;
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), offset + PAGE_SIZE, length))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<butil::IOPortal*>(NotNull()),
+        offset + PAGE_SIZE, length))
         .Times(1);
     EXPECT_EQ(CSErrorCode::Success,
               dataStore->ReadChunk(id,
@@ -2564,7 +2573,8 @@ TEST_F(CSDataStore_test, ReadChunkErrorTest1) {
     char buf[length];  // NOLINT
     memset(buf, 0, sizeof(buf));
     // test read chunk failed
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), offset + PAGE_SIZE, length))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<butil::IOPortal*>(NotNull()),
+        offset + PAGE_SIZE, length))
         .WillOnce(Return(-UT_ERRNO));
     EXPECT_EQ(CSErrorCode::InternalError,
               dataStore->ReadChunk(id,
@@ -2657,7 +2667,8 @@ TEST_F(CSDataStore_test, ReadSnapshotChunkTest2) {
     // test in range
     offset = PAGE_SIZE;
     length = 2 * PAGE_SIZE;
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), offset + PAGE_SIZE, length))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()),
+        offset + PAGE_SIZE, length))
         .Times(1);
     EXPECT_EQ(CSErrorCode::Success,
               dataStore->ReadSnapshotChunk(id,
@@ -2691,7 +2702,8 @@ TEST_F(CSDataStore_test, ReadSnapshotChunkTest3) {
     char writeBuf[length];  // NOLINT
     memset(writeBuf, 0, sizeof(writeBuf));
     // data in [PAGE_SIZE, 2*PAGE_SIZE) will be cow
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), offset + PAGE_SIZE, length))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()),
+        offset + PAGE_SIZE, length))
         .Times(1);
     EXPECT_CALL(*lfs_, Write(2, Matcher<const char*>(NotNull()),
                              offset + PAGE_SIZE, length))
@@ -2726,12 +2738,15 @@ TEST_F(CSDataStore_test, ReadSnapshotChunkTest3) {
     // test in range, read [0, 4*PAGE_SIZE)
     offset = 0;
     // read chunk in[0, PAGE_SIZE) and [3*PAGE_SIZE, 4*PAGE_SIZE)
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), PAGE_SIZE, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()),
+        PAGE_SIZE, PAGE_SIZE))
         .Times(1);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 4 * PAGE_SIZE, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()),
+        4 * PAGE_SIZE, PAGE_SIZE))
         .Times(1);
     // read snapshot in[PAGE_SIZE, 3*PAGE_SIZE)
-    EXPECT_CALL(*lfs_, Read(2, NotNull(), 2 * PAGE_SIZE, 2 * PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()),
+        2 * PAGE_SIZE, 2 * PAGE_SIZE))
         .Times(1);
     EXPECT_EQ(CSErrorCode::Success,
               dataStore->ReadSnapshotChunk(id,
@@ -2797,7 +2812,8 @@ TEST_F(CSDataStore_test, ReadSnapshotChunkErrorTest1) {
     char writeBuf[length];  // NOLINT
     memset(writeBuf, 0, sizeof(writeBuf));
     // data in [PAGE_SIZE, 2*PAGE_SIZE) will be cow
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), offset + PAGE_SIZE, length))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()),
+        offset + PAGE_SIZE, length))
         .Times(1);
     EXPECT_CALL(*lfs_, Write(2, Matcher<const char*>(NotNull()),
                              offset + PAGE_SIZE, length))
@@ -2824,7 +2840,8 @@ TEST_F(CSDataStore_test, ReadSnapshotChunkErrorTest1) {
     char readBuf[length];  // NOLINT
     memset(readBuf, 0, sizeof(readBuf));
     // read chunk failed
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), PAGE_SIZE, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()),
+        PAGE_SIZE, PAGE_SIZE))
         .WillOnce(Return(-UT_ERRNO));
     EXPECT_EQ(CSErrorCode::InternalError,
               dataStore->ReadSnapshotChunk(id,
@@ -2834,11 +2851,13 @@ TEST_F(CSDataStore_test, ReadSnapshotChunkErrorTest1) {
                                            length));
 
     // read snapshot failed
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), PAGE_SIZE, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), PAGE_SIZE, PAGE_SIZE))
         .Times(1);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 4 * PAGE_SIZE, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()),
+        4 * PAGE_SIZE, PAGE_SIZE))
         .Times(1);
-    EXPECT_CALL(*lfs_, Read(2, NotNull(), 2 * PAGE_SIZE, 2 * PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()),
+        2 * PAGE_SIZE, 2 * PAGE_SIZE))
         .WillOnce(Return(-UT_ERRNO));
     EXPECT_EQ(CSErrorCode::InternalError,
               dataStore->ReadSnapshotChunk(id,
@@ -2873,7 +2892,8 @@ TEST_F(CSDataStore_test, ReadSnapshotChunkErrorTest2) {
     memset(buf, 0, sizeof(buf));
     // test in range
     offset = PAGE_SIZE;
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), offset + PAGE_SIZE, length))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()),
+        offset + PAGE_SIZE, length))
         .WillOnce(Return(-UT_ERRNO));
     EXPECT_EQ(CSErrorCode::InternalError,
               dataStore->ReadSnapshotChunk(id,
@@ -2931,7 +2951,7 @@ TEST_F(CSDataStore_test, ReadChunkMetaDataTest2) {
     char buf[PAGE_SIZE];
     memset(buf, 0, PAGE_SIZE);
     // test chunk exists
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .Times(1);
     EXPECT_EQ(CSErrorCode::Success,
               dataStore->ReadChunkMetaPage(id, sn, buf));
@@ -3142,7 +3162,7 @@ TEST_F(CSDataStore_test, DeleteSnapshotChunkOrCorrectSnTest2) {
     FakeEnv();
     // set chunk1's correctedSn as 3
     FakeEncodeChunk(chunk1MetaPage, 3, 2);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1MetaPage,
                         chunk1MetaPage + PAGE_SIZE),
@@ -3183,7 +3203,7 @@ TEST_F(CSDataStore_test, DeleteSnapshotChunkOrCorrectSnTest3) {
     FakeEnv();
     // set chunk1's correctedSn as 0, sn as 3
     FakeEncodeChunk(chunk1MetaPage, 0, 3);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1MetaPage,
                         chunk1MetaPage + PAGE_SIZE),
@@ -3348,7 +3368,7 @@ TEST_F(CSDataStore_test, DeleteSnapshotChunkOrCorrectSnTest7) {
         .Times(1)
         .WillOnce(Return(4));
     // will read metapage
-    EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                         chunk3MetaPage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
@@ -3394,14 +3414,14 @@ TEST_F(CSDataStore_test, DeleteSnapshotChunkOrCorrectSnTest8) {
     FakeEnv();
     // fake read chunk1 metapage
     FakeEncodeChunk(chunk1MetaPage, 0, 2);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1MetaPage,
                         chunk1MetaPage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
     // fake read chunk1's snapshot1 metapage,chunk.sn==snap.sn
     FakeEncodeSnapshot(chunk1SnapMetaPage, 2);
-    EXPECT_CALL(*lfs_, Read(2, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1SnapMetaPage,
                         chunk1SnapMetaPage + PAGE_SIZE),
@@ -3445,14 +3465,14 @@ TEST_F(CSDataStore_test, DeleteSnapshotChunkOrCorrectSnTest9) {
     FakeEnv();
     // fake read chunk1 metapage
     FakeEncodeChunk(chunk1MetaPage, 2, 2);
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1MetaPage,
                         chunk1MetaPage + PAGE_SIZE),
                         Return(PAGE_SIZE)));
     // fake read chunk1's snapshot1 metapage,chunk.sn==snap.sn
     FakeEncodeSnapshot(chunk1SnapMetaPage, 3);
-    EXPECT_CALL(*lfs_, Read(2, NotNull(), 0, PAGE_SIZE))
+    EXPECT_CALL(*lfs_, Read(2, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
         .WillRepeatedly(DoAll(
                         SetArrayArgument<1>(chunk1SnapMetaPage,
                         chunk1SnapMetaPage + PAGE_SIZE),
@@ -3620,7 +3640,7 @@ TEST_F(CSDataStore_test, CreateCloneChunkTest) {
             .Times(1)
             .WillOnce(Return(4));
         // will read metapage
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
             .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                             chunk3MetaPage + PAGE_SIZE),
                             Return(PAGE_SIZE)));
@@ -3833,7 +3853,7 @@ TEST_F(CSDataStore_test, PasteChunkTest1) {
             .Times(1)
             .WillOnce(Return(4));
         // will read metapage
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
             .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                             chunk3MetaPage + PAGE_SIZE),
                             Return(PAGE_SIZE)));
@@ -4054,7 +4074,7 @@ TEST_F(CSDataStore_test, PasteChunkErrorTest1) {
             .Times(1)
             .WillOnce(Return(4));
         // will read metapage
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
             .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                             chunk3MetaPage + PAGE_SIZE),
                             Return(PAGE_SIZE)));
@@ -4159,7 +4179,7 @@ TEST_F(CSDataStore_test, GetHashErrorTest2) {
     off_t offset = 0;
     size_t length = PAGE_SIZE + CHUNK_SIZE;
     // test read chunk failed
-    EXPECT_CALL(*lfs_, Read(1, NotNull(), 0, 4096))
+    EXPECT_CALL(*lfs_, Read(1, Matcher<char*>(NotNull()), 0, 4096))
         .WillOnce(Return(-UT_ERRNO));
     EXPECT_EQ(CSErrorCode::InternalError,
               dataStore->GetChunkHash(id,
@@ -4227,7 +4247,7 @@ TEST_F(CSDataStore_test, CloneChunkUnAlignedTest) {
             .Times(1)
             .WillOnce(Return(4));
         // will read metapage
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
             .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                             chunk3MetaPage + PAGE_SIZE),
                             Return(PAGE_SIZE)));
@@ -4249,7 +4269,7 @@ TEST_F(CSDataStore_test, CloneChunkUnAlignedTest) {
     // read/write/paste offset are not aligned to pagesize
     {
         EXPECT_CALL(*lfs_, Write(4, Matcher<butil::IOBuf>(_), _, _)).Times(0);
-        EXPECT_CALL(*lfs_, Read(4, _, _, _)).Times(0);
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(_), _, _)).Times(0);
 
         ASSERT_EQ(CSErrorCode::InvalidArgError,
                   dataStore->WriteChunk(id, sn, buf, 1, length, nullptr));
@@ -4307,7 +4327,7 @@ TEST_F(CSDataStore_test, CloneChunkUnAlignedTest) {
     // read/write/paste length are not aligned to pagesize
     {
         EXPECT_CALL(*lfs_, Write(4, Matcher<butil::IOBuf>(_), _, _)).Times(0);
-        EXPECT_CALL(*lfs_, Read(4, _, _, _)).Times(0);
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(_), _, _)).Times(0);
 
         ASSERT_EQ(CSErrorCode::InvalidArgError,
                   dataStore->WriteChunk(id, sn, buf, 0, 1, nullptr));
@@ -4404,7 +4424,7 @@ TEST_F(CSDataStore_test, CloneChunkAlignedTest) {
             .Times(1)
             .WillOnce(Return(4));
         // will read metapage
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), 0, PAGE_SIZE))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()), 0, PAGE_SIZE))
             .WillOnce(DoAll(SetArrayArgument<1>(chunk3MetaPage,
                             chunk3MetaPage + PAGE_SIZE),
                             Return(PAGE_SIZE)));
@@ -4438,7 +4458,8 @@ TEST_F(CSDataStore_test, CloneChunkAlignedTest) {
 
     // read offset/length both aligned
     {
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), offset + PAGE_SIZE, length))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<butil::IOPortal*>(NotNull()),
+            offset + PAGE_SIZE, length))
             .Times(1)
             .WillOnce(Return(0));
         EXPECT_EQ(CSErrorCode::Success,
@@ -4453,7 +4474,8 @@ TEST_F(CSDataStore_test, CloneChunkAlignedTest) {
 
     // readspecificchunk offset/length both aligned
     {
-        EXPECT_CALL(*lfs_, Read(4, NotNull(), offset + PAGE_SIZE, length))
+        EXPECT_CALL(*lfs_, Read(4, Matcher<char*>(NotNull()),
+            offset + PAGE_SIZE, length))
             .Times(1)
             .WillOnce(Return(0));
         EXPECT_EQ(CSErrorCode::Success,
@@ -4495,7 +4517,7 @@ TEST_F(CSDataStore_test, NormalChunkAlignmentTest) {
 
     // read unaligned test
     {
-        EXPECT_CALL(*lfs_, Read(0, NotNull(), _, _))
+        EXPECT_CALL(*lfs_, Read(0, Matcher<char*>(NotNull()), _, _))
             .Times(0);
 
         EXPECT_EQ(CSErrorCode::InvalidArgError,
@@ -4524,7 +4546,7 @@ TEST_F(CSDataStore_test, NormalChunkAlignmentTest) {
 
     // read aligned test
     {
-        EXPECT_CALL(*lfs_, Read(3, _, _, _))
+        EXPECT_CALL(*lfs_, Read(3, Matcher<butil::IOPortal*>(_), _, _))
             .Times(AtLeast(1))
             .WillRepeatedly(Return(0));