[feat] tools-v2: add snapshot copyset --all

Signed-off-by: montaguelhz <1443171175@qq.com>

.bazelrc

@@ -1,7 +1,10 @@
+build --verbose_failures
+
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
 build --incompatible_blacklisted_protos_requires_proto_info=false
 build --copt=-fdiagnostics-color=always
+
 run --copt=-fdiagnostics-color=always

.gitignore

@@ -157,3 +157,12 @@ GPATH
 GRTAGS
 GTAGS
 core.*
+
+test/integration/*.conf
+test/integration/client/config/client.conf*
+test/integration/snapshotcloneserver/config/*.conf
+
+.pre-commit-config.yaml
+
+*.deb
+*.whl

WORKSPACE

@@ -75,11 +75,14 @@ bind(
 )
 
 #import the  gtest files.
-new_git_repository(
+http_archive(
     name = "com_google_googletest",
-    build_file = "//:thirdparties/gmock.BUILD",
-    remote = "https://github.com/google/googletest",
-    tag = "release-1.8.0",
+    urls = [
+        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
+        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
+    ],
+    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
+    strip_prefix = "googletest-release-1.12.1",
 )
 
 bind(

buildfs.sh

@@ -7,12 +7,22 @@ then
     exit
 fi
 
+if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
+then
+    bazelflags=''
+else
+    bazelflags='--copt -faligned-new'
+fi
+
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi
 
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
+--define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
+-DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
+
 if [ $? -ne 0 ]
 then
     echo "build curvefs failed"
@@ -34,4 +44,5 @@ then
     echo "mds_test failed"
     exit
 fi
-fi
+fi
+echo "end compile"

conf/chunkserver.conf

@@ -9,13 +9,20 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.block_size=4096
+
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +115,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/chunkserver.conf.example

@@ -13,9 +13,12 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +111,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/client.conf

@@ -168,13 +168,6 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
 
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
-
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@@ -188,4 +181,3 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
-

conf/cs_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/mds.conf

@@ -197,6 +197,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -237,3 +239,16 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
+
+#
+## poolset rules
+#
+# for backward compatibility, rules are applied for select poolset when creating file
+#
+# for example
+#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
+#
+# when creating file reqeust doesn't have poolset, above rules are used to select poolset
+# - if filename is /dir1/file, then poolset1 is select
+# - if filename is /dir1/sub/file, then sub is select
+mds.poolset.rules=

conf/py_client.conf

@@ -153,10 +153,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/snap_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

curve-ansible/roles/generate_config/defaults/main.yml

@@ -164,7 +164,6 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
-chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -244,8 +243,6 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
-client_alignment_common: 512
-client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -14,8 +14,6 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
-# minimum alignment for io request
-global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -175,10 +175,3 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume={{ client_alignment_common }}
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume={{ client_alignment_clone }}

curvefs/conf/client.conf

@@ -135,14 +135,14 @@ fuseClient.throttle.burstReadIopsSecs=180
 #### filesystem metadata
 # {
 # fs.disableXattr:
-#   if you want to get curvefs specified xattr,
-#   you can mount another fs with |fs.disableXattr| is true
+#   if you want to get better metadata performance,
+#   you can mount fs with |fs.disableXattr| is true
 #
 # fs.lookupCache.negativeTimeoutSec:
 #   entry which not found will be cached if |timeout| > 0
 fs.cto=true
 fs.maxNameLength=255
-fs.disableXattr=true
+fs.disableXattr=false
 fs.accessLogging=true
 fs.kernelCache.attrTimeoutSec=3600
 fs.kernelCache.dirAttrTimeoutSec=3600

curvefs/conf/mds.conf

@@ -6,6 +6,9 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
 mds.loglevel=0
+# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+mds.server.idleTimeoutSec=-1
 
 #
 # space options

curvefs/conf/metaserver.conf

@@ -128,16 +128,25 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000
 
-### apply queue options for each copyset
-### apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding worker queue by certain rules
-# number of apply queue workers for each, each worker will start a indepent thread
-applyqueue.worker_count=3
+#
+# Concurrent apply queue
+### concurrent apply queue options for each copyset
+### concurrent apply queue is used to isolate raft threads, each worker has its own queue
+### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
 
-# apply queue depth for each copyset
+# worker_count: number of apply queue workers for each, each worker will start a indepent thread
+# queue_depth: apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-applyqueue.queue_depth=1
+# write apply queue workers count
+applyqueue.write_worker_count=3
+# write apply queue depth
+applyqueue.write_queue_depth=1
+# read apply queue workers count
+applyqueue.read_worker_count=2
+# read apply queue depth
+applyqueue.read_queue_depth=1
+
 
 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@@ -145,6 +154,10 @@ applyqueue.queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto
 
+# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+server.idleTimeoutSec=-1
+
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need

curvefs/src/client/filesystem/rpc_client.cpp

@@ -104,9 +104,11 @@ CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
             continue;
         }
 
+        // NOTE: we can't use std::move() for attribute for hard link
+        // which will sharing inode attribute.
         dirEntry.ino = ino;
         dirEntry.name = std::move(dentry.name());
-        dirEntry.attr = std::move(iter->second);
+        dirEntry.attr = iter->second;
         (*entries)->Add(dirEntry);
     }
     return CURVEFS_ERROR::OK;

curvefs/src/client/fuse_client.cpp

@@ -219,6 +219,7 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                      struct fuse_conn_info *conn) {
     (void)userdata;
     (void)conn;
+    fs_->Run();
     return CURVEFS_ERROR::OK;
 }
 
@@ -368,10 +369,7 @@ CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
         if (option_.fileSystemOption.deferSyncOption.deferDirMtime) {
             inodeManager_->ShipToFlush(parentInodeWrapper);
         } else {
-            ret = parentInodeWrapper->SyncAttr();
-            if (ret != CURVEFS_ERROR::OK) {
-                return CURVEFS_ERROR::OK;
-            }
+            return parentInodeWrapper->SyncAttr();
         }
     }
 

curvefs/src/client/warmup/warmup_manager.cpp

@@ -473,7 +473,7 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
             }
             if (context->retCode == 0) {
                 VLOG(9) << "Get Object success: " << context->key;
-                PutObjectToCache(key, context->key, context->buf, context->len);
+                PutObjectToCache(key, context);
                 CollectMetrics(&warmupS3Metric_.warmupS3Cached, context->len,
                                start);
                 warmupS3Metric_.warmupS3CacheSize << context->len;
@@ -481,7 +481,6 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
                     VLOG(6) << "pendingReq is over";
                     cond.Signal();
                 }
-                delete[] context->buf;
                 return;
             }
             warmupS3Metric_.warmupS3Cached.eps.count << 1;
@@ -678,9 +677,8 @@ void WarmupManagerS3Impl::AddFetchS3objectsTask(fuse_ino_t key,
     }
 }
 
-void WarmupManagerS3Impl::PutObjectToCache(fuse_ino_t key,
-                                           const std::string &filename,
-                                           const char *data, uint64_t len) {
+void WarmupManagerS3Impl::PutObjectToCache(
+    fuse_ino_t key, const std::shared_ptr<GetObjectAsyncContext> &context) {
     ReadLockGuard lock(inode2ProgressMutex_);
     auto iter = FindWarmupProgressByKeyLocked(key);
     if (iter == inode2Progress_.end()) {
@@ -692,17 +690,21 @@ void WarmupManagerS3Impl::PutObjectToCache(fuse_ino_t key,
     iter->second.FinishedPlusOne();
     switch (iter->second.GetStorageType()) {
     case curvefs::client::common::WarmupStorageType::kWarmupStorageTypeDisk:
-        ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(filename, data,
-                                                                 len);
+        ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(
+            context->key, context->buf, context->len);
         if (ret < 0) {
             LOG_EVERY_SECOND(INFO)
-                << "write read directly failed, key: " << filename;
+                << "write read directly failed, key: " << context->key;
         }
+        delete[] context->buf;
         break;
     case curvefs::client::common::WarmupStorageType::kWarmupStorageTypeKvClient:
         if (kvClientManager_ != nullptr) {
-            kvClientManager_->Set(
-                std::make_shared<SetKVCacheTask>(filename, data, len));
+            kvClientManager_->Set(std::make_shared<SetKVCacheTask>(
+                context->key, context->buf, context->len,
+                [context](const std::shared_ptr<SetKVCacheTask> &) {
+                    delete[] context->buf;
+                }));
         }
         break;
     default:

curvefs/src/client/warmup/warmup_manager.h

@@ -413,8 +413,9 @@ class WarmupManagerS3Impl : public WarmupManager {
 
     void AddFetchS3objectsTask(fuse_ino_t key, std::function<void()> task);
 
-    void PutObjectToCache(fuse_ino_t key, const std::string &filename,
-                          const char *data, uint64_t len);
+    void
+    PutObjectToCache(fuse_ino_t key,
+                     const std::shared_ptr<GetObjectAsyncContext> &context);
 
  protected:
     std::deque<WarmupFilelist> warmupFilelistDeque_;

curvefs/src/mds/fs_storage.cpp

@@ -359,6 +359,15 @@ bool PersisKVStorage::LoadAllFs() {
 
         idToName_.emplace(fsInfo.fsid(), fsInfo.fsname());
 
+        // For compatibility when upgrading, the new field 'optional
+        // objectPrefix` in message `S3Info` needs to be set to the default
+        // value
+        if (fsInfo.fstype() == FSType::TYPE_S3) {
+            if (!fsInfo.detail().s3info().has_objectprefix()) {
+                fsInfo.mutable_detail()->mutable_s3info()->set_objectprefix(0);
+            }
+        }
+
         fs_.emplace(fsInfo.fsname(), std::move(fsInfo));
     }
 

curvefs/src/mds/mds.cpp

@@ -66,6 +66,8 @@ void MDS::InitOptions(std::shared_ptr<Configuration> conf) {
     conf_ = std::move(conf);
     conf_->GetValueFatalIfFail("mds.listen.addr", &options_.mdsListenAddr);
     conf_->GetValueFatalIfFail("mds.dummy.port", &options_.dummyPort);
+    conf_->GetValueFatalIfFail("mds.server.idleTimeoutSec",
+                               &options_.idleTimeoutSec);
 
     InitMetaServerOption(&options_.metaserverOptions);
     InitTopologyOption(&options_.topologyOptions);
@@ -275,6 +277,7 @@ void MDS::Run() {
 
     // start rpc server
     brpc::ServerOptions option;
+    option.idle_timeout_sec = options_.idleTimeoutSec;
     LOG_IF(FATAL, server.Start(options_.mdsListenAddr.c_str(), &option) != 0)
         << "start brpc server error";
     running_ = true;

curvefs/src/mds/mds.h

@@ -85,6 +85,8 @@ using ::curvefs::mds::space::MdsProxyOptions;
 struct MDSOptions {
     int dummyPort;
     std::string mdsListenAddr;
+    int idleTimeoutSec;
+
     MetaserverOptions metaserverOptions;
     // TODO(add EtcdConf): add etcd configure
 

curvefs/src/metaserver/copyset/apply_queue.cpp

@@ -1,135 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: curve
- * Date: Thu Sep  2 14:49:04 CST 2021
- * Author: wuhanqing
- */
-
-#include "curvefs/src/metaserver/copyset/apply_queue.h"
-
-#include <glog/logging.h>
-
-#include "absl/memory/memory.h"
-#include "absl/strings/str_cat.h"
-#include "curvefs/src/common/threading.h"
-#include "curvefs/src/metaserver/copyset/copyset_node.h"
-
-namespace curvefs {
-namespace metaserver {
-namespace copyset {
-
-using ::curvefs::common::SetThreadName;
-
-void ApplyQueue::StartWorkers() {
-    for (uint32_t i = 0; i < option_.workerCount; ++i) {
-        std::string name = [this, i]() -> std::string {
-            if (option_.copysetNode == nullptr) {
-                return "apply";
-            }
-
-            return absl::StrCat("apply", ":", option_.copysetNode->GetPoolId(),
-                                "_", option_.copysetNode->GetCopysetId(), ":",
-                                i);
-        }();
-        auto taskThread =
-            absl::make_unique<TaskWorker>(option_.queueDepth, std::move(name));
-        taskThread->Start();
-        workers_.emplace_back(std::move(taskThread));
-    }
-}
-
-bool ApplyQueue::Start(const ApplyQueueOption& option) {
-    if (running_) {
-        return true;
-    }
-
-    if (option.queueDepth < 1 || option.workerCount < 1) {
-        LOG(ERROR) << "ApplyQueue start failed, invalid argument, queue depth: "
-                   << option.queueDepth
-                   << ", worker count: " << option.workerCount;
-        return false;
-    }
-
-    option_ = option;
-
-    StartWorkers();
-    running_.store(true);
-    return true;
-}
-
-void ApplyQueue::Flush() {
-    if (!running_.load(std::memory_order_relaxed)) {
-        return;
-    }
-
-    curve::common::CountDownEvent event(option_.workerCount);
-
-    auto flush = [&event]() { event.Signal(); };
-
-    for (auto& worker : workers_) {
-        worker->tasks.Push(flush);
-    }
-
-    event.Wait();
-}
-
-void ApplyQueue::Stop() {
-    if (!running_.exchange(false)) {
-        return;
-    }
-
-    LOG(INFO) << "Going to stop apply queue";
-
-    for (auto& worker : workers_) {
-        worker->Stop();
-    }
-
-    workers_.clear();
-    LOG(INFO) << "Apply queue stopped";
-}
-
-void ApplyQueue::TaskWorker::Start() {
-    if (running.exchange(true)) {
-        return;
-    }
-
-    worker = std::thread(&TaskWorker::Work, this);
-}
-
-void ApplyQueue::TaskWorker::Stop() {
-    if (!running.exchange(false)) {
-        return;
-    }
-
-    auto wakeup = []() {};
-    tasks.Push(wakeup);
-
-    worker.join();
-}
-
-void ApplyQueue::TaskWorker::Work() {
-    SetThreadName(workerName_.c_str());
-
-    while (running.load(std::memory_order_relaxed)) {
-        tasks.Pop()();
-    }
-}
-
-}  // namespace copyset
-}  // namespace metaserver
-}  // namespace curvefs

curvefs/src/metaserver/copyset/apply_queue.h

@@ -1,98 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: curve
- * Date: Thu Sep  2 14:49:04 CST 2021
- * Author: wuhanqing
- */
-
-#ifndef CURVEFS_SRC_METASERVER_COPYSET_APPLY_QUEUE_H_
-#define CURVEFS_SRC_METASERVER_COPYSET_APPLY_QUEUE_H_
-
-#include <atomic>
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <thread>
-#include <utility>
-#include <vector>
-
-#include "include/curve_compiler_specific.h"
-#include "src/common/concurrent/count_down_event.h"
-#include "src/common/concurrent/task_queue.h"
-
-namespace curvefs {
-namespace metaserver {
-namespace copyset {
-
-class CopysetNode;
-
-struct ApplyQueueOption {
-    uint32_t workerCount = 1;
-    uint32_t queueDepth = 1;
-    CopysetNode* copysetNode = nullptr;
-};
-
-class CURVE_CACHELINE_ALIGNMENT ApplyQueue {
- public:
-    ApplyQueue() : option_(), running_(false), workers_() {}
-
-    bool Start(const ApplyQueueOption& option);
-
-    template <typename Func, typename... Args>
-    void Push(uint64_t hash, Func&& f, Args&&... args) {
-        workers_[hash % option_.workerCount]->tasks.Push(
-            std::forward<Func>(f), std::forward<Args>(args)...);
-    }
-
-    void Flush();
-
-    void Stop();
-
- private:
-    void StartWorkers();
-
-    struct TaskWorker {
-        TaskWorker(size_t cap, std::string workerName)
-            : running(false),
-              worker(),
-              tasks(cap),
-              workerName_(std::move(workerName)) {}
-
-        void Start();
-
-        void Stop();
-
-        void Work();
-
-        std::atomic<bool> running;
-        std::thread worker;
-        curve::common::TaskQueue tasks;
-        std::string workerName_;
-    };
-
- private:
-    ApplyQueueOption option_;
-    std::atomic<bool> running_;
-    std::vector<std::unique_ptr<TaskWorker>> workers_;
-};
-
-}  // namespace copyset
-}  // namespace metaserver
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_METASERVER_COPYSET_APPLY_QUEUE_H_

curvefs/src/metaserver/copyset/concurrent_apply_queue.cpp

@@ -0,0 +1,198 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * File Created: 20230521
+ * Author: Xinlong-Chen
+ */
+
+
+#include "curvefs/src/metaserver/copyset/concurrent_apply_queue.h"
+
+#include <algorithm>
+
+namespace curvefs {
+namespace metaserver {
+namespace copyset {
+bool ApplyQueue::Init(const ApplyOption &opt) {
+    if (start_) {
+        LOG(WARNING) << "concurrent module already start!";
+        return true;
+    }
+
+    if (false == CheckOptAndInit(opt)) {
+        return false;
+    }
+
+    start_ = true;
+    cond_.Reset(opt.rconcurrentsize + opt.wconcurrentsize);
+    InitThreadPool(ThreadPoolType::READ, rconcurrentsize_, rqueuedepth_);
+    InitThreadPool(ThreadPoolType::WRITE, wconcurrentsize_, wqueuedepth_);
+
+    if (!cond_.WaitFor(5000)) {
+        LOG(ERROR) << "init concurrent module's threads fail";
+        start_ = false;
+    }
+
+    LOG(INFO) << "Init concurrent module's threads success";
+    return start_;
+}
+
+bool ApplyQueue::CheckOptAndInit(const ApplyOption &opt) {
+    if (opt.rconcurrentsize <= 0 || opt.wconcurrentsize <= 0 ||
+        opt.rqueuedepth <= 0 || opt.wqueuedepth <= 0) {
+        LOG(INFO) << "init concurrent module fail, params must >=0"
+            << ", rconcurrentsize=" << opt.rconcurrentsize
+            << ", wconcurrentsize=" << opt.wconcurrentsize
+            << ", rqueuedepth=" << opt.rqueuedepth
+            << ", wconcurrentsize=" << opt.wqueuedepth;
+        return false;
+    }
+
+    wconcurrentsize_ = opt.wconcurrentsize;
+    wqueuedepth_ = opt.wqueuedepth;
+    rconcurrentsize_ = opt.rconcurrentsize;
+    rqueuedepth_ = opt.rqueuedepth;
+
+    return true;
+}
+
+void ApplyQueue::InitThreadPool(
+    ThreadPoolType type, int concurrent, int depth) {
+    for (int i = 0; i < concurrent; i++) {
+        auto asyncth = new (std::nothrow) TaskThread(depth);
+        CHECK(asyncth != nullptr) << "allocate failed!";
+
+        switch (type) {
+        case ThreadPoolType::READ:
+            rapplyMap_.insert(std::make_pair(i, asyncth));
+            break;
+
+        case ThreadPoolType::WRITE:
+            wapplyMap_.insert(std::make_pair(i, asyncth));
+            break;
+        }
+    }
+
+    for (int i = 0; i < concurrent; i++) {
+        switch (type) {
+        case ThreadPoolType::READ:
+            rapplyMap_[i]->th =
+                    std::thread(&ApplyQueue::Run, this, type, i);
+            break;
+
+        case ThreadPoolType::WRITE:
+            wapplyMap_[i]->th =
+                    std::thread(&ApplyQueue::Run, this, type, i);
+            break;
+        }
+    }
+}
+
+void ApplyQueue::Run(ThreadPoolType type, int index) {
+    cond_.Signal();
+    while (start_) {
+        switch (type) {
+        case ThreadPoolType::READ:
+            rapplyMap_[index]->tq.Pop()();
+            break;
+
+        case ThreadPoolType::WRITE:
+            wapplyMap_[index]->tq.Pop()();
+            break;
+        }
+    }
+}
+
+void ApplyQueue::Stop() {
+    if (!start_.exchange(false)) {
+        return;
+    }
+
+    LOG(INFO) << "stop ApplyQueue...";
+    auto wakeup = []() {};
+    for (auto iter : rapplyMap_) {
+        iter.second->tq.Push(wakeup);
+        iter.second->th.join();
+        delete iter.second;
+    }
+    rapplyMap_.clear();
+
+    for (auto iter : wapplyMap_) {
+        iter.second->tq.Push(wakeup);
+        iter.second->th.join();
+        delete iter.second;
+    }
+    wapplyMap_.clear();
+
+    LOG(INFO) << "stop ApplyQueue ok.";
+}
+
+void ApplyQueue::Flush() {
+    if (!start_.load(std::memory_order_relaxed)) {
+        return;
+    }
+
+    CountDownEvent event(wconcurrentsize_);
+    auto flushtask = [&event]() {
+        event.Signal();
+    };
+
+    for (int i = 0; i < wconcurrentsize_; i++) {
+        wapplyMap_[i]->tq.Push(flushtask);
+    }
+
+    event.Wait();
+}
+
+void ApplyQueue::FlushAll() {
+    if (!start_.load(std::memory_order_relaxed)) {
+        return;
+    }
+
+    CountDownEvent event(wconcurrentsize_ + rconcurrentsize_);
+    auto flushtask = [&event]() {
+        event.Signal();
+    };
+
+    for (int i = 0; i < wconcurrentsize_; i++) {
+        wapplyMap_[i]->tq.Push(flushtask);
+    }
+
+    for (int i = 0; i < rconcurrentsize_; i++) {
+        rapplyMap_[i]->tq.Push(flushtask);
+    }
+
+    event.Wait();
+}
+
+ThreadPoolType ApplyQueue::Schedule(OperatorType optype) {
+    switch (optype) {
+    case OperatorType::GetDentry:
+    case OperatorType::ListDentry:
+    case OperatorType::GetInode:
+    case OperatorType::BatchGetInodeAttr:
+    case OperatorType::BatchGetXAttr:
+    case OperatorType::GetVolumeExtent:
+        return ThreadPoolType::READ;
+    default:
+        return ThreadPoolType::WRITE;
+    }
+}
+}   // namespace copyset
+}   // namespace metaserver
+}   // namespace curvefs

curvefs/src/metaserver/copyset/concurrent_apply_queue.h

@@ -0,0 +1,152 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * File Created: 20230521
+ * Author: Xinlong-Chen
+ */
+
+#ifndef CURVEFS_SRC_METASERVER_COPYSET_CONCURRENT_APPLY_QUEUE_H_
+#define CURVEFS_SRC_METASERVER_COPYSET_CONCURRENT_APPLY_QUEUE_H_
+
+#include <bthread/condition_variable.h>
+#include <bthread/mutex.h>
+#include <glog/logging.h>
+
+#include <atomic>
+#include <thread>
+#include <unordered_map>
+#include <utility>
+
+#include "include/curve_compiler_specific.h"
+#include "src/common/concurrent/count_down_event.h"
+#include "src/common/concurrent/task_queue.h"
+#include "curvefs/src/metaserver/copyset/operator_type.h"
+
+namespace curvefs {
+namespace metaserver {
+namespace copyset {
+
+using curve::common::CountDownEvent;
+using curve::common::GenericTaskQueue;
+
+struct ApplyOption {
+    int wconcurrentsize = 3;
+    int wqueuedepth = 1;
+    int rconcurrentsize = 1;
+    int rqueuedepth = 1;
+    ApplyOption(int wsize, int wdepth, int rsize, int rdepth) :
+        wconcurrentsize(wsize),
+        wqueuedepth(wdepth),
+        rconcurrentsize(rsize),
+        rqueuedepth(rdepth) {}
+    ApplyOption() {}
+};
+
+enum class ThreadPoolType {READ, WRITE};
+
+/*
+TODO: this moudle is same as curvebs's ConcurrentApplyModule,
+      only Schedule function is different.
+      we can make ApplyQueue to a base class,
+      and define Schedule with virtual function,
+      derive class override Schedule.
+*/ 
+
+class CURVE_CACHELINE_ALIGNMENT ApplyQueue {
+ public:
+    ApplyQueue(): start_(false),
+                  rconcurrentsize_(0),
+                  rqueuedepth_(0),
+                  wconcurrentsize_(0),
+                  wqueuedepth_(0),
+                  cond_(0) {}
+
+    /**
+     * Init: initialize ApplyQueue
+     * @param[in] wconcurrentsize: num of write threads
+     * @param[in] wqueuedepth: depth of write queue in ervery thread
+     * @param[in] rconcurrentsizee: num of read threads
+     * @param[in] wqueuedephth: depth of read queue in every thread
+     */
+    bool Init(const ApplyOption &opt);
+
+    /**
+     * Push: apply task will be push to ApplyQueue
+     * @param[in] key: used to hash task to specified queue
+     * @param[in] optype: operation type defined in proto
+     * @param[in] f: task
+     * @param[in] args: param to excute task
+     */
+    template <class F, class... Args>
+    bool Push(uint64_t key, OperatorType optype, F&& f, Args&&... args) {
+        switch (Schedule(optype)) {
+            case ThreadPoolType::READ:
+                rapplyMap_[Hash(key, rconcurrentsize_)]->tq.Push(
+                        std::forward<F>(f), std::forward<Args>(args)...);
+                break;
+            case ThreadPoolType::WRITE:
+                wapplyMap_[Hash(key, wconcurrentsize_)]->tq.Push(
+                        std::forward<F>(f), std::forward<Args>(args)...);
+                break;
+        }
+
+        return true;
+    }
+
+    /**
+     * Flush: finish all task in write threads
+     */
+    void Flush();
+    void FlushAll();
+
+    void Stop();
+
+ private:
+    bool CheckOptAndInit(const ApplyOption &option);
+
+    void Run(ThreadPoolType type, int index);
+
+    static ThreadPoolType Schedule(OperatorType optype);
+
+    void InitThreadPool(ThreadPoolType type, int concorrent, int depth);
+
+    static int Hash(uint64_t key, int concurrent) {
+        return key % concurrent;
+    }
+
+ private:
+    struct TaskThread {
+        std::thread th;
+        GenericTaskQueue<bthread::Mutex, bthread::ConditionVariable> tq;
+        explicit TaskThread(size_t capacity) : tq(capacity) {}
+    };
+
+    std::atomic<bool> start_;
+    int rconcurrentsize_;
+    int rqueuedepth_;
+    int wconcurrentsize_;
+    int wqueuedepth_;
+    CountDownEvent cond_;
+    CURVE_CACHELINE_ALIGNMENT std::unordered_map<int, TaskThread*> wapplyMap_;
+    CURVE_CACHELINE_ALIGNMENT std::unordered_map<int, TaskThread*> rapplyMap_;
+};
+}   // namespace copyset
+}   // namespace metaserver
+}   // namespace curvefs
+
+#endif  // CURVEFS_SRC_METASERVER_COPYSET_CONCURRENT_APPLY_QUEUE_H_

curvefs/src/metaserver/copyset/config.h

@@ -31,7 +31,7 @@
 #include <memory>
 #include <string>
 
-#include "curvefs/src/metaserver/copyset/apply_queue.h"
+#include "curvefs/src/metaserver/copyset/concurrent_apply_queue.h"
 #include "curvefs/src/metaserver/copyset/trash.h"
 #include "curvefs/src/metaserver/storage/config.h"
 #include "src/fs/local_filesystem.h"
@@ -74,7 +74,7 @@ struct CopysetNodeOptions {
     uint32_t checkLoadMarginIntervalMs;
 
     // apply queue options
-    ApplyQueueOption applyQueueOption;
+    ApplyOption applyQueueOption;
 
     // filesystem adaptor
     curve::fs::LocalFileSystem* localFileSystem;

curvefs/src/metaserver/copyset/copyset_node.cpp

@@ -120,9 +120,8 @@ bool CopysetNode::Init(const CopysetNodeOptions& options) {
 
     // init apply queue
     applyQueue_ = absl::make_unique<ApplyQueue>();
-    options_.applyQueueOption.copysetNode = this;
-    if (!applyQueue_->Start(options_.applyQueueOption)) {
-        LOG(ERROR) << "Start apply queue failed";
+    if (!applyQueue_->Init(options_.applyQueueOption)) {
+        LOG(ERROR) << "init concurrent apply queue failed";
         return false;
     }
 
@@ -258,6 +257,7 @@ void CopysetNode::on_apply(braft::Iterator& iter) {
                           iter.index(), doneGuard.release(),
                           TimeUtility::GetTimeofDayUs());
             applyQueue_->Push(metaClosure->GetOperator()->HashCode(),
+                              metaClosure->GetOperator()->GetOperatorType(),
                               std::move(task));
             timer.stop();
             g_concurrent_apply_wait_latency << timer.u_elapsed();
@@ -268,10 +268,11 @@ void CopysetNode::on_apply(braft::Iterator& iter) {
             butil::Timer timer;
             timer.start();
             auto hashcode = metaOperator->HashCode();
+            auto type = metaOperator->GetOperatorType();
             auto task =
                 std::bind(&MetaOperator::OnApplyFromLog, metaOperator.release(),
                           TimeUtility::GetTimeofDayUs());
-            applyQueue_->Push(hashcode, std::move(task));
+            applyQueue_->Push(hashcode, type, std::move(task));
             timer.stop();
             g_concurrent_apply_from_log_wait_latency << timer.u_elapsed();
         }
@@ -426,6 +427,9 @@ int CopysetNode::on_snapshot_load(braft::SnapshotReader* reader) {
 }
 
 void CopysetNode::on_leader_start(int64_t term) {
+    LOG(INFO) << "Copyset: " << name_ << ", peer id: " << peerId_.to_string()
+          << " going to flush apply queue, term is " << term;
+    applyQueue_->Flush();
     leaderTerm_.store(term, std::memory_order_release);
 
     LOG(INFO) << "Copyset: " << name_ << ", peer id: " << peerId_.to_string()

curvefs/src/metaserver/copyset/copyset_node.h

@@ -31,7 +31,7 @@
 #include <vector>
 
 #include "curvefs/src/metaserver/common/types.h"
-#include "curvefs/src/metaserver/copyset/apply_queue.h"
+#include "curvefs/src/metaserver/copyset/concurrent_apply_queue.h"
 #include "curvefs/src/metaserver/copyset/conf_epoch_file.h"
 #include "curvefs/src/metaserver/copyset/config.h"
 #include "curvefs/src/metaserver/copyset/copyset_conf_change.h"
@@ -123,7 +123,7 @@ class CopysetNode : public braft::StateMachine {
 #ifdef UNIT_TEST
     void SetMetaStore(MetaStore* metastore) { metaStore_.reset(metastore); }
 
-    void FlushApplyQueue() { applyQueue_->Flush(); }
+    void FlushApplyQueue() { applyQueue_->FlushAll(); }
 
     void SetRaftNode(RaftNode* raftNode) { raftNode_.reset(raftNode); }
 #endif  // UNIT_TEST

curvefs/src/metaserver/copyset/meta_operator.cpp

@@ -107,7 +107,8 @@ void MetaOperator::FastApplyTask() {
     auto task =
         std::bind(&MetaOperator::OnApply, this, node_->GetAppliedIndex(),
                   new MetaOperatorClosure(this), TimeUtility::GetTimeofDayUs());
-    node_->GetApplyQueue()->Push(HashCode(), std::move(task));
+    node_->GetApplyQueue()->Push(HashCode(),
+                                 GetOperatorType(), std::move(task));
     timer.stop();
     g_concurrent_fast_apply_wait_latency << timer.u_elapsed();
 }

curvefs/src/metaserver/metaserver.cpp

@@ -68,7 +68,6 @@ using ::curve::fs::FileSystemType;
 using ::curve::fs::LocalFsFactory;
 using ::curve::fs::LocalFileSystemOption;
 
-using ::curvefs::metaserver::copyset::ApplyQueueOption;
 using ::curvefs::client::rpcclient::MetaServerClientImpl;
 using ::curvefs::client::rpcclient::ChannelManager;
 using ::curvefs::client::rpcclient::MetaCache;
@@ -100,6 +99,9 @@ void Metaserver::InitOptions(std::shared_ptr<Configuration> conf) {
             << "Parse bthread.worker_count to int failed, string value: "
             << value;
     }
+    conf_->GetValueFatalIfFail("server.idleTimeoutSec",
+                               &options_.idleTimeoutSec);
+
 
     InitBRaftFlags(conf);
 }
@@ -456,6 +458,7 @@ void Metaserver::Run() {
     if (options_.bthreadWorkerCount != -1) {
         option.num_threads = options_.bthreadWorkerCount;
     }
+    option.idle_timeout_sec = options_.idleTimeoutSec;
     LOG_IF(FATAL, server_->Start(listenAddr, &option) != 0)
         << "start internal brpc server error";
 
@@ -653,13 +656,18 @@ void Metaserver::InitCopysetNodeOptions() {
                       "copyset.check_loadmargin_interval_ms",
                       &copysetNodeOptions_.checkLoadMarginIntervalMs));
 
-    LOG_IF(FATAL, !conf_->GetUInt32Value(
-                      "applyqueue.worker_count",
-                      &copysetNodeOptions_.applyQueueOption.workerCount));
-    LOG_IF(FATAL, !conf_->GetUInt32Value(
-                      "applyqueue.queue_depth",
-                      &copysetNodeOptions_.applyQueueOption.queueDepth));
-
+    LOG_IF(FATAL, !conf_->GetIntValue(
+                      "applyqueue.write_worker_count",
+                      &copysetNodeOptions_.applyQueueOption.wconcurrentsize));
+    LOG_IF(FATAL, !conf_->GetIntValue(
+                      "applyqueue.write_queue_depth",
+                      &copysetNodeOptions_.applyQueueOption.wqueuedepth));
+    LOG_IF(FATAL, !conf_->GetIntValue(
+                      "applyqueue.read_worker_count",
+                      &copysetNodeOptions_.applyQueueOption.rconcurrentsize));
+    LOG_IF(FATAL, !conf_->GetIntValue(
+                      "applyqueue.read_queue_depth",
+                      &copysetNodeOptions_.applyQueueOption.rqueuedepth));
     LOG_IF(FATAL,
            !conf_->GetStringValue("copyset.trash.uri",
                                   &copysetNodeOptions_.trashOptions.trashUri));

curvefs/src/metaserver/metaserver.h

@@ -28,7 +28,6 @@
 #include <memory>
 #include <string>
 
-#include "curvefs/src/metaserver/copyset/apply_queue.h"
 #include "curvefs/src/metaserver/copyset/config.h"
 #include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
 #include "curvefs/src/metaserver/copyset/raft_cli_service2.h"
@@ -52,7 +51,6 @@ namespace metaserver {
 
 using ::curve::common::Configuration;
 using ::curvefs::metaserver::storage::StorageOptions;
-using ::curvefs::metaserver::copyset::ApplyQueue;
 using ::curvefs::metaserver::copyset::CopysetNodeManager;
 using ::curvefs::metaserver::copyset::CopysetNodeOptions;
 using ::curvefs::metaserver::copyset::CopysetServiceImpl;
@@ -70,6 +68,7 @@ struct MetaserverOptions {
     std::string externalIp;
     int externalPort;
     int bthreadWorkerCount = -1;
+    int idleTimeoutSec = -1;
     bool enableExternalServer;
 };
 

curvefs/src/metaserver/trash.cpp

@@ -208,6 +208,21 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
         clientAdaptorOption.objectPrefix = s3Info.objectprefix();
         s3Adaptor_->Reinit(clientAdaptorOption, s3Info.ak(), s3Info.sk(),
             s3Info.endpoint(), s3Info.bucketname());
+        ret = inodeStorage_->PaddingInodeS3ChunkInfo(item.fsId,
+          item.inodeId, inode.mutable_s3chunkinfomap());
+        if (ret != MetaStatusCode::OK) {
+            LOG(ERROR) << "GetInode chunklist fail, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId
+                << ", retCode = " << MetaStatusCode_Name(ret);
+            return ret;
+        }
+        if (inode.s3chunkinfomap().empty()) {
+            LOG(WARNING) << "GetInode chunklist empty, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId;
+            return MetaStatusCode::NOT_FOUND;
+        }
+        VLOG(9) << "DeleteInodeAndData, inode: "
+            << inode.ShortDebugString();
         int retVal = s3Adaptor_->Delete(inode);
         if (retVal != 0) {
             LOG(ERROR) << "S3ClientAdaptor delete s3 data failed"
@@ -216,7 +231,6 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
             return MetaStatusCode::S3_DELETE_ERR;
         }
     }
-
     ret = inodeStorage_->Delete(Key4Inode(item.fsId, item.inodeId));
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "Delete Inode fail, fsId = " << item.fsId

curvefs/test/client/client_operator_test.cpp

@@ -34,6 +34,7 @@ namespace curvefs {
 namespace client {
 
 using ::testing::SetArgPointee;
+using ::testing::DoAll;
 using rpcclient::MockMetaServerClient;
 using rpcclient::MockMdsClient;
 

curvefs/test/client/lease/lease_executor_test.cpp

@@ -36,7 +36,7 @@ using curvefs::mds::topology::PartitionTxId;
 using ::testing::AtLeast;
 using ::testing::Return;
 using ::testing::SetArgPointee;
-
+using ::testing::DoAll;
 
 namespace curvefs {
 namespace client {

curvefs/test/client/test_fuse_s3_client.cpp

@@ -21,8 +21,6 @@
  */
 
 
-#include <gmock/gmock-generated-actions.h>
-#include <gmock/gmock-more-actions.h>
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
 
@@ -75,6 +73,7 @@ namespace client {
 using ::curve::common::Configuration;
 using ::curvefs::mds::topology::PartitionTxId;
 using ::testing::_;
+using ::testing::DoAll;
 using ::testing::AtLeast;
 using ::testing::Contains;
 using ::testing::Invoke;

curvefs/test/client/test_fuse_volume_client.cpp

@@ -56,6 +56,7 @@ namespace client {
 using ::curve::common::Configuration;
 using ::curvefs::mds::topology::PartitionTxId;
 using ::testing::_;
+using ::testing::DoAll;
 using ::testing::Contains;
 using ::testing::Invoke;
 using ::testing::Return;
@@ -211,6 +212,8 @@ TEST_F(TestFuseVolumeClient, FuseOpInit_when_fs_exist) {
 
     ASSERT_EQ(fsInfo->fsid(), fsInfoExp.fsid());
     ASSERT_EQ(fsInfo->fsname(), fsInfoExp.fsname());
+
+    client_->GetFileSystem()->Destory();
 }
 
 TEST_F(TestFuseVolumeClient, FuseOpDestroy) {

curvefs/test/mds/fs_manager_test2.cpp

@@ -69,6 +69,7 @@ using ::testing::_;
 using ::testing::Invoke;
 using ::testing::Matcher;
 using ::testing::Return;
+using ::testing::DoAll;
 using ::testing::SaveArg;
 using ::testing::SetArgPointee;
 using ::curvefs::common::S3Info;

curvefs/test/mds/mock/mock_metaserver_client.h

@@ -23,7 +23,6 @@
 #ifndef CURVEFS_TEST_MDS_MOCK_MOCK_METASERVER_CLIENT_H_
 #define CURVEFS_TEST_MDS_MOCK_MOCK_METASERVER_CLIENT_H_
 
-#include <gmock/gmock-generated-function-mockers.h>
 #include <gmock/gmock.h>
 #include <set>
 #include <string>

curvefs/test/mds/persist_kvstorage_test.cpp

@@ -170,6 +170,10 @@ TEST_F(PersistKVStorageTest, TestInit) {
 
         EXPECT_FALSE(storage.Exist(3));
         EXPECT_FALSE(storage.Exist("foo"));
+
+        FsInfoWrapper wrapper;
+        EXPECT_EQ(FSStatusCode::OK, storage.Get("world", &wrapper));
+        EXPECT_TRUE(wrapper.GetFsDetail().s3info().has_objectprefix());
     }
 }
 
@@ -531,6 +535,5 @@ TEST_F(PersistKVStorageTest, TestDelete) {
         EXPECT_TRUE(storage.Exist(2));
     }
 }
-
 }  // namespace mds
 }  // namespace curvefs

curvefs/test/mds/topology/test_topology.cpp

@@ -36,6 +36,7 @@ using ::testing::_;
 using ::testing::AnyOf;
 using ::testing::Contains;
 using ::testing::SetArgPointee;
+using ::testing::DoAll;
 using ::curve::common::Configuration;
 
 class TestTopology : public ::testing::Test {

curvefs/test/metaserver/BUILD

@@ -26,7 +26,6 @@ cc_test(
             "mock_metaserver_s3.h",
             "metaserver_s3_adaptor_test.h",
             "metaserver_s3_adaptor_test.cpp",
-            "mock_metaserver_s3_adaptor.h",
             "metaserver_s3_test.cpp",
             "mock_s3compact_inode.h",
             "s3compact_test.cpp",

curvefs/test/metaserver/copyset/apply_queue_test.cpp

@@ -1,89 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: curve
- * Date: Thu Sep  2 14:49:04 CST 2021
- * Author: wuhanqing
- */
-
-#include "curvefs/src/metaserver/copyset/apply_queue.h"
-
-#include <gtest/gtest.h>
-
-#include <ctime>
-
-#include "src/common/concurrent/count_down_event.h"
-
-namespace curvefs {
-namespace metaserver {
-namespace copyset {
-
-using ::curve::common::CountDownEvent;
-
-TEST(ApplyQueueTest, StartAndStopTest) {
-    ApplyQueue applyQueue;
-
-    ApplyQueueOption option;
-    option.workerCount = 0;
-    option.queueDepth = 0;
-
-    EXPECT_FALSE(applyQueue.Start(option));
-
-    option.workerCount = 5;
-    option.queueDepth = 100 * 100 * 100;
-
-    ASSERT_TRUE(applyQueue.Start(option));
-    EXPECT_TRUE(applyQueue.Start(option));
-
-    std::atomic<bool> runned(false);
-    auto task = [&]() {
-        runned = true;
-    };
-
-    applyQueue.Push(time(nullptr), task);
-
-    while (!runned) {}
-
-    applyQueue.Stop();
-    applyQueue.Stop();
-}
-
-TEST(ApplyQueueTest, FlushTest) {
-    ApplyQueueOption option;
-    option.workerCount = 10;
-    option.queueDepth = 100 * 100;
-
-    ApplyQueue applyQueue;
-    ASSERT_TRUE(applyQueue.Start(option));
-
-    int taskCount = option.workerCount * option.queueDepth;
-    std::atomic<int> runned(0);
-
-    auto task = [&runned]() { runned.fetch_add(1, std::memory_order_relaxed); };
-
-    for (int i = 0; i < taskCount; ++i) {
-        applyQueue.Push(i, task);
-    }
-
-    applyQueue.Flush();
-    ASSERT_EQ(taskCount, runned.load(std::memory_order_relaxed));
-    applyQueue.Stop();
-}
-
-}  // namespace copyset
-}  // namespace metaserver
-}  // namespace curvefs

curvefs/test/metaserver/copyset/concurrent_apply_queue_test.cpp

@@ -0,0 +1,222 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * File Created: 20230521
+ * Author: Xinlong-Chen
+ */
+
+#include <gtest/gtest.h>
+
+#include <vector>
+#include <atomic>
+#include <functional>
+
+#include "src/common/timeutility.h"
+#include "curvefs/src/metaserver/copyset/concurrent_apply_queue.h"
+
+using curvefs::metaserver::copyset::ApplyQueue;
+using curvefs::metaserver::copyset::ApplyOption;
+using curvefs::metaserver::copyset::OperatorType;
+
+TEST(ApplyQueue, InitTest) {
+    ApplyQueue concurrentapply;
+
+    {
+        // 1. init with invalid write-concurrentsize
+        ApplyOption opt(-1, 1, 1, 1);
+        ASSERT_FALSE(concurrentapply.Init(opt));
+    }
+
+    {
+        // 2. init with invalid write-concurrentdepth
+        ApplyOption opt(1, -1, 1, 1);
+        ASSERT_FALSE(concurrentapply.Init(opt));
+    }
+
+    {
+        // 3. init with invalid read-concurrentsize
+        ApplyOption opt(1, 1, -1, 1);
+        ASSERT_FALSE(concurrentapply.Init(opt));
+    }
+
+    {
+        // 4. init with invalid read-concurrentdepth
+        ApplyOption opt(1, 1, 1, -1);
+        ASSERT_FALSE(concurrentapply.Init(opt));
+    }
+
+    {
+        // 5. double init
+        ApplyOption opt(1, 1, 1, 1);
+        // init with vaild params
+        ASSERT_TRUE(concurrentapply.Init(opt));
+        // re-init
+        ASSERT_TRUE(concurrentapply.Init(opt));
+    }
+
+    concurrentapply.Stop();
+    // re-stop
+    concurrentapply.Stop();
+}
+
+static void InitReadWriteTypeList(std::vector<OperatorType> *readTypeList,
+                                  std::vector<OperatorType> *writeTypeList) {
+    *readTypeList = {
+        OperatorType::GetDentry,
+        OperatorType::ListDentry,
+        OperatorType::GetInode,
+        OperatorType::BatchGetInodeAttr,
+        OperatorType::BatchGetXAttr,
+        OperatorType::GetVolumeExtent
+    };
+
+    auto IsRead = [&readTypeList](OperatorType type) -> bool {
+        for (auto &readType : *readTypeList) {
+            if (type == readType) {
+                return true;
+            }
+        }
+        return false;
+    };
+
+    for (uint32_t i = 0; i < static_cast<uint32_t>(OperatorType::OperatorTypeMax); ++i) {  // NOLINT
+        if (!IsRead(static_cast<OperatorType>(i))) {
+            writeTypeList->push_back(static_cast<OperatorType>(i));
+        }
+    }
+}
+
+OperatorType get_random_type(const std::vector<OperatorType> &type_list) {
+    return type_list[rand() % type_list.size()];
+}
+
+TEST(ApplyQueue, RunTest) {
+    std::vector<OperatorType> readTypeList;
+    std::vector<OperatorType> writeTypeList;
+    InitReadWriteTypeList(&readTypeList, &writeTypeList);
+
+    auto read_type  = get_random_type(readTypeList);
+    auto write_type = get_random_type(writeTypeList);
+
+    ApplyQueue concurrentapply;
+
+    int testw = 0;
+    auto wtask = [&testw]() {
+        std::this_thread::sleep_for(std::chrono::milliseconds(500));
+        testw++;
+    };
+
+    int testr = 0;
+    auto rtask = [&testr]() {
+        testr++;
+    };
+
+    // push write and read tasks
+    ApplyOption opt(1, 1, 1, 1);
+
+    ASSERT_TRUE(concurrentapply.Init(opt));
+
+    ASSERT_TRUE(concurrentapply.Push(1, read_type, rtask));
+    ASSERT_TRUE(concurrentapply.Push(1, write_type, wtask));
+
+    std::this_thread::sleep_for(std::chrono::milliseconds(200));
+    ASSERT_EQ(1, testr);
+    ASSERT_EQ(0, testw);
+    concurrentapply.Flush();
+    ASSERT_EQ(1, testw);
+
+    concurrentapply.Stop();
+}
+
+TEST(ApplyQueue, FlushTest) {
+    std::vector<OperatorType> readTypeList;
+    std::vector<OperatorType> writeTypeList;
+    InitReadWriteTypeList(&readTypeList, &writeTypeList);
+
+    ApplyQueue concurrentapply;
+    ApplyOption opt(2, 5000, 1, 1);
+    ASSERT_TRUE(concurrentapply.Init(opt));
+
+    std::atomic<uint32_t> testnum(0);
+    auto task = [&testnum]() {
+        testnum.fetch_add(1);
+    };
+
+    for (int i = 0; i < 5000; i++) {
+        auto write_type = get_random_type(writeTypeList);
+        concurrentapply.Push(i, write_type, task);
+    }
+
+    ASSERT_LE(testnum, 5000);
+    concurrentapply.Flush();
+    ASSERT_EQ(5000, testnum);
+
+    concurrentapply.Stop();
+}
+
+TEST(ApplyQueue, ConcurrentTest) {
+    std::vector<OperatorType> readTypeList;
+    std::vector<OperatorType> writeTypeList;
+    InitReadWriteTypeList(&readTypeList, &writeTypeList);
+
+    // interval flush when push
+    std::atomic<bool> stop(false);
+    std::atomic<uint32_t> testnum(0);
+    ApplyQueue concurrentapply;
+    ApplyOption opt(10, 1, 5, 2);
+    ASSERT_TRUE(concurrentapply.Init(opt));
+
+    auto push = [&concurrentapply, &stop, &testnum,
+                 &readTypeList, &writeTypeList]() {
+        auto task = [&testnum]() {
+            testnum.fetch_add(1);
+        };
+        while (!stop.load()) {
+            for (int i = 0; i < 10; i++) {
+                auto read_type  = get_random_type(readTypeList);
+                auto write_type = get_random_type(writeTypeList);
+                concurrentapply.Push(i, read_type, task);
+                concurrentapply.Push(i, write_type, task);
+            }
+        }
+    };
+
+    auto flush = [&concurrentapply, &stop, &testnum]() {
+        while (!stop.load()) {
+            std::this_thread::sleep_for(std::chrono::milliseconds(100));
+            concurrentapply.Flush();
+        }
+    };
+
+    std::thread t(push);
+    std::thread f(flush);
+
+    while (testnum.load() <= 1000000) {
+    }
+
+    stop.store(true);
+
+    std::cout << "thread exit, join" << std::endl;
+    t.join();
+    f.join();
+
+    concurrentapply.Flush();
+    ASSERT_GT(testnum, 1000000);
+    concurrentapply.Stop();
+}
+

curvefs/test/metaserver/copyset/copyset_node_test.cpp

@@ -111,8 +111,10 @@ TEST_F(CopysetNodeTest, TestInit) {
     // apply queue init failed
     {
         CopysetNodeOptions options = options_;
-        options.applyQueueOption.queueDepth = 0;
-        options.applyQueueOption.workerCount = 0;
+        options.applyQueueOption.wconcurrentsize = 0;
+        options.applyQueueOption.wqueuedepth = 0;
+        options.applyQueueOption.rconcurrentsize = 0;
+        options.applyQueueOption.rqueuedepth = 0;
         options.dataUri = "local:///mnt/data";
         options.localFileSystem = &mockfs_;
 

curvefs/test/metaserver/copyset/meta_operator_test.cpp

@@ -468,6 +468,17 @@ TEST_F(MetaOperatorTest, PropostTest_PropostTaskFailed) {
 
     CopysetNode node(poolId, copysetId, conf, &mockNodeManager_);
 
+    curve::fs::MockLocalFileSystem localFs;
+    CopysetNodeOptions options;
+    options.dataUri = "local:///mnt/data";
+    options.localFileSystem = &localFs;
+    options.storageOptions.type = "memory";
+
+    EXPECT_CALL(localFs, Mkdir(_))
+        .WillOnce(Return(0));
+
+    EXPECT_TRUE(node.Init(options));
+
     node.on_leader_start(1);
     node.UpdateAppliedIndex(101);
 

curvefs/test/metaserver/inode_storage_test.cpp

@@ -20,8 +20,6 @@
  * @Author: chenwei
  */
 
-#include <gmock/gmock-generated-matchers.h>
-#include <gmock/gmock-spec-builders.h>
 #include <gmock/gmock.h>
 #include <google/protobuf/util/message_differencer.h>
 #include <gtest/gtest.h>

curvefs/test/metaserver/mock/mock_kv_storage.h

@@ -23,7 +23,6 @@
 #ifndef CURVEFS_TEST_METASERVER_MOCK_MOCK_KV_STORAGE_H_
 #define CURVEFS_TEST_METASERVER_MOCK_MOCK_KV_STORAGE_H_
 
-#include <gmock/gmock-generated-function-mockers.h>
 #include <gmock/gmock.h>
 
 #include <memory>

curvefs/test/metaserver/trash_test.cpp

@@ -29,6 +29,7 @@
 #include "curvefs/test/metaserver/storage/utils.h"
 #include "src/fs/ext4_filesystem_impl.h"
 #include "curvefs/test/client/rpcclient/mock_mds_client.h"
+#include "curvefs/test/metaserver/mock_metaserver_s3_adaptor.h"
 
 using ::testing::AtLeast;
 using ::testing::StrEq;
@@ -105,7 +106,29 @@ class TestTrash : public ::testing::Test {
         inode.set_gid(0);
         inode.set_mode(0);
         inode.set_nlink(0);
-        inode.set_type(FsFileType::TYPE_FILE);
+        inode.set_type(FsFileType::TYPE_S3);
+        return inode;
+    }
+
+    Inode GenInodeHasChunks(uint32_t fsId, uint64_t inodeId) {
+        Inode inode;
+        inode.set_fsid(fsId);
+        inode.set_inodeid(inodeId);
+        inode.set_length(4096);
+        inode.set_ctime(0);
+        inode.set_ctime_ns(0);
+        inode.set_mtime(0);
+        inode.set_mtime_ns(0);
+        inode.set_atime(0);
+        inode.set_atime_ns(0);
+        inode.set_uid(0);
+        inode.set_gid(0);
+        inode.set_mode(0);
+        inode.set_nlink(0);
+        inode.set_type(FsFileType::TYPE_S3);
+
+        S3ChunkInfoList s3ChunkInfoList;
+        inode.mutable_s3chunkinfomap()->insert({0, s3ChunkInfoList});
         return inode;
     }
 
@@ -121,18 +144,17 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) {
     option.scanPeriodSec = 1;
     option.expiredAfterSec = 1;
     option.mdsClient = std::make_shared<MockMdsClient>();
-
+    option.s3Adaptor = std::make_shared<MockS3ClientAdaptor>();
     trashManager_->Init(option);
     trashManager_->Run();
-
     auto trash1 = std::make_shared<TrashImpl>(inodeStorage_);
     auto trash2 = std::make_shared<TrashImpl>(inodeStorage_);
     trashManager_->Add(1, trash1);
     trashManager_->Add(2, trash2);
 
-    inodeStorage_->Insert(GenInode(1, 1));
-    inodeStorage_->Insert(GenInode(1, 2));
-    inodeStorage_->Insert(GenInode(2, 1));
+    inodeStorage_->Insert(GenInodeHasChunks(1, 1));
+    inodeStorage_->Insert(GenInodeHasChunks(1, 2));
+    inodeStorage_->Insert(GenInodeHasChunks(2, 1));
 
     ASSERT_EQ(inodeStorage_->Size(), 3);
 
@@ -141,17 +163,42 @@ TEST_F(TestTrash, testAdd3ItemAndDelete) {
     trash2->Add(2, 1, 0);
 
     std::this_thread::sleep_for(std::chrono::seconds(5));
-
     std::list<TrashItem> list;
 
     trashManager_->ListItems(&list);
 
     ASSERT_EQ(0, list.size());
-
     ASSERT_EQ(inodeStorage_->Size(), 0);
 
     trashManager_->Fini();
 }
 
+TEST_F(TestTrash, testAdd3ItemAndNoDelete) {
+    TrashOption option;
+    option.scanPeriodSec = 1;
+    option.expiredAfterSec = 1;
+    option.mdsClient = std::make_shared<MockMdsClient>();
+    option.s3Adaptor = std::make_shared<MockS3ClientAdaptor>();
+    trashManager_->Init(option);
+    trashManager_->Run();
+
+    auto trash1 = std::make_shared<TrashImpl>(inodeStorage_);
+    trashManager_->Add(1, trash1);
+
+    inodeStorage_->Insert(GenInode(1, 1));
+    inodeStorage_->Insert(GenInode(1, 2));
+    inodeStorage_->Insert(GenInode(2, 1));
+    ASSERT_EQ(inodeStorage_->Size(), 3);
+    trash1->Add(1, 1, 0);
+    trash1->Add(1, 2, 0);
+    std::this_thread::sleep_for(std::chrono::seconds(5));
+    std::list<TrashItem> list;
+
+    trashManager_->ListItems(&list);
+    ASSERT_EQ(0, list.size());
+    ASSERT_EQ(inodeStorage_->Size(), 3);
+    trashManager_->Fini();
+}
+
 }  // namespace metaserver
 }  // namespace curvefs

curvefs_python/HOWTO.md

@@ -0,0 +1,5 @@
+1. `curvefs.py` and `curvefs_wrap.cxx` are generated by command `swig -c++ -python curvefs.i`.
+2. Revert `ListDir` and Delete `Opendir`/`Closedir` functions in `curvefs.py` if you don intend to make changes about these functions.
+3. Revert `_wrap_Read`/`_wrap_Listdir`/`_wrap_GetClusterId`/`_wrap_CBDClient_Read`/`_wrap_CBDClient_Listdir` if you don intend to make changes about these functions.
+4. :exclamation: C functions in `libcurvefs.h` are not recommended for use anymore.
+5. :exclamation: Types in `curve_type.h` are different from `include/client/*.h` even they have the similar name.

curvefs_python/cbd_client.cpp

@@ -26,9 +26,12 @@
 
 #include "src/client/libcurve_file.h"
 
-inline curve::client::UserInfo ToCurveClientUserInfo(UserInfo_t* userInfo) {
+namespace {
+inline curve::client::UserInfo ToCurveClientUserInfo(
+        const UserInfo_t* userInfo) {
     return curve::client::UserInfo(userInfo->owner, userInfo->password);
 }
+}
 
 CBDClient::CBDClient() : client_(new curve::client::FileClient()) {}
 
@@ -54,10 +57,17 @@ int CBDClient::Create(const char* filename, UserInfo_t* userInfo, size_t size) {
     return client_->Create(filename, ToCurveClientUserInfo(userInfo), size);
 }
 
-int CBDClient::Create2(const char* filename, UserInfo_t* userInfo, size_t size,
-                                    uint64_t stripeUnit, uint64_t stripeCount) {
-    return client_->Create2(filename, ToCurveClientUserInfo(userInfo),
-                                                 size, stripeUnit, stripeCount);
+int CBDClient::Create2(const CreateContext* context) {
+    curve::client::CreateFileContext internal;
+    internal.pagefile = true;
+    internal.name = context->name;
+    internal.user = ToCurveClientUserInfo(&context->user);
+    internal.length = context->length;
+    internal.poolset = context->poolset;
+    internal.stripeUnit = context->stripeUnit;
+    internal.stripeCount = context->stripeCount;
+
+    return client_->Create2(internal);
 }
 
 int CBDClient::Unlink(const char* filename, UserInfo_t* userInfo) {
@@ -166,3 +176,7 @@ int CBDClient::Rmdir(const char* dirpath, UserInfo_t* userInfo) {
 std::string CBDClient::GetClusterId() {
     return client_->GetClusterId();
 }
+
+std::vector<std::string> CBDClient::ListPoolset() {
+    return client_->ListPoolset();
+}

curvefs_python/cbd_client.h

@@ -25,6 +25,7 @@
 
 #include <memory>
 #include <string>
+#include <vector>
 
 #include "curvefs_python/curve_type.h"
 
@@ -48,8 +49,7 @@ class CBDClient {
     int Close(int fd);
 
     int Create(const char* filename, UserInfo_t* userInfo, size_t size);
-    int Create2(const char* filename, UserInfo_t* userInfo, size_t size,
-                                uint64_t stripeUnit, uint64_t stripeCount);
+    int Create2(const CreateContext* context);
     int Unlink(const char* filename, UserInfo_t* info);
     int DeleteForce(const char* filename, UserInfo_t* info);
     int Recover(const char* filename, UserInfo_t* info, uint64_t fileId);
@@ -76,6 +76,8 @@ class CBDClient {
 
     std::string GetClusterId();
 
+    std::vector<std::string> ListPoolset();
+
  private:
     std::unique_ptr<curve::client::FileClient> client_;
 };

curvefs_python/curve_type.h

@@ -27,6 +27,8 @@
 #include <stdint.h>
 #include <unistd.h>
 
+#include <string>
+
 #define CURVE_INODE_DIRECTORY 0
 #define CURVE_INODE_PAGEFILE 1
 #define CURVEINODE_APPENDFILE 2
@@ -134,4 +136,13 @@ typedef struct DirInfos {
     FileInfo_t*   fileinfo;
 } DirInfos_t;
 
+struct CreateContext {
+    std::string     name;
+    size_t          length;
+    UserInfo        user;
+    std::string     poolset;
+    uint64_t        stripeUnit;
+    uint64_t        stripeCount;
+};
+
 #endif  // CURVEFS_PYTHON_CURVE_TYPE_H_

curvefs_python/curvefs.py

@@ -313,6 +313,48 @@ class DirInfos_t(_object):
 DirInfos_t_swigregister = _curvefs.DirInfos_t_swigregister
 DirInfos_t_swigregister(DirInfos_t)
 
+class CreateContext(_object):
+    __swig_setmethods__ = {}
+    __setattr__ = lambda self, name, value: _swig_setattr(self, CreateContext, name, value)
+    __swig_getmethods__ = {}
+    __getattr__ = lambda self, name: _swig_getattr(self, CreateContext, name)
+    __repr__ = _swig_repr
+    __swig_setmethods__["name"] = _curvefs.CreateContext_name_set
+    __swig_getmethods__["name"] = _curvefs.CreateContext_name_get
+    if _newclass:
+        name = _swig_property(_curvefs.CreateContext_name_get, _curvefs.CreateContext_name_set)
+    __swig_setmethods__["length"] = _curvefs.CreateContext_length_set
+    __swig_getmethods__["length"] = _curvefs.CreateContext_length_get
+    if _newclass:
+        length = _swig_property(_curvefs.CreateContext_length_get, _curvefs.CreateContext_length_set)
+    __swig_setmethods__["user"] = _curvefs.CreateContext_user_set
+    __swig_getmethods__["user"] = _curvefs.CreateContext_user_get
+    if _newclass:
+        user = _swig_property(_curvefs.CreateContext_user_get, _curvefs.CreateContext_user_set)
+    __swig_setmethods__["poolset"] = _curvefs.CreateContext_poolset_set
+    __swig_getmethods__["poolset"] = _curvefs.CreateContext_poolset_get
+    if _newclass:
+        poolset = _swig_property(_curvefs.CreateContext_poolset_get, _curvefs.CreateContext_poolset_set)
+    __swig_setmethods__["stripeUnit"] = _curvefs.CreateContext_stripeUnit_set
+    __swig_getmethods__["stripeUnit"] = _curvefs.CreateContext_stripeUnit_get
+    if _newclass:
+        stripeUnit = _swig_property(_curvefs.CreateContext_stripeUnit_get, _curvefs.CreateContext_stripeUnit_set)
+    __swig_setmethods__["stripeCount"] = _curvefs.CreateContext_stripeCount_set
+    __swig_getmethods__["stripeCount"] = _curvefs.CreateContext_stripeCount_get
+    if _newclass:
+        stripeCount = _swig_property(_curvefs.CreateContext_stripeCount_get, _curvefs.CreateContext_stripeCount_set)
+
+    def __init__(self):
+        this = _curvefs.new_CreateContext()
+        try:
+            self.this.append(this)
+        except __builtin__.Exception:
+            self.this = this
+    __swig_destroy__ = _curvefs.delete_CreateContext
+    __del__ = lambda self: None
+CreateContext_swigregister = _curvefs.CreateContext_swigregister
+CreateContext_swigregister(CreateContext)
+
 
 def Init(path):
     return _curvefs.Init(path)
@@ -374,14 +416,14 @@ def Unlink(filename, info):
     return _curvefs.Unlink(filename, info)
 Unlink = _curvefs.Unlink
 
-def DeleteForce(filename, info):
-    return _curvefs.DeleteForce(filename, info)
-DeleteForce = _curvefs.DeleteForce
-
 def Recover(filename, info, fileId):
     return _curvefs.Recover(filename, info, fileId)
 Recover = _curvefs.Recover
 
+def DeleteForce(filename, info):
+    return _curvefs.DeleteForce(filename, info)
+DeleteForce = _curvefs.DeleteForce
+
 def Listdir(dirpath, info):
     return _curvefs.Listdir(dirpath, info)
 Listdir = _curvefs.Listdir
@@ -432,8 +474,8 @@ class CBDClient(_object):
     def Create(self, filename, userInfo, size):
         return _curvefs.CBDClient_Create(self, filename, userInfo, size)
 
-    def Create2(self, filename, userInfo, size, stripeUnit, stripeCount):
-        return _curvefs.CBDClient_Create2(self, filename, userInfo, size, stripeUnit, stripeCount)
+    def Create2(self, context):
+        return _curvefs.CBDClient_Create2(self, context)
 
     def Unlink(self, filename, info):
         return _curvefs.CBDClient_Unlink(self, filename, info)
@@ -479,6 +521,9 @@ class CBDClient(_object):
 
     def GetClusterId(self):
         return _curvefs.CBDClient_GetClusterId(self)
+
+    def ListPoolset(self):
+        return _curvefs.CBDClient_ListPoolset(self)
 CBDClient_swigregister = _curvefs.CBDClient_swigregister
 CBDClient_swigregister(CBDClient)
 

curvefs_python/curvefs_wrap.cxx

@@ -3022,22 +3022,24 @@ SWIG_Python_NonDynamicSetAttr(PyObject *obj, PyObject *name, PyObject *value) {
 
 #define SWIGTYPE_p_AioContext swig_types[0]
 #define SWIGTYPE_p_CBDClient swig_types[1]
-#define SWIGTYPE_p_DirInfos swig_types[2]
-#define SWIGTYPE_p_FileInfo swig_types[3]
-#define SWIGTYPE_p_UserInfo swig_types[4]
-#define SWIGTYPE_p_char swig_types[5]
-#define SWIGTYPE_p_f_p_AioContext__void swig_types[6]
-#define SWIGTYPE_p_int swig_types[7]
-#define SWIGTYPE_p_long_long swig_types[8]
-#define SWIGTYPE_p_short swig_types[9]
-#define SWIGTYPE_p_signed_char swig_types[10]
-#define SWIGTYPE_p_unsigned_char swig_types[11]
-#define SWIGTYPE_p_unsigned_int swig_types[12]
-#define SWIGTYPE_p_unsigned_long_long swig_types[13]
-#define SWIGTYPE_p_unsigned_short swig_types[14]
-#define SWIGTYPE_p_void swig_types[15]
-static swig_type_info *swig_types[17];
-static swig_module_info swig_module = {swig_types, 16, 0, 0, 0, 0};
+#define SWIGTYPE_p_CreateContext swig_types[2]
+#define SWIGTYPE_p_DirInfos swig_types[3]
+#define SWIGTYPE_p_FileInfo swig_types[4]
+#define SWIGTYPE_p_UserInfo swig_types[5]
+#define SWIGTYPE_p_char swig_types[6]
+#define SWIGTYPE_p_f_p_AioContext__void swig_types[7]
+#define SWIGTYPE_p_int swig_types[8]
+#define SWIGTYPE_p_long_long swig_types[9]
+#define SWIGTYPE_p_short swig_types[10]
+#define SWIGTYPE_p_signed_char swig_types[11]
+#define SWIGTYPE_p_std__vectorT_std__string_t swig_types[12]
+#define SWIGTYPE_p_unsigned_char swig_types[13]
+#define SWIGTYPE_p_unsigned_int swig_types[14]
+#define SWIGTYPE_p_unsigned_long_long swig_types[15]
+#define SWIGTYPE_p_unsigned_short swig_types[16]
+#define SWIGTYPE_p_void swig_types[17]
+static swig_type_info *swig_types[19];
+static swig_module_info swig_module = {swig_types, 18, 0, 0, 0, 0};
 #define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name)
 #define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name)
 
@@ -3615,29 +3617,6 @@ SWIG_From_unsigned_SS_long_SS_long  (unsigned long long value)
 #endif
 
 
-SWIGINTERN int
-SWIG_AsVal_unsigned_SS_int (PyObject * obj, unsigned int *val)
-{
-  unsigned long v;
-  int res = SWIG_AsVal_unsigned_SS_long (obj, &v);
-  if (SWIG_IsOK(res)) {
-    if ((v > UINT_MAX)) {
-      return SWIG_OverflowError;
-    } else {
-      if (val) *val = static_cast< unsigned int >(v);
-    }
-  }
-  return res;
-}
-
-
-SWIGINTERNINLINE PyObject*
-  SWIG_From_unsigned_SS_int  (unsigned int value)
-{
-  return PyInt_FromSize_t((size_t) value);
-}
-
-
 
 
 
@@ -3648,6 +3627,44 @@ SWIG_FromCharPtr(const char *cptr)
 }
 
 
+SWIGINTERN int
+SWIG_AsPtr_std_string (PyObject * obj, std::string **val) 
+{
+  char* buf = 0 ; size_t size = 0; int alloc = SWIG_OLDOBJ;
+  if (SWIG_IsOK((SWIG_AsCharPtrAndSize(obj, &buf, &size, &alloc)))) {
+    if (buf) {
+      if (val) *val = new std::string(buf, size - 1);
+      if (alloc == SWIG_NEWOBJ) delete[] buf;
+      return SWIG_NEWOBJ;
+    } else {
+      if (val) *val = 0;
+      return SWIG_OLDOBJ;
+    }
+  } else {
+    static int init = 0;
+    static swig_type_info* descriptor = 0;
+    if (!init) {
+      descriptor = SWIG_TypeQuery("std::string" " *");
+      init = 1;
+    }
+    if (descriptor) {
+      std::string *vptr;
+      int res = SWIG_ConvertPtr(obj, (void**)&vptr, descriptor, 0);
+      if (SWIG_IsOK(res) && val) *val = vptr;
+      return res;
+    }
+  }
+  return SWIG_ERROR;
+}
+
+
+SWIGINTERNINLINE PyObject *
+SWIG_From_std_string  (const std::string& s)
+{
+  return SWIG_FromCharPtrAndSize(s.data(), s.size());
+}
+
+
 SWIGINTERNINLINE int
 SWIG_AsVal_size_t (PyObject * obj, size_t *val)
 {
@@ -3670,9 +3687,18 @@ SWIG_AsVal_size_t (PyObject * obj, size_t *val)
 
 
 SWIGINTERNINLINE PyObject *
-SWIG_From_std_string  (const std::string& s)
-{
-  return SWIG_FromCharPtrAndSize(s.data(), s.size());
+SWIG_From_size_t  (size_t value)
+{    
+#ifdef SWIG_LONG_LONG_AVAILABLE
+  if (sizeof(size_t) <= sizeof(unsigned long)) {
+#endif
+    return SWIG_From_unsigned_SS_long  (static_cast< unsigned long >(value));
+#ifdef SWIG_LONG_LONG_AVAILABLE
+  } else {
+    /* assume sizeof(size_t) <= sizeof(unsigned long long) */
+    return SWIG_From_unsigned_SS_long_SS_long  (static_cast< unsigned long long >(value));
+  }
+#endif
 }
 
 #ifdef __cplusplus
@@ -4602,11 +4628,11 @@ SWIGINTERN PyObject *_wrap_FileInfo_t_fileStatus_get(PyObject *SWIGUNUSEDPARM(se
   int res1 = 0 ;
   PyObject * obj0 = 0 ;
   int result;
-
+  
   if (!PyArg_ParseTuple(args,(char *)"O:FileInfo_t_fileStatus_get",&obj0)) SWIG_fail;
   res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_fileStatus_get" "', argument " "1"" of type '" "FileInfo *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_fileStatus_get" "', argument " "1"" of type '" "FileInfo *""'"); 
   }
   arg1 = reinterpret_cast< FileInfo * >(argp1);
   result = (int) ((arg1)->fileStatus);
@@ -4620,25 +4646,25 @@ fail:
 SWIGINTERN PyObject *_wrap_FileInfo_t_stripeUnit_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *resultobj = 0;
   FileInfo *arg1 = (FileInfo *) 0 ;
-  uint32_t arg2 ;
+  uint64_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  unsigned int val2 ;
+  unsigned long long val2 ;
   int ecode2 = 0 ;
   PyObject * obj0 = 0 ;
   PyObject * obj1 = 0 ;
-
+  
   if (!PyArg_ParseTuple(args,(char *)"OO:FileInfo_t_stripeUnit_set",&obj0,&obj1)) SWIG_fail;
   res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeUnit_set" "', argument " "1"" of type '" "FileInfo *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeUnit_set" "', argument " "1"" of type '" "FileInfo *""'"); 
   }
   arg1 = reinterpret_cast< FileInfo * >(argp1);
-  ecode2 = SWIG_AsVal_unsigned_SS_int(obj1, &val2);
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(obj1, &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FileInfo_t_stripeUnit_set" "', argument " "2"" of type '" "uint32_t""'");
-  }
-  arg2 = static_cast< uint32_t >(val2);
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FileInfo_t_stripeUnit_set" "', argument " "2"" of type '" "uint64_t""'");
+  } 
+  arg2 = static_cast< uint64_t >(val2);
   if (arg1) (arg1)->stripeUnit = arg2;
   resultobj = SWIG_Py_Void();
   return resultobj;
@@ -4653,16 +4679,16 @@ SWIGINTERN PyObject *_wrap_FileInfo_t_stripeUnit_get(PyObject *SWIGUNUSEDPARM(se
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject * obj0 = 0 ;
-  uint32_t result;
-
+  uint64_t result;
+  
   if (!PyArg_ParseTuple(args,(char *)"O:FileInfo_t_stripeUnit_get",&obj0)) SWIG_fail;
   res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeUnit_get" "', argument " "1"" of type '" "FileInfo *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeUnit_get" "', argument " "1"" of type '" "FileInfo *""'"); 
   }
   arg1 = reinterpret_cast< FileInfo * >(argp1);
   result = (uint64_t) ((arg1)->stripeUnit);
-  resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result));
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
   return resultobj;
 fail:
   return NULL;
@@ -4672,25 +4698,25 @@ fail:
 SWIGINTERN PyObject *_wrap_FileInfo_t_stripeCount_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *resultobj = 0;
   FileInfo *arg1 = (FileInfo *) 0 ;
-  uint32_t arg2 ;
+  uint64_t arg2 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  unsigned int val2 ;
+  unsigned long long val2 ;
   int ecode2 = 0 ;
   PyObject * obj0 = 0 ;
   PyObject * obj1 = 0 ;
-
+  
   if (!PyArg_ParseTuple(args,(char *)"OO:FileInfo_t_stripeCount_set",&obj0,&obj1)) SWIG_fail;
   res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeCount_set" "', argument " "1"" of type '" "FileInfo *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeCount_set" "', argument " "1"" of type '" "FileInfo *""'"); 
   }
   arg1 = reinterpret_cast< FileInfo * >(argp1);
-  ecode2 = SWIG_AsVal_unsigned_SS_int(obj1, &val2);
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(obj1, &val2);
   if (!SWIG_IsOK(ecode2)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FileInfo_t_stripeCount_set" "', argument " "2"" of type '" "uint32_t""'");
-  }
-  arg2 = static_cast< uint32_t >(val2);
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "FileInfo_t_stripeCount_set" "', argument " "2"" of type '" "uint64_t""'");
+  } 
+  arg2 = static_cast< uint64_t >(val2);
   if (arg1) (arg1)->stripeCount = arg2;
   resultobj = SWIG_Py_Void();
   return resultobj;
@@ -4705,16 +4731,16 @@ SWIGINTERN PyObject *_wrap_FileInfo_t_stripeCount_get(PyObject *SWIGUNUSEDPARM(s
   void *argp1 = 0 ;
   int res1 = 0 ;
   PyObject * obj0 = 0 ;
-  uint32_t result;
-
+  uint64_t result;
+  
   if (!PyArg_ParseTuple(args,(char *)"O:FileInfo_t_stripeCount_get",&obj0)) SWIG_fail;
   res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_FileInfo, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeCount_get" "', argument " "1"" of type '" "FileInfo *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "FileInfo_t_stripeCount_get" "', argument " "1"" of type '" "FileInfo *""'"); 
   }
   arg1 = reinterpret_cast< FileInfo * >(argp1);
   result = (uint64_t) ((arg1)->stripeCount);
-  resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result));
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
   return resultobj;
 fail:
   return NULL;
@@ -5020,6 +5046,373 @@ SWIGINTERN PyObject *DirInfos_t_swigregister(PyObject *SWIGUNUSEDPARM(self), PyO
   return SWIG_Py_Void();
 }
 
+SWIGINTERN PyObject *_wrap_CreateContext_name_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  std::string *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject * obj0 = 0 ;
+  PyObject * obj1 = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"OO:CreateContext_name_set",&obj0,&obj1)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_name_set" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(obj1, &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "CreateContext_name_set" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "CreateContext_name_set" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  if (arg1) (arg1)->name = *arg2;
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_name_get(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject * obj0 = 0 ;
+  std::string *result = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"O:CreateContext_name_get",&obj0)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_name_get" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  result = (std::string *) & ((arg1)->name);
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_length_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  size_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  size_t val2 ;
+  int ecode2 = 0 ;
+  PyObject * obj0 = 0 ;
+  PyObject * obj1 = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"OO:CreateContext_length_set",&obj0,&obj1)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_length_set" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  ecode2 = SWIG_AsVal_size_t(obj1, &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "CreateContext_length_set" "', argument " "2"" of type '" "size_t""'");
+  } 
+  arg2 = static_cast< size_t >(val2);
+  if (arg1) (arg1)->length = arg2;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_length_get(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject * obj0 = 0 ;
+  size_t result;
+  
+  if (!PyArg_ParseTuple(args,(char *)"O:CreateContext_length_get",&obj0)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_length_get" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  result =  ((arg1)->length);
+  resultobj = SWIG_From_size_t(static_cast< size_t >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_user_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  UserInfo *arg2 = (UserInfo *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
+  PyObject * obj0 = 0 ;
+  PyObject * obj1 = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"OO:CreateContext_user_set",&obj0,&obj1)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_user_set" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  res2 = SWIG_ConvertPtr(obj1, &argp2,SWIGTYPE_p_UserInfo, 0 |  0 );
+  if (!SWIG_IsOK(res2)) {
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "CreateContext_user_set" "', argument " "2"" of type '" "UserInfo *""'"); 
+  }
+  arg2 = reinterpret_cast< UserInfo * >(argp2);
+  if (arg1) (arg1)->user = *arg2;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_user_get(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject * obj0 = 0 ;
+  UserInfo *result = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"O:CreateContext_user_get",&obj0)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_user_get" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  result = (UserInfo *)& ((arg1)->user);
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_UserInfo, 0 |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_poolset_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  std::string *arg2 = 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  int res2 = SWIG_OLDOBJ ;
+  PyObject * obj0 = 0 ;
+  PyObject * obj1 = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"OO:CreateContext_poolset_set",&obj0,&obj1)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_poolset_set" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  {
+    std::string *ptr = (std::string *)0;
+    res2 = SWIG_AsPtr_std_string(obj1, &ptr);
+    if (!SWIG_IsOK(res2)) {
+      SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "CreateContext_poolset_set" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    if (!ptr) {
+      SWIG_exception_fail(SWIG_ValueError, "invalid null reference " "in method '" "CreateContext_poolset_set" "', argument " "2"" of type '" "std::string const &""'"); 
+    }
+    arg2 = ptr;
+  }
+  if (arg1) (arg1)->poolset = *arg2;
+  resultobj = SWIG_Py_Void();
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return resultobj;
+fail:
+  if (SWIG_IsNewObj(res2)) delete arg2;
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_poolset_get(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject * obj0 = 0 ;
+  std::string *result = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"O:CreateContext_poolset_get",&obj0)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_poolset_get" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  result = (std::string *) & ((arg1)->poolset);
+  resultobj = SWIG_From_std_string(static_cast< std::string >(*result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_stripeUnit_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  uint64_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  unsigned long long val2 ;
+  int ecode2 = 0 ;
+  PyObject * obj0 = 0 ;
+  PyObject * obj1 = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"OO:CreateContext_stripeUnit_set",&obj0,&obj1)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_stripeUnit_set" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(obj1, &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "CreateContext_stripeUnit_set" "', argument " "2"" of type '" "uint64_t""'");
+  } 
+  arg2 = static_cast< uint64_t >(val2);
+  if (arg1) (arg1)->stripeUnit = arg2;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_stripeUnit_get(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject * obj0 = 0 ;
+  uint64_t result;
+  
+  if (!PyArg_ParseTuple(args,(char *)"O:CreateContext_stripeUnit_get",&obj0)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_stripeUnit_get" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  result = (uint64_t) ((arg1)->stripeUnit);
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_stripeCount_set(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  uint64_t arg2 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  unsigned long long val2 ;
+  int ecode2 = 0 ;
+  PyObject * obj0 = 0 ;
+  PyObject * obj1 = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"OO:CreateContext_stripeCount_set",&obj0,&obj1)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_stripeCount_set" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  ecode2 = SWIG_AsVal_unsigned_SS_long_SS_long(obj1, &val2);
+  if (!SWIG_IsOK(ecode2)) {
+    SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "CreateContext_stripeCount_set" "', argument " "2"" of type '" "uint64_t""'");
+  } 
+  arg2 = static_cast< uint64_t >(val2);
+  if (arg1) (arg1)->stripeCount = arg2;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_CreateContext_stripeCount_get(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject * obj0 = 0 ;
+  uint64_t result;
+  
+  if (!PyArg_ParseTuple(args,(char *)"O:CreateContext_stripeCount_get",&obj0)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CreateContext_stripeCount_get" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  result = (uint64_t) ((arg1)->stripeCount);
+  resultobj = SWIG_From_unsigned_SS_long_SS_long(static_cast< unsigned long long >(result));
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_new_CreateContext(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *result = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)":new_CreateContext")) SWIG_fail;
+  result = (CreateContext *)new CreateContext();
+  resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_CreateContext, SWIG_POINTER_NEW |  0 );
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *_wrap_delete_CreateContext(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CreateContext *arg1 = (CreateContext *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject * obj0 = 0 ;
+  
+  if (!PyArg_ParseTuple(args,(char *)"O:delete_CreateContext",&obj0)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CreateContext, SWIG_POINTER_DISOWN |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_CreateContext" "', argument " "1"" of type '" "CreateContext *""'"); 
+  }
+  arg1 = reinterpret_cast< CreateContext * >(argp1);
+  delete arg1;
+  resultobj = SWIG_Py_Void();
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
+SWIGINTERN PyObject *CreateContext_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *obj;
+  if (!PyArg_ParseTuple(args,(char*)"O:swigregister", &obj)) return NULL;
+  SWIG_TypeNewClientData(SWIGTYPE_p_CreateContext, SWIG_NewClientData(obj));
+  return SWIG_Py_Void();
+}
+
 SWIGINTERN PyObject *_wrap_Init(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *resultobj = 0;
   char *arg1 = (char *) 0 ;
@@ -5606,7 +5999,7 @@ SWIGINTERN PyObject *_wrap_Recover(PyObject *SWIGUNUSEDPARM(self), PyObject *arg
   PyObject * obj1 = 0 ;
   PyObject * obj2 = 0 ;
   int result;
-
+  
   if (!PyArg_ParseTuple(args,(char *)"OOO:Recover",&obj0,&obj1,&obj2)) SWIG_fail;
   res1 = SWIG_AsCharPtrAndSize(obj0, &buf1, NULL, &alloc1);
   if (!SWIG_IsOK(res1)) {
@@ -5615,13 +6008,13 @@ SWIGINTERN PyObject *_wrap_Recover(PyObject *SWIGUNUSEDPARM(self), PyObject *arg
   arg1 = reinterpret_cast< char * >(buf1);
   res2 = SWIG_ConvertPtr(obj1, &argp2,SWIGTYPE_p_UserInfo, 0 |  0 );
   if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Recover" "', argument " "2"" of type '" "UserInfo_t *""'");
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "Recover" "', argument " "2"" of type '" "UserInfo_t *""'"); 
   }
   arg2 = reinterpret_cast< UserInfo_t * >(argp2);
   ecode3 = SWIG_AsVal_unsigned_SS_long_SS_long(obj2, &val3);
   if (!SWIG_IsOK(ecode3)) {
     SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "Recover" "', argument " "3"" of type '" "uint64_t""'");
-  }
+  } 
   arg3 = static_cast< uint64_t >(val3);
   result = (int)Recover((char const *)arg1,arg2,arg3);
   resultobj = SWIG_From_int(static_cast< int >(result));
@@ -6010,7 +6403,7 @@ SWIGINTERN PyObject *_wrap_CBDClient_Open(PyObject *SWIGUNUSEDPARM(self), PyObje
   PyObject * obj1 = 0 ;
   PyObject * obj2 = 0 ;
   int result;
-
+  
   if (!PyArg_ParseTuple(args,(char *)"OOO:CBDClient_Open",&obj0,&obj1,&obj2)) SWIG_fail;
   res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CBDClient, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
@@ -6108,7 +6501,7 @@ SWIGINTERN PyObject *_wrap_CBDClient_Create(PyObject *SWIGUNUSEDPARM(self), PyOb
   ecode4 = SWIG_AsVal_size_t(obj3, &val4);
   if (!SWIG_IsOK(ecode4)) {
     SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "CBDClient_Create" "', argument " "4"" of type '" "size_t""'");
-  }
+  } 
   arg4 = static_cast< size_t >(val4);
   result = (int)(arg1)->Create((char const *)arg2,arg3,arg4);
   resultobj = SWIG_From_int(static_cast< int >(result));
@@ -6123,69 +6516,30 @@ fail:
 SWIGINTERN PyObject *_wrap_CBDClient_Create2(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *resultobj = 0;
   CBDClient *arg1 = (CBDClient *) 0 ;
-  char *arg2 = (char *) 0 ;
-  UserInfo_t *arg3 = (UserInfo_t *) 0 ;
-  size_t arg4 ;
-  uint32_t arg5 ;
-  uint32_t arg6 ;
+  CreateContext *arg2 = (CreateContext *) 0 ;
   void *argp1 = 0 ;
   int res1 = 0 ;
-  int res2 ;
-  char *buf2 = 0 ;
-  int alloc2 = 0 ;
-  void *argp3 = 0 ;
-  int res3 = 0 ;
-  size_t val4 ;
-  int ecode4 = 0 ;
-  unsigned int val5 ;
-  int ecode5 = 0 ;
-  unsigned int val6 ;
-  int ecode6 = 0 ;
+  void *argp2 = 0 ;
+  int res2 = 0 ;
   PyObject * obj0 = 0 ;
   PyObject * obj1 = 0 ;
-  PyObject * obj2 = 0 ;
-  PyObject * obj3 = 0 ;
-  PyObject * obj4 = 0 ;
-  PyObject * obj5 = 0 ;
   int result;
-
-  if (!PyArg_ParseTuple(args,(char *)"OOOOOO:CBDClient_Create2",&obj0,&obj1,&obj2,&obj3,&obj4,&obj5)) SWIG_fail;
+  
+  if (!PyArg_ParseTuple(args,(char *)"OO:CBDClient_Create2",&obj0,&obj1)) SWIG_fail;
   res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CBDClient, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CBDClient_Create2" "', argument " "1"" of type '" "CBDClient *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CBDClient_Create2" "', argument " "1"" of type '" "CBDClient *""'"); 
   }
   arg1 = reinterpret_cast< CBDClient * >(argp1);
-  res2 = SWIG_AsCharPtrAndSize(obj1, &buf2, NULL, &alloc2);
+  res2 = SWIG_ConvertPtr(obj1, &argp2,SWIGTYPE_p_CreateContext, 0 |  0 );
   if (!SWIG_IsOK(res2)) {
-    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "CBDClient_Create2" "', argument " "2"" of type '" "char const *""'");
+    SWIG_exception_fail(SWIG_ArgError(res2), "in method '" "CBDClient_Create2" "', argument " "2"" of type '" "CreateContext const *""'"); 
   }
-  arg2 = reinterpret_cast< char * >(buf2);
-  res3 = SWIG_ConvertPtr(obj2, &argp3,SWIGTYPE_p_UserInfo, 0 |  0 );
-  if (!SWIG_IsOK(res3)) {
-    SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "CBDClient_Create2" "', argument " "3"" of type '" "UserInfo_t *""'");
-  }
-  arg3 = reinterpret_cast< UserInfo_t * >(argp3);
-  ecode4 = SWIG_AsVal_size_t(obj3, &val4);
-  if (!SWIG_IsOK(ecode4)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "CBDClient_Create2" "', argument " "4"" of type '" "size_t""'");
-  }
-  arg4 = static_cast< size_t >(val4);
-  ecode5 = SWIG_AsVal_unsigned_SS_int(obj4, &val5);
-  if (!SWIG_IsOK(ecode5)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "CBDClient_Create2" "', argument " "5"" of type '" "uint32_t""'");
-  }
-  arg5 = static_cast< uint32_t >(val5);
-  ecode6 = SWIG_AsVal_unsigned_SS_int(obj5, &val6);
-  if (!SWIG_IsOK(ecode6)) {
-    SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "CBDClient_Create2" "', argument " "6"" of type '" "uint32_t""'");
-  }
-  arg6 = static_cast< uint32_t >(val6);
-  result = (int)(arg1)->Create2((char const *)arg2,arg3,arg4,arg5,arg6);
+  arg2 = reinterpret_cast< CreateContext * >(argp2);
+  result = (int)(arg1)->Create2((CreateContext const *)arg2);
   resultobj = SWIG_From_int(static_cast< int >(result));
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
   return resultobj;
 fail:
-  if (alloc2 == SWIG_NEWOBJ) delete[] buf2;
   return NULL;
 }
 
@@ -6275,6 +6629,7 @@ fail:
   return NULL;
 }
 
+
 SWIGINTERN PyObject *_wrap_CBDClient_Recover(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *resultobj = 0;
   CBDClient *arg1 = (CBDClient *) 0 ;
@@ -6295,11 +6650,11 @@ SWIGINTERN PyObject *_wrap_CBDClient_Recover(PyObject *SWIGUNUSEDPARM(self), PyO
   PyObject * obj2 = 0 ;
   PyObject * obj3 = 0 ;
   int result;
-
+  
   if (!PyArg_ParseTuple(args,(char *)"OOOO:CBDClient_Recover",&obj0,&obj1,&obj2,&obj3)) SWIG_fail;
   res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CBDClient, 0 |  0 );
   if (!SWIG_IsOK(res1)) {
-    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CBDClient_Recover" "', argument " "1"" of type '" "CBDClient *""'");
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CBDClient_Recover" "', argument " "1"" of type '" "CBDClient *""'"); 
   }
   arg1 = reinterpret_cast< CBDClient * >(argp1);
   res2 = SWIG_AsCharPtrAndSize(obj1, &buf2, NULL, &alloc2);
@@ -6309,13 +6664,13 @@ SWIGINTERN PyObject *_wrap_CBDClient_Recover(PyObject *SWIGUNUSEDPARM(self), PyO
   arg2 = reinterpret_cast< char * >(buf2);
   res3 = SWIG_ConvertPtr(obj2, &argp3,SWIGTYPE_p_UserInfo, 0 |  0 );
   if (!SWIG_IsOK(res3)) {
-    SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "CBDClient_Recover" "', argument " "3"" of type '" "UserInfo_t *""'");
+    SWIG_exception_fail(SWIG_ArgError(res3), "in method '" "CBDClient_Recover" "', argument " "3"" of type '" "UserInfo_t *""'"); 
   }
   arg3 = reinterpret_cast< UserInfo_t * >(argp3);
   ecode4 = SWIG_AsVal_unsigned_SS_long_SS_long(obj3, &val4);
   if (!SWIG_IsOK(ecode4)) {
     SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "CBDClient_Recover" "', argument " "4"" of type '" "uint64_t""'");
-  }
+  } 
   arg4 = static_cast< uint64_t >(val4);
   result = (int)(arg1)->Recover((char const *)arg2,arg3,arg4);
   resultobj = SWIG_From_int(static_cast< int >(result));
@@ -6326,6 +6681,7 @@ fail:
   return NULL;
 }
 
+
 SWIGINTERN PyObject *_wrap_CBDClient_Rename(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *resultobj = 0;
   CBDClient *arg1 = (CBDClient *) 0 ;
@@ -7005,6 +7361,34 @@ fail:
 }
 
 
+SWIGINTERN PyObject *_wrap_CBDClient_ListPoolset(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
+  PyObject *resultobj = 0;
+  CBDClient *arg1 = (CBDClient *) 0 ;
+  void *argp1 = 0 ;
+  int res1 = 0 ;
+  PyObject * obj0 = 0 ;
+  std::vector< std::string > result;
+  
+  if (!PyArg_ParseTuple(args,(char *)"O:CBDClient_ListPoolset",&obj0)) SWIG_fail;
+  res1 = SWIG_ConvertPtr(obj0, &argp1,SWIGTYPE_p_CBDClient, 0 |  0 );
+  if (!SWIG_IsOK(res1)) {
+    SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "CBDClient_ListPoolset" "', argument " "1"" of type '" "CBDClient *""'"); 
+  }
+  arg1 = reinterpret_cast< CBDClient * >(argp1);
+  result = (arg1)->ListPoolset();
+  resultobj = PyList_New(result.size());
+  if (resultobj == nullptr) {
+    SWIG_exception_fail(-1, "memory alloc failed"); 
+  }
+  for (size_t i = 0; i < result.size(); ++i) {
+    PyList_SetItem(resultobj, i, SWIG_From_std_string(result[i]));
+  }
+  return resultobj;
+fail:
+  return NULL;
+}
+
+
 SWIGINTERN PyObject *CBDClient_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) {
   PyObject *obj;
   if (!PyArg_ParseTuple(args,(char*)"O:swigregister", &obj)) return NULL;
@@ -7070,6 +7454,21 @@ static PyMethodDef SwigMethods[] = {
 	 { (char *)"new_DirInfos_t", _wrap_new_DirInfos_t, METH_VARARGS, NULL},
 	 { (char *)"delete_DirInfos_t", _wrap_delete_DirInfos_t, METH_VARARGS, NULL},
 	 { (char *)"DirInfos_t_swigregister", DirInfos_t_swigregister, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_name_set", _wrap_CreateContext_name_set, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_name_get", _wrap_CreateContext_name_get, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_length_set", _wrap_CreateContext_length_set, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_length_get", _wrap_CreateContext_length_get, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_user_set", _wrap_CreateContext_user_set, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_user_get", _wrap_CreateContext_user_get, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_poolset_set", _wrap_CreateContext_poolset_set, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_poolset_get", _wrap_CreateContext_poolset_get, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_stripeUnit_set", _wrap_CreateContext_stripeUnit_set, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_stripeUnit_get", _wrap_CreateContext_stripeUnit_get, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_stripeCount_set", _wrap_CreateContext_stripeCount_set, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_stripeCount_get", _wrap_CreateContext_stripeCount_get, METH_VARARGS, NULL},
+	 { (char *)"new_CreateContext", _wrap_new_CreateContext, METH_VARARGS, NULL},
+	 { (char *)"delete_CreateContext", _wrap_delete_CreateContext, METH_VARARGS, NULL},
+	 { (char *)"CreateContext_swigregister", CreateContext_swigregister, METH_VARARGS, NULL},
 	 { (char *)"Init", _wrap_Init, METH_VARARGS, NULL},
 	 { (char *)"Open4Qemu", _wrap_Open4Qemu, METH_VARARGS, NULL},
 	 { (char *)"Open", _wrap_Open, METH_VARARGS, NULL},
@@ -7085,8 +7484,8 @@ static PyMethodDef SwigMethods[] = {
 	 { (char *)"Rename", _wrap_Rename, METH_VARARGS, NULL},
 	 { (char *)"Extend", _wrap_Extend, METH_VARARGS, NULL},
 	 { (char *)"Unlink", _wrap_Unlink, METH_VARARGS, NULL},
-	 { (char *)"DeleteForce", _wrap_DeleteForce, METH_VARARGS, NULL},
 	 { (char *)"Recover", _wrap_Recover, METH_VARARGS, NULL},
+	 { (char *)"DeleteForce", _wrap_DeleteForce, METH_VARARGS, NULL},
 	 { (char *)"OpenDir", _wrap_OpenDir, METH_VARARGS, NULL},
 	 { (char *)"CloseDir", _wrap_CloseDir, METH_VARARGS, NULL},
 	 { (char *)"Listdir", _wrap_Listdir, METH_VARARGS, NULL},
@@ -7119,6 +7518,7 @@ static PyMethodDef SwigMethods[] = {
 	 { (char *)"CBDClient_Mkdir", _wrap_CBDClient_Mkdir, METH_VARARGS, NULL},
 	 { (char *)"CBDClient_Rmdir", _wrap_CBDClient_Rmdir, METH_VARARGS, NULL},
 	 { (char *)"CBDClient_GetClusterId", _wrap_CBDClient_GetClusterId, METH_VARARGS, NULL},
+	 { (char *)"CBDClient_ListPoolset", _wrap_CBDClient_ListPoolset, METH_VARARGS, NULL},
 	 { (char *)"CBDClient_swigregister", CBDClient_swigregister, METH_VARARGS, NULL},
 	 { NULL, NULL, 0, NULL }
 };
@@ -7128,6 +7528,7 @@ static PyMethodDef SwigMethods[] = {
 
 static swig_type_info _swigt__p_AioContext = {"_p_AioContext", "AioContext *|AioContext_t *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_CBDClient = {"_p_CBDClient", "CBDClient *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_CreateContext = {"_p_CreateContext", "CreateContext *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_DirInfos = {"_p_DirInfos", "DirInfos_t *|DirInfos *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_FileInfo = {"_p_FileInfo", "FileInfo *|FileInfo_t *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_UserInfo = {"_p_UserInfo", "UserInfo *|UserInfo_t *", 0, 0, (void*)0, 0};
@@ -7137,6 +7538,7 @@ static swig_type_info _swigt__p_int = {"_p_int", "intptr_t *|int *|int_least32_t
 static swig_type_info _swigt__p_long_long = {"_p_long_long", "int_least64_t *|int_fast64_t *|int64_t *|long long *|intmax_t *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_short = {"_p_short", "short *|int_least16_t *|int16_t *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_signed_char = {"_p_signed_char", "signed char *|int_least8_t *|int_fast8_t *|int8_t *", 0, 0, (void*)0, 0};
+static swig_type_info _swigt__p_std__vectorT_std__string_t = {"_p_std__vectorT_std__string_t", "std::vector< std::string > *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_unsigned_char = {"_p_unsigned_char", "unsigned char *|uint_least8_t *|uint_fast8_t *|uint8_t *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_unsigned_int = {"_p_unsigned_int", "uintptr_t *|uint_least32_t *|uint_fast32_t *|uint32_t *|unsigned int *|uint_fast16_t *", 0, 0, (void*)0, 0};
 static swig_type_info _swigt__p_unsigned_long_long = {"_p_unsigned_long_long", "uint_least64_t *|uint_fast64_t *|uint64_t *|unsigned long long *|uintmax_t *", 0, 0, (void*)0, 0};
@@ -7146,6 +7548,7 @@ static swig_type_info _swigt__p_void = {"_p_void", "void *", 0, 0, (void*)0, 0};
 static swig_type_info *swig_type_initial[] = {
   &_swigt__p_AioContext,
   &_swigt__p_CBDClient,
+  &_swigt__p_CreateContext,
   &_swigt__p_DirInfos,
   &_swigt__p_FileInfo,
   &_swigt__p_UserInfo,
@@ -7155,6 +7558,7 @@ static swig_type_info *swig_type_initial[] = {
   &_swigt__p_long_long,
   &_swigt__p_short,
   &_swigt__p_signed_char,
+  &_swigt__p_std__vectorT_std__string_t,
   &_swigt__p_unsigned_char,
   &_swigt__p_unsigned_int,
   &_swigt__p_unsigned_long_long,
@@ -7164,6 +7568,7 @@ static swig_type_info *swig_type_initial[] = {
 
 static swig_cast_info _swigc__p_AioContext[] = {  {&_swigt__p_AioContext, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_CBDClient[] = {  {&_swigt__p_CBDClient, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_CreateContext[] = {  {&_swigt__p_CreateContext, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_DirInfos[] = {  {&_swigt__p_DirInfos, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_FileInfo[] = {  {&_swigt__p_FileInfo, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_UserInfo[] = {  {&_swigt__p_UserInfo, 0, 0, 0},{0, 0, 0, 0}};
@@ -7173,6 +7578,7 @@ static swig_cast_info _swigc__p_int[] = {  {&_swigt__p_int, 0, 0, 0},{0, 0, 0, 0
 static swig_cast_info _swigc__p_long_long[] = {  {&_swigt__p_long_long, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_short[] = {  {&_swigt__p_short, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_signed_char[] = {  {&_swigt__p_signed_char, 0, 0, 0},{0, 0, 0, 0}};
+static swig_cast_info _swigc__p_std__vectorT_std__string_t[] = {  {&_swigt__p_std__vectorT_std__string_t, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_unsigned_char[] = {  {&_swigt__p_unsigned_char, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_unsigned_int[] = {  {&_swigt__p_unsigned_int, 0, 0, 0},{0, 0, 0, 0}};
 static swig_cast_info _swigc__p_unsigned_long_long[] = {  {&_swigt__p_unsigned_long_long, 0, 0, 0},{0, 0, 0, 0}};
@@ -7182,6 +7588,7 @@ static swig_cast_info _swigc__p_void[] = {  {&_swigt__p_void, 0, 0, 0},{0, 0, 0,
 static swig_cast_info *swig_cast_initial[] = {
   _swigc__p_AioContext,
   _swigc__p_CBDClient,
+  _swigc__p_CreateContext,
   _swigc__p_DirInfos,
   _swigc__p_FileInfo,
   _swigc__p_UserInfo,
@@ -7191,6 +7598,7 @@ static swig_cast_info *swig_cast_initial[] = {
   _swigc__p_long_long,
   _swigc__p_short,
   _swigc__p_signed_char,
+  _swigc__p_std__vectorT_std__string_t,
   _swigc__p_unsigned_char,
   _swigc__p_unsigned_int,
   _swigc__p_unsigned_long_long,

deploy/local/chunkserver/conf/chunkserver.conf.0

@@ -26,10 +26,9 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
-global.min_io_alignment=512
-
 #
 # MDS settings
 #
@@ -82,7 +81,7 @@ copyset.scan_rpc_timeout_ms=1000
 copyset.scan_rpc_retry_times=3
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync timer timeout interval
 copyset.synctimer_interval_ms=30000
 # check syncing interval

deploy/local/chunkserver/conf/chunkserver.conf.1

@@ -26,10 +26,9 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
-global.min_io_alignment=512
-
 #
 # MDS settings
 #
@@ -82,7 +81,7 @@ copyset.scan_rpc_timeout_ms=1000
 copyset.scan_rpc_retry_times=3
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync timer timeout interval
 copyset.synctimer_interval_ms=30000
 # check syncing interval

deploy/local/chunkserver/conf/chunkserver.conf.2

@@ -26,10 +26,9 @@ global.external_ip=127.0.0.1
 global.external_subnet=127.0.0.0/24
 global.chunk_size=16777216
 global.meta_page_size=4096
+global.block_size=4096
 global.location_limit=3000
 
-global.min_io_alignment=512
-
 #
 # MDS settings
 #
@@ -82,7 +81,7 @@ copyset.scan_rpc_timeout_ms=1000
 copyset.scan_rpc_retry_times=3
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync timer timeout interval
 copyset.synctimer_interval_ms=30000
 # check syncing interval

docker/debian9/Dockerfile

@@ -1,9 +1,7 @@
 FROM opencurvedocker/curve-base:debian9
 COPY --from=opencurvedocker/curve-base:curve-tgt-debian9 /curve-tgt/  /curve-tgt/
 COPY --from=opencurvedocker/curve-base:curve-tgt-debian9 /curve/curve-sdk /curve-tgt/curve-sdk
-RUN apt update \
-    && apt install -y kmod \
-    && cd /curve-tgt/curve-sdk \
+RUN cd /curve-tgt/curve-sdk \
     && cp -f lib/* /usr/lib \
     && cp -f bin/* /usr/bin \
     && mkdir -p /usr/curvefs \

docs/cn/gmock.md

@@ -23,23 +23,24 @@
 
 1. 从Turtle派生一个类MockTurtle。
 2. 使用Turtle的虚函数，计算它有多少参数。
-3. 在子类的public：部分，写MOCK_METHODn（）; （或MOCK_CONST_METHODn（）;如果你是一个const方法），其中n是参数的数量;如果你计数错误，产生一个一个编译器错误。
-4. 使用函数名作为宏的第一个参数，第二个参数是函数的类型。
+3. 在子类的public：部分，使用 `MOCK_METHOD` 创建 mock 函数
 
 例：
+```cpp
+#include "gmock/gmock.h"  // Brings in Google Mock.
 
-    #include "gmock/gmock.h"  // Brings in Google Mock.
-    class MockTurtle : public Turtle {
-     public:
-      ...
-      MOCK_METHOD0(PenUp, void());
-      MOCK_METHOD0(PenDown, void());
-      MOCK_METHOD1(Forward, void(int distance));
-      MOCK_METHOD1(Turn, void(int degrees));
-      MOCK_METHOD2(GoTo, void(int x, int y));
-      MOCK_CONST_METHOD0(GetX, int());
-      MOCK_CONST_METHOD0(GetY, int());
-    };
+class MockTurtle : public Turtle {
+ public:
+    // ...
+   MOCK_METHOD(void, PenUp, (), (override));
+   MOCK_METHOD(void, PenDown, (), (override));
+   MOCK_METHOD(void, Forward, (int), (override));
+   MOCK_METHOD(void, Turn, (int), (override));
+   MOCK_METHOD(void, GoTo, (int, int), (override));
+   MOCK_METHOD(int, GetX, (), (const, override));
+   MOCK_METHOD(int, GetY, (), (const, override));
+};
+```
 
 ### 在测试中使用mock类
 

include/BUILD

@@ -18,7 +18,7 @@ load("//:copts.bzl", "CURVE_DEFAULT_COPTS")
 
 cc_library(
     name = "include-common",
-    srcs = ["curve_compiler_specific.h"],
+    hdrs = ["curve_compiler_specific.h"],
     copts = CURVE_DEFAULT_COPTS,
     visibility = ["//visibility:public"],
 )

include/chunkserver/chunkserver_common.h

@@ -123,6 +123,12 @@ inline std::string ToGroupIdString(const LogicPoolID &logicPoolId,
 }
 #define ToGroupIdStr   ToGroupIdString
 
+// Meta page is header of chunkfile, and is used to store meta data of
+// chunkfile.
+// Currently, we need to ensure the atomicity of the meta page update, so set
+// its size to 4k.
+constexpr size_t kChunkfileMetaPageSize = 4096;
+
 }  // namespace chunkserver
 }  // namespace curve
 

include/client/libcurve.h

@@ -54,6 +54,7 @@ typedef struct FileStatInfo {
     int             fileStatus;
     uint64_t        stripeUnit;
     uint64_t        stripeCount;
+    uint32_t        blocksize;
 } FileStatInfo_t;
 
 // 存储用户信息
@@ -126,20 +127,6 @@ int Create(const char* filename,
            const C_UserInfo_t* userinfo,
            size_t size);
 
-/**
- * create file with stripe
- * @param: filename  file name
- * @param: userinfo  user info
- * @param: size      file size
- * @param: stripeUnit block in stripe size
- * @param: stripeCount stripe count in one stripe
- *
- * @return: success return 0, fail return less than 0
- */
-int Create2(const char* filename,
-           const C_UserInfo_t* userinfo,
-           size_t size, uint64_t stripeUnit, uint64_t stripeCount);
-
 /**
  * 同步模式读
  * @param: fd为当前open返回的文件描述符
@@ -382,6 +369,7 @@ inline bool operator==(const UserInfo& lhs, const UserInfo& rhs) {
 
 struct OpenFlags {
     bool exclusive;
+    std::string confPath;
 
     OpenFlags() : exclusive(true) {}
 };
@@ -448,10 +436,10 @@ class CurveClient {
 
     /**
      * 获取文件大小
-     * @param filename 文件名，格式为：文件名_用户名_
+     * @param fd 文件fd
      * @return 返回错误码
      */
-    virtual int64_t StatFile(const std::string& filename);
+    virtual int64_t StatFile(int fd, FileStatInfo* fileStat);
 
     /**
      * 异步读

include/client/libcurve_define.h

@@ -63,7 +63,7 @@ enum LIBCURVE_ERROR {
     // parameter error
     PARAM_ERROR             = 16,
     // internal error
-    INTERNAL_ERROR           = 17,
+    INTERNAL_ERROR          = 17,
     // CRC error
     CRC_ERROR               = 18,
     // parameter invalid

include/curve_compiler_specific.h

@@ -26,20 +26,13 @@
 #define CURVE_CACHELINE_SIZE 64
 #define CURVE_CACHELINE_ALIGNMENT alignas(CURVE_CACHELINE_SIZE)
 
-#if defined(COMPILER_GCC)
-#if defined(__cplusplus)
-#define CURVE_LIKELY(expr) \
-    (__builtin_expect(static_cast<bool>(expr), true))
-#define CURVE_UNLIKELY(expr) \
-    (__builtin_expect(static_cast<bool>(expr), false))
-#else
+#if defined(__GNUC__) || defined(__clang__)
 #define CURVE_LIKELY(expr) (__builtin_expect(!!(expr), 1))
 #define CURVE_UNLIKELY(expr) (__builtin_expect(!!(expr), 0))
-#endif
 #else
 #define CURVE_LIKELY(expr) (expr)
 #define CURVE_UNLIKELY(expr) (expr)
-#endif  // defined(COMPILER_GCC)
+#endif  // defined(__GNUC__) || defined(__clang__)
 
 #ifdef UNIT_TEST
 #define CURVE_MOCK virtual
@@ -55,4 +48,6 @@
 #define FALLTHROUGH_INTENDED ((void)0)
 #endif /* __GNUC__ >= 7 */
 
+#define CURVE_UNUSED __attribute__((__unused__))
+
 #endif  // INCLUDE_CURVE_COMPILER_SPECIFIC_H_

mk-deb.sh

@@ -108,13 +108,13 @@ function build_curvefs_python() {
         rm -rf ./bazel-bin/curvefs_python
 
         if [ "$1" = "release" ]; then
-            bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --copt -O2 -s \
+            bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --copt -O2 \
                 --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google \
                 --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --linkopt \
                 -L${dir}/curvefs_python/tmplib/ --copt -DCURVEVERSION=${curve_version} \
                 ${bazelflags}
         else
-            bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --compilation_mode=dbg -s \
+            bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --compilation_mode=dbg \
                 --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google \
                 --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --linkopt \
                 -L${dir}/curvefs_python/tmplib/ --copt -DCURVEVERSION=${curve_version} \
@@ -189,7 +189,7 @@ if [ "$1" = "debug" ]; then
         exit
     fi
 
-    bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --compilation_mode=dbg -s \
+    bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --compilation_mode=dbg \
         --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google \
         --copt \
         -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --linkopt \
@@ -211,7 +211,7 @@ else
         exit
     fi
 
-    bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --copt -O2 -s \
+    bazel build curvefs_python:curvefs --copt -DHAVE_ZLIB=1 --copt -O2 \
         --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google \
         --copt \
         -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --linkopt \

nbd

@@ -1 +1 @@
-Subproject commit 6231d3bc4954a5f2427148475cf39a2fdc4e3bad
+Subproject commit 631784c37f929727916a325737543de6a3ceff47

nebd/proto/client.proto

@@ -98,6 +98,7 @@ message FileInfo {
    required uint64 size = 1;
    required uint64 objSize = 2;
    required uint64 objNums = 3;
+   optional uint32 blocksize = 4;
 }
 
 message GetInfoRequest {

nebd/src/part1/libnebd.cpp

@@ -118,6 +118,10 @@ int64_t nebd_lib_filesize(int fd) {
     return GetFileSize4Nebd(fd);
 }
 
+int64_t nebd_lib_blocksize(int fd) {
+    return GetBlockSize4Nebd(fd);
+}
+
 int nebd_lib_resize(int fd, int64_t size) {
     return Extend4Nebd(fd, size);
 }

nebd/src/part1/libnebd.h

@@ -159,6 +159,8 @@ int nebd_lib_sync(int fd);
  */
 int64_t nebd_lib_filesize(int fd);
 
+int64_t nebd_lib_blocksize(int fd);
+
 /**
  *  @brief resize文件
  *  @param fd：文件的fd

nebd/src/part1/libnebd_file.cpp

@@ -51,6 +51,10 @@ int64_t GetFileSize4Nebd(int fd) {
     return nebd::client::nebdClient.GetFileSize(fd);
 }
 
+int64_t GetBlockSize4Nebd(int fd) {
+    return nebd::client::nebdClient.GetBlockSize(fd);
+}
+
 int Discard4Nebd(int fd, NebdClientAioContext* aioctx) {
     return nebd::client::nebdClient.Discard(fd, aioctx);
 }

nebd/src/part1/libnebd_file.h

@@ -62,6 +62,9 @@ int Extend4Nebd(int fd, int64_t newsize);
  *  @return 成功返回文件size，失败返回错误码
  */
 int64_t GetFileSize4Nebd(int fd);
+
+int64_t GetBlockSize4Nebd(int fd);
+
 /**
  *  @brief discard文件，异步函数
  *  @param fd：文件的fd

nebd/src/part1/nebd_client.cpp

@@ -320,6 +320,44 @@ int64_t NebdClient::GetFileSize(int fd) {
     return ret;
 }
 
+int64_t NebdClient::GetBlockSize(int fd) {
+    auto task = [&](brpc::Controller* cntl, brpc::Channel* channel,
+                    bool* rpcFailed) -> int64_t {
+        nebd::client::NebdFileService_Stub stub(channel);
+        nebd::client::GetInfoRequest request;
+        nebd::client::GetInfoResponse response;
+
+        request.set_fd(fd);
+        stub.GetInfo(cntl, &request, &response, nullptr);
+
+        *rpcFailed = cntl->Failed();
+        if (*rpcFailed) {
+            LOG(WARNING) << "GetBlockSize failed, error = " << cntl->ErrorText()
+                         << ", log id = " << cntl->log_id();
+            return -1;
+        } else {
+            if (response.retcode() != nebd::client::RetCode::kOK) {
+                LOG(ERROR) << "GetBlockSize failed, "
+                           << "retcode = " << response.retcode()
+                           << ",  retmsg = " << response.retmsg()
+                           << ", fd = " << fd
+                           << ", log id = " << cntl->log_id();
+                return -1;
+            } else {
+                return response.info().has_blocksize()
+                           ? response.info().blocksize()
+                           : kDefaultBlockSize;
+            }
+        }
+    };
+
+    int64_t ret = ExecuteSyncRpc(task);
+    if (ret < 0) {
+        LOG(ERROR) << "GetBlockSize failed, fd = " << fd;
+    }
+    return ret;
+}
+
 int NebdClient::Discard(int fd, NebdClientAioContext* aioctx) {
     auto task = [this, fd, aioctx]() {
         nebd::client::NebdFileService_Stub stub(&channel_);

nebd/src/part1/nebd_client.h

@@ -100,6 +100,8 @@ class NebdClient {
      */
     int64_t GetFileSize(int fd);
 
+    int64_t GetBlockSize(int fd);
+
     /**
      *  @brief discard文件，异步函数
      *  @param fd：文件的fd

nebd/src/part1/nebd_common.h

@@ -71,4 +71,6 @@ struct HeartbeatOption {
     int64_t rpcTimeoutMs;
 };
 
+constexpr uint32_t kDefaultBlockSize = 4096;
+
 #endif  // NEBD_SRC_PART1_NEBD_COMMON_H_

nebd/src/part2/define.h

@@ -105,6 +105,8 @@ struct NebdFileInfo {
     uint64_t obj_size;
     // object数量
     uint64_t num_objs;
+    // block size
+    uint32_t block_size;
 };
 
 using ExtendAttribute = std::map<std::string, std::string>;

nebd/src/part2/file_service.cpp

@@ -298,6 +298,7 @@ void NebdFileServiceImpl::GetInfo(
         info->set_size(fileInfo.size);
         info->set_objsize(fileInfo.obj_size);
         info->set_objnums(fileInfo.num_objs);
+        info->set_blocksize(fileInfo.block_size);
         response->set_retcode(RetCode::kOK);
         response->set_allocated_info(info);
     }

nebd/src/part2/request_executor_curve.cpp

@@ -34,50 +34,54 @@ using ::curve::client::UserInfo_t;
 const char* kSessionAttrKey = "session";
 const char* kOpenFlagsAttrKey = "openflags";
 
-curve::client::OpenFlags ConverToCurveOpenFlags(const OpenFlags* flags) {
+curve::client::OpenFlags ConverToCurveOpenFlags(
+    const OpenFlags* flags, const std::string& confPath) {
     curve::client::OpenFlags curveflags;
-
+    curveflags.confPath = confPath;
     if (!flags) {
         return curveflags;
     } else {
         if (flags->has_exclusive()) {
             curveflags.exclusive = flags->exclusive();
         }
-
         return curveflags;
     }
 }
 
-std::string FileNameParser::Parse(const std::string& fileName) {
+std::pair<std::string, std::string>
+FileNameParser::Parse(const std::string& fileName) {
+    std::string confPath;
     auto beginPos = fileName.find_first_of("/");
     if (beginPos == std::string::npos) {
         LOG(ERROR) << "error format fileName: " << fileName;
-        return "";
+        return std::make_pair("", "");
     }
     beginPos += 1;
 
     auto endPos = fileName.find_last_of(":");
     if (endPos == std::string::npos) {
         LOG(ERROR) << "error format fileName: " << fileName;
-        return "";
+        return std::make_pair("", "");
     }
 
     if (endPos < beginPos) {
         endPos = fileName.length();
+    } else if (endPos < fileName.length() - 1) {
+        confPath = fileName.substr(endPos + 1);
     }
 
     if (beginPos >= endPos) {
         LOG(ERROR) << "error format fileName: " << fileName;
-        return "";
+        return std::make_pair("", "");
     }
 
     auto length = endPos - beginPos;
     if (length <= 2) {
         LOG(ERROR) << "error format fileName: " << fileName;
-        return "";
+        return std::make_pair("", "");
     }
 
-    return fileName.substr(beginPos, length);
+    return std::make_pair(fileName.substr(beginPos, length), confPath);
 }
 
 void CurveRequestExecutor::Init(const std::shared_ptr<CurveClient> &client) {
@@ -86,17 +90,19 @@ void CurveRequestExecutor::Init(const std::shared_ptr<CurveClient> &client) {
 
 std::shared_ptr<NebdFileInstance> CurveRequestExecutor::Open(
     const std::string& filename, const OpenFlags* openFlags) {
-    std::string curveFileName = FileNameParser::Parse(filename);
-    if (curveFileName.empty()) {
+    auto curveFileInfo = FileNameParser::Parse(filename);
+    if (curveFileInfo.first.empty()) {
         return nullptr;
     }
 
-    int fd = client_->Open(curveFileName, ConverToCurveOpenFlags(openFlags));
+    int fd = client_->Open(
+        curveFileInfo.first,
+        ConverToCurveOpenFlags(openFlags, curveFileInfo.second));
 
     if (fd >= 0) {
         auto curveFileInstance = std::make_shared<CurveFileInstance>();
         curveFileInstance->fd = fd;
-        curveFileInstance->fileName = curveFileName;
+        curveFileInstance->fileName = curveFileInfo.first;
         curveFileInstance->xattr[kSessionAttrKey] = "";
 
         if (openFlags) {
@@ -113,8 +119,8 @@ std::shared_ptr<NebdFileInstance> CurveRequestExecutor::Open(
 std::shared_ptr<NebdFileInstance>
 CurveRequestExecutor::Reopen(const std::string& filename,
                              const ExtendAttribute& xattr) {
-    std::string curveFileName = FileNameParser::Parse(filename);
-    if (curveFileName.empty()) {
+    auto curveFileInfo = FileNameParser::Parse(filename);
+    if (curveFileInfo.first.empty()) {
         return nullptr;
     }
 
@@ -129,11 +135,13 @@ CurveRequestExecutor::Reopen(const std::string& filename,
         }
     }
 
-    int fd = client_->ReOpen(curveFileName, ConverToCurveOpenFlags(&flags));
+    int fd = client_->ReOpen(
+        curveFileInfo.first,
+        ConverToCurveOpenFlags(&flags, curveFileInfo.second));
     if (fd >= 0) {
         auto curveFileInstance = std::make_shared<CurveFileInstance>();
         curveFileInstance->fd = fd;
-        curveFileInstance->fileName = curveFileName;
+        curveFileInstance->fileName = curveFileInfo.first;
         curveFileInstance->xattr[kSessionAttrKey] = newSessionId;
         if (xattr.count(kOpenFlagsAttrKey)) {
             curveFileInstance->xattr[kOpenFlagsAttrKey] =
@@ -175,17 +183,20 @@ int CurveRequestExecutor::Extend(NebdFileInstance* fd, int64_t newsize) {
 
 int CurveRequestExecutor::GetInfo(
     NebdFileInstance* fd, NebdFileInfo* fileInfo) {
-    std::string fileName = GetFileNameFromNebdFileInstance(fd);
-    if (fileName.empty()) {
+    int curveFd = GetCurveFdFromNebdFileInstance(fd);
+    if (curveFd < 0) {
+        LOG(ERROR) << "Parse curve fd failed";
         return -1;
     }
 
-    int64_t size = client_->StatFile(fileName);
-    if (size < 0) {
+    FileStatInfo statInfo;
+    int64_t rc = client_->StatFile(curveFd, &statInfo);
+    if (rc < 0) {
         return -1;
     }
 
-    fileInfo->size = size;
+    fileInfo->size = statInfo.length;
+    fileInfo->block_size = statInfo.blocksize;
     return 0;
 }
 

nebd/src/part2/request_executor_curve.h

@@ -25,6 +25,7 @@
 
 #include <string>
 #include <memory>
+#include <utility>
 #include "nebd/src/part2/request_executor.h"
 #include "nebd/src/part2/define.h"
 #include "include/client/libcurve.h"
@@ -58,9 +59,12 @@ class FileNameParser {
      *    qemu "cbd:pool1//cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_:/etc/curve/client.conf" //NOLINT
      *    nbd  "cbd:pool1//cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_"  // NOLINT
      * @param[in] fileName
-     * @return 解析出的字符串: "/cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_" //NOLINT
+     * @return 解析结果
+     *  qemu "/cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_", "/etc/curve/client.conf" //NOLINT
+     *  nbd  "/cinder/volume-6f30d296-07f7-452e-a983-513191f8cd95_cinder_", "" //NOLINT
      */
-    static std::string Parse(const std::string& fileName);
+    static std::pair<std::string, std::string>
+            Parse(const std::string& fileName);
 };
 
 class CurveRequestExecutor : public NebdRequestExecutor {

nebd/test/part1/fake_file_service.cpp

@@ -157,6 +157,7 @@ void FakeNebdFileService::GetInfo(::google::protobuf::RpcController* controller,
     info->set_size(fileSize_);
     info->set_objsize(fileSize_);
     info->set_objnums(1);
+    info->set_blocksize(blockSize_);
     response->set_retcode(RetCode::kOK);
     response->set_retmsg("GetInfo OK");
     response->set_allocated_info(info);

nebd/test/part1/fake_file_service.h

@@ -27,13 +27,15 @@
 #include <brpc/controller.h>
 #include <string>
 #include "nebd/proto/client.pb.h"
+#include "nebd/src/part1/nebd_common.h"
 
 namespace nebd {
 namespace client {
 
 class FakeNebdFileService: public NebdFileService {
  public:
-    FakeNebdFileService() {}
+    explicit FakeNebdFileService(uint32_t blockSize = kDefaultBlockSize)
+        : fileSize_(0), blockSize_(blockSize) {}
 
     virtual ~FakeNebdFileService() {}
 
@@ -84,6 +86,7 @@ class FakeNebdFileService: public NebdFileService {
 
  private:
     int64_t fileSize_;
+    uint32_t blockSize_;
 };
 }  // namespace client
 }  // namespace nebd

nebd/test/part1/nebd_client_unittest.cpp

@@ -269,6 +269,7 @@ TEST_F(NebdFileClientTest, NoNebdServerTest) {
     }
     ASSERT_EQ(-1, Extend4Nebd(1, kFileSize));
     ASSERT_EQ(-1, GetFileSize4Nebd(1));
+    ASSERT_EQ(-1, GetBlockSize4Nebd(1));
     ASSERT_EQ(-1, GetInfo4Nebd(1));
     ASSERT_EQ(-1, InvalidCache4Nebd(1));
     ASSERT_EQ(-1, Close4Nebd(1));
@@ -292,6 +293,7 @@ TEST_F(NebdFileClientTest, CommonTest) {
     ASSERT_EQ(0, Extend4Nebd(fd, kFileSize));
     ASSERT_EQ(kFileSize, GetFileSize4Nebd(fd));
     ASSERT_EQ(kFileSize, GetInfo4Nebd(fd));
+    ASSERT_EQ(kDefaultBlockSize, GetBlockSize4Nebd(fd));
     ASSERT_EQ(0, InvalidCache4Nebd(fd));
 
     char buffer[kBufSize];
@@ -443,6 +445,17 @@ TEST_F(NebdFileClientTest, ResponseFailTest) {
         ASSERT_EQ(-1, GetFileSize4Nebd(1));
     }
 
+    {
+        GetInfoResponse response;
+        response.set_retcode(RetCode::kNoOK);
+        EXPECT_CALL(mockService, GetInfo(_, _, _, _))
+            .Times(1)
+            .WillOnce(DoAll(
+                SetArgPointee<2>(response),
+                Invoke(MockClientFunc<GetInfoRequest, GetInfoResponse>)));
+        ASSERT_EQ(-1, GetBlockSize4Nebd(1));
+    }
+
     {
         GetInfoResponse response;
         response.set_retcode(RetCode::kNoOK);

nebd/test/part1/nebd_lib_unittest.cpp

@@ -41,6 +41,7 @@ const char* kFileName = "nebd-lib-test-filename";
 const char* kNebdServerTestAddress = "./nebd-lib-unittest.sock";
 const char* kNebdClientConf = "./nebd/test/part1/nebd-lib-test.conf";
 const int64_t kFileSize = 10LL * 1024 * 1024 * 1024;
+const uint32_t kBlockSize = 4096;
 const int64_t kBufSize = 1024;
 
 namespace nebd {
@@ -89,7 +90,7 @@ class NebdLibTest : public ::testing::Test {
     }
 
     brpc::Server server;
-    FakeNebdFileService fakeService;
+    FakeNebdFileService fakeService{kBlockSize};
     MockNebdFileService mockService;
 };
 
@@ -106,6 +107,7 @@ TEST_F(NebdLibTest, CommonTest) {
     ASSERT_EQ(0, nebd_lib_resize(fd, kFileSize));
     ASSERT_EQ(kFileSize, nebd_lib_filesize(fd));
     ASSERT_EQ(kFileSize, nebd_lib_getinfo(fd));
+    ASSERT_EQ(kBlockSize, nebd_lib_blocksize(fd));
     ASSERT_EQ(0, nebd_lib_invalidcache(fd));
 
     ASSERT_EQ(-1, nebd_lib_pread(fd, 0, 0, 0));

nebd/test/part2/mock_curve_client.h

@@ -41,7 +41,7 @@ class MockCurveClient : public ::curve::client::CurveClient {
                  int(const std::string&, const ::curve::client::OpenFlags&));
     MOCK_METHOD1(Close, int(int));
     MOCK_METHOD2(Extend, int(const std::string&, int64_t));
-    MOCK_METHOD1(StatFile, int64_t(const std::string&));
+    MOCK_METHOD2(StatFile, int64_t(int, FileStatInfo*));
     MOCK_METHOD3(AioRead,
                  int(int, CurveAioContext*, curve::client::UserDataType));
     MOCK_METHOD3(AioWrite,

nebd/test/part2/test_request_executor_curve.cpp

@@ -36,6 +36,7 @@ using ::testing::_;
 using ::testing::SetArgPointee;
 using ::testing::DoAll;
 using ::testing::SaveArg;
+using ::testing::Invoke;
 
 class TestReuqestExecutorCurveClosure : public google::protobuf::Closure {
  public:
@@ -226,38 +227,48 @@ TEST_F(TestReuqestExecutorCurve, test_Extend) {
 TEST_F(TestReuqestExecutorCurve, test_GetInfo) {
     auto executor = CurveRequestExecutor::GetInstance();
     NebdFileInfo fileInfo;
-    std::string curveFilename("/cinder/volume-1234_cinder_");
+    int curveFd = 123;
 
     // 1. nebdFileIns不是CurveFileInstance类型, stat失败
     {
         auto nebdFileIns = new NebdFileInstance();
-        EXPECT_CALL(*curveClient_, StatFile(_)).Times(0);
+        EXPECT_CALL(*curveClient_, StatFile(curveFd, _)).Times(0);
         ASSERT_EQ(-1, executor.GetInfo(nebdFileIns, &fileInfo));
     }
 
-    // 2. nebdFileIns中的fileName为空, extend失败
+    // 2. nebdFileIns中的fd为空, stat失败
     {
         auto curveFileIns = new CurveFileInstance();
-        EXPECT_CALL(*curveClient_, StatFile(_)).Times(0);
+        EXPECT_CALL(*curveClient_, StatFile(curveFd, _)).Times(0);
         ASSERT_EQ(-1, executor.GetInfo(curveFileIns, &fileInfo));
     }
 
-    // 3. 调用curveclient的extend接口失败， extend失败
+
+    // 3. 调用curveclient的stat接口失败, stat失败
     {
         auto curveFileIns = new CurveFileInstance();
-        curveFileIns->fileName = curveFilename;
-        EXPECT_CALL(*curveClient_, StatFile(curveFilename))
+        curveFileIns->fd = curveFd;
+        EXPECT_CALL(*curveClient_, StatFile(curveFd, _))
             .WillOnce(Return(-1));
         ASSERT_EQ(-1, executor.GetInfo(curveFileIns, &fileInfo));
     }
 
-    // 4. extend成功
+    // 4. stat成功
     {
+        const uint64_t size = 10ull * 1024 * 1024 * 1024;
+        const uint32_t blocksize = 4096;
         auto curveFileIns = new CurveFileInstance();
-        curveFileIns->fileName = curveFilename;
-        EXPECT_CALL(*curveClient_, StatFile(curveFilename)).WillOnce(Return(1));
+        curveFileIns->fd = curveFd;
+        EXPECT_CALL(*curveClient_, StatFile(curveFd, _))
+            .WillOnce(Invoke(
+                [size, blocksize](int /*fd*/, FileStatInfo* info) {
+                    info->length = size;
+                    info->blocksize = blocksize;
+                    return 0;
+                }));
         ASSERT_EQ(0, executor.GetInfo(curveFileIns, &fileInfo));
-        ASSERT_EQ(1, fileInfo.size);
+        ASSERT_EQ(size, fileInfo.size);
+        ASSERT_EQ(blocksize, fileInfo.block_size);
     }
 }
 
@@ -470,20 +481,23 @@ TEST_F(TestReuqestExecutorCurve, test_InvalidCache) {
 
 TEST(TestFileNameParser, test_Parse) {
     std::string fileName("cbd:pool1//cinder/volume-1234_cinder_:/client.conf");
-    std::string res("/cinder/volume-1234_cinder_");
+    std::pair<std::string, std::string> res(
+        "/cinder/volume-1234_cinder_", "/client.conf");
+    ASSERT_EQ(res, FileNameParser::Parse(fileName));
+
+    fileName = "cbd:pool1//cinder/volume-1234_cinder_";
+    res = std::make_pair("/cinder/volume-1234_cinder_", "");
     ASSERT_EQ(res, FileNameParser::Parse(fileName));
 
     fileName = "cbd:pool1";
-    ASSERT_EQ("", FileNameParser::Parse(fileName));
-
-    fileName = "cbd:pool1//cinder/volume-1234_cinder_";
+    res = std::make_pair("", "");
     ASSERT_EQ(res, FileNameParser::Parse(fileName));
 
     fileName = "cbd:pool1//:";
-    ASSERT_EQ("", FileNameParser::Parse(fileName));
+    ASSERT_EQ(res, FileNameParser::Parse(fileName));
 
     fileName = "cbd:pool1//";
-    ASSERT_EQ("", FileNameParser::Parse(fileName));
+    ASSERT_EQ(res, FileNameParser::Parse(fileName));
 }
 
 

proto/nameserver2.proto

@@ -93,6 +93,8 @@ message FileInfo {
 
     optional    FileThrottleParams throttleParams = 17;
     optional    uint64      epoch = 18;
+    optional    string      poolset = 19;
+    optional    uint32      blocksize = 20;
 }
 
 // status code
@@ -166,7 +168,8 @@ enum  StatusCode {
     kRecoverFileError = 139;
     // epoch too old
     kEpochTooOld = 140;
-
+    // poolset doesn't exist
+    kPoolsetNotExist = 141;
     // 元数据存储错误
     kStorageError = 501;
     // 内部错误
@@ -202,6 +205,7 @@ message CreateFileRequest {
     required uint64     date = 6;
     optional uint64     stripeUnit = 7;
     optional uint64     stripeCount = 8;
+    optional string     poolset = 9;
 };
 
 message CreateFileResponse {
@@ -499,6 +503,7 @@ message  CreateCloneFileRequest {
     required string     cloneSource = 9;
     optional uint64     stripeUnit = 10;
     optional uint64     stripeCount = 11;
+    optional string     poolset = 12;
 }
 
 message CreateCloneFileResponse {

proto/snapshotcloneserver.proto

@@ -51,6 +51,7 @@ message SnapshotInfoData {
     required int32 status = 10;
     optional uint64 stripeUnit = 11;
     optional uint64 stripeCount = 12;
+    optional string poolset = 13;
 };
 
 message CloneInfoData {
@@ -66,6 +67,7 @@ message CloneInfoData {
     required bool isLazy = 10;
     required int32 nextStep = 11;
     required int32 status = 12;
+    optional string poolset = 13;
 };
 
 message HttpRequest {};
@@ -75,4 +77,3 @@ message HttpResponse {};
 service SnapshotCloneService{
     rpc default_method(HttpRequest) returns (HttpResponse);
 };
-

proto/topology.proto

@@ -61,6 +61,13 @@ message ClusterInfoData {
     required string clusterId = 1;
 }
 
+message PoolsetData {
+    required uint32 poolsetId = 1;
+    required string poolsetName = 2;
+    required string type = 3;
+    required string desc = 4;
+}
+
 message LogicalPoolData {
     required uint32 logicalPoolId = 1;
     required string logicalPoolName = 2;
@@ -79,6 +86,7 @@ message PhysicalPoolData {
     required uint32 physicalPoolId = 1;
     required string physicalPoolName = 2;
     required string desc = 3;
+    optional uint32 poolsetId = 4;
 }
 
 message ZoneData {
@@ -159,6 +167,8 @@ message ChunkServerRegistRequest {
     optional uint64 chunkFilepoolSize = 8;
     optional bool   useChunkFilePoolAsWalPool = 9;
     optional uint32 useChunkFilePoolAsWalPoolReserve = 10;
+    optional uint32 blockSize = 11;
+    optional uint32 chunkSize = 12;
 };
 
 message ChunkServerRegistResponse {
@@ -246,6 +256,7 @@ message ServerRegistRequest {
     optional uint32 physicalPoolID = 8;
     optional string physicalPoolName = 9; //physicalPoolName is unique
     required string desc = 10;
+    required string poolsetName = 11;
 }
 
 message ServerRegistResponse {
@@ -327,6 +338,8 @@ message PhysicalPoolInfo {
     required uint32 physicalPoolID = 1;
     required string physicalPoolName = 2;
     optional string desc = 3;
+    required uint32 poolsetId = 4;
+    required string poolsetName = 5;
 }
 
 message PhysicalPoolRequest {
@@ -334,6 +347,7 @@ message PhysicalPoolRequest {
     optional uint32 physicalPoolID = 1;
     optional string physicalPoolName = 2;
     optional string desc = 3;
+    optional string poolsetName = 4;
 }
 
 message PhysicalPoolResponse {
@@ -344,11 +358,43 @@ message PhysicalPoolResponse {
 message ListPhysicalPoolRequest {
 }
 
+message ListPhysicalPoolsInPoolsetRequest {
+    repeated uint32 poolsetId = 1;
+}
+
 message ListPhysicalPoolResponse {
     required sint32 statusCode = 1;
     repeated PhysicalPoolInfo physicalPoolInfos = 2;
 }
 
+//poolset message
+message PoolsetRequest {
+    //use either  poolsetID or poolsetName
+    optional uint32 poolsetID = 1;
+    optional string poolsetName = 2;
+    optional string type = 3;
+    optional string desc = 4;
+}
+
+message PoolsetInfo {
+    required uint32 poolsetID = 1;
+    required string poolsetName = 2;
+    required string type = 3;
+    optional string desc = 4;
+}
+
+message PoolsetResponse {
+    required sint32 statusCode = 1;
+    optional PoolsetInfo poolsetInfo = 2;
+}
+
+message ListPoolsetRequest {}
+
+message ListPoolsetResponse {
+    required sint32 statusCode = 1;
+    repeated PoolsetInfo poolsetInfos = 2;
+}
+
 //logicalpool message
 message LogicalPoolInfo {
     required uint32 logicalPoolID = 1;
@@ -514,7 +560,7 @@ service TopologyService {
     rpc DeleteChunkServer(DeleteChunkServerRequest) returns (DeleteChunkServerResponse);
     rpc SetChunkServer(SetChunkServerStatusRequest) returns (SetChunkServerStatusResponse);
 
-    rpc RegistServer(ServerRegistRequest)            returns (ServerRegistResponse);
+    rpc RegistServer(ServerRegistRequest)     returns (ServerRegistResponse);
     rpc GetServer(GetServerRequest)           returns (GetServerResponse);
     rpc DeleteServer(DeleteServerRequest)     returns (DeleteServerResponse);
     rpc ListZoneServer(ListZoneServerRequest) returns (ListZoneServerResponse);
@@ -524,11 +570,16 @@ service TopologyService {
     rpc GetZone(ZoneRequest)                  returns (ZoneResponse);
     rpc ListPoolZone(ListPoolZoneRequest)     returns (ListPoolZoneResponse);
 
+    rpc CreatePhysicalPool(PhysicalPoolRequest)              returns (PhysicalPoolResponse);
+    rpc DeletePhysicalPool(PhysicalPoolRequest)              returns (PhysicalPoolResponse);
+    rpc GetPhysicalPool(PhysicalPoolRequest)                 returns (PhysicalPoolResponse);
+    rpc ListPhysicalPool(ListPhysicalPoolRequest)            returns (ListPhysicalPoolResponse);
+    rpc ListPhysicalPoolsInPoolset(ListPhysicalPoolsInPoolsetRequest)  returns (ListPhysicalPoolResponse);
 
-    rpc CreatePhysicalPool(PhysicalPoolRequest)     returns (PhysicalPoolResponse);
-    rpc DeletePhysicalPool(PhysicalPoolRequest)     returns (PhysicalPoolResponse);
-    rpc GetPhysicalPool(PhysicalPoolRequest)        returns (PhysicalPoolResponse);
-    rpc ListPhysicalPool(ListPhysicalPoolRequest)   returns (ListPhysicalPoolResponse);
+    rpc CreatePoolset(PoolsetRequest)          returns (PoolsetResponse);
+    rpc DeletePoolset(PoolsetRequest)          returns (PoolsetResponse);
+    rpc GetPoolset(PoolsetRequest)             returns (PoolsetResponse);
+    rpc ListPoolset(ListPoolsetRequest)        returns (ListPoolsetResponse);
 
     rpc CreateLogicalPool(CreateLogicalPoolRequest)       returns (CreateLogicalPoolResponse);
     rpc DeleteLogicalPool(DeleteLogicalPoolRequest)    returns (DeleteLogicalPoolResponse);
@@ -545,4 +596,3 @@ service TopologyService {
     rpc SetCopysetsAvailFlag(SetCopysetsAvailFlagRequest) returns (SetCopysetsAvailFlagResponse);
     rpc ListUnAvailCopySets(ListUnAvailCopySetsRequest) returns (ListUnAvailCopySetsResponse);
 }
-

replace-curve-repo.sh

@@ -14,9 +14,6 @@ sed -i "s;https://zlib.net/zlib-1.2.11.tar.gz;https://curve-build.nos-eastchina1
 # protobuf
 sed -i "s;https://github.com/google/protobuf/archive/v3.6.1.3.zip;https://curve-build.nos-eastchina1.126.net/protobuf-3.6.1.3.zip;g" WORKSPACE
 
-# googletest
-sed -i "s;https://github.com/google/googletest;https://gitee.com/mirrors/googletest;g" WORKSPACE
-
 # gflags
 sed -i "s;https://github.com/gflags/gflags/archive/v2.2.2.tar.gz;https://curve-build.nos-eastchina1.126.net/gflags-2.2.2.tar.gz;g" WORKSPACE
 

robot/curve_robot.txt

@@ -1449,11 +1449,11 @@ read write iosize 512
     ${fd}   open libcurve file
     ${new_fd}       Convert To Integer    ${fd}
     ${rc}    write libcurve file error   ${new_fd}    length=512
-    ${expect_rc}    evaluate    int(512)
+    ${expect_rc}    evaluate    int(-22)
+    should be equal    ${rc}   ${expect_rc}
+    ${rc}    read libcurve file   ${new_fd}   length=512
+    ${expect_rc}    evaluate    int(-22)
     should be equal    ${rc}   ${expect_rc}
-    ${content}    read libcurve file   ${new_fd}   length=512
-    ${expect_content}    evaluate    str("a")*512
-    should be equal    ${content}   ${expect_content}
     [Teardown]	    file clean      ${new_fd}
 
 
@@ -1480,11 +1480,11 @@ read write iosize 512
     ${fd}   open libcurve file
     ${new_fd}       Convert To Integer    ${fd}
     ${rc}    write libcurve file error   ${new_fd}    length=512
-    ${expect_rc}    evaluate    int(512)
+    ${expect_rc}    evaluate    int(-22)
+    should be equal    ${rc}   ${expect_rc}
+    ${rc}    read libcurve file   ${new_fd}   length=512
+    ${expect_rc}    evaluate    int(-22)
     should be equal    ${rc}   ${expect_rc}
-    ${content}    read libcurve file   ${new_fd}   length=512
-    ${expect_content}    evaluate    str("a")*512
-    should be equal    ${content}   ${expect_content}
     [Teardown]	    file clean      ${new_fd}
 
 read write iosize 1k
@@ -1493,22 +1493,9 @@ read write iosize 1k
     ${fd}   open libcurve file
     ${new_fd}       Convert To Integer    ${fd}
     ${rc}    write libcurve file error   ${new_fd}    length=1024
-    ${expect_rc}    evaluate    int(1024)
-    should be equal    ${rc}   ${expect_rc}
-    ${content}    read libcurve file   ${new_fd}   length=1024
-    ${expect_content}    evaluate    str("a")*1024
-    should be equal    ${content}   ${expect_content}
-    [Teardown]	    file clean      ${new_fd}
-
-read write iosize not 512 alignment
-    [Tags]    P0    base    first release
-    [Setup]	file init
-    ${fd}   open libcurve file
-    ${new_fd}       Convert To Integer    ${fd}
-    ${rc}    write libcurve file error   ${new_fd}    length=511
     ${expect_rc}    evaluate    int(-22)
     should be equal    ${rc}   ${expect_rc}
-    ${rc}    read libcurve file    ${new_fd}    length=511
+    ${rc}    read libcurve file   ${new_fd}   length=1024
     ${expect_rc}    evaluate    int(-22)
     should be equal    ${rc}   ${expect_rc}
     [Teardown]	    file clean      ${new_fd}
@@ -2565,3 +2552,9 @@ check chunkserver should be exsits
 start mds with sleep
      start mds
 	 Sleep		5s
+
+
+
+
+
+

src/chunkserver/BUILD

@@ -48,6 +48,7 @@ cc_library(
         "//src/common:curve_s3_adapter",
         "//src/fs:lfs",
         "//src/client:curve_client",
+        "//include:include-common",
 	"//include/client:include_client",
 	"//proto:scan_cc_proto",
     ],

src/chunkserver/chunk_service.cpp

@@ -35,18 +35,23 @@
 #include "src/chunkserver/chunkserver_metrics.h"
 #include "src/chunkserver/op_request.h"
 #include "src/chunkserver/chunk_service_closure.h"
-
 #include "src/common/fast_align.h"
 
+#include "include/curve_compiler_specific.h"
+
 namespace curve {
 namespace chunkserver {
 
-ChunkServiceImpl::ChunkServiceImpl(ChunkServiceOptions chunkServiceOptions,
-    const std::shared_ptr<EpochMap> &epochMap) :
-    chunkServiceOptions_(chunkServiceOptions),
-    copysetNodeManager_(chunkServiceOptions.copysetNodeManager),
-    inflightThrottle_(chunkServiceOptions.inflightThrottle),
-    epochMap_(epochMap) {
+using ::curve::common::is_aligned;
+
+ChunkServiceImpl::ChunkServiceImpl(
+        const ChunkServiceOptions& chunkServiceOptions,
+        const std::shared_ptr<EpochMap>& epochMap)
+    : chunkServiceOptions_(chunkServiceOptions),
+      copysetNodeManager_(chunkServiceOptions.copysetNodeManager),
+      inflightThrottle_(chunkServiceOptions.inflightThrottle),
+      epochMap_(epochMap),
+      blockSize_(copysetNodeManager_->GetCopysetNodeOptions().blockSize) {
     maxChunkSize_ = copysetNodeManager_->GetCopysetNodeOptions().maxChunkSize;
 }
 
@@ -573,14 +578,13 @@ void ChunkServiceImpl::UpdateEpoch(RpcController *controller,
 }
 
 bool ChunkServiceImpl::CheckRequestOffsetAndLength(uint32_t offset,
-                                                   uint32_t len) {
+                                                   uint32_t len) const {
     // 检查offset+len是否越界
-    if (offset + len > maxChunkSize_) {
+    if (CURVE_UNLIKELY(offset + len > maxChunkSize_)) {
         return false;
     }
 
-    return common::is_aligned(offset, FLAGS_minIoAlignment) &&
-           common::is_aligned(len, FLAGS_minIoAlignment);
+    return is_aligned(offset, blockSize_) && is_aligned(len, blockSize_);
 }
 
 }  // namespace chunkserver

src/chunkserver/chunk_service.h

@@ -41,7 +41,7 @@ class CopysetNodeManager;
 
 class ChunkServiceImpl : public ChunkService {
  public:
-    explicit ChunkServiceImpl(ChunkServiceOptions chunkServiceOptions,
+    explicit ChunkServiceImpl(const ChunkServiceOptions& chunkServiceOptions,
         const std::shared_ptr<EpochMap> &epochMap);
     ~ChunkServiceImpl() {}
 
@@ -105,7 +105,7 @@ class ChunkServiceImpl : public ChunkService {
      * @param len[in]: op request' length
      * @return true，说明合法，否则返回false
      */
-    bool CheckRequestOffsetAndLength(uint32_t offset, uint32_t len);
+    bool CheckRequestOffsetAndLength(uint32_t offset, uint32_t len) const;
 
  private:
     ChunkServiceOptions chunkServiceOptions_;
@@ -114,6 +114,7 @@ class ChunkServiceImpl : public ChunkService {
     uint32_t            maxChunkSize_;
 
     std::shared_ptr<EpochMap> epochMap_;
+    uint32_t blockSize_;
 };
 
 }  // namespace chunkserver

src/chunkserver/chunkserver.cpp

@@ -59,7 +59,7 @@ DEFINE_string(chunkServerExternalIp, "127.0.0.1", "chunkserver external ip");
 DEFINE_int32(chunkServerPort, 8200, "chunkserver port");
 DEFINE_string(chunkServerStoreUri, "local://./0/", "chunkserver store uri");
 DEFINE_string(chunkServerMetaUri,
-    "local://./0/chunkserver.dat", "chunnkserver meata uri");
+    "local://./0/chunkserver.dat", "chunkserver meta uri");
 DEFINE_string(copySetUri, "local://./0/copysets", "copyset data uri");
 DEFINE_string(raftSnapshotUri, "curve://./0/copysets", "raft snapshot uri");
 DEFINE_string(raftLogUri, "curve://./0/copysets", "raft log uri");
@@ -109,12 +109,6 @@ int ChunkServer::Run(int argc, char** argv) {
     // ============================初始化各模块==========================//
     LOG(INFO) << "Initializing ChunkServer modules";
 
-    LOG_IF(FATAL, !conf.GetUInt32Value("global.min_io_alignment",
-                                       &FLAGS_minIoAlignment))
-        << "Failed to get global.min_io_alignment";
-    LOG_IF(FATAL, !common::is_aligned(FLAGS_minIoAlignment, 512))
-        << "minIoAlignment should align to 512";
-
     // 优先初始化 metric 收集模块
     ChunkServerMetricOptions metricOptions;
     InitMetricOptions(&conf, &metricOptions);
@@ -143,6 +137,7 @@ int ChunkServer::Run(int argc, char** argv) {
     InitChunkFilePoolOptions(&conf, &chunkFilePoolOptions);
     std::shared_ptr<FilePool> chunkfilePool =
             std::make_shared<FilePool>(fs);
+
     LOG_IF(FATAL, false == chunkfilePool->Initialize(chunkFilePoolOptions))
         << "Failed to init chunk file pool";
 
@@ -201,12 +196,14 @@ int ChunkServer::Run(int argc, char** argv) {
     registerOptions.useChunkFilePoolAsWalPool = useChunkFilePoolAsWalPool;
     registerOptions.fs = fs;
     registerOptions.chunkFilepool = chunkfilePool;
+    registerOptions.blockSize = chunkfilePool->GetFilePoolOpt().blockSize;
+    registerOptions.chunkSize = chunkfilePool->GetFilePoolOpt().fileSize;
     Register registerMDS(registerOptions);
     ChunkServerMetadata metadata;
     ChunkServerMetadata localMetadata;
     // 从本地获取meta
     std::string metaPath = UriParser::GetPathFromUri(
-        registerOptions.chunkserverMetaUri).c_str();
+        registerOptions.chunkserverMetaUri);
 
     auto epochMap = std::make_shared<EpochMap>();
     if (fs->FileExists(metaPath)) {
@@ -249,6 +246,13 @@ int ChunkServer::Run(int argc, char** argv) {
         FilePoolOptions poolOpt = walFilePool->GetFilePoolOpt();
         uint32_t maxWalSegmentSize = poolOpt.fileSize + poolOpt.metaPageSize;
         copysetNodeOptions.maxWalSegmentSize = maxWalSegmentSize;
+
+        if (poolOpt.getFileFromPool) {
+            // overwrite from file pool
+            copysetNodeOptions.maxChunkSize = poolOpt.fileSize;
+            copysetNodeOptions.metaPageSize = poolOpt.metaPageSize;
+            copysetNodeOptions.blockSize = poolOpt.blockSize;
+        }
     }
 
     // install snapshot的带宽限制
@@ -480,8 +484,15 @@ void ChunkServer::InitChunkFilePoolOptions(
     common::Configuration *conf, FilePoolOptions *chunkFilePoolOptions) {
     LOG_IF(FATAL, !conf->GetUInt32Value("global.chunk_size",
         &chunkFilePoolOptions->fileSize));
+
     LOG_IF(FATAL, !conf->GetUInt32Value("global.meta_page_size",
-        &chunkFilePoolOptions->metaPageSize));
+                                        &chunkFilePoolOptions->metaPageSize))
+        << "Not found `global.meta_page_size` in config file";
+
+    LOG_IF(FATAL, !conf->GetUInt32Value("global.block_size",
+                                        &chunkFilePoolOptions->blockSize))
+        << "Not found `global.block_size` in config file";
+
     LOG_IF(FATAL, !conf->GetUInt32Value("chunkfilepool.cpmeta_file_size",
         &chunkFilePoolOptions->metaFileSize));
     LOG_IF(FATAL, !conf->GetBoolValue(
@@ -586,10 +597,12 @@ void ChunkServer::InitCopysetNodeOptions(
         &copysetNodeOptions->recyclerUri));
     LOG_IF(FATAL, !conf->GetUInt32Value("global.chunk_size",
         &copysetNodeOptions->maxChunkSize));
+    LOG_IF(FATAL, !conf->GetUInt32Value("global.meta_page_size",
+        &copysetNodeOptions->metaPageSize));
+    LOG_IF(FATAL, !conf->GetUInt32Value("global.block_size",
+        &copysetNodeOptions->blockSize));
     LOG_IF(FATAL, !conf->GetUInt32Value("global.location_limit",
         &copysetNodeOptions->locationLimit));
-    LOG_IF(FATAL, !conf->GetUInt32Value("global.meta_page_size",
-        &copysetNodeOptions->pageSize));
     LOG_IF(FATAL, !conf->GetUInt32Value("copyset.load_concurrency",
         &copysetNodeOptions->loadConcurrency));
     LOG_IF(FATAL, !conf->GetUInt32Value("copyset.check_retrytimes",
@@ -870,10 +883,6 @@ void ChunkServer::LoadConfigFromCmdline(common::Configuration *conf) {
         conf->SetIntValue("copyset.load_concurrency",
             FLAGS_copysetLoadConcurrency);
     }
-
-    if (GetCommandLineFlagInfo("minIoAlignment", &info) && !info.is_default) {
-        conf->SetUInt32Value("global.min_io_alignment", FLAGS_minIoAlignment);
-    }
 }
 
 int ChunkServer::GetChunkServerMetaFromLocal(

src/chunkserver/clone_core.cpp

@@ -140,24 +140,24 @@ int CloneCore::CloneReadByLocalInfo(
     const ChunkRequest* request = readRequest->request_;
     off_t offset = request->offset();
     size_t length = request->size();
-    uint32_t pageSize = chunkInfo.pageSize;
+    const uint32_t blockSize = chunkInfo.blockSize;
 
-    // offset 和 length 必须与 pageSize 对齐
-    if (offset % pageSize != 0 || length % pageSize != 0) {
+    // offset and length must be aligned with blockSize
+    if (offset % blockSize != 0 || length % blockSize != 0) {
         LOG(ERROR) << "Invalid offset or length: "
                    << " logic pool id: " << request->logicpoolid()
                    << " copyset id: " << request->copysetid()
                    << " chunkid: " << request->chunkid()
                    << " offset: " << offset
                    << " length: " << length
-                   << " page size: " << pageSize;
+                   << " block size: " << blockSize;
         SetResponse(readRequest,
                     CHUNK_OP_STATUS::CHUNK_OP_STATUS_INVALID_REQUEST);
         return -1;
     }
 
-    uint32_t beginIndex = offset / pageSize;
-    uint32_t endIndex = (offset + length - 1) / pageSize;
+    uint32_t beginIndex = offset / blockSize;
+    uint32_t endIndex = (offset + length - 1) / blockSize;
 
     // 请求提交到CloneManager的时候，chunk一定是clone chunk
     // 但是由于有其他请求操作相同的chunk，此时chunk有可能已经被遍写过了
@@ -358,9 +358,9 @@ int CloneCore::ReadThenMerge(std::shared_ptr<ReadChunkRequest> readRequest,
 
     off_t offset = request->offset();
     size_t length = request->size();
-    uint32_t pageSize = chunkInfo.pageSize;
-    uint32_t beginIndex = offset / pageSize;
-    uint32_t endIndex = (offset + length - 1) / pageSize;
+    uint32_t blockSize = chunkInfo.blockSize;
+    uint32_t beginIndex = offset / blockSize;
+    uint32_t endIndex = (offset + length - 1) / blockSize;
     // 获取chunk文件已经写过和未被写过的区域
     std::vector<BitRange> copiedRanges;
     std::vector<BitRange> uncopiedRanges;
@@ -385,8 +385,8 @@ int CloneCore::ReadThenMerge(std::shared_ptr<ReadChunkRequest> readRequest,
     // 1.Read 对于已写过的区域，从chunk文件中读取
     CSErrorCode errorCode;
     for (auto& range : copiedRanges) {
-        readOff = range.beginIndex * pageSize;
-        readSize = (range.endIndex - range.beginIndex + 1) * pageSize;
+        readOff = range.beginIndex * blockSize;
+        readSize = (range.endIndex - range.beginIndex + 1) * blockSize;
         relativeOff = readOff - offset;
         errorCode = dataStore->ReadChunk(request->chunkid(),
                                          request->sn(),
@@ -407,8 +407,8 @@ int CloneCore::ReadThenMerge(std::shared_ptr<ReadChunkRequest> readRequest,
 
     // 2.Merge 对于未写过的区域，从源端下载的区域中拷贝出来进行merge
     for (auto& range : uncopiedRanges) {
-        readOff = range.beginIndex * pageSize;
-        readSize = (range.endIndex - range.beginIndex + 1) * pageSize;
+        readOff = range.beginIndex * blockSize;
+        readSize = (range.endIndex - range.beginIndex + 1) * blockSize;
         relativeOff = readOff - offset;
         cloneData->copy_to(chunkData + relativeOff, readSize, relativeOff);
     }

src/chunkserver/config_info.cpp

@@ -1,54 +0,0 @@
-/*
- *  Copyright (c) 2020 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: curve
- * Created Date: 19-2-28
- * Author: wudemiao
- */
-
-#include "src/chunkserver/config_info.h"
-
-#include "src/chunkserver/copyset_node_manager.h"
-#include "src/chunkserver/concurrent_apply/concurrent_apply.h"
-#include "src/chunkserver/datastore/file_pool.h"
-
-namespace curve {
-namespace chunkserver {
-
-CopysetNodeOptions::CopysetNodeOptions()
-    : electionTimeoutMs(1000),
-      snapshotIntervalS(3600),
-      catchupMargin(1000),
-      usercodeInPthread(false),
-      logUri("/log"),
-      raftMetaUri("/raft_meta"),
-      raftSnapshotUri("/raft_snapshot"),
-      chunkDataUri("/data"),
-      chunkSnapshotUri("/snapshot"),
-      recyclerUri("/recycler"),
-      port(8200),
-      maxChunkSize(16 * 1024 * 1024),
-      pageSize(4096),
-      concurrentapply(nullptr),
-      chunkFilePool(nullptr),
-      walFilePool(nullptr),
-      localFileSystem(nullptr),
-      snapshotThrottle(nullptr) {
-}
-
-}  // namespace chunkserver
-}  // namespace curve

src/chunkserver/config_info.h

@@ -89,7 +89,9 @@ struct CopysetNodeOptions {
     // WAL segment file size
     uint32_t maxWalSegmentSize;
     // chunk文件的page大小
-    uint32_t pageSize;
+    uint32_t metaPageSize;
+    // alignment for I/O request
+    uint32_t blockSize;
     // clone chunk的location长度限制
     uint32_t locationLimit;
 
@@ -146,6 +148,27 @@ struct ChunkServiceOptions {
     std::shared_ptr<InflightThrottle> inflightThrottle;
 };
 
+inline CopysetNodeOptions::CopysetNodeOptions()
+    : electionTimeoutMs(1000),
+      snapshotIntervalS(3600),
+      catchupMargin(1000),
+      usercodeInPthread(false),
+      logUri("/log"),
+      raftMetaUri("/raft_meta"),
+      raftSnapshotUri("/raft_snapshot"),
+      chunkDataUri("/data"),
+      chunkSnapshotUri("/snapshot"),
+      recyclerUri("/recycler"),
+      port(8200),
+      maxChunkSize(16 * 1024 * 1024),
+      metaPageSize(4096),
+      blockSize(4096),
+      concurrentapply(nullptr),
+      chunkFilePool(nullptr),
+      walFilePool(nullptr),
+      localFileSystem(nullptr),
+      snapshotThrottle(nullptr) {}
+
 }  // namespace chunkserver
 }  // namespace curve
 

src/chunkserver/copyset_node.cpp

@@ -126,7 +126,8 @@ int CopysetNode::Init(const CopysetNodeOptions &options) {
     DataStoreOptions dsOptions;
     dsOptions.baseDir = chunkDataApath_;
     dsOptions.chunkSize = options.maxChunkSize;
-    dsOptions.pageSize = options.pageSize;
+    dsOptions.metaPageSize = options.metaPageSize;
+    dsOptions.blockSize = options.blockSize;
     dsOptions.locationLimit = options.locationLimit;
     dsOptions.enableOdsyncWhenOpenChunkFile =
         options.enableOdsyncWhenOpenChunkFile;
@@ -208,7 +209,10 @@ int CopysetNode::Run() {
                    << "Copyset: " << GroupIdString();
         return -1;
     }
-    syncThread_.Run();
+
+    if (!enableOdsyncWhenOpenChunkFile_) {
+        syncThread_.Run();
+    }
 
     LOG(INFO) << "Run copyset success."
               << "Copyset: " << GroupIdString();