fix make curvebs image

Signed-off-by: Hanqing Wu <wuhanqing@corp.netease.com>

.bazelrc

@@ -0,0 +1,10 @@
+build --verbose_failures
+
+build --define=with_glog=true --define=libunwind=true
+build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
+build --cxxopt -Wno-error=format-security
+build --copt=-fdiagnostics-color=always
+
+build:gcc7-later --cxxopt -faligned-new
+
+run --copt=-fdiagnostics-color=always

.bazelversion

@@ -0,0 +1 @@
+0.17.2

.gitignore

@@ -78,3 +78,43 @@ bazel-testlogs
 
 # mac
 .DS_Store
+
+test/client/configs/*
+!test/client/configs/.gitkeep
+
+projects/*
+docker/curvebs
+docker/base/*
+!docker/base/Dockerfile
+!docker/base/Makefile
+
+# curvefs
+curvefs/devops/projects/
+curvefs/devops/ssh/
+curvefs/devops/inventory
+curvefs/BUILD_MODE
+*.pyc
+.facts/
+*retry
+
+curvefs/docker/curvefs
+curvefs/docker/base/*
+!curvefs/docker/base/Dockerfile
+!curvefs/docker/base/Makefile
+curvefs/BUILD_MODE
+
+.BUILD_MODE
+
+/external
+/bazel-*
+/compile_commands.json
+/.cache/
+
+test/integration/*.conf
+test/integration/client/config/client.conf*
+test/integration/snapshotcloneserver/config/*.conf
+
+.pre-commit-config.yaml
+
+*.deb
+*.whl

CHANGELOG-1.2.md

@@ -0,0 +1,33 @@
+# CHANGELOG of v1.2
+
+Previous change logs can be found at [CHANGELOG-1.1](https://github.com/opencurve/curve/blob/master/CHANGELOG-1.1.md)
+
+## new features
+
+- [When add a new pool, default new logic pool is DENY, need tool to enable it.](https://github.com/opencurve/curve/pull/221)
+- [When io error occurs, whether nebd-server discards the corresponding rpc request depends on configuration setting(discard default).](https://github.com/opencurve/curve/pull/212)
+- [When client read a unallocated space, it does not allocate the segment to improve the space utilization.](https://github.com/opencurve/curve/pull/172)
+- [Add data stripe feature.](https://github.com/opencurve/curve/pull/211)
+
+
+## optimization
+
+- Optimize ansible script, optimize build script, optimize log printing.
+- Translate some document and code comment from chinese to english.
+- [Curve_ops_tool statistics chunkserver capacity by pool.](https://github.com/opencurve/curve/pull/199)
+- Add a script for k8s to attach curve volume.
+- curve_ops_tool improve:
+    - [Check copyset consistency between chunkserver and copyset.](https://github.com/opencurve/curve/pull/184)
+    - [Support to take a snapshot of copyset.](https://github.com/opencurve/curve/pull/188)
+    - [Support to list the volume on copyset where majority copy is failed.](https://github.com/opencurve/curve/pull/233)
+- [Clean up the unit test temporary folder.](https://github.com/opencurve/curve/pull/206/)
+
+## bug fix
+
+- [Fix clone delete bug.](https://github.com/opencurve/curve/pull/176)
+- [Nbd unmap need wait thread exit.](https://github.com/opencurve/curve/pull/228)
+- [Mds need check file attach status.](https://github.com/opencurve/curve/pull/153)
+- [Fixed when disk fails, copyset reports an error, but sometimes the chunkserver does not exit.](https://github.com/opencurve/curve/pull/152)
+- [Fixed the direct_fd leak problem when wal write disk.](https://github.com/opencurve/curve/pull/113)
+- [Fixed the atomicity problem of GetFile when not using the file pool.](https://github.com/opencurve/curve/pull/195)
+- [Fixed in cluster_basic_test mds start error on InitEtcdClient() which produced coredump](https://github.com/opencurve/curve/pull/205)

Makefile

@@ -0,0 +1,24 @@
+# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
+
+.PHONY: list build install image clean
+
+prefix?= "$(PWD)/projects"
+release?= 0
+only?= "*"
+tag?= "curvebs:unknown"
+tgt_pkg?= ""
+
+list:
+	@bash util/build.sh --list
+
+build:
+	@bash util/build.sh --only=$(only) --release=$(release)
+
+install:
+	@bash util/install.sh --prefix=$(prefix) --only=$(only)
+
+image:
+	@bash util/image.sh $(tag) $(tgt_pkg)
+
+clean:
+	@bazel clean

WORKSPACE

@@ -189,3 +189,19 @@ new_http_archive(
     strip_prefix = "aws-checksums-0.1.5",
     build_file = "//:thirdparties/aws/aws-checksums.BUILD",
 )
+
+# C++ rules for Bazel.
+http_archive(
+    name = "rules_cc",
+    urls = ["https://github.com/bazelbuild/rules_cc/archive/9e10b8a6db775b1ecd358d8ddd3dab379a2c29a5.zip"],
+    strip_prefix = "rules_cc-9e10b8a6db775b1ecd358d8ddd3dab379a2c29a5",
+    sha256 = "954b7a3efc8752da957ae193a13b9133da227bdacf5ceb111f2e11264f7e8c95",
+)
+
+# abseil-cpp
+http_archive(
+  name = "com_google_absl",
+  urls = ["https://github.com/abseil/abseil-cpp/archive/refs/tags/20210324.2.tar.gz"],
+  strip_prefix = "abseil-cpp-20210324.2",
+  sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
+)

conf/chunkserver.conf

@@ -0,0 +1,241 @@
+#
+# Global settings
+#
+# log等级INFO=0/WARNING=1/ERROR=2/FATAL=3
+global.ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_addr} __CURVEADM_TEMPLATE__
+global.port=8200  # __CURVEADM_TEMPLATE__ ${service_port} __CURVEADM_TEMPLATE__
+global.subnet=127.0.0.0/24
+global.enable_external_server=true
+global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
+global.external_subnet=127.0.0.0/24
+# chunk大小，一般16MB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.chunk_size=16777216
+# chunk 元数据页大小，一般4KB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.block_size=4096
+
+# clone chunk允许的最长location长度
+global.location_limit=3000
+
+#
+# MDS settings
+#
+#支持mds多地址，以逗号分隔 127.0.0.1:6666,127.0.0.1:7777
+mds.listen.addr=127.0.0.1:6666  # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__
+# 向mds注册的最大重试次数
+mds.register_retries=100
+# 向mds注册的rpc超时时间，一般1000ms
+mds.register_timeout=1000
+# 向mds发送心跳的间隔，一般10s
+mds.heartbeat_interval=10
+# 向mds发送心跳的rpc超时间，一般1000ms
+mds.heartbeat_timeout=5000
+
+#
+# Chunkserver settings
+#
+# chunkserver主目录
+chunkserver.stor_uri=local://./0/  # __CURVEADM_TEMPLATE__ local://${prefix}/data __CURVEADM_TEMPLATE__
+# chunkserver元数据文件
+chunkserver.meta_uri=local://./0/chunkserver.dat  # __CURVEADM_TEMPLATE__ local://${prefix}/data/chunkserver.dat __CURVEADM_TEMPLATE__
+# disk类型
+chunkserver.disk_type=nvme
+# raft内部install snapshot带宽上限，一般20MB
+chunkserver.snapshot_throttle_throughput_bytes=20971520
+# check cycles是为了更精细的进行带宽控制，以snapshotThroughputBytes=100MB，
+# check cycles=10为例，它可以保证每1/10秒的带宽是10MB，且不累积，例如第1个
+# 1/10秒的带宽是10MB，但是就过期了，在第2个1/10秒依然只能用10MB的带宽，而
+# 不是20MB的带宽
+chunkserver.snapshot_throttle_check_cycles=4
+
+#
+# Testing purpose settings
+#
+test.create_testcopyset=false
+test.testcopyset_poolid=666
+test.testcopyset_copysetid=888888
+test.testcopyset_conf=127.0.0.1:8200:0,127.0.0.1:8201:0,127.0.0.1:8202:0
+
+#
+# Copyset settings
+#
+# 是否检查任期，一般检查
+copyset.check_term=true
+# 是否关闭raft配置变更的服务，一般不关闭
+copyset.disable_cli=false
+copyset.log_applied_task=false
+# raft选举超时时间，一般是5000ms
+copyset.election_timeout_ms=1000
+# raft打快照间隔，一般是1800s，也就是30分钟
+copyset.snapshot_interval_s=1800
+# add一个节点，add的节点首先以类似learner的角色拷贝数据
+# 在跟leader差距catchup_margin个entry的时候，leader
+# 会尝试将配置变更的entry进行提交(一般来说提交的entry肯定
+# 会commit&apply，catchup_margin较小可以大概率确保learner
+# 后续很快可以加入复制组
+copyset.catchup_margin=1000
+# copyset chunk数据目录
+copyset.chunk_data_uri=local://./0/copysets  # __CURVEADM_TEMPLATE__ local://${prefix}/data/copysets __CURVEADM_TEMPLATE__
+# raft wal log目录
+copyset.raft_log_uri=curve://./0/copysets  # __CURVEADM_TEMPLATE__ curve://${prefix}/data/copysets __CURVEADM_TEMPLATE__
+# raft元数据目录
+copyset.raft_meta_uri=local://./0/copysets  # __CURVEADM_TEMPLATE__ local://${prefix}/data/copysets __CURVEADM_TEMPLATE__
+# raft snapshot目录
+copyset.raft_snapshot_uri=curve://./0/copysets  # __CURVEADM_TEMPLATE__ curve://${prefix}/data/copysets __CURVEADM_TEMPLATE__
+# copyset回收目录
+copyset.recycler_uri=local://./0/recycler  # __CURVEADM_TEMPLATE__ local://${prefix}/data/recycler __CURVEADM_TEMPLATE__
+copyset.max_inflight_requests=5000
+# chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
+copyset.load_concurrency=10
+# chunkserver use how many threads to use copyset complete sync. 
+copyset.sync_concurrency=20
+# 检查copyset是否加载完成出现异常时的最大重试次数
+copyset.check_retrytimes=3
+# 当前peer的applied_index与leader上的committed_index差距小于该值
+# 则判定copyset已经加载完成
+copyset.finishload_margin=2000
+# 循环判定copyset是否加载完成的内部睡眠时间
+copyset.check_loadmargin_interval_ms=1000
+# scan copyset interval
+copyset.scan_interval_sec=5
+# the size each scan 4MB
+copyset.scan_size_byte=4194304
+# the follower send scanmap to leader rpc timeout
+copyset.scan_rpc_timeout_ms=1000
+# the follower send scanmap to leader rpc retry times
+copyset.scan_rpc_retry_times=3
+# the follower send scanmap to leader rpc retry interval
+copyset.scan_rpc_retry_interval_us=100000
+# enable O_DSYNC when open chunkfile
+copyset.enable_odsync_when_open_chunkfile=true
+# sync trigger seconds
+copyset.sync_trigger_seconds=25
+# sync chunk limit default = 2MB
+copyset.sync_chunk_limits=2097152
+# 30s if the sum of write > sync_threshold, let the sync_chunk_limits doubled.
+copyset.sync_threshold=65536
+# check syncing interval
+copyset.check_syncing_interval_ms=500
+
+#
+# Clone settings
+#
+# 禁止使用curveclient
+clone.disable_curve_client=false
+# 禁止使用s3adapter
+clone.disable_s3_adapter=false
+# 克隆的分片大小，一般1MB
+clone.slice_size=1048576
+# 读clone chunk时是否需要paste到本地
+# 该配置对recover chunk请求类型无效
+clone.enable_paste=false
+# 克隆的线程数量
+clone.thread_num=10
+# 克隆的队列深度
+clone.queue_depth=6000
+# curve用户名
+curve.root_username=root
+# curve密码
+curve.root_password=root_password
+# client配置文件
+curve.config_path=conf/cs_client.conf  # __CURVEADM_TEMPLATE__ ${prefix}/conf/cs_client.conf __CURVEADM_TEMPLATE__
+# s3配置文件
+s3.config_path=conf/s3.conf  # __CURVEADM_TEMPLATE__ ${prefix}/conf/s3.conf __CURVEADM_TEMPLATE__
+# Curve File time to live
+curve.curve_file_timeout_s=30
+
+#
+# Local FileSystem settings
+#
+# 是否开启使用renameat2，ext4内核3.15以后开始支持
+fs.enable_renameat2=true
+
+#
+# metrics settings
+# true means on, false means off
+#
+metric.onoff=true
+
+#
+# Storage engine settings
+#
+storeng.sync_write=false
+
+#
+# QoS settings
+#
+
+#
+# Concurrent apply module
+# 并发模块写线程的并发度，一般是10
+wconcurrentapply.size=10
+# 并发模块写线程的队列深度
+wconcurrentapply.queuedepth=1
+# 并发模块读线程的并发度，一般是5
+rconcurrentapply.size=5
+# 并发模块读线程的队列深度
+rconcurrentapply.queuedepth=1
+
+#
+# Chunkfile pool
+#
+# 是否开启从chunkfilepool获取chunk，一般是true
+chunkfilepool.enable_get_chunk_from_pool=true
+# chunkfilepool目录
+chunkfilepool.chunk_file_pool_dir=./0/  # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__
+# chunkfilepool meta文件路径
+chunkfilepool.meta_path=./chunkfilepool.meta  # __CURVEADM_TEMPLATE__ ${prefix}/data/chunkfilepool.meta __CURVEADM_TEMPLATE__
+# chunkfilepool meta文件大小
+chunkfilepool.cpmeta_file_size=4096
+# chunkfilepool get chunk最大重试次数
+chunkfilepool.retry_times=5
+# Enable clean chunk
+chunkfilepool.clean.enable=true
+# The bytes per write for cleaning chunk (max: 1MB)
+chunkfilepool.clean.bytes_per_write=4096
+# The throttle iops for cleaning chunk (4KB/IO)
+chunkfilepool.clean.throttle_iops=500
+
+#
+# WAL file pool
+#
+# walpool是否共用chunkfilepool，如果为true，从第三条开始配置无效
+walfilepool.use_chunk_file_pool=true
+# WALpool和ChunkFilePool共用时启用，在容量分配时会预留walpool的空间
+walfilepool.use_chunk_file_pool_reserve=15
+# 是否开启从walfilepool获取chunk，一般是true
+walfilepool.enable_get_segment_from_pool=true
+# walpool目录
+walfilepool.file_pool_dir=./0/  # __CURVEADM_TEMPLATE__ ${prefix}/data/walfilepool.meta __CURVEADM_TEMPLATE__
+# walpool meta文件路径
+walfilepool.meta_path=./walfilepool.meta  # __CURVEADM_TEMPLATE__ ${prefix}/data/walfilepool.meta __CURVEADM_TEMPLATE__
+# walpool meta文件大小
+walfilepool.segment_size=8388608
+# WAL metapage大小
+walfilepool.metapage_size=4096
+# WAL filepool 元数据文件大小
+walfilepool.meta_file_size=4096
+# WAL filepool get chunk最大重试次数
+walfilepool.retry_times=5
+
+#
+# trash settings
+#
+# chunkserver回收数据彻底删除的过期时间
+trash.expire_afterSec=300
+# chunkserver检查回收数据过期时间的周期
+trash.scan_periodSec=120
+
+# common option
+#
+# chunkserver 日志存放文件夹
+chunkserver.common.logDir=./  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__
+# 单元测试情况下
+# chunkserver.common.logDir=./runlog/

conf/chunkserver.conf.example

@@ -13,6 +13,11 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
 
 #
@@ -75,7 +80,7 @@ copyset.catchup_margin=1000
 # copyset chunk数据目录
 copyset.chunk_data_uri=local://./0/copysets
 # raft wal log目录
-copyset.raft_log_uri=local://./0/copysets
+copyset.raft_log_uri=curve://./0/copysets
 # raft元数据目录
 copyset.raft_meta_uri=local://./0/copysets
 # raft snapshot目录
@@ -85,6 +90,8 @@ copyset.recycler_uri=local://./0/recycler
 copyset.max_inflight_requests=5000
 # chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
 copyset.load_concurrency=10
+# chunkserver use how many threads to use copyset complete sync. 
+copyset.sync_concurrency=20
 # 检查copyset是否加载完成出现异常时的最大重试次数
 copyset.check_retrytimes=3
 # 当前peer的applied_index与leader上的committed_index差距小于该值
@@ -92,6 +99,16 @@ copyset.check_retrytimes=3
 copyset.finishload_margin=2000
 # 循环判定copyset是否加载完成的内部睡眠时间
 copyset.check_loadmargin_interval_ms=1000
+# enable O_DSYNC when open chunkfile
+copyset.enable_odsync_when_open_chunkfile=true
+# sync trigger seconds
+copyset.sync_trigger_seconds=25
+# sync chunk limit default = 2MB
+copyset.sync_chunk_limits=2097152
+# 30s if the sum of write > sync_threshold, let the sync_chunk_limits doubled.
+copyset.sync_threshold=65536
+# check syncing interval
+copyset.check_syncing_interval_ms=500
 
 #
 # Clone settings
@@ -167,8 +184,10 @@ chunkfilepool.retry_times=5
 #
 # WAL file pool
 #
-# walpool是否共用chunkfilepool，如果为true，则以下配置无效
+# walpool是否共用chunkfilepool，如果为true，从第三条开始配置无效
 walfilepool.use_chunk_file_pool=true
+# WALpool和ChunkFilePool共用时启用，在容量分配时会预留walpool的空间
+walfilepool.use_chunk_file_pool_reserve=15
 # 是否开启从walfilepool获取chunk，一般是true
 walfilepool.enable_get_segment_from_pool=true
 # walpool目录

conf/client.conf

@@ -26,6 +26,15 @@ mds.refreshTimesPerLease=4
 # mds RPC接口每次重试之前需要先睡眠一段时间
 mds.rpcRetryIntervalUS=100000
 
+# The normal retry times for trigger wait strategy
+mds.normalRetryTimesBeforeTriggerWait=3
+
+# Max retry time for IO-Path request
+mds.maxRetryMsInIOPath=86400000
+
+# Sleep interval for wait
+mds.waitSleepMs=10000
+
 #
 ################# metacache配置信息 ################
 #
@@ -123,7 +132,7 @@ global.fileIOSplitMaxSizeKB=64
 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
 global.logLevel=0
 # 设置log的路径
-global.logPath=/data/log/curve/
+global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ /curvebs/client/logs __CURVEADM_TEMPLATE__
 # 单元测试情况下
 # logpath=./runlog/
 
@@ -142,3 +151,18 @@ global.metricDummyServerStartPort=9000
 
 # 是否关闭健康检查: true/关闭 false/不关闭
 global.turnOffHealthCheck=true
+
+##### chunkserver client option #####
+# chunkserver client rpc timeout time
+csClientOpt.rpcTimeoutMs=500
+# chunkserver client rpc max try
+csClientOpt.rpcMaxTry=86400000
+# chunkserver client rpc retry interval
+csClientOpt.rpcIntervalUs=100000
+# chunkserver client rpc max timeout time
+csClientOpt.rpcMaxTimeoutMs=8000
+
+##### chunkserver broadcaster option #####
+# broad cast max machine num
+csBroadCasterOpt.broadCastMaxNum=200
+

conf/cs_client.conf

@@ -3,7 +3,7 @@
 #
 
 # mds的地址信息，对于mds集群，地址以逗号隔开
-mds.listen.addr=127.0.0.1:6666
+mds.listen.addr=127.0.0.1:6666  # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__
 
 # 初始化阶段向mds注册开关，默认为开
 mds.registerToMDS=false
@@ -26,6 +26,15 @@ mds.refreshTimesPerLease=4
 # mds RPC接口每次重试之前需要先睡眠一段时间
 mds.rpcRetryIntervalUS=100000
 
+# The normal retry times for trigger wait strategy
+mds.normalRetryTimesBeforeTriggerWait=3
+
+# Max retry time for IO-Path request
+mds.maxRetryMsInIOPath=86400000
+
+# Sleep interval for wait
+mds.waitSleepMs=10000
+
 #
 ################# metacache配置信息 ################
 #
@@ -129,7 +138,7 @@ global.fileIOSplitMaxSizeKB=64
 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
 global.logLevel=0
 # 设置log的路径
-global.logPath=/data/log/curve/
+global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__
 # 单元测试情况下
 # logpath=./runlog/
 

conf/etcd.conf

@@ -0,0 +1,113 @@
+# This is the configuration file for the etcd server.
+
+# Human-readable name for this member.
+name:  # __CURVEADM_TEMPLATE__ etcd${service_host_sequence}${service_replica_sequence} __CURVEADM_TEMPLATE__
+
+# Path to the data directory.
+data-dir:  # __CURVEADM_TEMPLATE__ ${prefix}/data __CURVEADM_TEMPLATE__
+
+# Path to the dedicated wal directory.
+wal-dir:  # __CURVEADM_TEMPLATE__ ${prefix}/data/wal __CURVEADM_TEMPLATE__
+
+# Number of committed transactions to trigger a snapshot to disk.
+snapshot-count: 10000
+
+# Time (in milliseconds) of a heartbeat interval.
+heartbeat-interval: 100
+
+# Time (in milliseconds) for an election to timeout.
+election-timeout: 1000
+quota-backend-bytes: 0
+
+# List of comma separated URLs to listen on for peer traffic.
+listen-peer-urls:  # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_port} __CURVEADM_TEMPLATE__
+
+# List of comma separated URLs to listen on for client traffic.
+listen-client-urls:  # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_client_port} __CURVEADM_TEMPLATE__
+
+# Maximum number of snapshot files to retain (0 is unlimited).
+max-snapshots: 5
+
+# Maximum number of wal files to retain (0 is unlimited).
+max-wals: 5
+
+# Comma-separated white list of origins for CORS (cross-origin resource sharing).
+cors:
+
+# List of this member's peer URLs to advertise to the rest of the cluster.
+# The URLs needed to be a comma-separated list.
+initial-advertise-peer-urls:  # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_port} __CURVEADM_TEMPLATE__
+
+# List of this member's client URLs to advertise to the public.
+# The URLs needed to be a comma-separated list.
+advertise-client-urls:  # __CURVEADM_TEMPLATE__ http://${service_addr}:${service_client_port} __CURVEADM_TEMPLATE__
+
+# Discovery URL used to bootstrap the cluster.
+discovery:
+
+# Valid values include 'exit', 'proxy'
+discovery-fallback: proxy
+
+# HTTP proxy to use for traffic to discovery service.
+discovery-proxy:
+
+# DNS domain used to bootstrap initial cluster.
+discovery-srv:
+
+# Initial cluster configuration for bootstrapping.
+initial-cluster:  # __CURVEADM_TEMPLATE__ ${cluster_etcd_http_addr} __CURVEADM_TEMPLATE__
+
+# Initial cluster token for the etcd cluster during bootstrap.
+initial-cluster-token: etcd-cluster
+
+# Initial cluster state ('new' or 'existing').
+initial-cluster-state: new
+
+# Reject reconfiguration requests that would cause quorum loss.
+strict-reconfig-check: False
+
+# Accept etcd V2 client requests
+enable-v2: True
+
+# Enable runtime profiling data via HTTP server
+enable-pprof: True
+
+# Valid values include 'on', 'readonly', 'off'
+proxy: 'off'
+
+# Time (in milliseconds) an endpoint will be held in a failed state.
+proxy-failure-wait: 5000
+
+# Time (in milliseconds) of the endpoints refresh interval.
+proxy-refresh-interval: 30000
+
+# Time (in milliseconds) for a dial to timeout.
+proxy-dial-timeout: 1000
+
+# Time (in milliseconds) for a write to timeout.
+proxy-write-timeout: 5000
+
+# Time (in milliseconds) for a read to timeout.
+proxy-read-timeout: 0
+
+# Enable debug-level logging for etcd.
+debug: False
+
+logger: zap
+
+# Specify 'stdout' or 'stderr' to skip journald logging even when running under systemd.
+log-outputs: [stderr]
+
+# Force to create a new one member cluster.
+force-new-cluster: False
+
+auto-compaction-mode: periodic
+auto-compaction-retention: "1"
+
+# Set level of detail for exported metrics, specify 'extensive' to include histogram metrics.
+metrics: extensive
+
+# Enable to run an additional Raft election phase.
+pre-vote: True
+
+enable-grpc-gateway: True

conf/mds.conf

@@ -1,16 +1,16 @@
 #
 # mds服务端口
 #
-mds.listen.addr=127.0.0.1:6666
-mds.dummy.listen.port=6667
+mds.listen.addr=127.0.0.1:6666  #__CURVEADM_TEMPLATE__ ${service_addr}:${service_port} __CURVEADM_TEMPLATE__
+mds.dummy.listen.port=6667  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__
 global.subnet=127.0.0.0/24
-global.port=6666
+global.port=6666  # __CURVEADM_TEMPLATE__ ${service_port} __CURVEADM_TEMPLATE__
 
 #
 # etcd相关配置
 #
 # etcd地址
-mds.etcd.endpoint=127.0.0.1:2379
+mds.etcd.endpoint=127.0.0.1:2379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__
 # client建立连接的超时时间
 mds.etcd.dailtimeoutMs=5000
 # client进行put/get/txn等操作的超时时间
@@ -135,10 +135,14 @@ mds.topology.CreateCopysetRpcRetryTimes=20
 mds.topology.CreateCopysetRpcRetrySleepTimeMs=1000
 # Topology模块刷新metric时间间隔
 mds.topology.UpdateMetricIntervalSec=60
-# 物理池使用百分比，即使用量超过这个值即不再往这个池分配
+#和mds.chunkserver.failure.tolerance设置有关,一个zone 标准配置20台节点，如果允许3台节点failover,
+#那么剩余17台机器需要承载原先20台机器的空间,17/20=0.85,即使用量超过这个值即不再往这个池分配,
+#具体分为来两种情况, 当不使用chunkfilepool,物理池限制使用百分比,当使用 chunkfilepool 进行chunkfilepool分配时需预留failover空间,
 mds.topology.PoolUsagePercentLimit=85
 # 多pool选pool策略 0:Random, 1:Weight
 mds.topology.choosePoolPolicy=0
+# enable LogicalPool ALLOW/DENY status
+mds.topology.enableLogicalPoolStatus=false
 
 #
 # copyset config
@@ -165,6 +169,14 @@ mds.copyset.scatterWidthFloatingPercentage=20
 #
 # curvefs的默认chunk size大小，16MB = 16*1024*1024 = 16777216
 mds.curvefs.defaultChunkSize=16777216
+# curvefs的默认segment size大小，1GB = 1*1024*1024*1024 = 1073741824
+mds.curvefs.defaultSegmentSize=1073741824
+# curvefs的默认最小文件大小，10GB = 10*1024*1024*1024 = 10737418240
+mds.curvefs.minFileLength=10737418240
+# curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
+mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -184,13 +196,25 @@ mds.chunkserverclient.updateLeaderRetryIntervalMs=5000
 # snapshotclone config
 #
 # snapshot clone server 地址
-mds.snapshotcloneclient.addr=127.0.0.1:5555
+mds.snapshotcloneclient.addr=127.0.0.1:5555  # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_proxy_addr} __CURVEADM_TEMPLATE__
 
 #
 # common options
 #
 # 日志存放文件夹
-mds.common.logDir=./
+mds.common.logDir=./  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__
 # 单元测试情况下
 # mds.common.logDir=./runlog/
 
+#
+## poolset rules
+#
+# for backward compatibility, rules are applied for select poolset when creating file
+#
+# for example
+#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
+#
+# when creating file reqeust doesn't have poolset, above rules are used to select poolset
+# - if filename is /dir1/file, then poolset1 is select
+# - if filename is /dir1/sub/file, then sub is select
+mds.poolset.rules=

conf/nginx.conf

@@ -0,0 +1,50 @@
+# -*- mode: nginx -*-
+# vim: set expandtab tabstop=4 shiftwidth=4:
+
+worker_processes  4;
+pid               ${prefix}/logs/nginx.pid;
+
+events {
+    worker_connections 65535;
+}
+
+http {
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for", "$http_range", "$request_time" '
+                      '"$upstream_addr" "-" "-" "-" ';
+
+    access_log  ${prefix}/logs/access.log  main;
+    error_log   ${prefix}/logs/error.log;
+
+    sendfile           on;
+    keepalive_timeout  65;
+
+    send_timeout              60;
+    proxy_read_timeout        60;
+    proxy_send_timeout        60;
+    proxy_connect_timeout     1;
+    proxy_max_temp_file_size  0m;
+
+    client_header_buffer_size    4k;
+    large_client_header_buffers  8 16k;
+    client_body_buffer_size      128k;
+    client_max_body_size         200m;
+    port_in_redirect             off;
+
+    resolver          119.29.29.29 223.5.5.5 valid=1s;
+    resolver_timeout  5s;
+
+    server {
+        listen ${service_addr}:${service_proxy_port};
+        location / {
+            proxy_pass http://curvebs.163.com;
+        }
+    }
+
+    upstream curvebs.163.com {
+        ${cluster_snapshotclone_nginx_upstream}
+    }
+}

conf/py_client.conf

@@ -26,6 +26,15 @@ mds.refreshTimesPerLease=4
 # mds RPC接口每次重试之前需要先睡眠一段时间
 mds.rpcRetryIntervalUS=100000
 
+# The normal retry times for trigger wait strategy
+mds.normalRetryTimesBeforeTriggerWait=3
+
+# Max retry time for IO-Path request
+mds.maxRetryMsInIOPath=86400000
+
+# Sleep interval for wait
+mds.waitSleepMs=10000
+
 #
 ################# metacache配置信息 ################
 #

conf/snap_client.conf

@@ -3,7 +3,7 @@
 #
 
 # mds的地址信息，对于mds集群，地址以逗号隔开
-mds.listen.addr=127.0.0.1:6666
+mds.listen.addr=127.0.0.1:6666  # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__
 
 # 初始化阶段向mds注册开关，默认为开
 mds.registerToMDS=false
@@ -26,6 +26,15 @@ mds.refreshTimesPerLease=4
 # mds RPC接口每次重试之前需要先睡眠一段时间
 mds.rpcRetryIntervalUS=100000
 
+# The normal retry times for trigger wait strategy
+mds.normalRetryTimesBeforeTriggerWait=3
+
+# Max retry time for IO-Path request
+mds.maxRetryMsInIOPath=86400000
+
+# Sleep interval for wait
+mds.waitSleepMs=10000
+
 #
 ################# metacache配置信息 ################
 #
@@ -129,7 +138,7 @@ global.fileIOSplitMaxSizeKB=64
 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
 global.logLevel=0
 # 设置log的路径
-global.logPath=/data/log/curve/
+global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__
 # 单元测试情况下
 # logpath=./runlog/
 

conf/snapshot_clone_server.conf

@@ -2,7 +2,7 @@
 # curvefs client options
 #
 # client配置文件位置
-client.config_path=conf/cs_client.conf
+client.config_path=conf/cs_client.conf  # __CURVEADM_TEMPLATE__ ${prefix}/conf/snap_client.conf __CURVEADM_TEMPLATE__
 # mds root 用户名
 mds.rootUser=root
 # mds root 密码
@@ -13,19 +13,19 @@ client.methodRetryTimeSec=300
 client.methodRetryIntervalMs=5000
 
 # 日志文件位置
-log.dir=./
+log.dir=./  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__
 
 #
 # S3 config path
 #
-s3.config_path=./conf/s3.conf
+s3.config_path=./conf/s3.conf  # __CURVEADM_TEMPLATE__ ${prefix}/conf/s3.conf __CURVEADM_TEMPLATE__
 #
 #server options
 #
 # for snapshot
-server.address=127.0.0.1:5556
+server.address=127.0.0.1:5556  # __CURVEADM_TEMPLATE__ ${service_addr}:${service_port} __CURVEADM_TEMPLATE__
 server.subnet=127.0.0.0/24
-server.port=5556
+server.port=5556  # __CURVEADM_TEMPLATE__ ${service_port} __CURVEADM_TEMPLATE__
 # 调用client异步方法重试总时间
 server.clientAsyncMethodRetryTimeSec=300
 # 调用client异步方法重试时间间隔
@@ -73,7 +73,7 @@ server.backEndReferenceFuncScanIntervalMs=3600000
 # etcd相关配置
 #
 # etcd地址
-etcd.endpoint=127.0.0.1:2379
+etcd.endpoint=127.0.0.1:2379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__
 # client建立连接的超时时间
 etcd.dailtimeoutMs=5000
 # client进行put/get/txn等操作的超时时间
@@ -98,4 +98,4 @@ leader.election.timeoutms=0
 #
 # dummyserver相关配置
 #
-server.dummy.listen.port=8081
+server.dummy.listen.port=8081   # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__

conf/tools.conf

@@ -1,14 +1,16 @@
 # mds地址
-mdsAddr=127.0.0.1:6666
+mdsAddr=127.0.0.1:6666  # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__
 # mds dummy port
-mdsDummyPort=6667
+mdsDummyPort=6700  # __CURVEADM_TEMPLATE__ ${cluster_mds_dummy_port} __CURVEADM_TEMPLATE__
 # 发送rpc的超时时间
 rpcTimeout=500
 # rpc重试次数
 rpcRetryTimes=5
 # etcd地址
-etcdAddr=127.0.0.1:2379
+etcdAddr=127.0.0.1:2379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__
 # snapshot clone server 地址
-snapshotCloneAddr=127.0.0.1:5555
+snapshotCloneAddr=  # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_addr} __CURVEADM_TEMPLATE__
 # snapshot clone server dummy port
-snapshotCloneDummyPort=8081
+snapshotCloneDummyPort=  # __CURVEADM_TEMPLATE__ ${cluster_snapshotclone_dummy_port} __CURVEADM_TEMPLATE__
+rootUserName=root
+rootUserPassword=root_password

curve-ansible/README.md

@@ -374,7 +374,6 @@ ansible-playbook rolling_update_nebd.yml -i client.ini --tags restart
 │   ├── get_curve_version_from_metric.yml                   # 从metric获取curve版本
 │   ├── get_distro_name.yml                                 # 获取系统版本
 │   ├── get_nebd_version_from_metric.yml                    # 从metric获取nebd版本
-│   ├── get_software_version_from_package_version.yml       # 从包版本获取软件版本
 │   ├── install_with_yum_apt.yml                            # 用apt或yum的方式安装
 │   ├── start_service.yml                                   # 启动服务
 │   ├── stop_service.yml                                    # 停止服务

curve-ansible/check_chunkserver.yml

@@ -18,6 +18,6 @@
 
 - name: check chunkserver config
   hosts: chunkservers
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tasks:
     - include_tasks: "common_tasks/check_chunkserver.yml"

curve-ansible/check_debain_package.yml

@@ -0,0 +1,61 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# Check curve debain package
+
+- name: check curve-mds debain package
+  hosts: mds
+  gather_facts: no
+  tags: mds
+  become: "{{ sudo_or_not | bool }}"
+  vars:
+    - package_name: curve-mds
+    - version: "{{ mds_package_version }}"
+  tasks:
+    - include_tasks: common_tasks/check_debain_package.yml
+
+- name: check curve-snapshotcloneserver debain package
+  hosts: snapshotclone
+  gather_facts: no
+  tags: snapshotclone
+  become: "{{ sudo_or_not | bool }}"
+  vars:
+    - package_name: curve-snapshotcloneserver
+    - version: "{{ snapshot_package_version }}"
+  tasks:
+    - include_tasks: common_tasks/check_debain_package.yml
+
+- name: check curve-chunkserver debain package
+  hosts: chunkservers
+  gather_facts: no
+  tags: chunkserver
+  become: "{{ sudo_or_not | bool }}"
+  vars:
+    - package_name: curve-chunkserver
+    - version: "{{ cs_package_version }}"
+  tasks:
+    - include_tasks: common_tasks/check_debain_package.yml
+
+- name: check curve-monitor debain package
+  hosts: monitor
+  gather_facts: no
+  tags: monitor
+  become: "{{ sudo_or_not | bool }}"
+  vars:
+    - package_name: curve-monitor
+    - version: "{{ monitor_package_version }}"
+  tasks:
+    - include_tasks: common_tasks/check_debain_package.yml

curve-ansible/check_mds.yml

@@ -18,6 +18,6 @@
 
 - name: check mds config
   hosts: mds
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tasks:
     - include_tasks: "common_tasks/check_mds.yml"

curve-ansible/clean_curve.yml

@@ -29,7 +29,7 @@
   hosts: etcd
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - etcd
   roles:
@@ -40,7 +40,7 @@
   hosts: mds
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - mds
   roles:
@@ -52,7 +52,7 @@
   hosts: chunkservers
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - chunkserver
   roles:
@@ -63,7 +63,7 @@
   hosts: snapshotclone
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - snapshotclone
     - never
@@ -75,7 +75,7 @@
   hosts: snapshotclone_nginx
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - snapshotclone_nginx
     - never

curve-ansible/clean_curve_sdk.yml

@@ -25,6 +25,7 @@
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
   roles:

curve-ansible/clean_nbd.yml

@@ -0,0 +1,32 @@
+---
+#
+#  Copyright (c) 2021 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# The deploy playbook of Curve
+
+- name: check_ansible_version
+  tags:
+    - always
+  import_playbook: check_ansible_version.yml
+
+- name: clean nbd
+  hosts: client
+  gather_facts: no
+  any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
+  tags:
+    - prepare
+  roles:
+    - { role: clean, module_name: nbd }

curve-ansible/clean_nebd.yml

@@ -28,6 +28,7 @@
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
   roles:

curve-ansible/client.ini

@@ -20,6 +20,9 @@ nebd_data_dir=/data/nebd
 nebd_log_dir=/data/log/nebd
 curve_sdk_log_dir=/data/log/curve
 py_client_config_path=/etc/curve/py_client.conf
+clean_log_when_clean=false
+curvetab_path=/etc/curve
+curve_service_path=/etc/systemd/system
 
 [mds:vars]
 mds_port=6666
@@ -34,3 +37,7 @@ ansible_connection=local
 wait_service_timeout=20
 curve_bin_dir=/usr/bin
 start_by_daemon=true
+install_with_deb=false
+sudo_or_not=True
+ansible_become_user=curve
+ansible_become_flags=-iu curve

curve-ansible/common_tasks/check_chunkserver.yml

@@ -103,10 +103,13 @@
     that:
       - firmware_version.stdout|version_compare('24.3.0-0050', '>=')
 
+- name: get chunkserver number
+  include_tasks: "get_chunkserver_number.yml"
+
 - name: check the disk schedule algorithm
   shell:  cat /sys/block/sd*/queue/scheduler  | grep "\[noop\]" | wc -l
   register: result
-  failed_when: result.stdout | int != chunkserver_num
+  failed_when: result.stdout | int < chunkserver_num | int
 
 - name: check if the disk cache closed
   shell:  sudo /sbin/hdparm -W /dev/sd*  | grep "write-caching"

curve-ansible/common_tasks/check_debain_package.yml

@@ -0,0 +1,31 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+#  Check curve debain package
+
+- name: check {{ package_name }} debain package
+  shell: apt-cache policy {{ package_name }} | grep {{ version }}
+  ignore_errors: true
+  register: has_debain_package
+
+- name: apt update
+  shell: sudo apt-get update
+  ignore_errors: true
+  when: has_debain_package.rc != 0
+
+- name: recheck {{ package_name }} debain package
+  shell: apt-cache policy {{ package_name }} | grep {{ version }}
+  when: has_debain_package.rc != 0

curve-ansible/common_tasks/check_if_nbd_exists_in_kernel.yml

@@ -16,7 +16,7 @@
 #
 
 - name: check if nbd exists in kernel
-  shell: /sbin/modprobe nbd
+  shell: sudo /sbin/modprobe nbd
   ignore_errors: true
   register: modprobe_res
 

curve-ansible/common_tasks/get_chunkserver_number.yml

@@ -0,0 +1,30 @@
+---
+#
+#  Copyright (c) 2021 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# Get chunkserver number
+
+- name: get chunkserver number not get disk automatically
+  set_fact:
+    chunkserver_num: "{{ disk_list | length }}"
+  when: not auto_get_disk_list | bool
+
+- name: get chunkserver number get disk automatically
+  block:
+    - shell: '{{ get_disk_list_cmd }} | wc -l'
+      register: result
+    - set_fact:
+        chunkserver_num: "{{ result.stdout }}"
+  when: auto_get_disk_list | bool

curve-ansible/common_tasks/install_with_yum_apt.yml

@@ -40,7 +40,15 @@
       register: result
     - name: install lib with yum
       shell: sudo yum -y install {{ yum_lib_name }}
+      register: install_result
       when: result.rc != 0
+    - name: install latest repository
+      shell: sudo yum -y install https://extras.getpagespeed.com/release-latest.rpm
+      register: repository
+      when: install_result.rc != 0
+    - name: retry install
+      shell: sudo yum -y install {{ yum_lib_name }}
+      when: install_result.rc != 0 and repository.rc != 0
   when:
     - "'CentOS' in distro"
 

curve-ansible/common_tasks/restart_service.yml

@@ -0,0 +1,72 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+- name: set sudo or not
+  set_fact:
+    sudo: "sudo"
+  when: command_need_sudo
+
+- name: set sudo no
+  set_fact:
+    sudo: ""
+  when: not command_need_sudo
+
+- name: restart service with daemon
+  block:
+    - name: restart by daemon
+      shell: chdir={{ ch_dir }} {{ sudo }} {{ daemon_script }} restart
+      register: outres
+    - debug:
+        msg: "{{ outres.stdout }}"
+  when: start_by_daemon | bool
+
+- name: restart service without daemon
+  block:
+    - name: stop service without daemon
+      ignore_errors: true
+      shell: >
+        "{{ sudo }}" killall {{ process_name }}
+    - name: wait until server is down
+      vars:
+        ping_port: "{{ service_port }}"
+        wait_timeout: "{{ wait_service_timeout }}"
+      include_tasks: common_tasks/wait_until_server_up.yml
+      when: service_port > 0
+    - name: wait nebd-server is down
+      shell: pidof nebd-server
+      register: pid
+      failed_when: pid.stdout != ""
+      when: service_port == 0
+    - name: start service without daemon
+      shell: >
+        chdir={{ ch_dir }} {{ sudo }} {{ start_cmd }}
+      async: "{{ service_async }}"
+      poll: "{{ service_poll }}"
+  when: not start_by_daemon | bool
+
+- name: wait until server is up
+  vars:
+    ping_port: "{{ service_port }}"
+    wait_timeout: "{{ wait_service_timeout }}"
+  include_tasks: common_tasks/wait_until_server_up.yml
+  when: service_port > 0
+
+- name: wait nebd-server is up
+  shell: pidof nebd-server
+  register: pid
+  failed_when: pid.stdout == ""
+  when: service_port == 0

curve-ansible/common_tasks/start_service.yml

@@ -23,7 +23,7 @@
 - name: set sudo no
   set_fact:
     sudo: ""
-  when: not {{ command_need_sudo }}
+  when: not command_need_sudo
 
 - name: start service with daemon
   block:
@@ -32,14 +32,14 @@
       register: outres
     - debug:
         msg: "{{ outres.stdout }}"
-  when: start_by_daemon
+  when: start_by_daemon | bool
 
 - name: start service without daemon
   shell: >
     chdir={{ ch_dir }} {{ sudo }} {{ start_cmd }}
   async: "{{ service_async }}"
   poll: "{{ service_poll }}"
-  when: not {{ start_by_daemon }}
+  when: not start_by_daemon | bool
 
 - name: wait until server is up
   vars:

curve-ansible/common_tasks/stop_service.yml

@@ -23,7 +23,7 @@
 - name: set sudo no
   set_fact:
     sudo: ""
-  when: not {{ command_need_sudo }}
+  when: not command_need_sudo
 
 - name: stop service with daemon
   block:
@@ -33,13 +33,13 @@
       register: stopres
     - debug:
         msg: "{{ stopres.stdout }}"
-  when: start_by_daemon
+  when: start_by_daemon | bool
 
 - name: stop service without daemon
   ignore_errors: true
   shell: >
     "{{ sudo }}" killall {{ process_name }}
-  when: not {{ start_by_daemon }}
+  when: not start_by_daemon | bool
 
 - name: wait until server is down
   ping_port: "{{ service_port }}"

curve-ansible/deploy_curve.yml

@@ -38,7 +38,7 @@
   hosts: all
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare_software_env
   roles:
@@ -49,7 +49,7 @@
   hosts: etcd
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - etcd
   roles:
@@ -60,11 +60,11 @@
   hosts: etcd
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - etcd
   roles:
-    - { role: start_service, service_name: etcd, command_need_sudo: "{{ etcd_need_sudo }}" }
+    - { role: start_service, service_name: etcd, command_need_sudo: "{{ etcd_need_sudo | bool }}" }
 
 
 ############################## deploy mds ##############################
@@ -72,7 +72,7 @@
   hosts: mds
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - mds
   roles:
@@ -93,11 +93,11 @@
   hosts: mds
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - mds
   roles:
-    - { role: start_service, service_name: mds, command_need_sudo: "{{ mds_need_sudo }}" }
+    - { role: start_service, service_name: mds, command_need_sudo: "{{ mds_need_sudo | bool }}" }
 
 
 ############################## create physical pool ##############################
@@ -117,7 +117,7 @@
   hosts: chunkservers
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - chunkserver
   roles:
@@ -127,19 +127,19 @@
     - { role: generate_config, template_name: chunkserver.conf, conf_path: "{{ chunkserver_conf_path }}",
                 tags: ["generate_config", "generage_chunkserver_conf"] }
     - { role: generate_config, template_name: client.conf, conf_path: "{{ chunkserver_client_config_path }}",
-                tags: ["generate_config", "generage_chunkserver_conf"] }
+                tags: ["generate_config", "generage_chunkserver_client_conf"] }
     - { role: generate_config, template_name: s3.conf, conf_path: "{{ chunkserver_s3_config_path }}",
-                tags: ["generate_config", "generage_s3_conf"] }
+                tags: ["generate_config", "generage_cs_s3_conf"] }
 
 - name: start chunkserver
   hosts: chunkservers
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - chunkserver
   roles:
-    - { role: start_service, service_name: chunkserver, command_need_sudo: "{{ chunkserver_need_sudo }}" }
+    - { role: start_service, service_name: chunkserver, command_need_sudo: "{{ chunkserver_need_sudo | bool }}" }
 
 
 ############################## create logical pool ##############################
@@ -157,7 +157,7 @@
   hosts: snapshotclone
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - snapshotclone
     - never
@@ -169,18 +169,18 @@
     - { role: generate_config, template_name: client.conf, conf_path: "{{ snap_client_config_path }}",
             tags: ["generate_config", "generate_snap_client_conf"] }
     - { role: generate_config, template_name: s3.conf, conf_path: "{{ snap_s3_config_path }}",
-                tags: ["generate_config", "generage_s3_conf"] }
+                tags: ["generate_config", "generage_snap_s3_conf"] }
 
 - name: start snapshotcloneserver
   hosts: snapshotclone
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - snapshotclone
     - never
   roles:
-    - { role: start_service, service_name: snapshotcloneserver, command_need_sudo: "{{ snapshot_need_sudo }}" }
+    - { role: start_service, service_name: snapshotcloneserver, command_need_sudo: "{{ snapshot_need_sudo | bool }}" }
 
 
 ############################## deploy snapshotcloneserver_nginx ##############################
@@ -188,7 +188,7 @@
   hosts: snapshotclone_nginx
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - snapshotclone_nginx
     - never
@@ -199,12 +199,20 @@
   hosts: snapshotclone_nginx
   any_errors_fatal: true
   gather_facts: no
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - snapshotclone_nginx
     - never
   roles:
-    - { role: install_package, package_name: curve-snapshotcloneserver-nginx,
-            tags: ["install_conf_package", "install_snapshotclone_nginx_conf_package"] }
+    - {
+        role: install_package,
+        package_name: curve-snapshotcloneserver-nginx,
+        package_version: "{{ snapshot_nginx_package_version }}",
+        tags: [
+            "install_conf_package",
+            "install_snapshotclone_nginx_conf_package"
+        ]
+      }
     - { role: generate_config, template_name: nginx.conf, conf_path: "{{ snapshot_nginx_conf_path }}",
             tags: ["generate_config", "generate_snap_nginx_conf"] }
     - { role: generate_config, template_name: nginx_config.lua, conf_path: "{{ snapshot_nginx_lua_conf_path}}",
@@ -215,7 +223,7 @@
   hosts: snapshotclone_nginx
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - snapshotclone_nginx
     - never
@@ -227,7 +235,7 @@
   hosts: mds[0]
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - rapid_leader_schedule
   roles:

curve-ansible/deploy_curve_sdk.yml

@@ -25,6 +25,7 @@
   hosts: client
   any_errors_fatal: true
   gather_facts: no
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare_software_env
   roles:
@@ -34,6 +35,7 @@
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
   vars:
@@ -44,4 +46,4 @@
     - { role: install_package, package_name: curve-sdk, package_version: "{{ sdk_package_version }}",
             tags: install_package }
     - { role: generate_config, template_name: client.conf, conf_path: "{{ py_client_config_path }}",
-            tags: generate_config }
+            tags: generate_config }

curve-ansible/deploy_monitor.yml

@@ -32,7 +32,7 @@
   hosts: monitor
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   roles:
     - { role: install_package, package_name: curve-monitor, package_version: "{{ monitor_package_version }}",
             tags: ["install_package", "install_monitor_package"] }
@@ -51,7 +51,7 @@
   hosts: monitor
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - start
   roles:
@@ -61,7 +61,7 @@
   hosts: monitor
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - grafana_settings
   roles:

curve-ansible/deploy_nbd.yml

@@ -25,6 +25,7 @@
   hosts: client
   any_errors_fatal: true
   gather_facts: yes
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - check_kernel_version
   tasks:
@@ -39,6 +40,7 @@
   hosts: client
   any_errors_fatal: true
   gather_facts: no
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare_software_env
   roles:
@@ -48,6 +50,7 @@
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
   roles:

curve-ansible/deploy_nebd.yml

@@ -25,6 +25,7 @@
   hosts: client
   any_errors_fatal: true
   gather_facts: no
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare_software_env
   roles:
@@ -34,6 +35,7 @@
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
   roles:
@@ -47,7 +49,8 @@
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - start
   roles:
-    - { role: start_service, service_name: nebd, command_need_sudo: "{{ nebd_need_sudo }}" }
+    - { role: start_service, service_name: nebd, command_need_sudo: "{{ nebd_need_sudo | bool }}" }

curve-ansible/deploy_walpool.yml

@@ -20,7 +20,7 @@
   hosts: chunkservers
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tasks:
     - name: init wal pool
       shell: chdir={{ deploy_dir }} sudo ./chunkserver_ctl.sh deploy-wal-pool all --noConfirm

curve-ansible/ping.yml

@@ -14,13 +14,11 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 #
+# Test the ssh connection
 
-- name: get software version from package_version
-  shell: echo {{ package_version }} | awk -F "+" '{print $1}'
-  register:
-    result
-  failed_when: result.stdout == ""
-
-- name: set software_version
-  set_fact:
-    software_version: "{{ result.stdout }}"
+- name: ping
+  hosts: all
+  gather_facts: no
+  tasks:
+    - name: check ssh
+      shell: echo "hello"

curve-ansible/release_disk_5%_space.yml

@@ -1,26 +0,0 @@
----
-    #
-    #  Copyright (c) 2020 NetEase Inc.
-    #
-    #  Licensed under the Apache License, Version 2.0 (the "License");
-    #  you may not use this file except in compliance with the License.
-    #  You may obtain a copy of the License at
-    #
-    #      http://www.apache.org/licenses/LICENSE-2.0
-    #
-    #  Unless required by applicable law or agreed to in writing, software
-    #  distributed under the License is distributed on an "AS IS" BASIS,
-    #  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    #  See the License for the specific language governing permissions and
-    #  limitations under the License.
-    #
-    # The deploy playbook of Curve
-
-- name: release 5% of disk
-  hosts: chunkservers
-  any_errors_fatal: true
-  gather_facts: no
-  become: "{{ sudo_or_not }}"
-  tasks:
-    - name: release reserved 5%
-      shell: for disk in `{{ get_disk_list_cmd }}`;do sudo tune2fs -m 0 /dev/${disk};done

curve-ansible/roles/clean/tasks/include/clean_chunkserver.yml

@@ -17,12 +17,12 @@
 
 - name: clean chunkserver with format disk
   include_tasks: "include/clean_chunkserver_with_disk_format.yml"
-  when: chunkserver_format_disk
+  when: chunkserver_format_disk | bool
 
 - name: clean chunkserver without format disk
   include_tasks: "include/clean_chunkserver_without_disk_format.yml"
-  when: not {{ chunkserver_format_disk }}
+  when: not chunkserver_format_disk | bool
 
 - name: clean log file
   shell: sudo rm -rf {{ chunkserver_data_dir }}/log/chunkserver*
-  when: clean_log_when_clean
+  when: clean_log_when_clean | bool

curve-ansible/roles/clean/tasks/include/clean_chunkserver_retain_chunkfilepool.yml

@@ -15,6 +15,9 @@
 #  limitations under the License.
 #
 
+- name: get chunkserver number
+  include_tasks: "common_tasks/get_chunkserver_number.yml"
+
 - name: clean chunkserver dat
   vars:
     end_seq: "{{ chunkserver_num | int - 1}}"
@@ -23,9 +26,9 @@
 
 - name: recycle copysets chunks and wals to same pool
   script: recycle_chunks.sh -d {{ chunkserver_data_dir }} -chunks chunkfilepool -wals chunkfilepool
-  when: walfilepool_use_chunk_file_pool
+  when: walfilepool_use_chunk_file_pool | bool
 
 - name: recycle copysets chunks and wals to different pool
   script: recycle_chunks.sh -d {{ chunkserver_data_dir }} -chunks chunkfilepool -wals walfilepool
-  when: not {{ walfilepool_use_chunk_file_pool }}
+  when: not walfilepool_use_chunk_file_pool | bool
 

curve-ansible/roles/clean/tasks/include/clean_chunkserver_totally.yml

@@ -15,6 +15,9 @@
 #  limitations under the License.
 #
 
+- name: get chunkserver number
+  include_tasks: "common_tasks/get_chunkserver_number.yml"
+
 - name: clean data under data dir
   vars:
     end_seq: "{{ chunkserver_num | int - 1}}"

curve-ansible/roles/clean/tasks/include/clean_chunkserver_with_disk_format.yml

@@ -17,8 +17,8 @@
 
 - name: clean chunkserver but retain chunkfilepool
   include_tasks: clean_chunkserver_retain_chunkfilepool.yml
-  when: retain_pool
+  when: retain_pool | bool
 
 - name: clean chunkserver totally
   include_tasks: clean_chunkserver_totally.yml
-  when: not {{ retain_pool }}
+  when: not retain_pool | bool

curve-ansible/roles/clean/tasks/include/clean_curve_sdk.yml

@@ -17,4 +17,4 @@
 
 - name: clean curve_sdk log dir
   shell: sudo rm -rf {{ curve_sdk_log_dir }}
-  when: clean_log_when_clean
+  when: clean_log_when_clean | bool

curve-ansible/roles/clean/tasks/include/clean_etcd.yml

@@ -23,4 +23,4 @@
 
 - name: clean etcd log dir
   shell: sudo rm -rf {{ etcd_log_dir }}
-  when: clean_log_when_clean
+  when: clean_log_when_clean | bool

curve-ansible/roles/clean/tasks/include/clean_mds.yml

@@ -17,4 +17,4 @@
 
 - name: clean mds log dir
   shell: sudo rm -rf {{ mds_log_dir }}
-  when: clean_log_when_clean
+  when: clean_log_when_clean | bool

curve-ansible/roles/clean/tasks/include/clean_nbd.yml

@@ -0,0 +1,23 @@
+---
+#
+#  Copyright (c) 2021 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+- name: clean nbd auto start at boot
+  block:
+    - name: disable service
+      shell: sudo systemctl disable map_curve_disk.service
+    - name: remove map_curve_disk.service
+      shell: sudo rm -f {{ curve_service_path }}/map_curve_disk.service && sudo rm -f {{ curvetab_path }}/curvetab

curve-ansible/roles/clean/tasks/include/clean_nebd.yml

@@ -20,4 +20,19 @@
 
 - name: clean nebd log dir
   shell: sudo rm -rf {{ nebd_log_dir }}
-  when: clean_log_when_clean
+  when: clean_log_when_clean | bool
+
+- name: clean nebd auto start at boot
+  block:
+    - name: get distro name
+      vars:
+        distro:
+      include_tasks: common_tasks/get_distro_name.yml
+    - name: disable on debain ubuntu
+      shell: sudo update-rc.d -f nebd-daemon remove
+      when: "'Ubuntu' in distro or 'Debian' in distro"
+    - name: disable on centos
+      shell: sudo chkconfig nebd-daemon off && sudo chkconfig --del nebd-daemon
+      when: "'CentOS' in distro"
+    - name: remove nebd-daemon in init.d
+      shell: sudo rm -f /etc/init.d/nebd-daemon

curve-ansible/roles/clean/tasks/include/clean_snapshotcloneserver.yml

@@ -17,4 +17,4 @@
 
 - name: clean snapshotcloneserver log dir
   shell: sudo rm -rf {{ snapshot_clone_server_log_dir }}
-  when: clean_log_when_clean
+  when: clean_log_when_clean | bool

curve-ansible/roles/clean/tasks/include/clean_snapshotcloneserver_nginx.yml

@@ -17,4 +17,4 @@
 
 - name: clean snapshotcloneserver nginx log dir
   shell: sudo rm -rf {{ snapshotcloneserver_nginx_dir }}/logs
-  when: clean_log_when_clean
+  when: clean_log_when_clean | bool

curve-ansible/roles/format_chunkserver/tasks/include/prepare_chunkserver_with_disk_format.yml

@@ -19,10 +19,10 @@
   shell: sudo ./chunkserver_ctl.sh deploy all --noConfirm
   async: 0
   poll: 3
-  when: not {{ retain_pool }}
+  when: not retain_pool | bool
 
 - name: format walfilepool
   shell: sudo ./chunkserver_ctl.sh deploy-wal-pool all --noConfirm
   async: 0
   poll: 3
-  when: not {{ retain_pool }} and not {{ walfilepool_use_chunk_file_pool }}
+  when: (not retain_pool | bool) and (not walfilepool_use_chunk_file_pool | bool)

curve-ansible/roles/format_chunkserver/tasks/include/prepare_chunkserver_without_disk_format.yml

@@ -18,6 +18,9 @@
 - name: clear data dir
   shell: sudo rm -rf {{ chunkserver_data_dir }}/chunkserver*
 
+- name: get chunkserver number
+  include_tasks: "common_tasks/get_chunkserver_number.yml"
+
 - name: create data dir and log dir for each chunkserver
   vars:
     end_seq: "{{ chunkserver_num | int - 1}}"

curve-ansible/roles/format_chunkserver/tasks/main.yml

@@ -17,8 +17,8 @@
 
 - name: deploy chunkserver with format disk
   include_tasks: "include/prepare_chunkserver_with_disk_format.yml"
-  when: chunkserver_format_disk
+  when: chunkserver_format_disk | bool
 
 - name: deploy chunkserver without format disk
   include_tasks: "include/prepare_chunkserver_without_disk_format.yml"
-  when: not chunkserver_format_disk
+  when: not chunkserver_format_disk | bool

curve-ansible/roles/generate_config/defaults/main.yml

@@ -19,6 +19,9 @@
 curve_root_username: root
 curve_root_password: root_password
 chunk_size: 16777216
+segment_size: 1073741824
+min_file_length: 10737418240
+max_file_length: 21990232555520
 file_expired_time_us: 5000000
 
 # mds配置默认值
@@ -60,6 +63,7 @@ mds_topology_create_copyset_rpc_retry_sleep_time_ms: 1000
 mds_topology_update_metric_interval_sec: 60
 mds_topology_pool_usage_percent_limit: 85
 mds_topology_choose_pool_policy: 0
+mds_topology_enable_logicalpool_status: true
 mds_copyset_copyset_retry_times: 10
 mds_copyset_scatterwidth_variance: 0
 mds_copyset_scatterwidth_standard_devation: 0
@@ -96,15 +100,18 @@ chunkserver_copyset_election_timeout_ms: 1000
 chunkserver_copyset_snapshot_interval_s: 1800
 chunkserver_copyset_catchup_margin: 1000
 chunkserver_copyset_chunk_data_uri: local://./0/copysets
-chunkserver_copyset_raft_log_uri: local://./0/copysets
+chunkserver_copyset_raft_log_uri: curve://./0/copysets
 chunkserver_copyset_raft_meta_uri: local://./0/copysets
-chunkserver_copyset_raft_snapshot_uri: local://./0/copysets
+chunkserver_copyset_raft_snapshot_uri: curve://./0/copysets
 chunkserver_copyset_recycler_uri: local://./0/recycler
 chunkserver_copyset_max_inflight_requests: 5000
 chunkserver_copyset_load_concurrency: 10
 chunkserver_copyset_check_retrytimes: 3
 chunkserver_copyset_finishload_margin: 2000
 chunkserver_copyset_check_loadmargin_interval_ms: 1000
+chunkserver_copyset_enable_odsync_when_open_chunkfile: false
+chunkserver_copyset_synctimer_interval_ms: 30000
+chunkserver_copyset_check_syncing_interval_ms: 500
 chunkserver_clone_slice_size: 1048576
 chunkserver_clone_enable_paste: false
 chunkserver_clone_thread_num: 10
@@ -177,6 +184,9 @@ client_mds_rpc_retry_interval_us: 100000
 client_metacache_get_leader_timeout_ms: 500
 client_metacache_get_leader_retry: 5
 client_metacache_rpc_retry_interval_us: 100000
+client_mds_normal_retry_times_before_trigger_wait: 3
+client_mds_max_retry_ms_in_io_path: 86400000
+client_mds_wait_sleep_ms: 10000
 client_schedule_queue_capacity: 1000000
 client_schedule_threadpool_size: 2
 client_isolation_task_queue_capacity: 1000000
@@ -199,6 +209,8 @@ client_log_path: /data/log/curve/
 client_metric_dummy_server_start_port: 9000
 client_turn_off_health_check: true
 client_session_map_path: ./session_map.json
+client_closefd_timeout_sec: 300
+client_closefd_time_interval_sec: 600
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/tasks/include/generate_config_with_template.yml

@@ -47,9 +47,13 @@
     - name: set backup true
       set_fact:
         backup: true
+    - name: check config file exist
+      stat:
+        path: "{{ conf_path }}"
+      register: file_status
     - name: backup config file
       shell: sudo cp {{ conf_path }} {{ conf_path }}.bak
-      when: backup is defined
+      when: backup is defined and file_status.stat.exists
     - name: mv config file
       shell: sudo mv /tmp/{{ template_name }} {{ conf_path }}
     - name: diff two files

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -98,6 +98,9 @@ copyset.check_retrytimes={{ chunkserver_copyset_check_retrytimes }}
 copyset.finishload_margin={{ chunkserver_copyset_finishload_margin }}
 # 循环判定copyset是否加载完成的内部睡眠时间
 copyset.check_loadmargin_interval_ms={{ chunkserver_copyset_check_loadmargin_interval_ms }}
+copyset.copyset_enable_odsync_when_open_chunkfile={{ chunkserver_copyset_enable_odsync_when_open_chunkfile }}
+copyset.copyset_synctimer_interval_ms={{ chunkserver_copyset_synctimer_interval_ms }}
+copyset.copyset_check_syncing_interval_ms={{ chunkserver_copyset_check_syncing_interval_ms }}
 
 #
 # Clone settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -32,6 +32,15 @@ mds.refreshTimesPerLease={{ client_mds_refresh_times_per_lease }}
 # mds RPC接口每次重试之前需要先睡眠一段时间
 mds.rpcRetryIntervalUS={{ client_mds_rpc_retry_interval_us }}
 
+# The normal retry times for trigger wait strategy
+mds.normalRetryTimesBeforeTriggerWait={{ client_mds_normal_retry_times_before_trigger_wait }}
+
+# Max retry time for IO-Path request (milliseconds)
+mds.maxRetryMsInIOPath={{ client_mds_max_retry_ms_in_io_path }}
+
+# Sleep interval for wait (milliseconds)
+mds.waitSleepMs={{ client_mds_wait_sleep_ms }}
+
 #
 ################# metacache配置信息 ################
 #
@@ -133,6 +142,14 @@ global.logPath={{ client_log_path }}
 # 单元测试情况下
 # logpath=./runlog/
 
+#
+################# read from source conf ###############
+#
+# close timeout fd which read from source file
+closefd.timeout={{ client_closefd_timeout_sec }}
+# timeInterval of backend thread iterates the fdMap to close the timeout fd
+closefd.timeInterval={{ client_closefd_time_interval_sec }}
+
 #
 ############### metric 配置信息 #############
 #

curve-ansible/roles/generate_config/templates/mds.conf.j2

@@ -147,6 +147,8 @@ mds.topology.UpdateMetricIntervalSec={{ mds_topology_update_metric_interval_sec
 mds.topology.PoolUsagePercentLimit={{ mds_topology_pool_usage_percent_limit }}
 # 多pool选pool策略 0:Random, 1:Weight
 mds.topology.choosePoolPolicy={{ mds_topology_choose_pool_policy }}
+# enable LogicalPool ALLOW/DENY status
+mds.topology.enableLogicalPoolStatus={{ mds_topology_enable_logicalpool_status}}
 
 #
 # copyset config
@@ -173,6 +175,12 @@ mds.copyset.scatterWidthFloatingPercentage={{ mds_copyset_scatterwidth_floating_
 #
 # curvefs的默认chunk size大小，16MB = 16*1024*1024 = 16777216
 mds.curvefs.defaultChunkSize={{ chunk_size }}
+# curvefs的默认segment size大小，1GB = 1*1024*1024*1024 = 1073741824
+mds.curvefs.defaultSegmentSize={{ segment_size }}
+# curvefs的默认最小文件大小，10GB = 10*1024*1024*1024 = 10737418240
+mds.curvefs.minFileLength={{ min_file_length }}
+# curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
+mds.curvefs.maxFileLength={{ max_file_length }}
 
 #
 # chunkseverclient config

curve-ansible/roles/generate_config/templates/tools.conf.j2

@@ -20,6 +20,7 @@ rpcRetryTimes={{ tool_rpc_retry_times }}
   {% set _ = etcd_address.append("%s:%s" % (etcd_ip, etcd_port)) -%}
 {% endfor -%}
 etcdAddr={{ etcd_address | join(',') }}
+{% if groups.snapshotclone is defined and groups.snapshotclone[0] is defined %}
 # snapshot clone server 地址
 {% set snap_address=[] -%}
 {% for host in groups.snapshotclone -%}
@@ -30,3 +31,6 @@ etcdAddr={{ etcd_address | join(',') }}
 snapshotCloneAddr={{ snap_address | join(',') }}
 # snapshot clone server dummy port
 snapshotCloneDummyPort={{ hostvars[groups.snapshotclone[0]].snapshot_dummy_port }}
+rootUserName={{ curve_root_username }}
+rootUserPassword={{ curve_root_password }}
+{% endif -%}

curve-ansible/roles/install_package/defaults/main.yml

@@ -30,6 +30,8 @@ local_nbd_package_path: ../../nbd-package
 snapshot_clone_server_log_dir: /data/log/curve/snapshotclone
 local_snapshotcloneserver_package_path: ../curve-snapshotcloneserver
 local_monitor_package_path: ../../curve-monitor
+curvetab_path: /etc/curve
+curve_service_path: /etc/systemd/system
 
 local_snapshotcloneserver_nginx_package_path: ../curve-nginx
 snapshotcloneserver_nginx_dir: /etc/curve/nginx
@@ -43,6 +45,8 @@ etcd_download_dir: /tmp
 install_etcd_dir: /usr/bin
 
 download_dir: /tmp
-jemalloc_download_url: https://github.com/jemalloc/jemalloc/releases/download/3.6.0/jemalloc-3.6.0.tar.bz2
+# jemalloc_download_url: https://github.com/jemalloc/jemalloc/releases/download/3.6.0/jemalloc-3.6.0.tar.bz2
+jemalloc_download_url: https://curve-build.nos-eastchina1.126.net/jemalloc-3.6.0.tar.bz2
 lib_install_prefix: /usr/local
-daemon_download_url: http://libslack.org/daemon/download/daemon-0.6.4.tar.gz
+# daemon_download_url: http://libslack.org/daemon/download/daemon-0.6.4.tar.gz
+daemon_download_url: https://curve-build.nos-eastchina1.126.net/daemon-0.6.4.tar.gz

curve-ansible/roles/install_package/tasks/include/install_curve-chunkserver.yml

@@ -15,37 +15,45 @@
 #  limitations under the License.
 #
 
-- name: install libunwind
-  vars:
-    apt_lib_name: libunwind8 libunwind8-dev
-    yum_lib_name: libunwind libunwind-devel
-  include_tasks: common_tasks/install_with_yum_apt.yml
+- name: install package via deb
+  include_tasks: "install_deb_package.yml"
+  when: install_with_deb | bool
 
-- name: install bzip2
-  vars:
-    apt_lib_name: bzip2
-    yum_lib_name: bzip2
-  include_tasks: common_tasks/install_with_yum_apt.yml
-
-- name: install jemalloc
-  vars:
-    lib_name: jemalloc
-    download_url: "{{ jemalloc_download_url }}"
-  include_tasks: install_jemalloc.yml
-
-- name: install psmisc
-  vars:
-    apt_lib_name: psmisc
-    yum_lib_name: psmisc
-  include_tasks: common_tasks/install_with_yum_apt.yml
-  when: chunkserver_format_disk
-
-- name: install chunkserver bin
-  vars:
-    remote_dir_name: "{{ curve_bin_dir }}"
-    local_file_path: "{{ local_chunkserver_package_path }}/bin/"
-    file_mode: 0755
-  include_tasks: copy_file_to_remote.yml
+- name: install package via tar
+  block:
+    - name: install libunwind
+      vars:
+        apt_lib_name: libunwind8
+        yum_lib_name: libunwind
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libunwind-dev
+      vars:
+        apt_lib_name: libunwind8-dev
+        yum_lib_name: libunwind-devel
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install bzip2
+      vars:
+        apt_lib_name: bzip2
+        yum_lib_name: bzip2
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install jemalloc
+      vars:
+        lib_name: jemalloc
+        download_url: "{{ jemalloc_download_url }}"
+      include_tasks: install_jemalloc.yml
+    - name: install psmisc
+      vars:
+        apt_lib_name: psmisc
+        yum_lib_name: psmisc
+      include_tasks: common_tasks/install_with_yum_apt.yml
+      when: chunkserver_format_disk | bool
+    - name: install chunkserver bin
+      vars:
+        remote_dir_name: "{{ curve_bin_dir }}"
+        local_file_path: "{{ local_chunkserver_package_path }}/bin/"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+  when: not install_with_deb | bool
 
 - name: generate chunkserver_ctl.sh
   vars:

curve-ansible/roles/install_package/tasks/include/install_curve-mds.yml

@@ -15,34 +15,47 @@
 #  limitations under the License.
 #
 
-- name: install libunwind
-  vars:
-    apt_lib_name: libunwind8 libunwind8-dev
-    yum_lib_name: libunwind libunwind-devel
-  include_tasks: common_tasks/install_with_yum_apt.yml
+- name: install package via deb
+  include_tasks: "install_deb_package.yml"
+  when: install_with_deb | bool
 
-- name: install libuuid
-  vars:
-    apt_lib_name: libuuid1 uuid-dev
-    yum_lib_name: libuuid libuuid-devel
-  include_tasks: common_tasks/install_with_yum_apt.yml
-
-- name: install daemon
-  include_tasks: install_daemon.yml
-
-- name: install mds bin
-  vars:
-    remote_dir_name: "{{ curve_bin_dir }}"
-    local_file_path: "{{ local_mds_package_path }}/bin/"
-    file_mode: 0755
-  include_tasks: copy_file_to_remote.yml
-
-- name: install mds lib
-  vars:
-    remote_dir_name: "{{ curve_lib_dir }}"
-    local_file_path: "{{ local_mds_package_path }}/lib/"
-    file_mode: 0644
-  include_tasks: copy_file_to_remote.yml
+- name: install package via tar
+  block:
+    - name: install libunwind
+      vars:
+        apt_lib_name: libunwind8
+        yum_lib_name: libunwind
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libunwind-dev
+      vars:
+        apt_lib_name: libunwind8-dev
+        yum_lib_name: libunwind-devel
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libuuid
+      vars:
+        apt_lib_name: libuuid1
+        yum_lib_name: libuuid
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libuuid-dev
+      vars:
+        apt_lib_name: uuid-dev
+        yum_lib_name: libuuid-devel
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install daemon
+      include_tasks: install_daemon.yml
+    - name: install mds bin
+      vars:
+        remote_dir_name: "{{ curve_bin_dir }}"
+        local_file_path: "{{ local_mds_package_path }}/bin/"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: install mds lib
+      vars:
+        remote_dir_name: "{{ curve_lib_dir }}"
+        local_file_path: "{{ local_mds_package_path }}/lib/"
+        file_mode: 0644
+      include_tasks: copy_file_to_remote.yml
+  when: not install_with_deb | bool
 
 - name: generate mds-daemon.sh
   template: src=mds-daemon.sh.j2 dest={{ deploy_dir }}/mds-daemon.sh mode=0755

curve-ansible/roles/install_package/tasks/include/install_curve-monitor.yml

@@ -15,27 +15,27 @@
 #  limitations under the License.
 #
 
-- name: pull prom/prometheus:latest
-  shell: sudo docker image pull prom/prometheus:latest
+- name: install package via deb
+  include_tasks: "install_deb_package.yml"
+  when: install_with_deb | bool
 
-- name: pull grafana/grafana:latest
-  shell: sudo docker image pull grafana/grafana:latest
-
-- name: promoon/reporter:latest
-  shell: sudo docker image pull promoon/reporter:latest
-
-- name: install jq
-  vars:
-    apt_lib_name: jq
-    yum_lib_name: jq
-  include_tasks: common_tasks/install_with_yum_apt.yml
-
-- name: install monitor work space
-  vars:
-    remote_dir_name: "{{ monitor_work_dir }}"
-    local_file_path: "{{ local_monitor_package_path }}/"
-    file_mode: 0777
-  include_tasks: copy_file_to_remote.yml
-
-- name: generate curve-monitor.sh
-  template: src=curve-monitor.sh.j2 dest={{ deploy_dir }}/curve-monitor.sh mode=0755
+- name: install package via tar
+  block:
+    - name: pull prom/prometheus:latest
+      shell: sudo docker image pull prom/prometheus:latest
+    - name: pull grafana/grafana:latest
+      shell: sudo docker image pull grafana/grafana:latest
+    - name: promoon/reporter:latest
+      shell: sudo docker image pull promoon/reporter:latest
+    - name: install jq
+      vars:
+        apt_lib_name: jq
+        yum_lib_name: jq
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install monitor work space
+      vars:
+        remote_dir_name: "{{ monitor_work_dir }}"
+        local_file_path: "{{ local_monitor_package_path }}/"
+        file_mode: 0777
+      include_tasks: copy_file_to_remote.yml
+  when: not install_with_deb | bool

curve-ansible/roles/install_package/tasks/include/install_curve-nbd.yml

@@ -15,21 +15,63 @@
 #  limitations under the License.
 #
 
-- name: install libunwind
-  vars:
-    apt_lib_name: libunwind8 libunwind8-dev
-    yum_lib_name: libunwind libunwind-devel
-  include_tasks: common_tasks/install_with_yum_apt.yml
+- name: install package via deb
+  include_tasks: "install_deb_package.yml"
+  when: install_with_deb | bool
 
-- name: install libnl
-  vars:
-    apt_lib_name: libnl-3-dev libnl-genl-3-dev
-    yum_lib_name: libnl3 libnl3-devel
-  include_tasks: common_tasks/install_with_yum_apt.yml
-
-- name: install nbd bin
-  vars:
-    remote_dir_name: "{{ curve_bin_dir }}"
-    local_file_path: "{{ local_nbd_package_path }}/bin/"
-    file_mode: 0755
-  include_tasks: copy_file_to_remote.yml
+- name: install package via tar
+  block:
+    - name: install libunwind
+      vars:
+        apt_lib_name: libunwind8
+        yum_lib_name: libunwind
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libunwind-dev
+      vars:
+        apt_lib_name: libunwind8-dev
+        yum_lib_name: libunwind-devel
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libnl
+      vars:
+        apt_lib_name: libnl-3-dev
+        yum_lib_name: libnl3
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libnl-dev
+      vars:
+        apt_lib_name: libnl-genl-3-dev
+        yum_lib_name: libnl3-devel
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install nbd bin
+      vars:
+        remote_dir_name: "{{ curve_bin_dir }}"
+        local_file_path: "{{ local_nbd_package_path }}/bin/"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: check curvetab exist
+      stat:
+        path: "{{ curvetab_path }}/curvetab"
+      register: file_status
+    - name: install curvetab if not exist
+      vars:
+        remote_dir_name: "{{ curvetab_path }}"
+        local_file_path: "{{ local_nbd_package_path }}/etc/curvetab"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+      when: not file_status.stat.exists
+    - name: install map_curve_disk.sh
+      vars:
+        remote_dir_name: "{{ curve_bin_dir }}"
+        local_file_path: "{{ local_nbd_package_path }}/bin/map_curve_disk.sh"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: add nbd auto start at boot
+      block:
+        - name: install map_curve_disk.service
+          vars:
+            remote_dir_name: "{{ curve_service_path }}"
+            local_file_path: "{{ local_nbd_package_path }}/etc/map_curve_disk.service"
+            file_mode: 0755
+          include_tasks: copy_file_to_remote.yml
+        - name: enable service
+          shell: sudo systemctl daemon-reload && sudo systemctl enable map_curve_disk.service && sudo systemctl start map_curve_disk.service
+  when: not install_with_deb | bool

curve-ansible/roles/install_package/tasks/include/install_curve-sdk.yml

@@ -15,41 +15,48 @@
 #  limitations under the License.
 #
 
-- name: install libunwind
-  vars:
-    apt_lib_name: libunwind8 libunwind8-dev
-    yum_lib_name: libunwind libunwind-devel
-  include_tasks: common_tasks/install_with_yum_apt.yml
+- name: install package via deb
+  include_tasks: "install_deb_package.yml"
+  when: install_with_deb | bool
 
-- name: install curve-sdk bin
-  vars:
-    remote_dir_name: "{{ curve_bin_dir }}"
-    local_file_path: "{{ local_curve_sdk_package_path }}/bin/"
-    file_mode: 0755
-  include_tasks: copy_file_to_remote.yml
-
-- name: install curve-sdk lib
-  vars:
-    remote_dir_name: "{{ curve_lib_dir }}"
-    local_file_path: "{{ local_curve_sdk_package_path }}/lib/"
-    file_mode: 0644
-  include_tasks: copy_file_to_remote.yml
-
-- name: install curve-sdk include
-  vars:
-    remote_dir_name: "{{ curve_include_dir }}"
-    local_file_path: "{{ local_curve_sdk_package_path }}/include/"
-    file_mode: 0644
-  include_tasks: copy_file_to_remote.yml
-
-- name: install curve-sdk curvefs
-  vars:
-    remote_dir_name: "{{ curvefs_dir }}"
-    local_file_path: "{{ local_curve_sdk_package_path }}/curvefs/"
-    file_mode: 0644
-  include_tasks: copy_file_to_remote.yml
-
-- name: create log dir
-  vars:
-    dir_name: "{{ curve_sdk_log_dir }}"
-  include_tasks: common_tasks/create_dir.yml
+- name: install package via tar
+  block:
+    - name: install libunwind
+      vars:
+        apt_lib_name: libunwind8
+        yum_lib_name: libunwind
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libunwind-dev
+      vars:
+        apt_lib_name: libunwind8-dev
+        yum_lib_name: libunwind-devel
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install curve-sdk bin
+      vars:
+        remote_dir_name: "{{ curve_bin_dir }}"
+        local_file_path: "{{ local_curve_sdk_package_path }}/bin/"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: install curve-sdk lib
+      vars:
+        remote_dir_name: "{{ curve_lib_dir }}"
+        local_file_path: "{{ local_curve_sdk_package_path }}/lib/"
+        file_mode: 0644
+      include_tasks: copy_file_to_remote.yml
+    - name: install curve-sdk include
+      vars:
+        remote_dir_name: "{{ curve_include_dir }}"
+        local_file_path: "{{ local_curve_sdk_package_path }}/include/"
+        file_mode: 0644
+      include_tasks: copy_file_to_remote.yml
+    - name: install curve-sdk curvefs
+      vars:
+        remote_dir_name: "{{ curvefs_dir }}"
+        local_file_path: "{{ local_curve_sdk_package_path }}/curvefs/"
+        file_mode: 0644
+      include_tasks: copy_file_to_remote.yml
+    - name: create log dir
+      vars:
+        dir_name: "{{ curve_sdk_log_dir }}"
+      include_tasks: common_tasks/create_dir.yml
+  when: not install_with_deb | bool

curve-ansible/roles/install_package/tasks/include/install_curve-snapshotcloneserver-nginx.yml

@@ -15,9 +15,14 @@
 #  limitations under the License.
 #
 
-- name: install snapshotcloneserver_nginx
+- name: install package via deb
+  include_tasks: "install_deb_package.yml"
+  when: install_with_deb | bool
+
+- name: install snapshotcloneserver_nginx via tar
   vars:
     remote_dir_name: "{{ snapshotcloneserver_nginx_dir }}"
     local_file_path: "{{ local_snapshotcloneserver_nginx_package_path }}/"
     file_mode: 0644
-  include_tasks: copy_file_to_remote.yml
+  include_tasks: copy_file_to_remote.yml
+  when: not install_with_deb | bool

curve-ansible/roles/install_package/tasks/include/install_curve-snapshotcloneserver.yml

@@ -15,34 +15,47 @@
 #  limitations under the License.
 #
 
-- name: install libunwind
-  vars:
-    apt_lib_name: libunwind8 libunwind8-dev
-    yum_lib_name: libunwind libunwind-devel
-  include_tasks: common_tasks/install_with_yum_apt.yml
+- name: install package via deb
+  include_tasks: "install_deb_package.yml"
+  when: install_with_deb | bool
 
-- name: install libuuid
-  vars:
-    apt_lib_name: libuuid1 uuid-dev
-    yum_lib_name: libuuid libuuid-devel
-  include_tasks: common_tasks/install_with_yum_apt.yml
-
-- name: install daemon
-  include_tasks: install_daemon.yml
-
-- name: install snapshotcloneserver bin
-  vars:
-    remote_dir_name: "{{ curve_bin_dir }}"
-    local_file_path: "{{ local_snapshotcloneserver_package_path }}/bin/"
-    file_mode: 0755
-  include_tasks: copy_file_to_remote.yml
-
-- name: install snapshotcloneserver lib
-  vars:
-    remote_dir_name: "{{ curve_lib_dir}}"
-    local_file_path: "{{ local_snapshotcloneserver_package_path }}/lib/"
-    file_mode: 0644
-  include_tasks: copy_file_to_remote.yml
+- name: install package via tar
+  block:
+    - name: install libunwind
+      vars:
+        apt_lib_name: libunwind8
+        yum_lib_name: libunwind
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libunwind-dev
+      vars:
+        apt_lib_name: libunwind8-dev
+        yum_lib_name: libunwind-devel
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libuuid
+      vars:
+        apt_lib_name: libuuid1
+        yum_lib_name: libuuid
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libuuid-dev
+      vars:
+        apt_lib_name: uuid-dev
+        yum_lib_name: libuuid-devel
+      include_tasks: common_tasks/install_with_yum_apt.yml  
+    - name: install daemon
+      include_tasks: install_daemon.yml
+    - name: install snapshotcloneserver bin
+      vars:
+        remote_dir_name: "{{ curve_bin_dir }}"
+        local_file_path: "{{ local_snapshotcloneserver_package_path }}/bin/"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: install snapshotcloneserver lib
+      vars:
+        remote_dir_name: "{{ curve_lib_dir}}"
+        local_file_path: "{{ local_snapshotcloneserver_package_path }}/lib/"
+        file_mode: 0644
+      include_tasks: copy_file_to_remote.yml
+  when: not install_with_deb | bool
 
 - name: generate snapshot-daemon.sh
-  template: src=snapshot-daemon.sh.j2 dest={{ deploy_dir }}/snapshot-daemon.sh mode=0755
+  template: src=snapshot-daemon.sh.j2 dest={{ deploy_dir }}/snapshot-daemon.sh mode=0755

curve-ansible/roles/install_package/tasks/include/install_curve-tools.yml

@@ -15,18 +15,23 @@
 #  limitations under the License.
 #
 
-- name: install tool bin
-  vars:
-    remote_dir_name: "{{ curve_bin_dir }}"
-    local_file_path: "{{ local_tool_package_path }}/bin/"
-    file_mode: 0755
-  include_tasks: copy_file_to_remote.yml
+- name: install package via deb
+  include_tasks: "install_deb_package.yml"
+  when: install_with_deb | bool
 
-- name: pip install requests
-  ignore_errors: true
-  shell: sudo pip install requests
-  register: pip_requests_res
-
-- name: pip2 install requests
-  shell: sudo pip2 install requests
-  when: pip_requests_res.failed
+- name: install package via tar
+  block:
+    - name: install tool bin
+      vars:
+        remote_dir_name: "{{ curve_bin_dir }}"
+        local_file_path: "{{ local_tool_package_path }}/bin/"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: pip install requests
+      ignore_errors: true
+      shell: sudo pip install requests
+      register: pip_requests_res
+    - name: pip2 install requests
+      shell: sudo pip2 install requests
+      when: pip_requests_res.failed
+  when: not install_with_deb | bool

curve-ansible/roles/install_package/tasks/include/install_etcd-daemon.yml

@@ -0,0 +1,19 @@
+---
+#
+#  Copyright (c) 2021 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+- name: generate etcd-daemon.sh
+  template: src=etcd-daemon.sh.j2 dest={{ deploy_dir }}/etcd-daemon.sh mode=0755

curve-ansible/roles/install_package/tasks/include/install_nebd.yml

@@ -15,67 +15,93 @@
 #  limitations under the License.
 #
 
-- name: install libunwind
-  vars:
-    apt_lib_name: libunwind8 libunwind8-dev
-    yum_lib_name: libunwind libunwind-devel
-  include_tasks: common_tasks/install_with_yum_apt.yml
+- name: install package via deb
+  include_tasks: "install_deb_package.yml"
+  when: install_with_deb | bool
 
-- name: install bzip2
-  vars:
-    apt_lib_name: bzip2
-    yum_lib_name: bzip2
-  include_tasks: common_tasks/install_with_yum_apt.yml
-
-- name: install jemalloc
-  vars:
-    lib_name: jemalloc
-    download_url: "{{ jemalloc_download_url }}"
-  include_tasks: install_jemalloc.yml
-
-- name: install daemon
-  include_tasks: install_daemon.yml
-
-- name: prepare date dir
-  vars:
-    dir_name: "{{ nebd_data_dir }}"
-  include_tasks: common_tasks/create_dir.yml
-
-- name: prepare log dir
-  vars:
-    dir_name: "{{ nebd_log_dir }}/{{ item }}"
-  include_tasks: common_tasks/create_dir.yml
-  with_items:
-    - client
-    - server
-
-- name: prepare lock dir
-  vars:
-    dir_name: "{{ nebd_data_dir }}/lock"
-  include_tasks: common_tasks/create_dir.yml
-
-- name: install nebd bin
-  vars:
-    remote_dir_name: "{{ curve_bin_dir }}"
-    local_file_path: "{{ local_nebd_package_path }}/bin/"
-    file_mode: 0755
-  include_tasks: copy_file_to_remote.yml
-
-- name: install nebd lib
-  vars:
-    remote_dir_name: "{{ curve_lib_dir }}/nebd"
-    local_file_path: "{{ local_nebd_package_path }}/lib/"
-    file_mode: 0644
-  include_tasks: copy_file_to_remote.yml
-
-- name: generate nebd-daemon
-  vars:
-    jemalloc_path: "{{ lib_install_prefix }}/lib/libjemalloc.so.1"
-  local_action: template src=nebd-daemon.j2 dest=/tmp/nebd-daemon mode=0755
-
-- name: copy nebd-daemon
-  vars:
-    remote_dir_name: "{{ curve_bin_dir }}"
-    local_file_path: "/tmp/nebd-daemon"
-    file_mode: 0755
-  include_tasks: copy_file_to_remote.yml
+- name: install package via tar
+  block:
+    - name: install libunwind
+      vars:
+        apt_lib_name: libunwind8
+        yum_lib_name: libunwind
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install libunwind-dev
+      vars:
+        apt_lib_name: libunwind8-dev
+        yum_lib_name: libunwind-devel
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install bzip2
+      vars:
+        apt_lib_name: bzip2
+        yum_lib_name: bzip2
+      include_tasks: common_tasks/install_with_yum_apt.yml
+    - name: install jemalloc
+      vars:
+        lib_name: jemalloc
+        download_url: "{{ jemalloc_download_url }}"
+      include_tasks: install_jemalloc.yml
+    - name: install daemon
+      include_tasks: install_daemon.yml
+    - name: prepare date dir
+      vars:
+        dir_name: "{{ nebd_data_dir }}"
+      include_tasks: common_tasks/create_dir.yml
+    - name: prepare log dir
+      vars:
+        dir_name: "{{ nebd_log_dir }}/{{ item }}"
+      include_tasks: common_tasks/create_dir.yml
+      with_items:
+        - client
+        - server
+    - name: prepare lock dir
+      vars:
+        dir_name: "{{ nebd_data_dir }}/lock"
+      include_tasks: common_tasks/create_dir.yml
+    - name: install nebd bin
+      vars:
+        remote_dir_name: "{{ curve_bin_dir }}"
+        local_file_path: "{{ local_nebd_package_path }}/bin/"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: install nebd include
+      vars:
+        remote_dir_name: "{{ curve_include_dir }}"
+        local_file_path: "{{ local_nebd_package_path }}/include/"
+        file_mode: 0644
+      include_tasks: copy_file_to_remote.yml
+    - name: install nebd lib
+      vars:
+        remote_dir_name: "{{ curve_lib_dir }}"
+        local_file_path: "{{ local_nebd_package_path }}/lib/"
+        file_mode: 0644
+      include_tasks: copy_file_to_remote.yml
+    - name: generate nebd-daemon
+      vars:
+        jemalloc_path: "{{ lib_install_prefix }}/lib/libjemalloc.so.1"
+      local_action: template src=nebd-daemon.j2 dest=/tmp/nebd-daemon mode=0755
+    - name: copy nebd-daemon
+      vars:
+        remote_dir_name: "{{ curve_bin_dir }}"
+        local_file_path: "/tmp/nebd-daemon"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: add nebd auto start at boot
+      block:
+        - name: get distro name
+          vars:
+            distro:
+          include_tasks: common_tasks/get_distro_name.yml
+        - name: copy nebd-daemon to init.d
+          vars:
+            remote_dir_name: "/etc/init.d"
+            local_file_path: "/tmp/nebd-daemon"
+            file_mode: 0755
+          include_tasks: copy_file_to_remote.yml
+        - name: enable on debain ubuntu
+          shell: sudo update-rc.d nebd-daemon defaults
+          when: "'Ubuntu' in distro or 'Debian' in distro"
+        - name: enable on centos
+          shell: sudo chkconfig --add nebd-daemon && sudo chkconfig nebd-daemon on
+          when: "'CentOS' in distro"
+  when: not install_with_deb | bool

curve-ansible/roles/install_package/tasks/main.yml

@@ -18,10 +18,5 @@
 - name: set curve_lib_dir
   include_tasks: "include/set_curve_lib_dir.yml"
 
-- name: install package with deb
-  include_tasks: "include/install_deb_package.yml"
-  when: install_with_deb
-
-- name: install package without deb
+- name: install package
   include_tasks: "include/install_{{ package_name }}.yml"
-  when: not install_with_deb

curve-ansible/roles/install_package/templates/chunkserver_ctl.sh.j2

@@ -3,6 +3,7 @@
 # default config path
 confPath={{ chunkserver_conf_path }}
 dataDir={{ chunkserver_data_dir }}
+raftLogProtocol={{ chunkserver_raft_log_procotol }}
 source ./chunkserver_deploy.sh
 
 # 使用方式
@@ -42,10 +43,17 @@ function help() {
     echo "    [-nc|--noConfirm]  if specified, deploy no need to confirm"
 }
 
+function ip_value() {
+  echo $1 | awk -F '[:/.]' '{
+    mask = (2 ^ 8)
+    printf ("%.0f", (($1 * mask + $2) * mask + $3) * mask + $4)
+  }'
+}
+
 # 从subnet获取ip
 function get_ip_from_subnet() {
     subnet=$1
-    prefix=`echo $subnet|awk -F/ '{print $1}'|awk -F. '{printf "%d", ($1*(2^24))+($2*(2^16))+($3*(2^8))+$4}'`
+    prefix=$(ip_value $subnet)
     mod=`echo $subnet|awk -F/ '{print $2}'`
     mask=$((2**32-2**(32-$mod)))
     # 对prefix再取一次模，为了支持10.182.26.50/22这种格式
@@ -54,7 +62,7 @@ function get_ip_from_subnet() {
     for i in `/sbin/ifconfig -a|grep inet|grep -v inet6|awk '{print $2}'|tr -d "addr:"`
     do
         # 把ip转换成整数
-        ip_int=`echo $i|awk -F. '{printf "%d\n", ($1*(2^24))+($2*(2^16))+($3*(2^8))+$4}'`
+        ip_int=$(ip_value $i)
         if [ $(($ip_int&$mask)) -eq $prefix ]
         then
             ip=$i
@@ -125,7 +133,7 @@ function start_one() {
         return 1
     fi
 
-    ps -efl|grep -w "${dataDir}/chunkserver$1"|grep -v grep
+    ps -efl|grep -E "curve-chunkserver .*${dataDir}/chunkserver$1 "|grep -v grep
     if [ $? -eq 0 ]
     then
         echo "chunkserver$1 is already active!"
@@ -162,7 +170,7 @@ function start_one() {
             -chunkServerStoreUri=local://${dataDir}/chunkserver$1/ \
             -copySetUri=local://${dataDir}/chunkserver$1/copysets \
             -raftSnapshotUri=curve://${dataDir}/chunkserver$1/copysets \
-            -raftLogUri=curve://${dataDir}/chunkserver$1/copysets \
+            -raftLogUri=${raftLogProtocol}://${dataDir}/chunkserver$1/copysets \
             -recycleUri=local://${dataDir}/chunkserver$1/recycler \
             -raft_sync_segments=true \
             -graceful_quit_on_sigterm=true \
@@ -191,7 +199,7 @@ function stop() {
     fi
 
     echo "kill chunkserver $1"
-    kill `ps -efl|grep -w ${dataDir}/chunkserver$1|grep -v grep|awk '{print $4}'`
+    kill `ps -efl|grep -E "curve-chunkserver .*${dataDir}/chunkserver$1 "|grep -v grep|awk '{print $4}'`
 }
 
 function restart() {
@@ -227,7 +235,7 @@ function wait_stop() {
     while [ $retry_times -le 3 ]
     do
         ((retry_times=$retry_times+1))
-        ps -efl|grep -w "${dataDir}/chunkserver$1"|grep -v grep > /dev/null 2>&1
+        ps -efl|grep -E "curve-chunkserver .*${dataDir}/chunkserver$1 "|grep -v grep > /dev/null 2>&1
         if [ $? -eq 0 ]
         then
             sleep 1
@@ -237,11 +245,11 @@ function wait_stop() {
         fi
     done
     # 如果进程还在，就kill -9
-    ps -efl|grep -w "${dataDir}/chunkserver$1"|grep -v grep > /dev/null 2>&1
+    ps -efl|grep -E "curve-chunkserver .*${dataDir}/chunkserver$1 "|grep -v grep > /dev/null 2>&1
     if [ $? -eq 0 ]
     then
         echo "The process of chunkserver$1 still exists after 3s, now kill -9 it"
-        kill -9 `ps -efl|grep -w ${dataDir}/chunkserver$1|grep -v grep|awk '{print $4}'`
+        kill -9 `ps -efl|grep -E "curve-chunkserver .*${dataDir}/chunkserver$1 "|grep -v grep|awk '{print $4}'`
     fi
 }
 
@@ -263,7 +271,7 @@ function status() {
 }
 
 function status_one() {
-    ps -efl|grep -w "${dataDir}/chunkserver$1"|grep -v grep > /dev/null 2>&1
+    ps -efl|grep -E "curve-chunkserver .*${dataDir}/chunkserver$1 "|grep -v grep > /dev/null 2>&1
     if [ $? -eq 0 ]
     then
         echo "chunkserver$1 is active!"
@@ -290,19 +298,26 @@ function deploy() {
         return 1
     fi
 
-    if [[ "$1" = "all" && $# -gt 1 ]]
-    then
-        do_confirm_if_need $2
-    fi
-
     if [ "$1" = "all" ]
     then
-        deploy_all;
+        if [ $# -gt 1 ]
+        then
+            do_confirm_if_need $2
+        else
+            do_confirm
+        fi
+        deploy_all
         return $?
     fi
 
     if [ $# -ge 2 ]
     then
+        if [ $# -gt 2 ]
+        then
+            do_confirm_if_need $3
+        else
+            do_confirm
+        fi
         deploy_one $@;
         return $?
     fi
@@ -326,19 +341,21 @@ function deploy-wal-pool() {
         return 1
     fi
 
-    if [[ "$1" = "all" && $# -gt 1 ]]
+    if [ $# -gt 1 ]
     then
         do_confirm_if_need $2
+    else
+        do_confirm
     fi
 
     if [ "$1" = "all" ]
     then
-        walfile_pool_prep;
+        walfile_pool_prep
         return $?
+    else
+        deploy_one_walfile_pool $1
+        wait
     fi
-
-    deploy_one_walfile_pool $1
-    wait
 }
 
 

curve-ansible/roles/install_package/templates/chunkserver_deploy.sh.j2

@@ -14,7 +14,7 @@ function do_confirm {
   fi
 }
 
-{% if not auto_get_disk_list %}
+{% if not auto_get_disk_list | bool %}
 {% set dlist=[] -%}
 {% for disk in disk_list -%}
   {% set _ = dlist.append("%s" % (disk)) -%}
@@ -172,6 +172,14 @@ done
 wait
 }
 
+function release_disk_reserved_space {
+  disks=`{{ get_disk_list_cmd }}`
+  for disk in ${disks}
+  do
+    sudo tune2fs -m 0 /dev/$disk
+  done
+}
+
 function deploy_one_walfile_pool {
   curve-format -allocatePercent={{ wal_segment_alloc_percent }} \
   -filePoolDir=/data/chunkserver$1/walfilepool \
@@ -203,12 +211,14 @@ function usage {
 
 function deploy_all {
     deploy_prep;
+    record_diskinfo;
     chunk_dir_prep;
     disk_format;
     mount_dir;
     fstab_record;
     meta_record;
     chunkfile_pool_prep;
+    release_disk_reserved_space;
 }
 
 function deploy_one {
@@ -278,5 +288,7 @@ function deploy_one {
   -fileSize={{ chunk_size }} \
   -fileSystemPath=$dirname/chunkfilepool  &
   wait
+  # release disk reserved space
+  sudo tune2fs -m 0 $diskname
   exit
 }

curve-ansible/roles/install_package/templates/curve-monitor.sh.j2

@@ -1,66 +0,0 @@
-#!/bin/sh
-
-#sh update_dashboard.sh
-#echo "update dashboards success!"
-
-WORKDIR={{ monitor_work_dir }}
-
-if [ ! -d $WORKDIR ]; then
-  echo "${WORKDIR} not exists"
-  exit 1
-fi
-
-cd $WORKDIR
-chmod -R 777 prometheus
-chmod -R 777 grafana
-
-start() {
-    echo "==========start==========="
-
-    echo "" > monitor.log
-
-    stdbuf -oL python target_json.py >> monitor.log 2>&1 &
-    echo "start prometheus targets service success!"
-
-    docker-compose up >> monitor.log 2>&1 &
-    echo "start metric system success!"
-}
-
-stop() {
-    echo "===========stop============"
-
-    docker-compose down
-
-    ID=`(ps -ef | grep "target_json.py"| grep -v "grep") | awk '{print $2}'`
-    for id in $ID
-    do
-        kill -9 $id
-	echo "killed $id"
-    done
-}
-
-restart() {
-    stop
-    echo "sleeping........."
-    sleep 3
-    start
-}
-
-case "$1" in
-    'start')
-        start
-        ;;
-    'stop')
-        stop
-        ;;
-    'status')
-        status
-        ;;
-    'restart')
-        restart
-        ;;
-    *)
-    echo "usage: $0 {start|stop|restart}"
-    exit 1
-        ;;
-    esac

curve-ansible/roles/install_package/templates/etcd-daemon.sh.j2

@@ -10,7 +10,7 @@ logDir={{ etcd_log_dir }}
 logPath=${logDir}/etcd.log
 
 # pidfile
-pidFile={{ deploy_dir }}/etcd.pid
+pidFile=${HOME}/etcd.pid
 
 # daemon log
 daemonLog=${logDir}/daemon-etcd.log

curve-ansible/roles/install_package/templates/mds-daemon.sh.j2

@@ -13,7 +13,7 @@ logPath={{ mds_log_dir }}
 mdsAddr=
 
 # pidfile
-pidFile={{ deploy_dir }}/curve-mds.pid
+pidFile=${HOME}/curve-mds.pid
 
 # daemon log
 daemonLog=${logPath}/curve-mds-daemon.log
@@ -21,6 +21,13 @@ daemonLog=${logPath}/curve-mds-daemon.log
 # console output
 consoleLog=${logPath}/curve-mds-console.log
 
+function ip_value() {
+  echo $1 | awk -F '[:/.]' '{
+    mask = (2 ^ 8)
+    printf ("%.0f", (($1 * mask + $2) * mask + $3) * mask + $4)
+  }'
+}
+
 # 启动mds
 function start_mds() {
   # 检查daemon
@@ -88,7 +95,7 @@ function start_mds() {
   then
         subnet=`cat $confPath|grep global.subnet|awk -F"=" '{print $2}'`
         port=`cat $confPath|grep global.port|awk -F"=" '{print $2}'`
-        prefix=`echo $subnet|awk -F/ '{print $1}'|awk -F. '{printf "%d", ($1*(2^24))+($2*(2^16))+($3*(2^8))+$4}'`
+        prefix=$(ip_value $subnet)
         mod=`echo $subnet|awk -F/ '{print $2}'`
         mask=$((2**32-2**(32-$mod)))
         ip=
@@ -99,7 +106,7 @@ function start_mds() {
         for i in `/sbin/ifconfig -a|grep inet|grep -v inet6|awk '{print $2}'|tr -d "addr:"`
         do
                 # 把ip转换成整数
-                ip_int=`echo $i|awk -F. '{printf "%d\n", ($1*(2^24))+($2*(2^16))+($3*(2^8))+$4}'`
+                ip_int=$(ip_value $i)
                 if [ $(($ip_int&$mask)) -eq $prefix ]
                 then
                     ip=$i

curve-ansible/roles/install_package/templates/nebd-daemon.j2

@@ -10,261 +10,305 @@
 # Description:       Start the nebd-daemon service and associated helpers
 ### END INIT INFO
 
-# 检查脚本执行是否通过root权限执行
-if [[ $(id -u) -ne 0 ]]
-then
-    echo "Please run with sudo"
-    exit 1
+NEBD_SERVER_BIN={{ curve_bin_dir }}/nebd-server
+STARTED_AS_ROOT=false
+USERS_FILE=/etc/nebd/users
+USERS=()
+
+if [[ $(id -u) -eq 0 ]]; then
+    STARTED_AS_ROOT=true
 fi
 
-# nebd-server路径
-bin={{ curve_bin_dir }}/nebd-server
-
-# 默认配置文件
-confPath={{ nebd_server_config_path }}
-
-# 日志文件路径
-baseLogPath={{ nebd_log_dir }}
-logPath=${baseLogPath}/server
-
-# pidfile
-pidFile=${baseLogPath}/nebd-server.pid
-
-# daemon log
-daemonLog=${baseLogPath}/nebd-server-daemon.log
-
-# console output
-consoleLog=${baseLogPath}/nebd-server-console.log
-
 function get_nebd_server_pid() {
-    local tmp=`pidof nebd-server`
+    local tmp=$(pgrep -u ${1} nebd-server)
     echo $tmp
 }
 
-# 启动nebd-server
-function start() {
-    # 检查daemon
-    if ! type daemon &> /dev/null
-    then
-        echo "No daemon installed"
-        exit 1
+function list_users() {
+    if [ -f ${USERS_FILE} ]; then
+        IFS=$'\n' read -d '' -r -a TMP_USERS <${USERS_FILE}
+        USERS=("${TMP_USERS[@]}")
     fi
 
-    # 检查nebd-server
-    if [ ! -f ${bin} ]
-    then
-        echo "No nebd-server installed"
-        exit 1
-    fi
-
-    # 检查配置文件
-    if [ ! -f ${confPath} ]
-    then
-        echo "Not found nebd-server.conf, Path is ${confPath}"
-        exit 1
-    fi
-
-    # 判断是否已经通过daemon启动了nebd-server
-    daemon --name nebd-server --pidfile ${pidFile} --running
-    if [ $? -eq 0 ]
-    then
-        echo "Already started nebd-server by daemon"
-        exit 0
-    fi
-
-    # 创建logPath
-    mkdir -p ${logPath} > /dev/null 2>&1
-    if [ $? -ne 0 ]
-    then
-        echo "Create log dir failed: ${logPath}"
-        exit 1
-    fi
-
-    # 检查logPath是否有写权限
-    if [ ! -w ${logPath} ]
-    then
-        echo "Write permission denied: ${logPath}"
-        exit 1
-    fi
-
-    # 检查consoleLog是否可写或者是否能够创建
-    touch ${consoleLog} > /dev/null 2>&1
-    if [ $? -ne 0 ]
-    then
-        echo "Can't Write or Create console Log: ${consoleLog}"
-        exit 1
-    fi
-
-    # 检查daemonLog是否可写或者是否能够创建
-    touch ${daemonLog} > /dev/null 2>&1
-    if [ $? -ne 0 ]
-    then
-        echo "Can't Write or Create daemon logfile: ${daemonLog}"
-        exit 1
-    fi
-
-    # 检查jemalloc库文件
-    jemallocpath=`find /usr/ -name "libjemalloc.so*" -print -quit`
-    if [ "${jemallocpath}" = "" ]
-    then
-        echo "Not found jemalloc library in /usr/"
-        exit 1
-    fi
-
-    # 记录运行daemon前的daemonLog的行数
-    line1=`cat ${daemonLog} | wc -l`
-
-    LD_PRELOAD=${jemallocpath} daemon --name nebd-server --core --inherit \
-           --respawn --attempts 10 --delay 10 --acceptable 10\
-           --pidfile ${pidFile} \
-           --errlog ${daemonLog} \
-           --output ${consoleLog} \
-           -- ${bin} -confPath=${confPath} -log_dir=${logPath} -graceful_quit_on_sigterm=true -stderrthreshold=3
-
-    # sleep 1秒检测daemonLog是否有变化，如果有变化，说明启动遇到问题
-    sleep 1
-    line2=`cat ${daemonLog} | wc -l`
-    if [ $line1 != $line2 ]
-    then
-        echo "start nebd-server met error!"
-        stop
+    # always add root user for compatible
+    if [[ ! "${USERS[@]}" =~ "root" ]]; then
+        USERS=("root" "${USERS[@]}")
     fi
 }
 
-# 停止daemon进程，且停止nebd-server
-function stop() {
-    # 判断是否已经通过daemon启动了nebd-server
-    daemon --name nebd-server --pidfile ${pidFile} --running
-    if [ $? -ne 0 ]
-    then
-        echo "Didn't start nebd-server by daemon"
-        exit 0
+function create_dir_if_not_exists() {
+    if [ ! -d ${1} ]; then
+        mkdir -p ${1} >/dev/null 2>&1
+        chown ${2} ${1} >/dev/null 2>&1
+        chmod 777 ${1} >/dev/null 2>&1
+    fi
+}
+
+function prepare_env() {
+    PREPARE_RES=
+
+    cat /etc/passwd | awk -F':' '{ print $1 }' | grep $1 >/dev/null 2>&1
+    if [ $? -ne 0 ]; then
+        PREPARE_RES="user ${1} not found in '/etc/passwd'"
+        # echo "user '${1}' not found in '/etc/passwd'"
+        return
     fi
 
-    daemon --name nebd-server --pidfile ${pidFile} --stop
-    if [ $? -ne 0 ]
-    then
-        echo "stop may not success!"
+    if [ "${1}" == "root" ]; then
+        DAEMON_NAME=nebd-server
+        META_DATA_PATH=/data/nebd
+        DATA_LOCK_PATH=${META_DATA_PATH}/lock
+        BASE_LOG_PATH=/data/log/nebd
+        CONF_PATH=/etc/nebd/nebd-server.conf
+    else
+        DAEMON_NAME=nebd-server-${1}
+        META_DATA_PATH=/data/nebd/nebd-${1}
+        DATA_LOCK_PATH=${META_DATA_PATH}/lock
+        BASE_LOG_PATH=/data/log/nebd/nebd-${1}
+        CONF_PATH=/etc/nebd/nebd-server-${1}.conf
+    fi
+
+    PID_FILE=${BASE_LOG_PATH}/nebd-server.pid
+    DAEMON_LOG=${BASE_LOG_PATH}/nebd-server-daemon.log
+    CONSOLE_LOG=${BASE_LOG_PATH}/nebd-server-console.log
+    LOG_PATH=${BASE_LOG_PATH}/server
+    NEBD_CLIENT_LOG_PATH=${BASE_LOG_PATH}/client
+
+    # check config file
+    if [ ! -f ${CONF_PATH} ]; then
+        PREPARE_RES="not found config file '${CONF_PATH}'"
+        # echo "not found config file '${CONF_PATH}'"
+        return
+    fi
+
+    create_dir_if_not_exists ${META_DATA_PATH} ${1}
+    create_dir_if_not_exists ${DATA_LOCK_PATH} ${1}
+    create_dir_if_not_exists ${LOG_PATH} ${1}
+    create_dir_if_not_exists ${NEBD_CLIENT_LOG_PATH} ${1}
+}
+
+function start_one() {
+    prepare_env $1
+
+    if [ "${PREPARE_RES}" != "" ]; then
+        echo "$1: prepare env failed, ${PREPARE_RES}"
+        return
+    fi
+
+    # check whether already started by daemon
+    daemon --name ${DAEMON_NAME} --pidfile ${PID_FILE} --running
+    if [ $? -eq 0 ]; then
+        echo "$1: already started nebd-server by daemon"
+        return
+    fi
+
+    # record number of line before start
+    line1=$(cat ${DAEMON_LOG} | wc -l)
+
+    USEROPTS=
+    if $STARTED_AS_ROOT; then
+        USEROPTS="--user ${1}"
+    fi
+
+    LD_PRELOAD=${jemallocpath} daemon --name ${DAEMON_NAME} --core --inherit \
+        --respawn --attempts 10 --delay 10 --acceptable 10 --pidfile ${PID_FILE} \
+        --errlog ${DAEMON_LOG} \
+        --output ${CONSOLE_LOG} ${USEROPTS} \
+        -- ${NEBD_SERVER_BIN} -confPath=${CONF_PATH} -log_dir=${LOG_PATH} -graceful_quit_on_sigterm=true -stderrthreshold=3
+
+    # sleep a while, check whether start successful
+    sleep 1
+    line2=$(cat ${DAEMON_LOG} | wc -l)
+    if [ $line1 != $line2 ]; then
+        echo "$1: start nebd-server met error!"
+        stop_one $1
+    fi
+}
+
+function stop_one() {
+    prepare_env $1
+
+    if [ "${PREPARE_RES}" != "" ]; then
+        echo "$1: prepare env failed, ${PREPARE_RES}"
+        return
+    fi
+
+    # 判断是否已经通过daemon启动了nebd-server
+    daemon --name ${DAEMON_NAME} --pidfile ${PID_FILE} --running
+    if [ $? -ne 0 ]; then
+        echo "$1: didn't start nebd-server by daemon"
+        return
+    fi
+
+    daemon --name ${DAEMON_NAME} --pidfile ${PID_FILE} --stop
+    if [ $? -ne 0 ]; then
+        echo "$1: stop may not success!"
     else
         # wait 3s
         retry_times=0
-        while [ $retry_times -le 3 ]
-        do
-            ((retry_times=$retry_times+1))
-            pidof nebd-server > /dev/null 2>&1
-            if [ $? -eq 0 ]
-            then
+        while [ $retry_times -le 3 ]; do
+            ((retry_times = $retry_times + 1))
+            pgrep -u $1 nebd-server >/dev/null 2>&1
+            if [ $? -eq 0 ]; then
                 sleep 1
             else
                 break
             fi
         done
 
-        pidof nebd-server > /dev/null 2>&1
-        if [ $? -eq 0 ]
-        then
-            echo "nebd-server still exists after 3s, now kill it with SIGKILL"
-            kill -9 `pidof nebd-server`
+        pgrep -u $1 nebd-server >/dev/null 2>&1
+        if [ $? -eq 0 ]; then
+            echo "$1: nebd-server still exists after 3s, now kill it with SIGKILL"
+            kill -9 $(pgrep -u $1 nebd-server)
         fi
 
-        echo "nebd-server exit success!"
-        echo "daemon exit success!"
+        echo "$1: nebd-server exit success, daemon exit success!"
     fi
 }
 
-# restart
-function restart() {
-    # 判断是否已经通过daemon启动了nebd-server
-    daemon --name nebd-server --pidfile ${pidFile} --running
-    if [ $? -ne 0 ]
-    then
-        echo "Didn't start nebd-server by daemon"
-        exit 1
+function restart_one() {
+    prepare_env $1
+
+    if [ "${PREPARE_RES}" != "" ]; then
+        echo "$1: prepare env failed, ${PREPARE_RES}"
+        return
     fi
 
-    old_pid=$(get_nebd_server_pid)
+    # 判断是否已经通过daemon启动了nebd-server
+    daemon --name ${DAEMON_NAME} --pidfile ${PID_FILE} --running
+    if [ $? -ne 0 ]; then
+        echo "$1: didn't start nebd-server by daemon"
+        return
+    fi
 
-    daemon --name nebd-server --pidfile ${pidFile} --restart
+    old_pid=$(get_nebd_server_pid $1)
+
+    daemon --name ${DAEMON_NAME} --pidfile ${PID_FILE} --restart
 
     sleep 1
-    new_pid=$(get_nebd_server_pid)
+    new_pid=$(get_nebd_server_pid $1)
 
-    if [ $old_pid == $new_pid ]
-    then
-        echo "restart nebd-server may failed, now kill it with SIGKILL"
+    if [ $old_pid == $new_pid ]; then
+        echo "$1: restart nebd-server may failed, now kill it with SIGKILL"
         kill -9 $old_pid
     fi
 }
 
+function status_one() {
+    prepare_env $1
+
+    if [ "${PREPARE_RES}" != "" ]; then
+        echo "$1: prepare env failed, ${PREPARE_RES}"
+        return
+    fi
+
+    daemon --name ${DAEMON_NAME} --pidfile ${PID_FILE} --running
+    if [ $? -ne 0 ]; then
+        echo "$1: didn't start nebd-server by daemon"
+    else
+        echo "$1: nebd-server is running by daemon"
+    fi
+}
+
+# start nebd-server
+function start() {
+    if ! $STARTED_AS_ROOT; then
+        echo "nebd-daemon start should run as root"
+        exit 1
+    fi
+
+    list_users
+    for u in "${USERS[@]}"; do
+        start_one $u
+    done
+}
+
+# stop daemon and nebd-server
+function stop() {
+    if ! $STARTED_AS_ROOT; then
+        echo "nebd-daemon stop should run as root"
+        exit 1
+    fi
+
+    list_users
+    for u in "${USERS[@]}"; do
+        stop_one $u
+    done
+}
+
+# restart
+function restart() {
+    if ! $STARTED_AS_ROOT; then
+        echo "nebd-daemon restart should run as root"
+        exit 1
+    fi
+
+    list_users
+    for u in "${USERS[@]}"; do
+        restart_one $u
+    done
+}
+
 # status
 function status() {
-    daemon --name nebd-server --pidfile ${pidFile} --running
-    if [ $? -ne 0 ]
-    then
-        echo "Didn't start nebd-server by daemon"
-    else
-        echo "nebd-server is running by daemon"
+    if ! $STARTED_AS_ROOT; then
+        echo "nebd-daemon status should run as root"
+        exit 1
     fi
+
+    list_users
+    for u in "${USERS[@]}"; do
+        status_one $u
+    done
 }
 
 # 使用方式
 function usage() {
     echo "Usage:"
-    echo "  nebd-daemon start -- start deamon process and watch on nebd-server process"
-    echo "    [-c|--confPath path]    conf path"
-    echo "    [-l|--logPath  path]    log path"
-    echo "  nebd-daemon stop  -- stop daemon process and nebd-server"
-    echo "  nebd-daemon restart -- restart nebd-server"
-    echo "  nebd-daemon status -- show if the nebd-server is running by daemon"
-    echo "Examples:"
-    echo "  nebd-daemon start -c /etc/nebd/nebd-server.conf -l ${HOME}/"
+    echo "  nebd-daemon start -- start deamon process and watch on nebd-server process for all instance"
+    echo "  nebd-daemon start-one -- start deamon process and watch on nebd-server process for current user's instance"
+
+    echo "  nebd-daemon stop  -- stop daemon process and nebd-server for all instance"
+    echo "  nebd-daemon stop-one  -- stop daemon process and nebd-server for current user's instance"
+
+    echo "  nebd-daemon restart -- restart nebd-server for all instance"
+    echo "  nebd-daemon restart-one -- restart nebd-server for current user's instance"
+
+    echo "  nebd-daemon status -- show if the nebd-server is running by daemon for all instance"
+    echo "  nebd-daemon status-one -- show if the nebd-server is running by daemon for current user's instance"
 }
 
 # 检查参数启动参数，最少1个
-if [ $# -lt 1 ]
-then
+if [ $# -lt 1 ]; then
     usage
     exit
 fi
 
 case $1 in
 "start")
-    shift # pass first argument
-
-    # 解析参数
-    while [[ $# -gt 1 ]]
-    do
-        key=$1
-
-        case $key in
-        -c|--confPath)
-            confPath=`realpath $2`
-            shift # pass key
-            shift # pass value
-            ;;
-        -l|--logPath)
-            logPath=`realpath $2`
-            shift # pass key
-            shift # pass value
-            ;;
-        *)
-            usage
-            exit
-            ;;
-        esac
-    done
-
     start
     ;;
+"start-one")
+    start_one ${USER}
+    ;;
 "stop")
     stop
     ;;
+"stop-one")
+    stop_one ${USER}
+    ;;
 "restart")
     restart
     ;;
+"restart-one")
+    restart_one ${USER}
+    ;;
 "status")
     status
     ;;
+"status-one")
+    status_one ${USER}
+    ;;
 *)
     usage
     ;;

curve-ansible/roles/install_package/templates/snapshot-daemon.sh.j2

@@ -13,7 +13,7 @@ logPath={{ snapshot_clone_server_log_dir }}
 serverAddr=
 
 # pidfile
-pidFile={{ deploy_dir }}/curve-snapshot.pid
+pidFile=${HOME}/curve-snapshot.pid
 
 # daemon log
 daemonLog=${logPath}/curve-snapshot-daemon.log
@@ -21,6 +21,13 @@ daemonLog=${logPath}/curve-snapshot-daemon.log
 # console output
 consoleLog=${logPath}/curve-snapshot-console.log
 
+function ip_value() {
+  echo $1 | awk -F '[:/.]' '{
+    mask = (2 ^ 8)
+    printf ("%.0f", (($1 * mask + $2) * mask + $3) * mask + $4)
+  }'
+}
+
 # 启动snapshotcloneserver
 function start_server() {
     # 检查daemon
@@ -88,7 +95,7 @@ function start_server() {
     then
         subnet=`cat $confPath|grep server.subnet|awk -F"=" '{print $2}'`
         port=`cat $confPath|grep server.port|awk -F"=" '{print $2}'`
-        prefix=`echo $subnet|awk -F/ '{print $1}'|awk -F. '{printf "%d", ($1*(2^24))+($2*(2^16))+($3*(2^8))+$4}'`
+        prefix=$(ip_value $subnet)
         mod=`echo $subnet|awk -F/ '{print $2}'`
         mask=$((2**32-2**(32-$mod)))
         ip=
@@ -99,7 +106,7 @@ function start_server() {
         for i in `/sbin/ifconfig -a|grep inet|grep -v inet6|awk '{print $2}'|tr -d "addr:"`
         do
                 # 把ip转换成整数
-                ip_int=`echo $i|awk -F. '{printf "%d\n", ($1*(2^24))+($2*(2^16))+($3*(2^8))+$4}'`
+                ip_int=$(ip_value $i)
                 if [ $(($ip_int&$mask)) -eq $prefix ]
                 then
                     ip=$i
@@ -173,7 +180,7 @@ function show_status() {
     fi
 
     # 查询leader的IP
-    leaderAddr=`tac ${consoleLog}|grep -m 1 -B 1000000 "Logging before InitGoogleLogging()"|grep "leader"|grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}"|head -n1`
+    leaderAddr=`tac ${consoleLog}|grep -a -m 1 -B 1000000 "Logging before InitGoogleLogging()"|grep "leader"|grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}"|head -n1`
 
     # 如果load configuration之后的日志，没有leader相关日志
     # 那么leaderAddr为空, snapshotcloneserver应该没有起来

curve-ansible/roles/prepare_software_env/tasks/main.yml

@@ -15,6 +15,12 @@
 #  limitations under the License.
 #
 
+- name: install gcc
+  vars:
+    apt_lib_name: gcc
+    yum_lib_name: gcc
+  include_tasks: common_tasks/install_with_yum_apt.yml
+
 - name: get gcc version
   shell: gcc -dumpversion
   register: gcc_version

curve-ansible/roles/restart_service/tasks/include/restart_chunkserver.yml

@@ -15,17 +15,20 @@
 #  limitations under the License.
 #
 
+- name: get chunkserver number
+  include_tasks: "common_tasks/get_chunkserver_number.yml"
+
 - name: Determine need restart or not
   include_tasks: append_need_restart_cs_list.yml
   with_sequence: start=0 end={{ chunkserver_num | int - 1}}
-  when: not restart_directly
+  when: not restart_directly | bool
 
 - name: set restart_cs_list as all
   vars:
     all_cs: all
   set_fact:
     restart_cs_list: "{{ restart_cs_list + [all_cs] }}"
-  when: restart_directly
+  when: restart_directly | bool
 
 - name: set sudo
   set_fact:
@@ -48,11 +51,11 @@
   delay: "{{ restart_chunkserver_check_delay }}"
   retries: "{{ restart_chunkserver_check_times }}"
   with_items: "{{ restart_cs_res.results }}"
-  when: restart_cs_list != "" or restart_directly
+  when: restart_cs_list != "" or restart_directly | bool
 
 - name: check chunkserver version after restart
   vars:
     metric_port: "{{ chunkserver_base_port | int + item | int }}"
   include_tasks: check_version_after_restart.yml
   with_items: "{{ restart_cs_list }}"
-  when: restart_cs_list != "" and not restart_directly
+  when: restart_cs_list != "" and not restart_directly | bool

curve-ansible/roles/restart_service/tasks/include/restart_etcd.yml

@@ -15,8 +15,12 @@
 #  limitations under the License.
 #
 
-- name: stop etcd
-  include_tasks: roles/stop_service/tasks/include/stop_etcd.yml
-
-- name: start etcd
-  include_tasks: roles/start_service/tasks/include/start_etcd.yml
+- name: restart etcd
+  vars:
+    ch_dir: "{{ deploy_dir }}"
+    daemon_script: ./etcd-daemon.sh
+    service_port: "{{ etcd_listen_client_port }}"
+    process_name: etcd
+    start_cmd: >
+      nohup {{ install_etcd_dir }}/etcd --config-file {{ etcd_config_path }} > etcd-console-$(date -d "today" +"%Y%m%d_%H%M%S").log 2>&1 &
+  include_tasks: common_tasks/restart_service.yml

curve-ansible/roles/restart_service/tasks/include/restart_mds.yml

@@ -31,18 +31,23 @@
       set_fact:
         need_restart: true
       when: curve_version != expected_curve_version
-  when: not restart_directly
+  when: not restart_directly | bool
 
 - name: restart mds
-  block:
-    - name: stop mds
-      include_tasks: roles/stop_service/tasks/include/stop_mds.yml
-    - name: start mds
-      include_tasks: roles/start_service/tasks/include/start_mds.yml
-  when: need_restart or restart_directly
+  vars:
+    ch_dir: "{{ deploy_dir }}"
+    daemon_script: ./mds-daemon.sh
+    service_port: "{{ mds_dummy_port }}"
+    process_name: curve-mds
+    start_cmd: >
+      nohup "{{ curve_bin_dir }}"/curve-mds -confPath={{ mds_config_path }}
+      -mdsAddr={{ ansible_ssh_host }}:{{ mds_port }} -log_dir={{ mds_log_dir }} -graceful_quit_on_sigterm=true
+      -stderrthreshold=3 > mds-console.log 2>&1 &
+  include_tasks: common_tasks/restart_service.yml
+  when: need_restart or restart_directly | bool
 
 - name: check mds version after restart
   vars:
         metric_port: "{{ mds_dummy_port }}"
   include_tasks: check_version_after_restart.yml
-  when: not restart_directly
+  when: not restart_directly | bool

curve-ansible/roles/restart_service/tasks/include/restart_nebd.yml

@@ -16,14 +16,13 @@
 #
 
 - name: restart nebd-server
-  block:
-    - name: stop nebd-server
-      vars:
-        ch_dir: "{{ deploy_dir }}"
-        daemon_script: nebd-daemon
-        service_port: "{{ nebd_port }}"
-        process_name: >
-          {{ curve_bin_dir }}/nebd-server
-      include_tasks: common_tasks/stop_service.yml
-    - name: start nebd-server
-      include_tasks: roles/start_service/tasks/include/start_nebd.yml
+  vars:
+    ch_dir: "{{ deploy_dir }}"
+    daemon_script: nebd-daemon
+    service_port: 0
+    process_name: nebd-server
+    start_cmd: >
+      nohup {{ curve_bin_dir }}/nebd-server -confPath={{ nebd_server_config_path }}
+        -log_dir={{ nebd_log_dir }}/server -graceful_quit_on_sigterm=true -stderrthreshold=3
+        > nebd-console.log 2>&1 &
+  include_tasks: common_tasks/restart_service.yml

curve-ansible/roles/restart_service/tasks/include/restart_snapshotclone.yml

@@ -31,18 +31,22 @@
       set_fact:
         need_restart: true
       when: curve_version != expected_curve_version
-  when: not restart_directly
+  when: not restart_directly | bool
 
 - name: restart snapshot-clone
-  block:
-    - name: stop snapshot-clone
-      include_tasks: roles/stop_service/tasks/include/stop_snapshotcloneserver.yml
-    - name: start snapshot-clone
-      include_tasks: roles/start_service/tasks/include/start_snapshotcloneserver.yml
-  when: need_restart or restart_directly
+  vars:
+    ch_dir: "{{ deploy_dir }}"
+    daemon_script: ./snapshot-daemon.sh
+    service_port: "{{ snapshot_dummy_port }}"
+    process_name: curve-snapshotcloneserver
+    start_cmd: >
+      nohup {{ curve_bin_dir }}/curve-snapshotcloneserver -conf={{ snapshot_config_path }} -addr={{ ansible_ssh_host }}:{{ snapshot_port }}
+      -log_dir={{ snapshot_clone_server_log_dir }} -stderrthreshold=3 -graceful_quit_on_sigterm=true > snapshot-console.log 2>&1 &
+  include_tasks: common_tasks/restart_service.yml
+  when: need_restart or restart_directly | bool
 
 - name: check snapshotclone version after restart
   vars:
         metric_port: "{{ snapshot_dummy_port }}"
   include_tasks: check_version_after_restart.yml
-  when: not restart_directly
+  when: not restart_directly | bool

curve-ansible/roles/start_service/tasks/include/start_chunkserver.yml

@@ -32,6 +32,9 @@
   debug:
     var: start_chunkserver_res.stdout_lines
 
+- name: get chunkserver number
+  include_tasks: "common_tasks/get_chunkserver_number.yml"
+
 - name: wait until chunkserver is up
   vars:
     end_seq: "{{ chunkserver_num | int - 1}}"

curve-ansible/roles/start_service/tasks/include/start_etcd.yml

@@ -21,5 +21,5 @@
     daemon_script: ./etcd-daemon.sh
     service_port: "{{ etcd_listen_client_port }}"
     start_cmd: >
-      nohup {{ install_etcd_dir }}/etcd --config-file {{ etcd_config_path }} > etcd-console.log 2>&1 &
+      nohup {{ install_etcd_dir }}/etcd --config-file {{ etcd_config_path }} > etcd-console-$(date -d "today" +"%Y%m%d_%H%M%S").log 2>&1 &
   include_tasks: common_tasks/start_service.yml

curve-ansible/roles/start_service/tasks/include/start_monitor.yml

@@ -16,7 +16,7 @@
 #
 
 - name: start monitor
-  shell:  chdir={{ deploy_dir }} sudo ./curve-monitor.sh start
+  shell:  chdir={{ monitor_work_dir }} sudo ./curve-monitor.sh start
 
 - name: wait until prometheus is up
   vars:

curve-ansible/roles/start_service/tasks/include/start_nebd.yml

@@ -22,11 +22,11 @@
 
 - name: start nebd with daemon
   shell: chdir={{ deploy_dir }} {{ sudo }} nebd-daemon start
-  when: start_by_daemon
+  when: start_by_daemon | bool
 
 - name: start nebd without daemon
   shell: >
     chdir={{ deploy_dir }} {{ sudo }} nohup {{ curve_bin_dir }}/nebd-server -confPath={{ nebd_server_config_path }}
     -log_dir={{ nebd_log_dir }}/server -graceful_quit_on_sigterm=true -stderrthreshold=3
     > nebd-console.log 2>&1 &
-  when: not {{ start_by_daemon }}
+  when: not start_by_daemon | bool

curve-ansible/roles/stop_service/tasks/include/stop_chunkserver.yml

@@ -24,6 +24,9 @@
   ignore_errors: true
   shell:  chdir={{ deploy_dir }} {{ sudo }} ./chunkserver_ctl.sh stop all
 
+- name: get chunkserver number
+  include_tasks: "common_tasks/get_chunkserver_number.yml"
+
 - name: wait until chunkserver is down
   vars:
     end_seq: "{{ chunkserver_num | int - 1}}"

curve-ansible/roles/stop_service/tasks/include/stop_etcd.yml

@@ -20,6 +20,5 @@
     ch_dir: "{{ deploy_dir }}"
     daemon_script: ./etcd-daemon.sh
     service_port: "{{ etcd_listen_client_port }}"
-    process_name: >
-      "{{ install_etcd_dir }}"/etcd
+    process_name: etcd
   include_tasks: common_tasks/stop_service.yml

curve-ansible/roles/stop_service/tasks/include/stop_mds.yml

@@ -20,6 +20,5 @@
     ch_dir: "{{ deploy_dir }}"
     daemon_script: ./mds-daemon.sh
     service_port: "{{ mds_dummy_port }}"
-    process_name: >
-      "{{ curve_bin_dir }}"/curve-mds
+    process_name: curve-mds
   include_tasks: common_tasks/stop_service.yml

curve-ansible/roles/stop_service/tasks/include/stop_nebd.yml

@@ -22,9 +22,10 @@
 
 - name: stop nebd with daemon
   shell: chdir={{ deploy_dir }} {{ sudo }} nebd-daemon stop
-  when: start_by_daemon
+  when: start_by_daemon | bool
 
 - name: stop nebd without daemon
   ignore_errors: true
-  shell: killall {{ curve_bin_dir }}/nebd-server
-  when: not {{ start_by_daemon }}
+  shell: >
+    "{{ sudo }}" killall nebd-server
+  when: not start_by_daemon | bool

curve-ansible/roles/stop_service/tasks/include/stop_snapshotcloneserver.yml

@@ -20,6 +20,5 @@
     ch_dir: "{{ deploy_dir }}"
     daemon_script: ./snapshot-daemon.sh
     service_port: "{{ snapshot_dummy_port }}"
-    process_name: >
-      "{{ curve_bin_dir }}"/curve-snapshotcloneserver
+    process_name: curve-snapshotcloneserver
   include_tasks: common_tasks/stop_service.yml

curve-ansible/rolling_update_curve.yml

@@ -26,13 +26,14 @@
   hosts: etcd
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - etcd
     - never
   roles:
     - { role: install_package, package_name: etcd, tags: install_etcd }
     - { role: generate_config, template_name: etcd.conf.yml, conf_path: "{{ etcd_config_path }}", tags: update_etcd_config }
+    - { role: install_package, package_name: etcd-daemon, tags: install_etcd_daemon }
 
 - name: set etcd leader and follower list
   hosts: etcd
@@ -48,7 +49,7 @@
   hosts: etcd_servers_followers, etcd_servers_leader
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   serial: 1
   tags:
     - etcd
@@ -57,16 +58,16 @@
   pre_tasks:
     - pause:
         prompt: "Confirm update etcd in {{ inventory_hostname }}. ENTER to continue or 'CTRL-C A' to quit"
-      when: need_confirm
+      when: need_confirm | bool
   roles:
-    - { role: restart_service, service_name: etcd, command_need_sudo: "{{ etcd_need_sudo }}"}
+    - { role: restart_service, service_name: etcd, command_need_sudo: "{{ etcd_need_sudo | bool }}"}
 
 ############################## rolling update mds ##############################
 - name: prepare mds
   hosts: mds
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
     - mds
@@ -99,7 +100,7 @@
   hosts: mds_servers_followers, mds_servers_leader
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   serial: 1
   tags:
     - restart_mds
@@ -108,15 +109,11 @@
   pre_tasks:
     - pause:
         prompt: "Confirm restart mds in {{ inventory_hostname }}. ENTER to continue or CTRL-C A to quit"
-      when: need_confirm
-    - vars:
-        package_version: "{{ mds_package_version }}"
-        software_version:
-      include_tasks: "common_tasks/get_software_version_from_package_version.yml"
+      when: need_confirm | bool
   # 重启mds
   roles:
-    - { role: restart_service, service_name: mds, expected_curve_version: "{{ software_version }}",
-        command_need_sudo: "{{ mds_need_sudo }}"}
+    - { role: restart_service, service_name: mds, expected_curve_version: "{{ mds_package_version }}",
+        command_need_sudo: "{{ mds_need_sudo | bool }}"}
 
 
 ############################## rolling update chunkserver ##############################
@@ -125,7 +122,7 @@
   hosts: chunkservers
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
     - chunkserver
@@ -137,7 +134,7 @@
     - { role: generate_config, template_name: client.conf, conf_path: "{{ chunkserver_client_config_path }}",
                 tags: ["generate_config", "generage_chunkserver_client_conf"] }
     - { role: generate_config, template_name: s3.conf, conf_path: "{{ chunkserver_s3_config_path }}",
-                tags: ["generate_config", "generage_s3_conf"] }
+                tags: ["generate_config", "generage_cs_s3_conf"] }
 
 # 逐个重启chunkserver，每重启完一个需要等待copyset健康
 # 继续操作下一个的的时候还需要一个命令行交互确认
@@ -145,7 +142,7 @@
   hosts: chunkservers
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   serial: "{{ groups.zone1 | length }}"
   tags:
     - restart
@@ -154,14 +151,10 @@
   pre_tasks:
     - pause:
         prompt: "Confirm restart chunkserver in {{ play_hosts | join(',') }}. ENTER to continue or CTRL-C A to quit"
-      when: need_confirm
-    - vars:
-        package_version: "{{ cs_package_version }}"
-        software_version:
-      include_tasks: "common_tasks/get_software_version_from_package_version.yml"
+      when: need_confirm | bool
   roles:
-    - { role: restart_service, service_name: chunkserver, expected_curve_version: "{{ software_version }}",
-        command_need_sudo: "{{ chunkserver_need_sudo }}"}
+    - { role: restart_service, service_name: chunkserver, expected_curve_version: "{{ cs_package_version }}",
+        command_need_sudo: "{{ chunkserver_need_sudo | bool }}"}
   tasks:
     - include_tasks: "common_tasks/wait_copysets_status_healthy.yml"
 
@@ -171,7 +164,7 @@
   hosts: snapshotclone
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
     - snapshotclone
@@ -183,7 +176,7 @@
     - { role: generate_config, template_name: client.conf, conf_path: "{{ snap_client_config_path }}",
             tags: ["generate_config", "generate_snap_client_conf"] }
     - { role: generate_config, template_name: s3.conf, conf_path: "{{ snap_s3_config_path }}",
-                tags: ["generate_config", "generage_s3_conf"] }
+                tags: ["generate_config", "generage_snap_s3_conf"] }
 
 - name: set leader and follower list
   hosts: snapshotclone
@@ -200,7 +193,7 @@
   hosts: snapshot_servers_followers, snapshot_servers_leader
   any_errors_fatal: true
   gather_facts: no
-  become: "{{ sudo_or_not }}"
+  become: "{{ sudo_or_not | bool }}"
   serial: 1
   tags:
     - restart
@@ -209,13 +202,8 @@
   pre_tasks:
     - pause:
         prompt: "Confirm restart snapshotclone in {{ inventory_hostname }}. ENTER to continue or CTRL-C A to quit"
-      when: need_confirm
-    - vars:
-        package_version: "{{ snapshot_package_version }}"
-        software_version:
-      include_tasks: "common_tasks/get_software_version_from_package_version.yml"
-
+      when: need_confirm | bool
   # 重启snapshot clone
   roles:
-    - { role: restart_service, service_name: snapshotclone, expected_curve_version: "{{ software_version }}",
-        command_need_sudo: "{{ snapshot_need_sudo }}" }
+    - { role: restart_service, service_name: snapshotclone, expected_curve_version: "{{ snapshot_package_version }}",
+        command_need_sudo: "{{ snapshot_need_sudo | bool }}" }

curve-ansible/rolling_update_curve_sdk.yml

@@ -25,6 +25,7 @@
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
   roles:

curve-ansible/rolling_update_nbd.yml

@@ -25,6 +25,7 @@
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
   roles:

curve-ansible/rolling_update_nebd.yml

@@ -25,34 +25,32 @@
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - prepare
   roles:
     - { role: install_package, package_name: nebd, package_version: "{{ nebd_package_version }}",
             tags: ["install_package"] }
-    - { role: generate_config, template_name: client.conf, conf_path: "{{ client_config_path }}", when: "not {{ update_config_with_puppet }}" }
-    - { role: generate_config, template_name: nebd-client.conf, conf_path: "{{ nebd_client_config_path }}", when: "not {{ update_config_with_puppet }}" }
-    - { role: generate_config, template_name: nebd-server.conf, conf_path: "{{ nebd_server_config_path }}", when: "not {{ update_config_with_puppet }}" }
+    - { role: generate_config, template_name: client.conf, conf_path: "{{ client_config_path }}", when: not update_config_with_puppet | bool }
+    - { role: generate_config, template_name: nebd-client.conf, conf_path: "{{ nebd_client_config_path }}", when: not update_config_with_puppet | bool }
+    - { role: generate_config, template_name: nebd-server.conf, conf_path: "{{ nebd_server_config_path }}", when: not update_config_with_puppet | bool }
   tasks:
     - name: update config with puppet
       include_tasks: "common_tasks/update_config_with_puppet.yml"
-      when: update_config_with_puppet
+      when: update_config_with_puppet | bool
 
 - name: restart nebd server
   hosts: client
   gather_facts: no
   any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
   tags:
     - restart
 
   pre_tasks:
     - pause:
         prompt: "Confirm restart nebd-server in {{ inventory_hostname }}. ENTER to continue or CTRL-C A to quit"
-      when: need_confirm
-    - vars:
-        package_version: "{{ nebd_package_version }}"
-        software_version:
-      include_tasks: "common_tasks/get_software_version_from_package_version.yml"
+      when: need_confirm | bool
   roles:
-    - { role: restart_service, service_name: nebd, expected_nebd_version: "{{ software_version }}",
-        command_need_sudo: "{{ nebd_need_sudo }}" }
+    - { role: restart_service, service_name: nebd, expected_nebd_version: "{{ nebd_package_version }}",
+        command_need_sudo: "{{ nebd_need_sudo | bool }}" }