[feat]curve: run deamon after start

Signed-off-by: Cyber-SiKu <Cyber-SiKu@outlook.com>

.bazelversion

@@ -0,0 +1 @@
+0.17.2

CHANGELOG-1.0.md

@@ -0,0 +1,18 @@
+See [code changes](https://github.com/opencurve/curve/compare/v0.1.4...release1.0).
+
+ ## The new features:
+- Support external ip for ChunkServer to divide inter-cluster communication and communication with clients.
+- The Curve_ops_tool supports printing space-info for each logical pool.
+- Add snapshot clone tool.
+- The curve-ansible adds deploy_monitor.yml.
+
+## optimization
+- Braft completely non-intrusive modifications.
+- Replace HMacSha256 implementation.
+
+## bug fix
+- Fixed the bug that chunkserver_deploy.sh not able to delete the previous fstab record when deploy one.
+- Fixed mds abort when exiting.
+- Fixed curvefs_tool (curve) missing support for password.
+- Fixed the bug that data from Etcd would not placed in LRU cache.
+- Fixed ChunkServer not aborting when read returns an internal error.

Makefile

@@ -0,0 +1,24 @@
+# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
+
+.PHONY: list build install image clean
+
+prefix?= "$(PWD)/projects"
+release?= 0
+only?= "*"
+tag?= "curvebs:unknown"
+tgt_pkg?= ""
+
+list:
+	@bash util/build.sh --list
+
+build:
+	@bash util/build.sh --only=$(only) --release=$(release)
+
+install:
+	@bash util/install.sh --prefix=$(prefix) --only=$(only)
+
+image:
+	@bash util/image.sh $(tag) $(tgt_pkg)
+
+clean:
+	@bazel clean

conf/client.conf

@@ -123,7 +123,7 @@ global.fileIOSplitMaxSizeKB=64
 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
 global.logLevel=0
 # 设置log的路径
-global.logPath=/data/log/curve/
+global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ /curvebs/client/logs __CURVEADM_TEMPLATE__
 # 单元测试情况下
 # logpath=./runlog/
 

conf/mds.conf

@@ -165,6 +165,12 @@ mds.copyset.scatterWidthFloatingPercentage=20
 #
 # curvefs的默认chunk size大小，16MB = 16*1024*1024 = 16777216
 mds.curvefs.defaultChunkSize=16777216
+# curvefs的默认segment size大小，1GB = 1*1024*1024*1024 = 1073741824
+mds.curvefs.defaultSegmentSize=1073741824
+# curvefs的默认最小文件大小，10GB = 10*1024*1024*1024 = 10737418240
+mds.curvefs.minFileLength=10737418240
+# curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
+mds.curvefs.maxFileLength=21990232555520
 
 #
 # chunkseverclient config

curve-ansible/README.md

@@ -176,6 +176,12 @@ ansible-playbook deploy_curve.yml -i server.ini --tags snapshotclone
 ansible-playbook deploy_curve.yml -i server.ini --tags snapshotclone_nginx
 ```
 
+#### 1.10 部署监控
+一键部署监控命令：
+```shell
+ansible-playbook deploy_monitor.yml -i server.ini
+```
+
 ### 2、集群升级
 目前curve的升级流程为先升级mds，后升级chunkserver和快照克隆最后升级client。使用ansible需要指定一台主控机，我们规定主控机为mds节点之一。ansible-playbook同时需要yml文件和inventory文件，yml文件规定了要做哪些操作，inventory指定了机器列表并定义了一些变量。yml文件在curve仓库的curve-ansible目录中，inventory每个环境一份，由用户自行管理。
 
@@ -326,6 +332,7 @@ ansible-playbook stop_nebd_server.yml -i client.ini
 ├── common_tasks                                            # 放置可以复用的代码（可以理解为各种函数）
 │   ├── check_chunkserver.yml                               # 检查chunkserver机器配置
 │   ├── check_cluster_healthy_status.yml                    # 检查集群健康状态
+│   ├── check_docker_exists.yml                             # 检查是否安装了docker
 │   ├── check_if_nbd_exists_in_kernel.yml                   # 检查内核是否有nbd模块
 │   ├── check_mds.yml                                       # 检查mds机器配置
 │   ├── create_dir.yml                                      # 创建目录
@@ -341,7 +348,6 @@ ansible-playbook stop_nebd_server.yml -i client.ini
 │   ├── wait_until_server_down.yml                          # 等待直到server停掉
 │   └── wait_until_server_up.yml                            # 等待直到server起来
 ├── group_vars                                              # 组变量
-│   ├── all.yml                                             # 属于所有host的变量
 │   ├── chunkservers.yml                                    # 属于chunkserver的变量
 │   └── mds.yml                                             # 属于mds的变量
 ├── host_vars                                               # 主机变量, 其中的文件名要合inventory中定义的主机名一致
@@ -394,6 +400,7 @@ ansible-playbook stop_nebd_server.yml -i client.ini
 │   │   │   │   ├── copy_file_to_remote.yml                 # 将文件拷贝到远端
 │   │   │   │   ├── install_curve-chunkserver.yml           # 安装chunkserver
 │   │   │   │   ├── install_curve-mds.yml                   # 安装mds
+│   │   │   │   ├── install_curve-monitor.yml
 │   │   │   │   ├── install_curve-nbd.yml                   # 安装nbd
 │   │   │   │   ├── install_curve-sdk.yml                   # 安装curve-sdk
 │   │   │   │   ├── install_curve-snapshotcloneserver-nginx.yml  # 安装快照克隆使用的Nginx
@@ -434,10 +441,15 @@ ansible-playbook stop_nebd_server.yml -i client.ini
 │   │   ├── templates                                       # 配置文件的模板
 │   │   │   ├── chunkserver.conf.j2
 │   │   │   ├── client.conf.j2
+│   │   │   ├── docker-compose.yml.j2                       # docker config for curve monitor
 │   │   │   ├── etcd.conf.yml.j2
+│   │   │   ├── grafana.ini.j2
 │   │   │   ├── mds.conf.j2
 │   │   │   ├── nebd-client.conf.j2
 │   │   │   ├── nebd-server.conf.j2
+│   │   │   ├── nginx_config.lua.j2
+│   │   │   ├── nginx.conf.j2
+│   │   │   ├── prometheus.yml.j2
 │   │   │   ├── s3.conf.j2
 │   │   │   ├── snapshot_clone_server.conf.j2
 │   │   │   ├── snapshot_tools.conf.j2
@@ -445,6 +457,9 @@ ansible-playbook stop_nebd_server.yml -i client.ini
 │   │   │   └── topo.json.j2
 │   │   └── vars
 │   │       └── main.yml
+│   ├── grafana_settings                                  # set grafana datasource and dashboard
+│   │   └── tasks
+│   │       └── main.yml
 │   ├── stop_service                                       # 停止服务的role
 │   │   ├── tasks
 │   │   │   ├── include
@@ -485,6 +500,7 @@ ansible-playbook stop_nebd_server.yml -i client.ini
 ├── deploy_curve.yml                                        # 一键部署curve集群
 ├── deploy_etcd.yml                                         # 部署etcd
 ├── deploy_mds.yml                                          # 部署mds
+├── deploy_monitor.yml                                      # 部署监控服务
 ├── deploy_nbd.yml                                          # 部署nbd
 ├── deploy_nebd.yml                                         # 部署nebd
 ├── deploy_snapshotcloneserver_nginx.yml                    # 部署快照克隆所用Nginx

curve-ansible/clean_nbd.yml

@@ -0,0 +1,32 @@
+---
+#
+#  Copyright (c) 2021 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# The deploy playbook of Curve
+
+- name: check_ansible_version
+  tags:
+    - always
+  import_playbook: check_ansible_version.yml
+
+- name: clean nbd
+  hosts: client
+  gather_facts: no
+  any_errors_fatal: true
+  become: "{{ sudo_or_not | bool }}"
+  tags:
+    - prepare
+  roles:
+    - { role: clean, module_name: nbd }

curve-ansible/client.ini

@@ -19,6 +19,8 @@ nebd_server_config_path=/etc/nebd/nebd-server.conf
 nebd_data_dir=/data/nebd
 nebd_log_dir=/data/log/nebd
 curve_sdk_log_dir=/data/log/curve
+curvetab_path=/etc/curve
+curve_service_path=/etc/systemd/system
 
 [mds:vars]
 mds_port=6666
@@ -30,3 +32,4 @@ ansible_ssh_port=22
 lib_install_prefix=/usr/local
 bin_install_prefix=/usr
 ansible_connection=local
+wait_service_timeout=20

curve-ansible/common_tasks/check_docker_exists.yml

@@ -0,0 +1,25 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+- name: check docker exist
+  shell: docker -v
+  register: docker_res
+
+- name: fail when docker not exist
+  fail:
+    msg: "Docker should be installed on this machine!"
+  when: docker_res.rc != 0

curve-ansible/deploy_curve.yml

@@ -89,7 +89,7 @@
             tags: ["install_package", "install_tool_package"] }
     - { role: generate_config, template_name: mds.conf, conf_path: "{{ mds_config_path }}",
             tags: ["generate_config", "generage_mds_conf"] }
-    - { role: generate_config, template_name: tools.conf, conf_path: "{{ tool_config_path }}",
+    - { role: generate_config, template_name: tools.conf, conf_path: "{{ curve_ops_tool_config }}",
             tags: ["generate_config", "generage_tools_conf"] }
     - { role: generate_config, template_name: topo.json, conf_path: "{{ topo_file_path }}",
             tags: ["generate_config", "generage_topo_json"] }
@@ -202,6 +202,16 @@
 
 
 ############################## deploy snapshotcloneserver_nginx ##############################
+- name: check docker exist
+  hosts: snapshotclone_nginx
+  any_errors_fatal: true
+  gather_facts: no
+  tags:
+    - snapshotclone_nginx
+    - never
+  tasks:
+    - include_tasks: common_tasks/check_docker_exists.yml
+
 - name: prepare snapshotcloneserver_nginx
   hosts: snapshotclone_nginx
   any_errors_fatal: true

curve-ansible/deploy_monitor.yml

@@ -0,0 +1,74 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+# The deploy playbook of Curve
+
+- name: check_ansible_version
+  tags:
+    - always
+  import_playbook: check_ansible_version.yml
+
+- name: check docker exist
+  hosts: monitor
+  any_errors_fatal: true
+  gather_facts: no
+  tasks:
+    - include_tasks: common_tasks/check_docker_exists.yml
+
+- name: prepare monitor
+  hosts: monitor
+  any_errors_fatal: true
+  gather_facts: no
+  become: yes
+  become_user: "{{ sudo_user }}"
+  become_flags: -iu {{ sudo_user }}
+  roles:
+    - { role: install_package, package_name: curve-monitor, package_version: "{{ monitor_package_version }}",
+            tags: ["install_package", "install_monitor_package"] }
+    - { role: install_package, package_name: curve-tools, package_version: "{{ tool_package_version }}",
+            tags: ["install_package", "install_tool_package"] }
+    - { role: generate_config, template_name: docker-compose.yml, conf_path: "{{ monitor_work_dir }}/docker-compose.yml",
+            tags: ["generate_config", "generage_docker_compose"] }
+    - { role: generate_config, template_name: grafana.ini, conf_path: "{{ monitor_work_dir }}/grafana/grafana.ini",
+            tags: ["generate_config", "generage_grafana_ini"] }
+    - { role: generate_config, template_name: prometheus.yml, conf_path: "{{ monitor_work_dir }}/prometheus/prometheus.yml",
+            tags: ["generate_config", "generage_prometheus_yml"] }
+    - { role: generate_config, template_name: tools.conf, conf_path: "{{ curve_ops_tool_config }}",
+            tags: ["generate_config", "generage_tools_conf"] }
+
+- name: start monitor
+  hosts: monitor
+  any_errors_fatal: true
+  gather_facts: no
+  become: yes
+  become_user: "{{ sudo_user }}"
+  become_flags: -iu {{ sudo_user }}
+  tags:
+    - start
+  roles:
+    - { role: start_service, service_name: monitor }
+
+- name: set grafana data source and dashboards
+  hosts: monitor
+  any_errors_fatal: true
+  gather_facts: no
+  become: yes
+  become_user: "{{ sudo_user }}"
+  become_flags: -iu {{ sudo_user }}
+  tags:
+    - grafana_settings
+  roles:
+    - { role: grafana_settings }

curve-ansible/roles/clean/tasks/include/clean_nbd.yml

@@ -0,0 +1,23 @@
+---
+#
+#  Copyright (c) 2021 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+- name: clean nbd auto start at boot
+  block:
+    - name: disable service
+      shell: sudo systemctl disable map_curve_disk.service
+    - name: remove map_curve_disk.service
+      shell: sudo rm -f {{ curve_service_path }}/map_curve_disk.service && sudo rm -f {{ curvetab_path }}/curvetab

curve-ansible/roles/generate_config/defaults/main.yml

@@ -19,6 +19,9 @@
 curve_root_username: root
 curve_root_password: root_password
 chunk_size: 16777216
+segment_size: 1073741824
+min_file_length: 10737418240
+max_file_length: 21990232555520
 file_expired_time_us: 5000000
 
 # mds配置默认值
@@ -81,7 +84,7 @@ chunkserver_register_timeout: 1000
 chunkserver_heartbeat_interval: 10
 chunkserver_heartbeat_timeout: 5000
 chunkserver_stor_uri: local://./0/
-chunkserver_meta_uri: local://./0/chunkserver_dat
+chunkserver_meta_uri: local://./0/chunkserver.dat
 chunkserver_disk_type: nvme
 chunkserver_snapshot_throttle_throughput_bytes: 20971520
 chunkserver_snapshot_throttle_check_cycles: 4

curve-ansible/roles/generate_config/templates/docker-compose.yml.j2

@@ -0,0 +1,39 @@
+version: '2.0'
+
+services:
+
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus/:/etc/prometheus/:rw
+      - ./prometheus/data:/prometheus:rw
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time={{ monitor_retention_time }}'
+      - '--storage.tsdb.retention.size={{ monitor_retention_size }}'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+      - '--web.listen-address=:{{ prometheus_listen_port }}'
+    network_mode: host
+
+  grafana:
+    image: grafana/grafana
+    depends_on:
+      - prometheus
+    network_mode: host
+    volumes:
+      - ./grafana/data:/var/lib/grafana:rw
+      - ./grafana/grafana.ini:/etc/grafana/grafana.ini:rw
+    environment:
+      - GF_INSTALL_PLUGINS=grafana-piechart-panel
+      - GF_SECURITY_ADMIN_USER={{ grafana_username }}
+      - GF_SECURITY_ADMIN_PASSWORD={{ grafana_password }}
+
+  reporter:
+    image: promoon/reporter:latest
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - /etc/timezone:/etc/timezone:ro
+      - ./grafana/report:/tmp/report:rw
+    network_mode: host

curve-ansible/roles/generate_config/templates/grafana.ini.j2

@@ -0,0 +1,597 @@
+#
+#     Copyright (c) 2020 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+##################### Grafana Configuration Example #####################
+#
+# Everything has defaults so you only need to uncomment things you want to
+# change
+
+# possible values : production, development
+;app_mode = production
+
+# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
+;instance_name = ${HOSTNAME}
+
+#################################### Paths ####################################
+[paths]
+# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
+;data = /var/lib/grafana
+
+# Temporary files in `data` directory older than given duration will be removed
+;temp_data_lifetime = 24h
+
+# Directory where grafana can store logs
+;logs = /var/log/grafana
+
+# Directory where grafana will automatically scan and look for plugins
+;plugins = /var/lib/grafana/plugins
+
+# folder that contains provisioning config files that grafana will apply on startup and while running.
+;provisioning = conf/provisioning
+
+#################################### Server ####################################
+[server]
+# Protocol (http, https, socket)
+;protocol = http
+
+# The ip address to bind to, empty will bind to all interfaces
+;http_addr =
+
+# The http port  to use
+;http_port = {{ grafana_listen_port }}
+
+# The public facing domain name used to access grafana from a browser
+;domain = localhost
+
+# Redirect to correct domain if host header does not match domain
+# Prevents DNS rebinding attacks
+;enforce_domain = false
+
+# The full public facing url you use in browser, used for redirects and emails
+# If you use reverse proxy and sub path specify full url (with sub path)
+;root_url = http://localhost:3000
+
+# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons.
+;serve_from_sub_path = false
+
+# Log web requests
+;router_logging = false
+
+# the path relative working path
+;static_root_path = public
+
+# enable gzip
+;enable_gzip = false
+
+# https certs & key file
+;cert_file =
+;cert_key =
+
+# Unix socket path
+;socket =
+
+#################################### Database ####################################
+[database]
+# You can configure the database connection by specifying type, host, name, user and password
+# as separate properties or as on string using the url properties.
+
+# Either "mysql", "postgres" or "sqlite3", it's your choice
+;type = sqlite3
+;host = 127.0.0.1:3306
+;name = grafana
+;user = root
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+;password =
+
+# Use either URL or the previous fields to configure the database
+# Example: mysql://user:secret@host:port/database
+;url =
+
+# For "postgres" only, either "disable", "require" or "verify-full"
+;ssl_mode = disable
+
+# For "sqlite3" only, path relative to data_path setting
+;path = grafana.db
+
+# Max idle conn setting default is 2
+;max_idle_conn = 2
+
+# Max conn setting default is 0 (mean not set)
+;max_open_conn =
+
+# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
+;conn_max_lifetime = 14400
+
+# Set to true to log the sql calls and execution times.
+;log_queries =
+
+# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared)
+;cache_mode = private
+
+#################################### Cache server #############################
+[remote_cache]
+# Either "redis", "memcached" or "database" default is "database"
+;type = database
+
+# cache connectionstring options
+# database: will use Grafana primary database.
+# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0`. Only addr is required.
+# memcache: 127.0.0.1:11211
+;connstr =
+
+#################################### Data proxy ###########################
+[dataproxy]
+
+# This enables data proxy logging, default is false
+;logging = false
+
+# How long the data proxy should wait before timing out default is 30 (seconds)
+;timeout = 30
+
+# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
+;send_user_header = false
+
+#################################### Analytics ####################################
+[analytics]
+# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
+# No ip addresses are being tracked, only simple counters to track
+# running instances, dashboard and error counts. It is very helpful to us.
+# Change this option to false to disable reporting.
+;reporting_enabled = true
+
+# Set to false to disable all checks to https://grafana.net
+# for new vesions (grafana itself and plugins), check is used
+# in some UI views to notify that grafana or plugin update exists
+# This option does not cause any auto updates, nor send any information
+# only a GET request to http://grafana.com to get latest versions
+;check_for_updates = true
+
+# Google Analytics universal tracking code, only enabled if you specify an id here
+;google_analytics_ua_id =
+
+# Google Tag Manager ID, only enabled if you specify an id here
+;google_tag_manager_id =
+
+#################################### Security ####################################
+[security]
+# default admin user, created on startup
+;admin_user = admin
+
+# default admin password, can be changed before first start of grafana,  or in profile settings
+;admin_password = admin
+
+# used for signing
+;secret_key = SW2YcwTIb9zpOOhoPsMm
+
+# disable gravatar profile images
+;disable_gravatar = false
+
+# data source proxy whitelist (ip_or_domain:port separated by spaces)
+;data_source_proxy_whitelist =
+
+# disable protection against brute force login attempts
+;disable_brute_force_login_protection = false
+
+# set to true if you host Grafana behind HTTPS. default is false.
+;cookie_secure = false
+
+# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict" and "none"
+;cookie_samesite = lax
+
+# set to true if you want to allow browsers to render Grafana in a <frame>, <iframe>, <embed> or <object>. default is false.
+;allow_embedding = false
+
+# Set to true if you want to enable http strict transport security (HSTS) response header.
+# This is only sent when HTTPS is enabled in this configuration.
+# HSTS tells browsers that the site should only be accessed using HTTPS.
+# The default version will change to true in the next minor release, 6.3.
+;strict_transport_security = false
+
+# Sets how long a browser should cache HSTS. Only applied if strict_transport_security is enabled.
+;strict_transport_security_max_age_seconds = 86400
+
+# Set to true if to enable HSTS preloading option. Only applied if strict_transport_security is enabled.
+;strict_transport_security_preload = false
+
+# Set to true if to enable the HSTS includeSubDomains option. Only applied if strict_transport_security is enabled.
+;strict_transport_security_subdomains = false
+
+# Set to true to enable the X-Content-Type-Options response header.
+# The X-Content-Type-Options response HTTP header is a marker used by the server to indicate that the MIME types advertised
+# in the Content-Type headers should not be changed and be followed. The default will change to true in the next minor release, 6.3.
+;x_content_type_options = false
+
+# Set to true to enable the X-XSS-Protection header, which tells browsers to stop pages from loading
+# when they detect reflected cross-site scripting (XSS) attacks. The default will change to true in the next minor release, 6.3.
+;x_xss_protection = false
+
+#################################### Snapshots ###########################
+[snapshots]
+# snapshot sharing options
+;external_enabled = true
+;external_snapshot_url = https://snapshots-origin.raintank.io
+;external_snapshot_name = Publish to snapshot.raintank.io
+
+# remove expired snapshot
+;snapshot_remove_expired = true
+
+#################################### Dashboards History ##################
+[dashboards]
+# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
+;versions_to_keep = 20
+
+#################################### Users ###############################
+[users]
+# disable user signup / registration
+;allow_sign_up = true
+
+# Allow non admin users to create organizations
+;allow_org_create = true
+
+# Set to true to automatically assign new users to the default organization (id 1)
+;auto_assign_org = true
+
+# Default role new users will be automatically assigned (if disabled above is set to true)
+;auto_assign_org_role = Viewer
+
+# Background text for the user field on the login page
+;login_hint = email or username
+;password_hint = password
+
+# Default UI theme ("dark" or "light")
+;default_theme = dark
+
+# External user management, these options affect the organization users view
+;external_manage_link_url =
+;external_manage_link_name =
+;external_manage_info =
+
+# Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
+;viewers_can_edit = false
+
+# Editors can administrate dashboard, folders and teams they create
+;editors_can_admin = false
+
+[auth]
+# Login cookie name
+;login_cookie_name = grafana_session
+
+# The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days,
+;login_maximum_inactive_lifetime_days = 7
+
+# The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days.
+;login_maximum_lifetime_days = 30
+
+# How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes.
+;token_rotation_interval_minutes = 10
+
+# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
+;disable_login_form = false
+
+# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false
+;disable_signout_menu = false
+
+# URL to redirect the user to after sign out
+;signout_redirect_url =
+
+# Set to true to attempt login with OAuth automatically, skipping the login screen.
+# This setting is ignored if multiple OAuth providers are configured.
+;oauth_auto_login = false
+
+#################################### Anonymous Auth ######################
+[auth.anonymous]
+# enable anonymous access
+;enabled = false
+
+# specify organization name that should be used for unauthenticated users
+;org_name = Main Org.
+
+# specify role for unauthenticated users
+;org_role = Viewer
+
+#################################### Github Auth ##########################
+[auth.github]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email,read:org
+;auth_url = https://github.com/login/oauth/authorize
+;token_url = https://github.com/login/oauth/access_token
+;api_url = https://api.github.com/user
+;team_ids =
+;allowed_organizations =
+
+#################################### Google Auth ##########################
+[auth.google]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_client_id
+;client_secret = some_client_secret
+;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
+;auth_url = https://accounts.google.com/o/oauth2/auth
+;token_url = https://accounts.google.com/o/oauth2/token
+;api_url = https://www.googleapis.com/oauth2/v1/userinfo
+;allowed_domains =
+
+#################################### Generic OAuth ##########################
+[auth.generic_oauth]
+;enabled = false
+;name = OAuth
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email,read:org
+;auth_url = https://foo.bar/login/oauth/authorize
+;token_url = https://foo.bar/login/oauth/access_token
+;api_url = https://foo.bar/user
+;team_ids =
+;allowed_organizations =
+;tls_skip_verify_insecure = false
+;tls_client_cert =
+;tls_client_key =
+;tls_client_ca =
+
+; Set to true to enable sending client_id and client_secret via POST body instead of Basic authentication HTTP header
+; This might be required if the OAuth provider is not RFC6749 compliant, only supporting credentials passed via POST payload
+;send_client_credentials_via_post = false
+
+#################################### SAML Auth ###########################
+;[auth.saml] # Enterprise only
+;enabled = false
+;private_key =
+;private_key_path =
+;certificate =
+;certificate_path =
+;idp_metadata =
+;idp_metadata_path =
+;idp_metadata_url =
+;max_issue_delay = 90s
+;metadata_valid_duration = 48h
+
+#################################### Grafana.com Auth ####################
+[auth.grafana_com]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email
+;allowed_organizations =
+
+#################################### Auth Proxy ##########################
+[auth.proxy]
+;enabled = false
+;header_name = X-WEBAUTH-USER
+;header_property = username
+;auto_sign_up = true
+;ldap_sync_ttl = 60
+;whitelist = 192.168.1.1, 192.168.2.1
+;headers = Email:X-User-Email, Name:X-User-Name
+
+#################################### Basic Auth ##########################
+[auth.basic]
+;enabled = true
+
+#################################### Auth LDAP ##########################
+[auth.ldap]
+;enabled = false
+;config_file = /etc/grafana/ldap.toml
+;allow_sign_up = true
+
+# LDAP backround sync (Enterprise only)
+# At 1 am every day
+;sync_cron = "0 0 1 * * *"
+;active_sync_enabled = true
+
+#################################### SMTP / Emailing ##########################
+[smtp]
+;enabled = false
+;host = localhost:25
+;user =
+# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
+;password =
+;cert_file =
+;key_file =
+;skip_verify = false
+;from_address = admin@grafana.localhost
+;from_name = Grafana
+# EHLO identity in SMTP dialog (defaults to instance_name)
+;ehlo_identity = dashboard.example.com
+
+[emails]
+;welcome_email_on_sign_up = false
+
+#################################### Logging ##########################
+[log]
+# Either "console", "file", "syslog". Default is console and  file
+# Use space to separate multiple modes, e.g. "console file"
+;mode = console file
+
+# Either "debug", "info", "warn", "error", "critical", default is "info"
+;level = info
+
+# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
+;filters =
+
+# For "console" mode only
+[log.console]
+;level =
+
+# log line format, valid options are text, console and json
+;format = console
+
+# For "file" mode only
+[log.file]
+;level =
+
+# log line format, valid options are text, console and json
+;format = text
+
+# This enables automated log rotate(switch of following options), default is true
+;log_rotate = true
+
+# Max line number of single file, default is 1000000
+;max_lines = 1000000
+
+# Max size shift of single file, default is 28 means 1 << 28, 256MB
+;max_size_shift = 28
+
+# Segment log daily, default is true
+;daily_rotate = true
+
+# Expired days of log file(delete after max days), default is 7
+;max_days = 7
+
+[log.syslog]
+;level =
+
+# log line format, valid options are text, console and json
+;format = text
+
+# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
+;network =
+;address =
+
+# Syslog facility. user, daemon and local0 through local7 are valid.
+;facility =
+
+# Syslog tag. By default, the process' argv[0] is used.
+;tag =
+
+#################################### Alerting ############################
+[alerting]
+# Disable alerting engine & UI features
+;enabled = true
+# Makes it possible to turn off alert rule execution but alerting UI is visible
+;execute_alerts = true
+
+# Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state)
+;error_or_timeout = alerting
+
+# Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
+;nodata_or_nullvalues = no_data
+
+# Alert notifications can include images, but rendering many images at the same time can overload the server
+# This limit will protect the server from render overloading and make sure notifications are sent out quickly
+;concurrent_render_limit = 5
+
+
+# Default setting for alert calculation timeout. Default value is 30
+;evaluation_timeout_seconds = 30
+
+# Default setting for alert notification timeout. Default value is 30
+;notification_timeout_seconds = 30
+
+# Default setting for max attempts to sending alert notifications. Default value is 3
+;max_attempts = 3
+
+#################################### Explore #############################
+[explore]
+# Enable the Explore section
+;enabled = true
+
+#################################### Internal Grafana Metrics ##########################
+# Metrics available at HTTP API Url /metrics
+[metrics]
+# Disable / Enable internal metrics
+;enabled           = true
+
+# Publish interval
+;interval_seconds  = 10
+
+# Send internal metrics to Graphite
+[metrics.graphite]
+# Enable by setting the address setting (ex localhost:2003)
+;address =
+;prefix = prod.grafana.%(instance_name)s.
+
+#################################### Distributed tracing ############
+[tracing.jaeger]
+# Enable by setting the address sending traces to jaeger (ex localhost:6831)
+;address = localhost:6831
+# Tag that will always be included in when creating new spans. ex (tag1:value1,tag2:value2)
+;always_included_tag = tag1:value1
+# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote
+;sampler_type = const
+# jaeger samplerconfig param
+# for "const" sampler, 0 or 1 for always false/true respectively
+# for "probabilistic" sampler, a probability between 0 and 1
+# for "rateLimiting" sampler, the number of spans per second
+# for "remote" sampler, param is the same as for "probabilistic"
+# and indicates the initial sampling rate before the actual one
+# is received from the mothership
+;sampler_param = 1
+# Whether or not to use Zipkin propagation (x-b3- HTTP headers).
+;zipkin_propagation = false
+# Setting this to true disables shared RPC spans.
+# Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
+;disable_shared_zipkin_spans = false
+
+#################################### Grafana.com integration  ##########################
+# Url used to import dashboards directly from Grafana.com
+[grafana_com]
+;url = https://grafana.com
+
+#################################### External image storage ##########################
+[external_image_storage]
+# Used for uploading images to public servers so they can be included in slack/email messages.
+# you can choose between (s3, webdav, gcs, azure_blob, local)
+;provider =
+
+[external_image_storage.s3]
+;bucket =
+;region =
+;path =
+;access_key =
+;secret_key =
+
+[external_image_storage.webdav]
+;url =
+;public_url =
+;username =
+;password =
+
+[external_image_storage.gcs]
+;key_file =
+;bucket =
+;path =
+
+[external_image_storage.azure_blob]
+;account_name =
+;account_key =
+;container_name =
+
+[external_image_storage.local]
+# does not require any configuration
+
+[rendering]
+# Options to configure external image rendering server like https://github.com/grafana/grafana-image-renderer
+;server_url =
+;callback_url =
+
+[enterprise]
+# Path to a valid Grafana Enterprise license.jwt file
+;license_path =
+
+[panels]
+# If set to true Grafana will allow script tags in text panels. Not recommended as it enable XSS vulnerabilities.
+;disable_sanitize_html = false
+
+[plugins]
+;enable_alpha = false
+;app_tls_skip_verify_insecure = false

curve-ansible/roles/generate_config/templates/mds.conf.j2

@@ -173,6 +173,12 @@ mds.copyset.scatterWidthFloatingPercentage={{ mds_copyset_scatterwidth_floating_
 #
 # curvefs的默认chunk size大小，16MB = 16*1024*1024 = 16777216
 mds.curvefs.defaultChunkSize={{ chunk_size }}
+# curvefs的默认segment size大小，1GB = 1*1024*1024*1024 = 1073741824
+mds.curvefs.defaultSegmentSize={{ segment_size }}
+# curvefs的默认最小文件大小，10GB = 10*1024*1024*1024 = 10737418240
+mds.curvefs.minFileLength={{ min_file_length }}
+# curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
+mds.curvefs.maxFileLength={{ max_file_length }}
 
 #
 # chunkseverclient config

curve-ansible/roles/generate_config/templates/prometheus.yml.j2

@@ -0,0 +1,53 @@
+#
+#     Copyright (c) 2020 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+# my global config
+global:
+  scrape_interval:     {{ prometheus_scrape_interval }} # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: {{ prometheus_evaluation_interval }} # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+      # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+    - targets: ['localhost:9090']
+
+  - job_name: 'curve_metrics'
+
+    file_sd_configs:
+    - files: ['*.json']
+

curve-ansible/roles/generate_config/templates/tools.conf.j2

@@ -7,7 +7,7 @@
 {% endfor -%}
 mdsAddr={{ mds_address | join(',') }}
 # mds dummy port
-mdsDummyPort={{ mds_dummy_port }}
+mdsDummyPort={{ hostvars[groups.mds[0]].mds_dummy_port }}
 # 发送rpc的超时时间
 rpcTimeout={{ tool_rpc_timeout }}
 # rpc重试次数

curve-ansible/roles/grafana_settings/tasks/main.yml

@@ -0,0 +1,36 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+- name: set grafana datasource
+  grafana_datasource:
+    grafana_url: "http://localhost:{{ grafana_listen_port }}"
+    grafana_user: "{{ grafana_username }}"
+    grafana_password: "{{ grafana_password }}"
+    access: proxy
+    ds_type: prometheus
+    is_default: true
+    name: Prometheus
+    org_id: 1
+    url: "localhost:{{ prometheus_listen_port }}"
+
+- name: set grafana dashboard
+  shell: >
+   cat {{ item }} | jq '. * {overwrite: true, dashboard: {id: null}}' |
+   curl -s -X POST -H "Content-Type: application/json" --user {{ grafana_username }}:{{ grafana_password }}
+   localhost:{{ grafana_listen_port }}/api/dashboards/db -d @-
+  with_fileglob:
+    - "{{ monitor_work_dir }}/grafana/dashboards/*"

curve-ansible/roles/install_package/defaults/main.yml

@@ -29,6 +29,9 @@ local_nebd_package_path: ../../nebd-package
 local_nbd_package_path: ../../nbd-package
 snapshot_clone_server_log_dir: /data/log/curve/snapshotclone
 local_snapshotcloneserver_package_path: ../curve-snapshotcloneserver
+local_monitor_package_path: ../../curve-monitor
+curvetab_path: /etc/curve
+curve_service_path: /etc/systemd/system
 
 local_snapshotcloneserver_nginx_package_path: ../curve-nginx
 snapshotcloneserver_nginx_dir: /etc/curve/nginx

curve-ansible/roles/install_package/tasks/include/install_curve-monitor.yml

@@ -0,0 +1,41 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+- name: pull prom/prometheus:latest
+  shell: sudo docker image pull prom/prometheus:latest
+
+- name: pull grafana/grafana:latest
+  shell: sudo docker image pull grafana/grafana:latest
+
+- name: promoon/reporter:latest
+  shell: sudo docker image pull promoon/reporter:latest
+
+- name: install jq
+  vars:
+    apt_lib_name: jq
+    yum_lib_name: jq
+  include_tasks: common_tasks/install_with_yum_apt.yml
+
+- name: install monitor work space
+  vars:
+    remote_dir_name: "{{ monitor_work_dir }}"
+    local_file_path: "{{ local_monitor_package_path }}/"
+    file_mode: 0777
+  include_tasks: copy_file_to_remote.yml
+
+- name: generate curve-monitor.sh
+  template: src=curve-monitor.sh.j2 dest={{ deploy_dir }}/curve-monitor.sh mode=0755

curve-ansible/roles/install_package/tasks/include/install_curve-nbd.yml

@@ -33,3 +33,34 @@
     local_file_path: "{{ local_nbd_package_path }}/bin/"
     file_mode: 0755
   include_tasks: copy_file_to_remote.yml
+
+- name: check curvetab exist
+  stat:
+    path: "{{ curvetab_path }}/curvetab"
+  register: file_status
+
+- name: install curvetab if not exist
+  vars:
+    remote_dir_name: "{{ curvetab_path }}"
+    local_file_path: "{{ local_nbd_package_path }}/etc/curvetab"
+    file_mode: 0755
+  include_tasks: copy_file_to_remote.yml
+  when: not file_status.stat.exists
+
+- name: install map_curve_disk.sh
+  vars:
+    remote_dir_name: "{{ curve_bin_dir }}"
+    local_file_path: "{{ local_nbd_package_path }}/bin/map_curve_disk.sh"
+    file_mode: 0755
+  include_tasks: copy_file_to_remote.yml
+
+- name: add nbd auto start at boot
+  block:
+    - name: install map_curve_disk.service
+      vars:
+        remote_dir_name: "{{ curve_service_path }}"
+        local_file_path: "{{ local_nbd_package_path }}/etc/map_curve_disk.service"
+        file_mode: 0755
+      include_tasks: copy_file_to_remote.yml
+    - name: enable service
+      shell: sudo systemctl daemon-reload && sudo systemctl enable map_curve_disk.service && sudo systemctl start map_curve_disk.service

curve-ansible/roles/install_package/templates/curve-monitor.sh.j2

@@ -0,0 +1,66 @@
+#!/bin/sh
+
+#sh update_dashboard.sh
+#echo "update dashboards success!"
+
+WORKDIR={{ monitor_work_dir }}
+
+if [ ! -d $WORKDIR ]; then
+  echo "${WORKDIR} not exists"
+  exit 1
+fi
+
+cd $WORKDIR
+chmod -R 777 prometheus
+chmod -R 777 grafana
+
+start() {
+    echo "==========start==========="
+
+    echo "" > monitor.log
+
+    stdbuf -oL python target_json.py >> monitor.log 2>&1 &
+    echo "start prometheus targets service success!"
+
+    docker-compose up >> monitor.log 2>&1 &
+    echo "start metric system success!"
+}
+
+stop() {
+    echo "===========stop============"
+
+    docker-compose down
+
+    ID=`(ps -ef | grep "target_json.py"| grep -v "grep") | awk '{print $2}'`
+    for id in $ID
+    do
+        kill -9 $id
+	echo "killed $id"
+    done
+}
+
+restart() {
+    stop
+    echo "sleeping........."
+    sleep 3
+    start
+}
+
+case "$1" in
+    'start')
+        start
+        ;;
+    'stop')
+        stop
+        ;;
+    'status')
+        status
+        ;;
+    'restart')
+        restart
+        ;;
+    *)
+    echo "usage: $0 {start|stop|restart}"
+    exit 1
+        ;;
+    esac

curve-ansible/roles/install_package/templates/nebd-daemon.j2

@@ -37,7 +37,7 @@ daemonLog=${baseLogPath}/nebd-server-daemon.log
 consoleLog=${baseLogPath}/nebd-server-console.log
 
 function get_nebd_server_pid() {
-    local tmp=`ps -ef | grep nebd-server | grep -v grep | grep -v daemon | awk '{print $2}'`
+    local tmp=`pidof nebd-server`
     echo $tmp
 }
 
@@ -103,11 +103,11 @@ function start() {
         exit 1
     fi
 
-    jemallocpath={{ jemalloc_path }}
     # 检查jemalloc库文件
-    if [ ! -f ${jemallocpath} ]
+    jemallocpath=`find /usr/ -name "libjemalloc.so*" -print -quit`
+    if [ "${jemallocpath}" = "" ]
     then
-        echo "Not found jemalloc library, Path is ${jemallocpath}"
+        echo "Not found jemalloc library in /usr/"
         exit 1
     fi
 
@@ -146,6 +146,27 @@ function stop() {
     then
         echo "stop may not success!"
     else
+        # wait 3s
+        retry_times=0
+        while [ $retry_times -le 3 ]
+        do
+            ((retry_times=$retry_times+1))
+            pidof nebd-server > /dev/null 2>&1
+            if [ $? -eq 0 ]
+            then
+                sleep 1
+            else
+                break
+            fi
+        done
+
+        pidof nebd-server > /dev/null 2>&1
+        if [ $? -eq 0 ]
+        then
+            echo "nebd-server still exists after 3s, now kill it with SIGKILL"
+            kill -9 `pidof nebd-server`
+        fi
+
         echo "nebd-server exit success!"
         echo "daemon exit success!"
     fi
@@ -170,7 +191,7 @@ function restart() {
 
     if [ $old_pid == $new_pid ]
     then
-        echo "restart nebd-server may failed, try kill -9 nebd-server"
+        echo "restart nebd-server may failed, now kill it with SIGKILL"
         kill -9 $old_pid
     fi
 }

curve-ansible/roles/restart_service/defaults/main.yml

@@ -0,0 +1,18 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+check_health: false

curve-ansible/roles/restart_service/tasks/include/restart_etcd.yml

@@ -20,5 +20,4 @@
     ch_dir: "{{ deploy_dir }}"
     daemon_script: ./etcd-daemon.sh
     service_port: "{{ etcd_listen_client_port }}"
-    check_health: true
   include_tasks: restart_by_daemon.yml

curve-ansible/roles/restart_service/tasks/include/restart_mds.yml

@@ -32,6 +32,5 @@
     ch_dir: "{{ deploy_dir }}"
     daemon_script: ./mds-daemon.sh
     service_port: "{{ mds_dummy_port }}"
-    check_health: true
   include_tasks: restart_by_daemon.yml
   when: need_restart

curve-ansible/roles/restart_service/tasks/include/restart_nebd.yml

@@ -23,7 +23,7 @@
 
 - name: get nebd version
   vars:
-    service_port: "{{ nebd_port }}"
+    metric_port: "{{ nebd_port }}"
     nebd_version:
   include_tasks: common_tasks/get_nebd_version_from_metric.yml
 
@@ -41,7 +41,6 @@
   vars:
     ch_dir: "{{ deploy_dir }}"
     daemon_script: nebd-daemon
-    ping_port: "{{ nebd_port }}"
-    check_health: false
+    service_port: "{{ nebd_port }}"
   include_tasks: restart_by_daemon.yml
   when: need_restart

curve-ansible/roles/restart_service/tasks/include/restart_snapshotclone.yml

@@ -32,6 +32,5 @@
     ch_dir: "{{ deploy_dir }}"
     daemon_script: ./snapshot-daemon.sh
     service_port: "{{ snapshot_dummy_port }}"
-    check_health: true
   include_tasks: restart_by_daemon.yml
   when: need_restart

curve-ansible/roles/start_service/tasks/include/start_by_daemon.yml

@@ -24,6 +24,7 @@
   shell: chdir={{ ch_dir }} {{ sudo }} {{ daemon_script }} start
 
 - name: wait until server is up
-  ping_port: "{{ service_port }}"
-  wait_timeout: "{{ wait_service_timeout }}"
+  vars:
+    ping_port: "{{ service_port }}"
+    wait_timeout: "{{ wait_service_timeout }}"
   include_tasks: common_tasks/wait_until_server_up.yml

curve-ansible/roles/start_service/tasks/include/start_monitor.yml

@@ -0,0 +1,31 @@
+---
+#
+#  Copyright (c) 2020 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+- name: start monitor
+  shell:  chdir={{ deploy_dir }} sudo ./curve-monitor.sh start
+
+- name: wait until prometheus is up
+  vars:
+    ping_port: "{{ prometheus_listen_port }}"
+    wait_timeout: "{{ wait_service_timeout }}"
+  include_tasks: common_tasks/wait_until_server_up.yml
+
+- name: wait until grafana is up
+  vars:
+    ping_port: "{{ grafana_listen_port }}"
+    wait_timeout: "{{ wait_service_timeout }}"
+  include_tasks: common_tasks/wait_until_server_up.yml

curve-ansible/rolling_update_curve.yml

@@ -82,7 +82,7 @@
             tags: ["install_package", "install_tool_package"] }
     - { role: generate_config, template_name: mds.conf, conf_path: "{{ mds_config_path }}",
             tags: ["generate_config", "generage_mds_conf"] }
-    - { role: generate_config, template_name: tools.conf, conf_path: "{{ tool_config_path }}",
+    - { role: generate_config, template_name: tools.conf, conf_path: "{{ curve_ops_tool_config }}",
             tags: ["generate_config", "generage_tools_conf"] }
     - { role: generate_config, template_name: topo.json, conf_path: "{{ topo_file_path }}",
             tags: ["generate_config", "generage_topo_json"] }
@@ -142,7 +142,7 @@
     - { role: generate_config, template_name: chunkserver.conf, conf_path: "{{ chunkserver_conf_path }}",
                 tags: ["generate_config", "generage_chunkserver_conf"] }
     - { role: generate_config, template_name: client.conf, conf_path: "{{ chunkserver_client_config_path }}",
-                tags: ["generate_config", "generage_chunkserver_conf"] }
+                tags: ["generate_config", "generage_chunkserver_client_conf"] }
     - { role: generate_config, template_name: s3.conf, conf_path: "{{ chunkserver_s3_config_path }}",
                 tags: ["generate_config", "generage_s3_conf"] }
 

curve-ansible/rolling_update_nebd.yml

@@ -21,28 +21,19 @@
     - always
   import_playbook: check_ansible_version.yml
 
-- name: update config
+- name: prepare nebd
   hosts: client
   gather_facts: no
   any_errors_fatal: true
   tags:
-    - update_config
+    - prepare
   roles:
+    - { role: install_package, package_name: nebd, package_version: "{{ nebd_package_version }}",
+            tags: ["install_package"] }
     - { role: generate_config, template_name: client.conf, conf_path: "{{ client_config_path }}" }
     - { role: generate_config, template_name: nebd-client.conf, conf_path: "{{ nebd_client_config_path }}" }
     - { role: generate_config, template_name: nebd-server.conf, conf_path: "{{ nebd_server_config_path }}" }
 
-- name: update package
-  hosts: client
-  gather_facts: no
-  any_errors_fatal: true
-  tags:
-    - update_package
-  tasks:
-    - vars:
-        update_component: "nebd={{ nebd_package_version }}"
-      include_tasks: "common_tasks/update_package.yml"
-
 - name: restart nebd server
   hosts: client
   gather_facts: no

curve-ansible/server.ini

@@ -14,6 +14,9 @@ localhost ansible_ssh_host=127.0.0.1
 [chunkservers]
 localhost ansible_ssh_host=127.0.0.1
 
+[monitor]
+localhost ansible_ssh_host=127.0.0.1
+
 [mds:vars]
 mds_dummy_port=6667
 mds_port=6666
@@ -24,7 +27,6 @@ tool_package_version="0.0.6.1+160be351"
 # 启动命令是否用sudo
 mds_need_sudo=true
 mds_config_path=/etc/curve/mds.conf
-tool_config_path=/etc/curve/tools.conf
 mds_log_dir=/data/log/curve/mds
 topo_file_path=/etc/curve/topo.json
 
@@ -49,7 +51,6 @@ snapshot_need_sudo=true
 snapshot_config_path=/etc/curve/snapshot_clone_server.conf
 snap_s3_config_path=/etc/curve/s3.conf
 snap_client_config_path=/etc/curve/snap_client.conf
-snap_tool_config_path=/etc/curve/snapshot_tools.conf
 client_register_to_mds=false
 client_chunkserver_op_max_retry=50
 client_chunkserver_max_rpc_timeout_ms=16000
@@ -98,11 +99,24 @@ nginx_docker_external_port=5555
 snapshot_nginx_name=curve-snapshotclone-nginx
 snapshot_nginx_vip=127.0.0.1
 
+[monitor:vars]
+monitor_work_dir=/etc/curve/monitor
+monitor_retention_time=7d
+monitor_retention_size=256GB
+prometheus_listen_port=9090
+prometheus_scrape_interval=3s
+prometheus_evaluation_interval=15s
+grafana_username=curve
+grafana_password=123
+grafana_listen_port=3000
+monitor_package_version="0.0.6.1.1+7af4d6a4"
+
 [all:vars]
 need_confirm=true
 curve_ops_tool_config=/etc/curve/tools.conf
+snap_tool_config_path=/etc/curve/snapshot_tools.conf
 need_update_config=true
-wait_service_timeout=10
+wait_service_timeout=20
 sudo_user=curve
 deploy_dir=/home/curve
 s3_ak=""
@@ -115,3 +129,4 @@ curve_root_password=root_password
 lib_install_prefix=/usr/local
 bin_install_prefix=/usr
 ansible_connection=local
+check_health=false

docker/Dockerfile

@@ -0,0 +1,11 @@
+FROM opencurvedocker/curvebs-base:debian9
+ENV TZ=Asia/Shanghai
+RUN mkdir -p /curvebs /etc/curve /etc/nebd /core /curve/init.d/
+COPY curvebs /curvebs
+COPY entrypoint.sh /
+COPY curvebs/tools/sbin/curve_ops_tool curvebs/nbd/sbin/curve-nbd /usr/bin/
+RUN ([ -f /curvebs/curvebs_tgt.deb ] && dpkg -i /curvebs/curvebs_tgt.deb && rm /curvebs/curvebs_tgt.deb) \
+    && mv /curvebs/lib /usr/lib/nebd \
+    && chmod a+x /entrypoint.sh \
+    && apt update && apt install -y kmod
+ENTRYPOINT ["/entrypoint.sh"]

docker/entrypoint.sh

@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
+
+############################  GLOBAL VARIABLES
+g_role=""
+g_args=""
+g_prefix=""
+g_preexec="/curvebs/tools-v2/sbin/daemon"
+g_binary=""
+g_start_args=""
+
+############################  BASIC FUNCTIONS
+function msg() {
+    printf '%b' "$1" >&2
+}
+
+function success() {
+    msg "\33[32m[✔]\33[0m ${1}${2}"
+}
+
+function die() {
+    msg "\33[31m[✘]\33[0m ${1}${2}"
+    exit 1
+}
+
+############################ FUNCTIONS
+function usage () {
+    cat << _EOC_
+Usage:
+    entrypoint.sh --role=ROLE
+    entrypoint.sh --role=ROLE --args=ARGS
+
+Examples:
+    entrypoint.sh --role=etcd
+    entrypoint.sh --role=client --args="-o default_permissions"
+_EOC_
+}
+
+function get_options() {
+    local long_opts="role:,args:,help"
+    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
+    eval set -- "${args}"
+    while true
+    do
+        case "$1" in
+            -r|--role)
+                g_role=$2
+                shift 2
+                ;;
+            -a|--args)
+                g_args=$2
+                shift 2
+                ;;
+            -h)
+                usage
+                exit 1
+                ;;
+            --)
+                shift
+                break
+                ;;
+            *)
+                exit 1
+                ;;
+        esac
+    done
+}
+
+function prepare() {
+    g_prefix="/curvebs/$g_role"
+    conf_path="$g_prefix/conf/$g_role.conf"
+
+    case $g_role in
+        etcd)
+            g_binary="$g_prefix/sbin/etcd"
+            g_start_args="--config-file $conf_path"
+            ;;
+        mds)
+            g_binary="$g_prefix/sbin/curvebs-mds"
+            g_start_args="--confPath $conf_path"
+            ;;
+        chunkserver)
+            g_binary="$g_prefix/sbin/curvebs-chunkserver"
+            g_start_args="--conf=$conf_path"
+            ;;
+        snapshotclone)
+            g_preexec="/usr/sbin/nginx -c $g_prefix/conf/nginx.conf"
+            g_binary="$g_prefix/sbin/curvebs-snapshotclone"
+            g_start_args="--conf=$conf_path"
+            ;;
+        nebd)
+            g_binary="$g_prefix/sbin/nebd-server"
+            g_start_args="-confPath=$g_prefix/conf/nebd-server.conf -log_dir=$g_prefix/logs"
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+
+    if [ "$g_args" != "" ]; then
+        g_start_args=$g_args
+    fi
+}
+
+function create_directory() {
+    chmod 700 "$g_prefix/data"
+    if [ "$g_role" == "etcd" ]; then
+        mkdir -p "$g_prefix/data/wal"
+    elif [ "$g_role" == "client" ]; then
+        mkdir -p "$g_prefix/mnt"
+    fi
+}
+
+function main() {
+    get_options "$@"
+
+    prepare
+    create_directory
+    [[ $(command -v crontab) ]] && cron
+    [[ ! -z $g_preexec ]] && $g_preexec &
+    exec $g_binary $g_start_args
+}
+
+############################  MAIN()
+main "$@"

docs/cn/deploy.md

@@ -65,6 +65,7 @@ ansible是一款自动化运维工具，curve-ansible 是基于 ansible playbook
    $ pip install --upgrade pip
    $ pip install --upgrade setuptools
    ```
+4. 确保源里面有以下几个包：net-tools, openssl>=1.1.1, perf, perl-podlators, make
 
 ##### Debian9环境准备具体步骤
 1. root用户登录机器，创建curve用户
@@ -88,6 +89,7 @@ ansible是一款自动化运维工具，curve-ansible 是基于 ansible playbook
    $ pip install --upgrade pip
    $ pip install --upgrade setuptools
    ```
+4. 确保源里面有以下几个包：net-tools, openssl>=1.1.1, perf, perl-podlators, make
 
 #### 实施部署
 
@@ -153,7 +155,10 @@ ansible是一款自动化运维工具，curve-ansible 是基于 ansible playbook
       curve-nbd list-mapped
    ```
 
-
+9. 部署监控（可选）。关于监控体系的文档见[curve监控体系文档](./monitor.md)，监控的部署需要用到docker编排，首先请先确保安装了docker和docker-compose。其次，需要在curve包的同一级目录下下载并解压curve-monitor的tar包。
+   ```
+   ansible-playbook deploy_monitor.yml -i server.ini
+   ```
 
 ## 多机部署
 
@@ -196,7 +201,7 @@ ansible是一款自动化运维工具，curve-ansible 是基于 ansible playbook
    $ su curve  # 切换到curve用户
    $ sudo ls  # 测试sudo是否正确配置
    ```
-3. 检查其他依赖，未安装的需要手动安装：net-tools, openssl-1.1.1, perf, perl-podlators, make, gcc6.1, libstdc++.so.6.22
+3. 确保源里面有以下几个包：net-tools, openssl>=1.1.1, perf, perl-podlators, make
 
 下面的步骤只需要在中控机上执行：
 1. curve用户下配置ssh登陆到所有机器（包括自己），假设三台机器的ip分别为10.192.100.1,10.192.100.2,10.192.100.3
@@ -216,6 +221,7 @@ ansible是一款自动化运维工具，curve-ansible 是基于 ansible playbook
    $ ansible-playbook  # 如果没有报错的话说明安装成功，报错的话执行下面两步
    $ pip install --upgrade pip
    $ pip install --upgrade setuptools
+3. 确保源里面有以下几个包：net-tools, openssl>=1.1.1, perf, perl-podlators, make
 
 #### Debian9环境准备步骤
 下面这些步骤要三台机器都操作：

docs/cn/k8s_csi_interface.md

@@ -1,4 +1,4 @@
-目前，curve可以通过CSI插件的方式对接Kubernetes。本文给出的是CSI插件开发指导说明。内部使用的Curve CSI插件目前在联调测试中，后续会开源出来。
+目前，curve可以通过CSI插件的方式对接Kubernetes。本文给出的是CSI插件开发指导说明。curve csi插件源码请见[curve-csi](https://github.com/opencurve/curve-csi)。
 
 ## Curve Interface
 

mk-tar.sh

@@ -325,22 +325,8 @@ fi
 # README
 
 # curve-monitor
-mkdir -p build/curve-monitor/bin
-if [ $? -ne 0 ]
-then
-	exit
-fi
-mkdir -p build/curve-monitor/etc/curve/monitor
-if [ $? -ne 0 ]
-then
-	exit
-fi
-cp monitor/curve-monitor.sh build/curve-monitor/bin/curve-monitor.sh
-if [ $? -ne 0 ]
-then
-	exit
-fi
-cp -r monitor/* build/curve-monitor/etc/curve/monitor
+mkdir -p build/curve-monitor
+cp -r monitor/* build/curve-monitor/
 if [ $? -ne 0 ]
 then
 	exit
@@ -360,7 +346,11 @@ cp bazel-bin/nebd/src/part2/nebd-server build/nebd-package/bin
 
 # step 3.2 prepare for curve-nbd package
 mkdir -p build/nbd-package/bin
+mkdir -p build/nbd-package/etc
 cp bazel-bin/nbd/src/curve-nbd build/nbd-package/bin
+cp nbd/nbd-package/usr/bin/map_curve_disk.sh build/nbd-package/bin
+cp nbd/nbd-package/etc/curve/curvetab build/nbd-package/etc
+cp nbd/nbd-package/etc/systemd/system/map_curve_disk.service build/nbd-package/etc
 
 #step4 获取git提交版本信息
 commit_id=`git show --abbrev-commit HEAD|head -n 1|awk '{print $2}'`

monitor/grafana/dashboards/chunkserver.json

@@ -1,4 +1,5 @@
 {
+  "dashboard": {
   "annotations": {
     "list": [
       {
@@ -7833,4 +7834,5 @@
   "title": "ChunkServer",
   "uid": "4r3JKSGWz",
   "version": 3
+}
 }

monitor/grafana/dashboards/client.json

@@ -1,4 +1,5 @@
 {
+  "dashboard": {
   "annotations": {
     "list": [
       {
@@ -2446,4 +2447,5 @@
   "title": "Client",
   "uid": "DCy23HnWk",
   "version": 5
+}
 }

monitor/grafana/dashboards/etcd.json

@@ -1,4 +1,5 @@
 {
+  "dashboard": {
   "annotations": {
     "list": [
       {
@@ -2515,4 +2516,5 @@
   "title": "Etcd",
   "uid": "EuLA2DVZz",
   "version": 135
+}
 }

monitor/grafana/dashboards/mds.json

@@ -1,4 +1,5 @@
 {
+  "dashboard": {
   "annotations": {
     "list": [
       {
@@ -2738,4 +2739,5 @@
   "title": "MDS",
   "uid": "EpuJscnWk",
   "version": 7
+}
 }

monitor/grafana/dashboards/report.json

@@ -1,4 +1,5 @@
 {
+  "dashboard": {
   "annotations": {
     "list": [
       {
@@ -3739,4 +3740,5 @@
   "title": "Report",
   "uid": "nj1Iq_2Wk",
   "version": 14
+}
 }

monitor/grafana/dashboards/snapshotcloneserver.json

@@ -1,4 +1,5 @@
 {
+  "dashboard": {
   "annotations": {
     "list": [
       {
@@ -1052,4 +1053,5 @@
   "title": "SnapshotCloneServer",
   "uid": "TZzWso-Wz",
   "version": 2
+}
 }

nbd/nbd-package/DEBIAN/postinst

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+chmod +x /usr/bin/map_curve_disk.sh
+
+systemctl daemon-reload
+systemctl enable map_curve_disk.service
+systemctl start map_curve_disk.service
+
+CURVETAB=/etc/curve/curvetab
+CURVETABBAK=/etc/curve/curvetab-bak
+
+if [ -f ${CURVETABBAK} ]; then
+    cp -f -p ${CURVETABBAK} ${CURVETAB}
+    rm -f ${CURVETABBAK}
+fi
+
+exit 0

nbd/nbd-package/DEBIAN/preinst

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+CURVETAB=/etc/curve/curvetab
+CURVETABBAK=/etc/curve/curvetab-bak
+
+if [ -f ${CURVETAB} ]; then
+    cp -f -p ${CURVETAB} ${CURVETABBAK}
+fi
+
+exit 0

nbd/nbd-package/DEBIAN/prerm

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -e
+
+systemctl disable map_curve_disk.service
+
+exit 0

nbd/nbd-package/etc/curve/curvetab

@@ -0,0 +1,9 @@
+#
+# /etc/curvetab
+# use for auto map and mount curve disk when boot
+#
+# this file record the correspondence of curve nbd images/ devices
+# add record when 'curve-nbd map' and 'curve-nbd unmap'
+#
+# format: device    image   mount
+# /dev/nbd0   cbd:pool//curvefile_test_   /curve

nbd/nbd-package/etc/systemd/system/map_curve_disk.service

@@ -0,0 +1,22 @@
+[Unit]
+Description=curve-nbd auto map service
+Before=kubelet.service
+After=remote-fs.target
+After=network-online.target
+After=nebd-daemon.service
+Wants=network-online.target
+
+[Service]
+Type=forking
+Restart=no
+TimeoutSec=5min
+IgnoreSIGPIPE=no
+KillMode=process
+GuessMainPID=no
+RemainAfterExit=yes
+SuccessExitStatus=5 6
+ExecStart=/usr/bin/map_curve_disk.sh start
+ExecStop=/usr/bin/map_curve_disk.sh stop
+
+[Install]
+WantedBy=multi-user.target

nbd/nbd-package/usr/bin/map_curve_disk.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# default confpath
+# file format is:dealflag \t device \t image \t mountpoint(option)
+# +	/dev/nbd0	cbd:pool//curvefile_test_	/test
+confPath=/etc/curve/curvetab
+
+# check sudo
+if [[ $(id -u) -ne 0 ]]
+then
+    echo "Please run with sudo"
+    exit 1
+fi
+
+# usage
+function usage() {
+    echo "Usage: ./map_curve_disk.sh start"
+    echo "  -c/--confPath: set the confPath (default /etc/curve/curvetab)"
+    echo "    file format is:dealflag \t device \t image \t mountpoint(option)"
+    echo "    example: +	/dev/nbd0	cbd:pool//curvefile_test_	/test"
+    echo "  -h/--help: get the script usage"
+    echo "Examples:"
+    echo "  ./map_curve_disk.sh start //use the default configuration"
+    echo "  ./map_curve_disk.sh start --confPath yourConfPath //use the new confPath"
+}
+
+# remove extra records
+function dealcurvetab() {
+    if [ ! -f ${confPath} ]
+    then
+        echo "ERROR: not found configuration file, confPath is ${confPath}!"
+        exit
+    fi
+
+    declare -A recordmap
+    while read line
+    do
+        flag=$(echo $line | awk '{print $1}')
+        device=$(echo $line | awk '{print $2}')
+        if [ "$flag" = "+" ]
+        then
+            recordmap["$device"]=$line
+        elif [ "$flag" = "-" ]
+        then
+            unset recordmap[$device]
+        fi
+    done < ${confPath}
+    for key in ${!recordmap[@]}; do
+	echo ${recordmap[$key]} >> ${confPath}.bak
+    done
+}
+
+# map the curve disk based on curvetab
+function map() {
+    if [ ! -f ${confPath}.bak ]
+    then
+        echo "ERROR: not found configuration file, confPath is ${confPath}.bak!"
+        exit
+    fi
+
+    cat ${confPath}.bak | grep -v '#' | while read line
+    do
+        device=$(echo $line | awk '{print $2}')
+        image=$(echo $line | awk '{print $3}')
+        mount=$(echo $line | awk '{print $4}')
+        curve-nbd map $image --device $device
+        if [ "$mount" != "" ]
+        then
+            mount $device $mount
+        fi
+    done
+    mv ${confPath}.bak ${confPath}
+}
+
+if [ $# -lt 1 ]
+then
+    usage
+    exit
+fi
+
+case $1 in
+"start")
+    shift # pass first argument
+
+    while [[ $# -gt 1 ]]
+    do
+        key=$1
+
+        case $key in
+        -c|--confPath)
+            confPath=`realpath $2`
+            shift # pass key
+            shift # pass value
+            ;;
+        *)
+            usage
+            exit
+            ;;
+        esac
+    done
+    dealcurvetab
+    map
+    ;;
+*)
+    usage
+    ;;
+esac

nbd/src/define.h

@@ -61,6 +61,7 @@ namespace nbd {
 #define PROCESS_NAME "curve-nbd"
 #define NBD_PATH_PREFIX "/sys/block/nbd"
 #define DEV_PATH_PREFIX "/dev/nbd"
+#define CURVETAB_PATH "/etc/curve/curvetab"
 
 using std::cerr;
 

nbd/src/main.cpp

@@ -93,6 +93,24 @@ static void Usage() {
         << std::endl;
 }
 
+// use for record image and device info to auto map when boot
+static int AddRecord(int flag) {
+    std::string record;
+    int fd = open(CURVETAB_PATH, O_WRONLY | O_APPEND);
+    if (fd < 0) {
+        std::cerr << "curve-nbd: open curvetab file failed.";
+        return -EINVAL;
+    }
+    if (1 == flag) {
+        record = "+\t" + nbdConfig->devpath + "\t" + nbdConfig->imgname + "\n";
+    } else if (-1 == flag) {
+        record = "-\t" + nbdConfig->devpath + "\n";
+    }
+    write(fd, record.c_str(), record.size());
+    close(fd);
+    return 0;
+}
+
 static int NBDConnect() {
     int waitConnectPipe[2];
 
@@ -138,6 +156,10 @@ static int NBDConnect() {
     if (ret < 0) {
         ::write(waitConnectPipe[1], &connectionRes, sizeof(connectionRes));
     } else {
+        ret = AddRecord(1);
+        if (0 != ret) {
+            return ret;
+        }
         connectionRes = 0;
         ::write(waitConnectPipe[1], &connectionRes, sizeof(connectionRes));
         nbdTool->RunServerUntilQuit();
@@ -190,6 +212,10 @@ static int CurveNbdMain(int argc, const char* argv[]) {
                 return -EINVAL;
             }
 
+            r = AddRecord(-1);
+            if (0 != r) {
+                return r;
+            }
             break;
         }
         case Command::List: {

nebd/etc/nebd/nebd-client.conf

@@ -1,8 +1,8 @@
 # part2 socket file address
-nebdserver.serverAddress=/data/nebd/nebd.sock
+nebdserver.serverAddress=/data/nebd/nebd.sock  # __CURVEADM_TEMPLATE__ ${prefix}/data/nebd.sock __CURVEADM_TEMPLATE__
 
 # 文件锁路径
-metacache.fileLockPath=/data/nebd/lock
+metacache.fileLockPath=/data/nebd/lock  # __CURVEADM_TEMPLATE__ ${prefix}/data/lock __CURVEADM_TEMPLATE__
 
 # 同步rpc的最大重试次数
 request.syncRpcMaxRetryTimes=50
@@ -23,4 +23,4 @@ heartbeat.intervalS=5
 heartbeat.rpcTimeoutMs=500
 
 # 日志路径
-log.path=/data/log/nebd/client
+log.path=/data/log/nebd/client   # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__

nebd/etc/nebd/nebd-server.conf

@@ -1,11 +1,11 @@
 # curve-client配置文件地址
-curveclient.confPath=/etc/curve/client.conf
+curveclient.confPath=/etc/curve/client.conf  # __CURVEADM_TEMPLATE__ ${prefix}/conf/client.conf __CURVEADM_TEMPLATE__
 
 #brpc server监听端口
-listen.address=/data/nebd/nebd.sock
+listen.address=/data/nebd/nebd.sock  # __CURVEADM_TEMPLATE__ ${prefix}/data/nebd.sock __CURVEADM_TEMPLATE__
 
 #元数据文件地址,包含文件名
-meta.file.path=/data/nebd/nebdserver.meta
+meta.file.path=/data/nebd/nebdserver.meta  # __CURVEADM_TEMPLATE__ ${prefix}/data/nebdserver.meta __CURVEADM_TEMPLATE__
 
 #心跳超时时间
 heartbeat.timeout.sec=30

nebd/nebd-package/DEBIAN/control

@@ -1,7 +1,7 @@
 Package: nebd
 Section:
 Priority: optional
-Depends: libunwind8, libjemalloc1(= 3.6.0-9.1)
+Depends: libunwind8, libjemalloc1(= 3.6.0-9.1) | libjemalloc2(>= 5.1.0-3)
 Suggests:
 Architecture:amd64
 Installed-Size:

nebd/nebd-package/usr/bin/nebd-daemon

@@ -37,7 +37,7 @@ daemonLog=${baseLogPath}/nebd-server-daemon.log
 consoleLog=${baseLogPath}/nebd-server-console.log
 
 function get_nebd_server_pid() {
-    local tmp=`ps -ef | grep nebd-server | grep -v grep | grep -v daemon | awk '{print $2}'`
+    local tmp=`pidof nebd-server`
     echo $tmp
 }
 
@@ -103,11 +103,11 @@ function start() {
         exit 1
     fi
 
-    jemallocpath=/usr/lib/x86_64-linux-gnu/libjemalloc.so.1
     # 检查jemalloc库文件
-    if [ ! -f ${jemallocpath} ]
+    jemallocpath=`find /usr/ -name "libjemalloc.so*" -print -quit`
+    if [ "${jemallocpath}" = "" ]
     then
-        echo "Not found jemalloc library, Path is ${jemallocpath}"
+        echo "Not found jemalloc library in /usr/"
         exit 1
     fi
 
@@ -146,6 +146,27 @@ function stop() {
     then
         echo "stop may not success!"
     else
+        # wait 3s
+        retry_times=0
+        while [ $retry_times -le 3 ]
+        do
+            ((retry_times=$retry_times+1))
+            pidof nebd-server > /dev/null 2>&1
+            if [ $? -eq 0 ]
+            then
+                sleep 1
+            else
+                break
+            fi
+        done
+
+        pidof nebd-server > /dev/null 2>&1
+        if [ $? -eq 0 ]
+        then
+            echo "nebd-server still exists after 3s, now kill it with SIGKILL"
+            kill -9 `pidof nebd-server`
+        fi
+
         echo "nebd-server exit success!"
         echo "daemon exit success!"
     fi
@@ -170,7 +191,7 @@ function restart() {
 
     if [ $old_pid == $new_pid ]
     then
-        echo "restart nebd-server may failed, try kill -9 nebd-server"
+        echo "restart nebd-server may failed, now kill it with SIGKILL"
         kill -9 $old_pid
     fi
 }

proto/nameserver2.proto

@@ -345,6 +345,11 @@ message ProtoSession {
     // other useful infos
 };
 
+message CloneSourceSegment {
+    required uint64 segmentSize = 1;
+    repeated uint64 allocatedSegmentOffset = 2;
+}
+
 message OpenFileRequest {
     required string fileName = 1;
     required string owner = 2;
@@ -363,6 +368,7 @@ message OpenFileResponse {
     required StatusCode statusCode = 1;
     optional ProtoSession protoSession = 2;
     optional FileInfo   fileInfo = 3;
+    optional CloneSourceSegment cloneSourceSegment = 4;
 };
 
 message CloseFileRequest {

robot/curve_init.txt

@@ -0,0 +1,21 @@
+*** Settings ***
+Library			  Collections
+#Library           RequestsLibrary
+Library           Process
+#Library           SSHLibrary
+
+Suite Setup       init
+Suite Teardown    clean
+
+
+*** Test Cases ***
+inject cases
+    [Tags]    P0    base   init
+     log  hello word
+
+*** Keywords ***
+init 
+     log  init cluster
+
+clean
+     log   clean

src/chunkserver/raftsnapshot/curve_file_service.cpp

@@ -67,8 +67,6 @@ void CurveFileService::get_file(::google::protobuf::RpcController* controller,
          * 的uri format为:remote://ip:port/reader_id，所以使用ENXIO
          * 代表reader id不存在的错误
          */
-        LOG(ERROR) << "reader " << request->reader_id()
-                   << " not found in reader map";
         cntl->SetFailed(ENXIO, "Fail to find reader=%" PRId64,
                                     request->reader_id());
         return;
@@ -82,7 +80,6 @@ void CurveFileService::get_file(::google::protobuf::RpcController* controller,
               << request->count();
 
     if (request->count() <= 0 || request->offset() < 0) {
-        LOG(ERROR) << "request count or offset elegal";
         cntl->SetFailed(brpc::EREQUEST, "Invalid request=%s",
                         request->ShortDebugString().c_str());
         return;
@@ -160,8 +157,6 @@ void CurveFileService::get_file(::google::protobuf::RpcController* controller,
                                 &read_count,
                                 &is_eof);
         if (rc != 0) {
-            LOG(ERROR) << "Fail to read file " << reader->path() << "/"
-                       << request->filename() << " error code: " << rc;
             cntl->SetFailed(rc, "Fail to read from path=%s filename=%s : %s",
                             reader->path().c_str(),
                             request->filename().c_str(), berror(rc));

src/client/chunk_closure.cpp

@@ -279,7 +279,7 @@ void ClientClosure::Run() {
 
         default:
             needRetry = true;
-            LOG_EVERY_N(ERROR, 10) << OpTypeToString(reqCtx_->optype_)
+            LOG(WARNING) << OpTypeToString(reqCtx_->optype_)
                 << " failed for UNKNOWN reason, " << *reqCtx_
                 << ", status="
                 << curve::chunkserver::CHUNK_OP_STATUS_Name(
@@ -309,7 +309,7 @@ void ClientClosure::OnRpcFailed() {
         MetricHelper::IncremTimeOutRPCCount(fileMetric_, reqCtx_->optype_);
     }
 
-    LOG_EVERY_N(ERROR, 10) << OpTypeToString(reqCtx_->optype_)
+    LOG_EVERY_SECOND(WARNING) << OpTypeToString(reqCtx_->optype_)
         << " failed, error code: "
         << cntl_->ErrorCode()
         << ", error: " << cntl_->ErrorText()
@@ -444,7 +444,7 @@ void ClientClosure::OnRetry() {
         failReqOpt_.chunkserverMaxRetryTimesBeforeConsiderSuspend) {
         reqDone_->SetSuspendRPCFlag();
         MetricHelper::IncremIOSuspendNum(fileMetric_);
-        LOG(WARNING) << "IO Retried "
+        LOG(ERROR) << "IO Retried "
                     << failReqOpt_.chunkserverMaxRetryTimesBeforeConsiderSuspend
                     << " times, set suspend flag! " << *reqCtx_
                     << ", IO id = " << reqDone_->GetIOTracker()->GetID()

src/client/client_common.cpp

@@ -35,6 +35,8 @@ const char* OpTypeToString(OpType optype) {
         return "ReadSnapshot";
     case OpType::DELETE_SNAP:
         return "DeleteSnapshot";
+    case OpType::CREATE_CLONE:
+        return "CreateCloneChunk";
     case OpType::RECOVER_CHUNK:
         return "RecoverChunk";
     case OpType::GET_CHUNK_INFO:
@@ -47,4 +49,3 @@ const char* OpTypeToString(OpType optype) {
 
 }  // namespace client
 }  // namespace curve
-

src/client/client_common.h

@@ -31,6 +31,7 @@
 #include <string>
 #include <atomic>
 #include <vector>
+#include <unordered_set>
 
 #include "include/client/libcurve.h"
 #include "src/common/net_common.h"
@@ -137,6 +138,23 @@ typedef struct SegmentInfo {
     LogicalPoolCopysetIDInfo lpcpIDInfo;
 } SegmentInfo_t;
 
+struct CloneSourceInfo {
+    std::string name;
+    uint64_t length = 0;
+    uint64_t segmentSize = 0;
+    std::unordered_set<uint64_t> allocatedSegmentOffsets;
+
+    CloneSourceInfo() = default;
+
+    CloneSourceInfo(const CloneSourceInfo& other)
+        : name(other.name),
+          length(other.length),
+          segmentSize(other.segmentSize),
+          allocatedSegmentOffsets(other.allocatedSegmentOffsets) {}
+
+    bool IsSegmentAllocated(uint64_t offset) const;
+};
+
 typedef struct FInfo {
     uint64_t        id;
     uint64_t        parentid;
@@ -153,8 +171,7 @@ typedef struct FInfo {
     std::string     filename;
     std::string     fullPathName;
     FileStatus      filestatus;
-    std::string     cloneSource;
-    uint64_t        cloneLength{0};
+    CloneSourceInfo sourceInfo;
 
     FInfo() {
         id = 0;
@@ -218,6 +235,7 @@ struct ChunkServerAddr {
 };
 
 const char* OpTypeToString(OpType optype);
+
 struct ClusterContext {
     std::string clusterId;
 };
@@ -273,6 +291,36 @@ class ClientDummyServerInfo {
     bool register_ = false;
 };
 
+inline void TrivialDeleter(void* ptr) {}
+
+inline const char* FileStatusToName(FileStatus status) {
+    switch (status) {
+        case FileStatus::Created:
+            return "Created";
+        case FileStatus::Deleting:
+            return "Deleting";
+        case FileStatus::Cloning:
+            return "Cloning";
+        case FileStatus::CloneMetaInstalled:
+            return "CloneMetaInstalled";
+        case FileStatus::Cloned:
+            return "Cloned";
+        case FileStatus::BeingCloned:
+            return "BeingCloned";
+        default:
+            return "Unknown";
+    }
+}
+
+inline bool CloneSourceInfo::IsSegmentAllocated(uint64_t offset) const {
+    if (length == 0) {
+        return false;
+    }
+
+    uint64_t segmentOffset = offset / segmentSize * segmentSize;
+    return allocatedSegmentOffsets.count(segmentOffset) != 0;
+}
+
 }   // namespace client
 }   // namespace curve
 

src/client/client_metric.h

@@ -134,7 +134,7 @@ struct FileMetric {
 
 // 用于全局mds接口统计信息调用信息统计
 struct MDSClientMetric {
-    const std::string prefix = "curve mds client";
+    std::string prefix;
 
     // mds的地址信息
     std::string metaserverAddr;
@@ -174,8 +174,12 @@ struct MDSClientMetric {
     // 切换mds server总次数
     bvar::Adder<uint64_t> mdsServerChangeTimes;
 
-    MDSClientMetric()
-        : metaserverAddress(prefix, "current_metaserver_addr", GetStringValue,
+    explicit MDSClientMetric(const std::string& prefix_ = "")
+        : prefix(!prefix_.empty()
+                     ? prefix_
+                     : "curve_mds_client_" +
+                           std::to_string(reinterpret_cast<uint64_t>(this))),
+          metaserverAddress(prefix, "current_metaserver_addr", GetStringValue,
                             &metaserverAddr),
           mdsServerChangeTimes(prefix, "mds_server_change_times"),
           openFile(prefix, "openFile"),

src/client/io_tracker.cpp

@@ -294,11 +294,17 @@ void IOTracker::Done() {
         MetricHelper::IncremUserEPSCount(fileMetric_, type_);
         if (type_ == OpType::READ || type_ == OpType::WRITE) {
             LOG(ERROR) << "file [" << fileMetric_->filename << "]"
-                    << ", IO Error, OpType = " << static_cast<int>(type_)
+                    << ", IO Error, OpType = " << OpTypeToString(type_)
                     << ", offset = " << offset_
                     << ", length = " << length_;
         } else {
-            LOG(ERROR) << ", IO Error, OpType = " << static_cast<int>(type_);
+            if (OpType::CREATE_CLONE == type_ &&
+                LIBCURVE_ERROR::EXISTS == errcode_) {
+                // if CreateCloneChunk return Exists
+                // no error should be reported
+            } else {
+                LOG(ERROR) << "IO Error, OpType = " << OpTypeToString(type_);
+            }
         }
     }
 

src/client/lease_executor.cpp

@@ -97,14 +97,14 @@ bool LeaseExecutor::RefreshLease() {
         return true;
     } else if (LIBCURVE_ERROR::AUTHFAIL == ret) {
         iomanager_->LeaseTimeoutBlockIO();
-        LOG(WARNING) << "Refresh session auth fail, block io. "
-                     << "session id = " << leasesession_.sessionID
-                     << ", filename = " << fullFileName_;
+        LOG(ERROR) << "Refresh session auth fail, block io. "
+                   << "session id = " << leasesession_.sessionID
+                   << ", filename = " << fullFileName_;
         return true;
     }
 
     if (response.status == LeaseRefreshResult::Status::OK) {
-        CheckNeedUpdateVersion(response.finfo.seqnum);
+        CheckNeedUpdateFileInfo(response.finfo);
         failedrefreshcount_.store(0);
         isleaseAvaliable_.store(true);
         iomanager_->RefeshSuccAndResumeIO();
@@ -112,12 +112,12 @@ bool LeaseExecutor::RefreshLease() {
     } else if (response.status == LeaseRefreshResult::Status::NOT_EXIST) {
         iomanager_->LeaseTimeoutBlockIO();
         isleaseAvaliable_.store(false);
-        LOG(WARNING) << "session or file not exists, no longer refresh!"
-                     << ", sessionid = " << leasesession_.sessionID
-                     << ", filename = " << fullFileName_;
+        LOG(ERROR) << "session or file not exists, no longer refresh!"
+                   << ", sessionid = " << leasesession_.sessionID
+                   << ", filename = " << fullFileName_;
         return false;
     } else {
-        LOG(WARNING) << "Refresh session failed, filename = " << fullFileName_;
+        LOG(ERROR) << "Refresh session failed, filename = " << fullFileName_;
         return true;
     }
     return true;
@@ -146,14 +146,27 @@ void LeaseExecutor::IncremRefreshFailed() {
     }
 }
 
-void LeaseExecutor::CheckNeedUpdateVersion(uint64_t newversion) {
-    const uint64_t currentFileSn = iomanager_->GetLatestFileSn();
+void LeaseExecutor::CheckNeedUpdateFileInfo(const FInfo& fileInfo) {
+    MetaCache* metaCache = iomanager_->GetMetaCache();
 
-    DVLOG(9) << "new file version = " << newversion
-        << ", current version = " << currentFileSn
-        << ", filename = " << fullFileName_;
-    if (newversion > currentFileSn) {
-        iomanager_->SetLatestFileSn(newversion);
+    uint64_t currentFileSn = metaCache->GetLatestFileSn();
+    uint64_t newSn = fileInfo.seqnum;
+    if (newSn > currentFileSn) {
+        LOG(INFO) << "Update file sn, new file sn = " << newSn
+                  << ", current sn = " << currentFileSn
+                  << ", filename = " << fullFileName_;
+        metaCache->SetLatestFileSn(newSn);
+    }
+
+    FileStatus currentFileStatus = metaCache->GetLatestFileStatus();
+    FileStatus newFileStatus = fileInfo.filestatus;
+    if (newFileStatus != currentFileStatus) {
+        LOG(INFO) << "Update file status, new status = "
+                  << FileStatusToName(newFileStatus)
+                  << ", current file status = "
+                  << FileStatusToName(currentFileStatus)
+                  << ", filename = " << fullFileName_;
+        metaCache->SetLatestFileStatus(newFileStatus);
     }
 }
 

src/client/lease_executor.h

@@ -24,6 +24,8 @@
 #define SRC_CLIENT_LEASE_EXECUTOR_H_
 
 #include <brpc/periodic_task.h>
+#include <bthread/condition_variable.h>
+#include <bthread/mutex.h>
 
 #include <memory>
 #include <string>
@@ -127,10 +129,11 @@ class LeaseExecutor {
     void IncremRefreshFailed();
 
     /**
-     * 每次续约会携带新的文件信息，该函数检查当前文件是否需要更新版本信息
-     * @param: newversion是lease续约后从mds端携带回来的版本号
+     * @brief Updating local file information consistent with MDS record.
+     *        Currently, only seqnum and file status are updated
+     * @param fileInfo Latest file information returned by refresh session RPC
      */
-    void CheckNeedUpdateVersion(uint64_t newversion);
+    void CheckNeedUpdateFileInfo(const FInfo& fileInfo);
 
  private:
     // 与mds进行lease续约的文件名
@@ -197,7 +200,7 @@ class RefreshSessionTask : public brpc::PeriodicTask {
      *         false 停止执行当前任务
      */
     bool OnTriggeringTask(timespec* next_abstime) override {
-        std::lock_guard<std::mutex> lk(stopMtx_);
+        std::lock_guard<bthread::Mutex> lk(stopMtx_);
         if (stopped_) {
             return false;
         }
@@ -210,7 +213,7 @@ class RefreshSessionTask : public brpc::PeriodicTask {
      * @brief 停止再次执行当前任务
      */
     void Stop() {
-        std::lock_guard<std::mutex> lk(stopMtx_);
+        std::lock_guard<bthread::Mutex> lk(stopMtx_);
         stopped_ = true;
     }
 
@@ -218,7 +221,7 @@ class RefreshSessionTask : public brpc::PeriodicTask {
      * @brief 任务停止后调用
      */
     void OnDestroyingTask() override {
-        std::unique_lock<std::mutex> lk(terminatedMtx_);
+        std::unique_lock<bthread::Mutex> lk(terminatedMtx_);
         terminated_ = true;
         terminatedCv_.notify_one();
     }
@@ -227,8 +230,10 @@ class RefreshSessionTask : public brpc::PeriodicTask {
      * @brief 等待任务退出
      */
     void WaitTaskExit() {
-        std::unique_lock<std::mutex> lk(terminatedMtx_);
-        terminatedCv_.wait(lk, [this]() { return terminated_ == true; });
+        std::unique_lock<bthread::Mutex> lk(terminatedMtx_);
+        while (terminated_ != true) {
+            terminatedCv_.wait(lk);
+        }
     }
 
     /**
@@ -244,11 +249,11 @@ class RefreshSessionTask : public brpc::PeriodicTask {
     uint64_t refreshIntervalUs_;
 
     bool stopped_;
-    std::mutex stopMtx_;
+    bthread::Mutex stopMtx_;
 
     bool terminated_;
-    std::mutex terminatedMtx_;
-    std::condition_variable terminatedCv_;
+    bthread::Mutex terminatedMtx_;
+    bthread::ConditionVariable terminatedCv_;
 };
 
 }   // namespace client

src/client/libcurve_file.cpp

@@ -174,7 +174,8 @@ int FileClient::Open(const std::string& filename,
 
     int ret = fileserv->Open(filename, userinfo, sessionId);
     if (ret != LIBCURVE_ERROR::OK) {
-        LOG(ERROR) << "Open file fail, retCode = " << ret;
+        LOG(ERROR) << "Open file failed, filename: " << filename
+                   << ", retCode: " << ret;
         fileserv->UnInitialize();
         delete fileserv;
         return ret;
@@ -261,6 +262,8 @@ int FileClient::Create(const std::string& filename,
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
         ret = mdsClient_->CreateFile(filename, userinfo, size);
+        LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
+            << "Create file failed, filename: " << filename << ", ret: " << ret;
     } else {
         LOG(ERROR) << "global mds client not inited!";
         return -LIBCURVE_ERROR::FAILED;
@@ -355,6 +358,10 @@ int FileClient::Rename(const UserInfo_t& userinfo,
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
         ret = mdsClient_->RenameFile(userinfo, oldpath, newpath);
+        LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
+            << "Rename failed, OldPath: " << oldpath
+            << ", NewPath: " << newpath
+            << ", ret: " << ret;
     } else {
         LOG(ERROR) << "global mds client not inited!";
         return -LIBCURVE_ERROR::FAILED;
@@ -367,6 +374,10 @@ int FileClient::Extend(const std::string& filename,
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
         ret = mdsClient_->Extend(filename, userinfo, newsize);
+        LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
+            << "Extend failed, filename: " << filename
+            << ", NewSize: " << newsize
+            << ", ret: " << ret;
     } else {
         LOG(ERROR) << "global mds client not inited!";
         return -LIBCURVE_ERROR::FAILED;
@@ -379,6 +390,10 @@ int FileClient::Unlink(const std::string& filename,
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
         ret = mdsClient_->DeleteFile(filename, userinfo, deleteforce);
+        LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
+            << "Unlink failed, filename: " << filename
+            << ", force: " << deleteforce
+            << ", ret: " << ret;
     } else {
         LOG(ERROR) << "global mds client not inited!";
         return -LIBCURVE_ERROR::FAILED;
@@ -392,6 +407,8 @@ int FileClient::StatFile(const std::string& filename,
     int ret;
     if (mdsClient_ != nullptr) {
         ret = mdsClient_->GetFileInfo(filename, userinfo, &fi);
+        LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
+            << "StatFile failed, filename: " << filename << ", ret" << ret;
     } else {
         LOG(ERROR) << "global mds client not inited!";
         return -LIBCURVE_ERROR::FAILED;
@@ -420,6 +437,9 @@ int FileClient::Listdir(const std::string& dirpath,
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
         ret = mdsClient_->Listdir(dirpath, userinfo, filestatVec);
+        LOG_IF(ERROR,
+               ret != LIBCURVE_ERROR::OK && ret != LIBCURVE_ERROR::NOTEXIST)
+            << "Listdir failed, Path: " << dirpath << ", ret: " << ret;
     } else {
         LOG(ERROR) << "global mds client not inited!";
         return -LIBCURVE_ERROR::FAILED;
@@ -431,6 +451,8 @@ int FileClient::Mkdir(const std::string& dirpath, const UserInfo_t& userinfo) {
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
         ret = mdsClient_->CreateFile(dirpath, userinfo, 0, false);
+        LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
+            << "Create file failed, filename: " << dirpath << ", ret: " << ret;
     } else {
         LOG(ERROR) << "global mds client not inited!";
         return -LIBCURVE_ERROR::FAILED;
@@ -442,6 +464,8 @@ int FileClient::Rmdir(const std::string& dirpath, const UserInfo_t& userinfo) {
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
         ret = mdsClient_->DeleteFile(dirpath, userinfo);
+        LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
+            << "Rmdir failed, Path: " << dirpath << ", ret: " << ret;
     } else {
         LOG(ERROR) << "global mds client not inited!";
         return -LIBCURVE_ERROR::FAILED;
@@ -454,6 +478,8 @@ int FileClient::ChangeOwner(const std::string& filename,
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
         ret = mdsClient_->ChangeOwner(filename, newOwner, userinfo);
+        LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
+            << "ChangeOwner failed, filename: " << filename << ", ret: " << ret;
     } else {
         LOG(ERROR) << "global mds client not inited!";
         return -LIBCURVE_ERROR::FAILED;
@@ -550,6 +576,7 @@ std::string FileClient::GetClusterId() {
         return clsctx.clusterId;
     }
 
+    LOG(ERROR) << "GetClusterId failed, ret: " << ret;
     return {};
 }
 

src/client/libcurve_snapshot.cpp

@@ -112,7 +112,7 @@ int SnapshotClient::GetSnapShot(const std::string& filename,
     if (ret == LIBCURVE_ERROR::OK && !infomap.empty()) {
         auto it = infomap.begin();
         if (it->first != seq) {
-            LOG(ERROR) << "Snapshot info not found with seqnum = " << seq;
+            LOG(WARNING) << "Snapshot info not found with seqnum = " << seq;
             return -LIBCURVE_ERROR::NOTEXIST;
         }
         *snapinfo = it->second;
@@ -138,11 +138,11 @@ int SnapshotClient::GetSnapshotSegmentInfo(const std::string& filename,
                                         seq, offset, segInfo);
 
     if (ret != LIBCURVE_ERROR::OK) {
-        LOG(ERROR) << "GetSnapshotSegmentInfo failed, ret = " << ret;
+        LOG(WARNING) << "GetSnapshotSegmentInfo failed, ret = " << ret;
         return -ret;
     }
 
-    for (auto iter : segInfo->chunkvec) {
+    for (const auto& iter : segInfo->chunkvec) {
         iomanager4chunk_.GetMetaCache()->UpdateChunkInfoByID(iter.cid_, iter);
     }
 

src/client/mds_client.cpp

@@ -70,7 +70,8 @@ using curve::mds::topology::GetChunkServerListInCopySetsResponse;
 
 namespace curve {
 namespace client {
-MDSClient::MDSClient() {
+MDSClient::MDSClient(const std::string& metricPrefix)
+    : mdsClientMetric_(metricPrefix) {
     inited_   = false;
 }
 
@@ -232,7 +233,7 @@ int MDSClient::MDSRPCExcutor::ExcuteTask(
     brpc::Channel channel;
     int ret = channel.Init(mdsaddr.c_str(), nullptr);
     if (ret != 0) {
-        LOG(WARNING) << "Init channel failed! addr = " << mdsaddr.c_str();
+        LOG(WARNING) << "Init channel failed! addr = " << mdsaddr;
         // 返回EHOSTDOWN给上层调用者，促使其切换mds
         return -EHOSTDOWN;
     }
@@ -265,8 +266,8 @@ LIBCURVE_ERROR MDSClient::OpenFile(const std::string& filename,
         LIBCURVE_ERROR retcode = LIBCURVE_ERROR::FAILED;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-            << "OpenFile: filename = " << filename.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+            << "OpenFile: filename = " << filename
             << ", owner = " << userinfo.owner
             << ", errocde = " << retcode
             << ", error msg = " << StatusCode_Name(stcode)
@@ -279,10 +280,25 @@ LIBCURVE_ERROR MDSClient::OpenFile(const std::string& filename,
             lease->leaseTime     = leasesession.leasetime();
             lease->createTime    = leasesession.createtime();
 
-            FileInfo finfo = response.fileinfo();
-            ServiceHelper::ProtoFileInfo2Local(&finfo, fi);
+            const curve::mds::FileInfo& protoFileInfo = response.fileinfo();
+            ServiceHelper::ProtoFileInfo2Local(protoFileInfo, fi);
+
+            if (protoFileInfo.has_clonesource() &&
+                protoFileInfo.filestatus() ==
+                    curve::mds::FileStatus::kFileCloneMetaInstalled) {
+                if (!response.has_clonesourcesegment()) {
+                    LOG(WARNING) << filename
+                                 << " has clone source and status is "
+                                    "CloneMetaInstalled, but response does not "
+                                    "contains clone source segment";
+                    return retcode;
+                }
+
+                ServiceHelper::ProtoCloneSourceInfo2Local(response,
+                                                          &fi->sourceInfo);
+            }
         } else {
-            LOG(ERROR) << "mds response has no file info or session info!";
+            LOG(WARNING) << "mds response has no file info or session info!";
             return LIBCURVE_ERROR::FAILED;
         }
 
@@ -311,8 +327,8 @@ LIBCURVE_ERROR MDSClient::CreateFile(const std::string& filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-            << "CreateFile: filename = " << filename.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+            << "CreateFile: filename = " << filename
             << ", owner = " << userinfo.owner
             << ", is nomalfile: " << normalFile
             << ", errocde = " << retcode
@@ -343,8 +359,8 @@ LIBCURVE_ERROR MDSClient::CloseFile(const std::string& filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-            << "CloseFile: filename = " << filename.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+            << "CloseFile: filename = " << filename
             << ", owner = " << userinfo.owner
             << ", sessionid = " << sessionid
             << ", errocde = " << retcode
@@ -371,15 +387,14 @@ LIBCURVE_ERROR MDSClient::GetFileInfo(const std::string& filename,
         }
 
         if (response.has_fileinfo()) {
-            FileInfo finfo = response.fileinfo();
-            ServiceHelper::ProtoFileInfo2Local(&finfo, fi);
+            ServiceHelper::ProtoFileInfo2Local(response.fileinfo(), fi);
         }
 
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-            << "GetFileInfo: filename = " << filename.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+            << "GetFileInfo: filename = " << filename
             << ", owner = " << uinfo.owner
             << ", errocde = " << retcode
             << ", error msg = " << StatusCode_Name(stcode)
@@ -410,8 +425,7 @@ LIBCURVE_ERROR MDSClient::CreateSnapShot(const std::string& filename,
              stcode == StatusCode::kFileUnderSnapShot) &&
             hasinfo) {
             FInfo_t* fi = new (std::nothrow) FInfo_t;
-            curve::mds::FileInfo finfo = response.snapshotfileinfo();
-            ServiceHelper::ProtoFileInfo2Local(&finfo, fi);
+            ServiceHelper::ProtoFileInfo2Local(response.snapshotfileinfo(), fi);
             *seq = fi->seqnum;
             delete fi;
             if (stcode == StatusCode::kOK) {
@@ -426,14 +440,14 @@ LIBCURVE_ERROR MDSClient::CreateSnapShot(const std::string& filename,
 
         if (hasinfo) {
             FInfo_t fi;
-            ServiceHelper::ProtoFileInfo2Local(&response.snapshotfileinfo(), &fi);  // NOLINT
+            ServiceHelper::ProtoFileInfo2Local(response.snapshotfileinfo(), &fi);  // NOLINT
             *seq = fi.seqnum;
         }
 
         LIBCURVE_ERROR retcode;
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                << "CreateSnapShot: filename = " << filename.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                << "CreateSnapShot: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", errocde = " << retcode
                 << ", error msg = " << StatusCode_Name(stcode)
@@ -460,8 +474,8 @@ LIBCURVE_ERROR MDSClient::DeleteSnapShot(const std::string& filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                << "DeleteSnapShot: filename = " << filename.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                << "DeleteSnapShot: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", seqnum = " << seq << ", errocde = " << retcode
                 << ", error msg = " << StatusCode_Name(stcode)
@@ -489,8 +503,8 @@ LIBCURVE_ERROR MDSClient::ListSnapShot(const std::string& filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                << "ListSnapShot: filename = " << filename.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                << "ListSnapShot: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", errocde = " << retcode
                 << ", error msg = " << StatusCode_Name(stcode);
@@ -501,8 +515,7 @@ LIBCURVE_ERROR MDSClient::ListSnapShot(const std::string& filename,
 
         for (int i = 0; i < response.fileinfo_size(); i++) {
             FInfo_t tempInfo;
-            FileInfo finfo = response.fileinfo(i);
-            ServiceHelper::ProtoFileInfo2Local(&finfo, &tempInfo);
+            ServiceHelper::ProtoFileInfo2Local(response.fileinfo(i), &tempInfo);
             snapif->insert(std::make_pair(tempInfo.seqnum, tempInfo));
         }
 
@@ -522,7 +535,7 @@ LIBCURVE_ERROR MDSClient::GetSnapshotSegmentInfo(const std::string& filename,
     auto task = RPCTaskDefine {
         GetOrAllocateSegmentResponse response;
         mdsClientBase_.GetSnapshotSegmentInfo(filename, userinfo, seq, offset,
-                                             &response, cntl, channel);
+                                              &response, cntl, channel);
         if (cntl->Failed()) {
             LOG(WARNING) << "get snap file segment info failed, errcorde = "
                 << cntl->ErrorCode()
@@ -533,8 +546,8 @@ LIBCURVE_ERROR MDSClient::GetSnapshotSegmentInfo(const std::string& filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                << "GetSnapshotSegmentInfo: filename = " << filename.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                << "GetSnapshotSegmentInfo: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", offset = " << offset
                 << ", seqnum = " << seq
@@ -542,7 +555,7 @@ LIBCURVE_ERROR MDSClient::GetSnapshotSegmentInfo(const std::string& filename,
                 << ", error msg = " << StatusCode_Name(stcode);
 
         if (stcode != StatusCode::kOK) {
-            LOG(ERROR) << "GetSnapshotSegmentInfo return error, "
+            LOG(WARNING) << "GetSnapshotSegmentInfo return error, "
                        << ", filename = " << filename
                        << ", seq = " << seq;
             return retcode;
@@ -561,7 +574,7 @@ LIBCURVE_ERROR MDSClient::GetSnapshotSegmentInfo(const std::string& filename,
 
         int chunksNum = pfs.chunks_size();
         if (chunksNum == 0) {
-            LOG(ERROR) << "mds allocate segment, but no chunk!";
+            LOG(WARNING) << "mds allocate segment, but no chunk!";
             return LIBCURVE_ERROR::FAILED;
         }
 
@@ -597,18 +610,19 @@ LIBCURVE_ERROR MDSClient::RefreshSession(const std::string& filename,
         }
 
         StatusCode stcode = response.statuscode();
-        LOG_IF(WARNING, stcode != StatusCode::kOK)
-            << "RefreshSession NOT OK: filename = "
-            << filename.c_str() << ", owner = "
-            << userinfo.owner << ", sessionid = "
-            << sessionid << ", status code = "
-            << StatusCode_Name(stcode);
-
-        LOG_EVERY_N(INFO, 10) << "RefreshSession returned: filename = "
-            << filename.c_str() << ", owner = "
-            << userinfo.owner << ", sessionid = "
-            << sessionid << ", status code = "
-            << StatusCode_Name(stcode);
+        if (stcode != StatusCode::kOK) {
+            LOG(WARNING)
+                << "RefreshSession NOT OK: filename = " << filename
+                << ", owner = " << userinfo.owner
+                << ", sessionid = " << sessionid
+                << ", status code = " << StatusCode_Name(stcode);
+        } else {
+            LOG_EVERY_SECOND(INFO)
+                << "RefreshSession returned: filename = " << filename
+                << ", owner = " << userinfo.owner
+                << ", sessionid = " << sessionid
+                << ", status code = " << StatusCode_Name(stcode);
+        }
 
         switch (stcode) {
             case StatusCode::kSessionNotExist:
@@ -621,8 +635,8 @@ LIBCURVE_ERROR MDSClient::RefreshSession(const std::string& filename,
                 break;
             case StatusCode::kOK:
                 if (response.has_fileinfo()) {
-                    FileInfo finfo = response.fileinfo();
-                    ServiceHelper::ProtoFileInfo2Local(&finfo, &resp->finfo);
+                    ServiceHelper::ProtoFileInfo2Local(response.fileinfo(),
+                                                       &resp->finfo);
                     resp->status = LeaseRefreshResult::Status::OK;
                 } else {
                     LOG(WARNING) << "session response has no fileinfo!";
@@ -630,7 +644,7 @@ LIBCURVE_ERROR MDSClient::RefreshSession(const std::string& filename,
                 }
                 if (nullptr != lease) {
                     if (!response.has_protosession()) {
-                        LOG(ERROR) << "session response has no protosession";
+                        LOG(WARNING) << "session response has no protosession";
                         return LIBCURVE_ERROR::FAILED;
                     }
                     ProtoSession leasesession = response.protosession();
@@ -671,8 +685,8 @@ LIBCURVE_ERROR MDSClient::CheckSnapShotStatus(const std::string& filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-            << "CheckSnapShotStatus: filename = " << filename.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+            << "CheckSnapShotStatus: filename = " << filename
             << ", owner = " << userinfo.owner
             << ", seqnum = " << seq << ", errocde = " << retcode
             << ", error msg = " << StatusCode_Name(stcode);
@@ -732,10 +746,10 @@ LIBCURVE_ERROR MDSClient::GetServerList(const LogicPoolID& logicalpooid,
             }
             cpinfoVec->push_back(copysetseverl);
             DVLOG(9) << "copyset id : " << copysetseverl.cpid_
-                     << ", peer info : " << copyset_peer.c_str();
+                     << ", peer info : " << copyset_peer;
         }
 
-        LOG_IF(ERROR, response.statuscode() != 0)
+        LOG_IF(WARNING, response.statuscode() != 0)
             << "GetServerList failed"
             << ", errocde = " << response.statuscode()
             << ", log id = " << cntl->log_id();
@@ -752,10 +766,9 @@ LIBCURVE_ERROR MDSClient::GetClusterInfo(ClusterContext* clsctx) {
         mdsClientBase_.GetClusterInfo(&response, cntl, channel);
 
         if (cntl->Failed()) {
-            LOG(ERROR)  << "get cluster info from mds failed, status code = "
-                << cntl->ErrorCode()
-                << ", error content: "
-                << cntl->ErrorText();
+            LOG(WARNING) << "get cluster info from mds failed, status code = "
+                       << cntl->ErrorCode()
+                       << ", error content: " << cntl->ErrorText();
             return -cntl->ErrorCode();
         }
 
@@ -789,18 +802,19 @@ LIBCURVE_ERROR MDSClient::CreateCloneFile(const std::string& source,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
             << "CreateCloneFile: source = " << source
             << ", destination = " << destination
-            << ", owner = " << userinfo.owner.c_str() << ", seqnum = " << sn
+            << ", owner = " << userinfo.owner << ", seqnum = " << sn
             << ", size = " << size << ", chunksize = " << chunksize
             << ", errocde = " << retcode
             << ", error msg = " << StatusCode_Name(stcode)
             << ", log id = " << cntl->log_id();
 
         if (stcode == StatusCode::kOK) {
-            FileInfo finfo = response.fileinfo();
-            ServiceHelper::ProtoFileInfo2Local(&finfo, fileinfo);
+            ServiceHelper::ProtoFileInfo2Local(response.fileinfo(), fileinfo);
+            fileinfo->sourceInfo.name = response.fileinfo().clonesource();
+            fileinfo->sourceInfo.length = response.fileinfo().clonelength();
         }
 
         return retcode;
@@ -835,9 +849,9 @@ LIBCURVE_ERROR MDSClient::SetCloneFileStatus(const std::string &filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                << "SetCloneFileStatus failed, filename = " << filename.c_str()
-                << ", owner = " << userinfo.owner.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                << "SetCloneFileStatus failed, filename = " << filename
+                << ", owner = " << userinfo.owner
                 << ", filestatus = " << static_cast<int>(filestatus)
                 << ", fileID = " << fileID
                 << ", errocde = " << retcode
@@ -858,7 +872,7 @@ LIBCURVE_ERROR MDSClient::GetOrAllocateSegment(bool allocate, uint64_t offset,
                                             &response, cntl, channel);
         if (cntl->Failed()) {
             mdsClientMetric_.getOrAllocateSegment.eps.count << 1;
-            LOG_EVERY_SECOND(ERROR)
+            LOG(WARNING)
                 << "allocate segment failed, error code = "
                 << cntl->ErrorCode()
                 << ", error content:" << cntl->ErrorText()
@@ -869,7 +883,7 @@ LIBCURVE_ERROR MDSClient::GetOrAllocateSegment(bool allocate, uint64_t offset,
         auto statuscode = response.statuscode();
         switch (statuscode) {
             case StatusCode::kOwnerAuthFail:
-                LOG(ERROR) << "GetOrAllocateSegment Auth failed!";
+                LOG(WARNING) << "GetOrAllocateSegment Auth failed!";
                 return LIBCURVE_ERROR::AUTHFAIL;
                 break;
             case StatusCode::kSegmentNotAllocated:
@@ -888,7 +902,7 @@ LIBCURVE_ERROR MDSClient::GetOrAllocateSegment(bool allocate, uint64_t offset,
 
         int chunksNum = pfs.chunks_size();
         if (allocate && chunksNum <= 0) {
-            LOG(ERROR) << "MDS allocate segment, but no chunkinfo!";
+            LOG(WARNING) << "MDS allocate segment, but no chunkinfo!";
             return LIBCURVE_ERROR::FAILED;
         }
 
@@ -923,12 +937,12 @@ LIBCURVE_ERROR MDSClient::RenameFile(const UserInfo_t& userinfo,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                << "RenameFile: origin = " << origin.c_str()
-                << ", destination = " << destination.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                << "RenameFile: origin = " << origin
+                << ", destination = " << destination
                 << ", originId = " << originId
                 << ", destinationId = " << destinationId
-                << ", owner = " << userinfo.owner.c_str()
+                << ", owner = " << userinfo.owner
                 << ", errocde = " << retcode
                 << ", error msg = "
                 << StatusCode_Name(stcode)
@@ -959,9 +973,9 @@ LIBCURVE_ERROR MDSClient::Extend(const std::string& filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                    << "Extend: filename = " << filename.c_str()
-                    << ", owner = " << userinfo.owner.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                    << "Extend: filename = " << filename
+                    << ", owner = " << userinfo.owner
                     << ", newsize = " << newsize
                     << ", errocde = " << retcode
                     << ", error msg = " << StatusCode_Name(stcode)
@@ -990,9 +1004,9 @@ LIBCURVE_ERROR MDSClient::DeleteFile(const std::string& filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                << "DeleteFile: filename = " << filename.c_str()
-                << ", owner = " << userinfo.owner.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                << "DeleteFile: filename = " << filename
+                << ", owner = " << userinfo.owner
                 << ", errocde = " << retcode
                 << ", error msg = " << StatusCode_Name(stcode)
                 << ", log id = " << cntl->log_id();
@@ -1020,10 +1034,10 @@ LIBCURVE_ERROR MDSClient::ChangeOwner(const std::string& filename,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                << "ChangeOwner: filename = " << filename.c_str()
-                << ", owner = " << userinfo.owner.c_str()
-                << ", new owner = " << newOwner.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                << "ChangeOwner: filename = " << filename
+                << ", owner = " << userinfo.owner
+                << ", new owner = " << newOwner
                 << ", errocde = " << retcode
                 << ", error msg = " << StatusCode_Name(stcode)
                 << ", log id = " << cntl->log_id();
@@ -1051,9 +1065,9 @@ LIBCURVE_ERROR MDSClient::Listdir(const std::string& dirpath,
         LIBCURVE_ERROR retcode;
         StatusCode stcode = response.statuscode();
         MDSStatusCode2LibcurveError(stcode, &retcode);
-        LOG_IF(ERROR, retcode != LIBCURVE_ERROR::OK)
-                << "Listdir: filename = " << dirpath.c_str()
-                << ", owner = " << userinfo.owner.c_str()
+        LOG_IF(WARNING, retcode != LIBCURVE_ERROR::OK)
+                << "Listdir: filename = " << dirpath
+                << ", owner = " << userinfo.owner
                 << ", errocde = " << retcode
                 << ", error msg = " << StatusCode_Name(stcode)
                 << ", log id = " << cntl->log_id();
@@ -1072,7 +1086,7 @@ LIBCURVE_ERROR MDSClient::Listdir(const std::string& dirpath,
                 memcpy(filestat.owner, finfo.owner().c_str(), NAME_MAX_SIZE);
                 memset(filestat.filename, 0, NAME_MAX_SIZE);
                 memcpy(filestat.filename, finfo.filename().c_str(),
-                NAME_MAX_SIZE);
+                       NAME_MAX_SIZE);
                 filestatVec->push_back(filestat);
             }
         }
@@ -1116,7 +1130,7 @@ LIBCURVE_ERROR MDSClient::GetChunkServerInfo(const ChunkServerAddr& csAddr,
         }
 
         int statusCode = response.statuscode();
-        LOG_IF(ERROR, statusCode != 0)
+        LOG_IF(WARNING, statusCode != 0)
                 << "GetChunkServerInfo: errocde = " << statusCode
                 << ", log id = " << cntl->log_id();
 
@@ -1163,7 +1177,7 @@ LIBCURVE_ERROR MDSClient::ListChunkServerInServer(
         }
 
         int statusCode = response.statuscode();
-        LOG_IF(ERROR, statusCode != 0)
+        LOG_IF(WARNING, statusCode != 0)
             << "ListChunkServerInServer failed, "
             << "errorcode = " << response.statuscode()
             << ", chunkserver ip = " << serverIp

src/client/mds_client.h

@@ -57,7 +57,7 @@ struct LeaseRefreshResult;
 // MDSClient是client与MDS通信的唯一窗口
 class MDSClient {
  public:
-    MDSClient();
+    explicit MDSClient(const std::string& metricPrefix = "");
     ~MDSClient() = default;
     using RPCFunc = std::function<int(int, uint64_t,
                     brpc::Channel*, brpc::Controller*)>;

src/client/mds_client_base.cpp

@@ -42,7 +42,7 @@ void MDSClientBase::OpenFile(const std::string& filename,
     request.set_clientversion(curve::common::CurveVersion());
     FillUserInfo<OpenFileRequest>(&request, userinfo);
 
-    LOG(INFO) << "OpenFile: filename = " << filename.c_str()
+    LOG(INFO) << "OpenFile: filename = " << filename
               << ", owner = " << userinfo.owner
               << ", log id = " << cntl->log_id();
 
@@ -68,7 +68,7 @@ void MDSClientBase::CreateFile(const std::string& filename,
 
     FillUserInfo<CreateFileRequest>(&request, userinfo);
 
-    LOG(INFO) << "CreateFile: filename = " << filename.c_str()
+    LOG(INFO) << "CreateFile: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", is nomalfile: " << normalFile
                 << ", log id = " << cntl->log_id();
@@ -88,7 +88,7 @@ void MDSClientBase::CloseFile(const std::string& filename,
     request.set_sessionid(sessionid);
     FillUserInfo<curve::mds::CloseFileRequest>(&request, userinfo);
 
-    LOG(INFO) << "CloseFile: filename = " << filename.c_str()
+    LOG(INFO) << "CloseFile: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", sessionid = " << sessionid
                 << ", log id = " << cntl->log_id();
@@ -106,9 +106,9 @@ void MDSClientBase::GetFileInfo(const std::string& filename,
     request.set_filename(filename);
     FillUserInfo<curve::mds::GetFileInfoRequest>(&request, userinfo);
 
-    LOG(INFO) << "GetFileInfo: filename = " << filename.c_str()
-                << ", owner = " << userinfo.owner
-                << ", log id = " << cntl->log_id();
+    LOG_EVERY_SECOND(INFO) << "GetFileInfo: filename = " << filename
+                           << ", owner = " << userinfo.owner
+                           << ", log id = " << cntl->log_id();
 
     curve::mds::CurveFSService_Stub stub(channel);
     stub.GetFileInfo(cntl, &request, response, nullptr);
@@ -123,7 +123,7 @@ void MDSClientBase::CreateSnapShot(const std::string& filename,
     request.set_filename(filename);
     FillUserInfo<::curve::mds::CreateSnapShotRequest>(&request, userinfo);
 
-    LOG(INFO) << "CreateSnapShot: filename = " << filename.c_str()
+    LOG(INFO) << "CreateSnapShot: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", log id = " << cntl->log_id();
 
@@ -142,7 +142,7 @@ void MDSClientBase::DeleteSnapShot(const std::string& filename,
     request.set_filename(filename);
     FillUserInfo<::curve::mds::DeleteSnapShotRequest>(&request, userinfo);
 
-    LOG(INFO) << "DeleteSnapShot: filename = " << filename.c_str()
+    LOG(INFO) << "DeleteSnapShot: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", seqnum = " << seq
                 << ", log id = " << cntl->log_id();
@@ -164,7 +164,7 @@ void MDSClientBase::ListSnapShot(const std::string& filename,
     request.set_filename(filename);
     FillUserInfo<ListSnapShotFileInfoRequest>(&request, userinfo);
 
-    LOG(INFO) << "ListSnapShot: filename = " << filename.c_str()
+    LOG(INFO) << "ListSnapShot: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", seqnum = " << [seq] () {
                     std::string data("[ ");
@@ -195,7 +195,7 @@ void MDSClientBase::GetSnapshotSegmentInfo(const std::string& filename,
     request.set_seqnum(seq);
     FillUserInfo<GetOrAllocateSegmentRequest>(&request, userinfo);
 
-    LOG(INFO) << "GetSnapshotSegmentInfo: filename = " << filename.c_str()
+    LOG(INFO) << "GetSnapshotSegmentInfo: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", offset = " << offset
                 << ", seqnum = " << seq
@@ -226,7 +226,7 @@ void MDSClientBase::RefreshSession(const std::string& filename,
 
     FillUserInfo<ReFreshSessionRequest>(&request, userinfo);
 
-    LOG_EVERY_N(INFO, 10) << "RefreshSession: filename = " << filename.c_str()
+    LOG_EVERY_N(INFO, 10) << "RefreshSession: filename = " << filename
                           << ", owner = " << userinfo.owner
                           << ", sessionid = " << sessionid
                           << ", log id = " << cntl->log_id();
@@ -246,7 +246,7 @@ void MDSClientBase::CheckSnapShotStatus(const std::string& filename,
     request.set_filename(filename);
     FillUserInfo<CheckSnapShotStatusRequest>(&request, userinfo);
 
-    LOG(INFO) << "CheckSnapShotStatus: filename = " << filename.c_str()
+    LOG(INFO) << "CheckSnapShotStatus: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", seqnum = " << seq
                 << ", log id = " << cntl->log_id();
@@ -299,7 +299,7 @@ void MDSClientBase::CreateCloneFile(const std::string& source,
 
     LOG(INFO) << "CreateCloneFile: source = " << source
               << ", destination = " << destination
-              << ", owner = " << userinfo.owner.c_str() << ", seqnum = " << sn
+              << ", owner = " << userinfo.owner << ", seqnum = " << sn
               << ", size = " << size << ", chunksize = " << chunksize
               << ", log id = " << cntl->log_id();
 
@@ -322,8 +322,8 @@ void MDSClientBase::SetCloneFileStatus(const std::string &filename,
     }
     FillUserInfo<SetCloneFileStatusRequest>(&request, userinfo);
 
-    LOG(INFO) << "SetCloneFileStatus: filename = " << filename.c_str()
-                << ", owner = " << userinfo.owner.c_str()
+    LOG(INFO) << "SetCloneFileStatus: filename = " << filename
+                << ", owner = " << userinfo.owner
                 << ", filestatus = " << static_cast<int>(filestatus)
                 << ", fileID = " << fileID
                 << ", log id = " << cntl->log_id();
@@ -350,7 +350,7 @@ void MDSClientBase::GetOrAllocateSegment(bool allocate,
     FillUserInfo<GetOrAllocateSegmentRequest>(&request, fi->userinfo);
 
     LOG(INFO) << "GetOrAllocateSegment: allocate = " << allocate
-                << ", owner = " << fi->owner.c_str()
+                << ", owner = " << fi->owner
                 << ", offset = " << offset
                 << ", segment offset = " << seg_offset
                 << ", log id = " << cntl->log_id();
@@ -376,11 +376,11 @@ void MDSClientBase::RenameFile(const UserInfo_t& userinfo,
     }
     FillUserInfo<RenameFileRequest>(&request, userinfo);
 
-    LOG(INFO) << "RenameFile: origin = " << origin.c_str()
-              << ", destination = " << destination.c_str()
+    LOG(INFO) << "RenameFile: origin = " << origin
+              << ", destination = " << destination
               << ", originId = " << originId
               << ", destinationId = " << destinationId
-              << ", owner = " << userinfo.owner.c_str()
+              << ", owner = " << userinfo.owner
               << ", log id = " << cntl->log_id();
 
     curve::mds::CurveFSService_Stub stub(channel);
@@ -398,8 +398,8 @@ void MDSClientBase::Extend(const std::string& filename,
     request.set_newsize(newsize);
     FillUserInfo<ExtendFileRequest>(&request, userinfo);
 
-    LOG(INFO) << "Extend: filename = " << filename.c_str()
-              << ", owner = " << userinfo.owner.c_str()
+    LOG(INFO) << "Extend: filename = " << filename
+              << ", owner = " << userinfo.owner
               << ", newsize = " << newsize
               << ", log id = " << cntl->log_id();
 
@@ -422,8 +422,8 @@ void MDSClientBase::DeleteFile(const std::string& filename,
     }
     FillUserInfo<DeleteFileRequest>(&request, userinfo);
 
-    LOG(INFO) << "DeleteFile: filename = " << filename.c_str()
-                << ", owner = " << userinfo.owner.c_str()
+    LOG(INFO) << "DeleteFile: filename = " << filename
+                << ", owner = " << userinfo.owner
                 << ", log id = " << cntl->log_id();
 
     curve::mds::CurveFSService_Stub stub(channel);
@@ -444,9 +444,9 @@ void MDSClientBase::ChangeOwner(const std::string& filename,
     request.set_rootowner(userinfo.owner);
     request.set_signature(CalcSignature(userinfo, date));
 
-    LOG(INFO) << "ChangeOwner: filename = " << filename.c_str()
-                << ", operator owner = " << userinfo.owner.c_str()
-                << ", new owner = " << newOwner.c_str()
+    LOG(INFO) << "ChangeOwner: filename = " << filename
+                << ", operator owner = " << userinfo.owner
+                << ", new owner = " << newOwner
                 << ", log id = " << cntl->log_id();
 
     curve::mds::CurveFSService_Stub stub(channel);
@@ -463,8 +463,8 @@ void MDSClientBase::Listdir(const std::string& dirpath,
 
     FillUserInfo<::curve::mds::ListDirRequest>(&request, userinfo);
 
-    LOG(INFO) << "Listdir: filename = " << dirpath.c_str()
-                << ", owner = " << userinfo.owner.c_str()
+    LOG(INFO) << "Listdir: filename = " << dirpath
+                << ", owner = " << userinfo.owner
                 << ", log id = " << cntl->log_id();
 
     curve::mds::CurveFSService_Stub stub(channel);

src/client/metacache.cpp

@@ -125,7 +125,7 @@ int MetaCache::GetLeader(LogicPoolID logicPoolId,
     }
 
     if (ret == -1) {
-        LOG_EVERY_N(ERROR, 10) << "get leader failed after retry!"
+        LOG(WARNING) << "get leader failed after retry!"
             << ", copyset id = " << copysetId
             << ", logicpool id = " << logicPoolId;
         return -1;
@@ -149,7 +149,7 @@ int MetaCache::UpdateLeaderInternal(LogicPoolID logicPoolId,
         getLeaderInfo, &leaderaddr, &csid, fm);
 
     if (ret == -1) {
-        LOG_EVERY_N(ERROR, 10) << "get leader failed!"
+        LOG(WARNING) << "get leader failed!"
             << ", copyset id = " << copysetId
             << ", logicpool id = " << logicPoolId;
         return -1;
@@ -304,7 +304,7 @@ int MetaCache::SetServerUnstable(const std::string& serverIp) {
     std::vector<ChunkServerID> csIds;
     int ret = mdsclient_->ListChunkServerInServer(serverIp, &csIds);
     if (ret != LIBCURVE_ERROR::OK) {
-        LOG(WARNING) << "ListChunkServer failed";
+        LOG(ERROR) << "ListChunkServer failed, server ip: " << serverIp;
         return -1;
     }
 

src/client/metacache.h

@@ -237,6 +237,12 @@ class MetaCache {
         fileInfo_.seqnum = newSn;
     }
 
+    FileStatus GetLatestFileStatus() const { return fileInfo_.filestatus; }
+
+    void SetLatestFileStatus(FileStatus status) {
+        fileInfo_.filestatus = status;
+    }
+
     /**
      * 获取对应的copyset的LeaderMayChange标志
      */

src/client/request_context.h

@@ -34,19 +34,22 @@ namespace client {
 
 struct RequestSourceInfo {
     std::string cloneFileSource;
-    uint64_t cloneFileOffset{0};
+    uint64_t cloneFileOffset = 0;
+    bool valid = false;
 
     RequestSourceInfo() = default;
     RequestSourceInfo(const std::string& source, uint64_t offset)
-        : cloneFileSource(source), cloneFileOffset(offset) {}
+        : cloneFileSource(source), cloneFileOffset(offset), valid(true) {}
+
+    bool IsValid() const { return valid; }
 };
 
 inline std::ostream& operator<<(std::ostream& os,
                                 const RequestSourceInfo& location) {
-    if (location.cloneFileSource.empty()) {
-        os << "empty";
-    } else {
+    if (location.IsValid()) {
         os << location.cloneFileSource << ":" << location.cloneFileOffset;
+    } else {
+        os << "empty";
     }
 
     return os;

src/client/request_sender.cpp

@@ -37,6 +37,36 @@ namespace client {
 
 static void EmptyDeleter(void* ptr) {}
 
+using curve::chunkserver::ChunkRequest;
+using curve::chunkserver::ChunkResponse;
+using curve::chunkserver::ChunkService_Stub;
+using curve::chunkserver::GetChunkInfoRequest;
+using curve::chunkserver::GetChunkInfoResponse;
+using curve::common::TimeUtility;
+using ::google::protobuf::Closure;
+
+inline void RequestSender::UpdateRpcRPS(ClientClosure* done,
+                                        OpType type) const {
+    RequestClosure* request = static_cast<RequestClosure*>(done->GetClosure());
+    MetricHelper::IncremRPCRPSCount(request->GetMetric(), type);
+}
+
+inline void RequestSender::SetRpcStuff(
+    ClientClosure* done, brpc::Controller* cntl,
+    google::protobuf::Message* rpcResponse) const {
+    RequestClosure* request = static_cast<RequestClosure*>(done->GetClosure());
+    request->SetStartTime(TimeUtility::GetTimeofDayUs());
+
+    cntl->set_timeout_ms(
+        std::max(request->GetNextTimeoutMS(),
+                 iosenderopt_.failRequestOpt.chunkserverRPCTimeoutMS));
+
+    done->SetCntl(cntl);
+    done->SetResponse(rpcResponse);
+    done->SetChunkServerID(chunkServerId_);
+    done->SetChunkServerEndPoint(serverEndPoint_);
+}
+
 int RequestSender::Init(const IOSenderOption_t& ioSenderOpt) {
     if (0 != channel_.Init(serverEndPoint_, NULL)) {
         LOG(ERROR) << "failed to init channel to server, id: " << chunkServerId_
@@ -49,7 +79,7 @@ int RequestSender::Init(const IOSenderOption_t& ioSenderOpt) {
     return 0;
 }
 
-int RequestSender::ReadChunk(ChunkIDInfo idinfo,
+int RequestSender::ReadChunk(const ChunkIDInfo& idinfo,
                              uint64_t sn,
                              off_t offset,
                              size_t length,
@@ -57,20 +87,11 @@ int RequestSender::ReadChunk(ChunkIDInfo idinfo,
                              const RequestSourceInfo& sourceInfo,
                              ClientClosure *done) {
     brpc::ClosureGuard doneGuard(done);
-
-    RequestClosure* rc = static_cast<RequestClosure*>(done->GetClosure());
-    MetricHelper::IncremRPCRPSCount(rc->GetMetric(), OpType::READ);
-    rc->SetStartTime(TimeUtility::GetTimeofDayUs());
-
     brpc::Controller *cntl = new brpc::Controller();
-    cntl->set_timeout_ms(
-    std::max(rc->GetNextTimeoutMS(),
-        iosenderopt_.failRequestOpt.chunkserverRPCTimeoutMS));
-    done->SetCntl(cntl);
     ChunkResponse *response = new ChunkResponse();
-    done->SetResponse(response);
-    done->SetChunkServerID(chunkServerId_);
-    done->SetChunkServerEndPoint(serverEndPoint_);
+
+    UpdateRpcRPS(done, OpType::READ);
+    SetRpcStuff(done, cntl, response);
 
     ChunkRequest request;
     request.set_optype(curve::chunkserver::CHUNK_OP_TYPE::CHUNK_OP_READ);
@@ -80,7 +101,7 @@ int RequestSender::ReadChunk(ChunkIDInfo idinfo,
     request.set_offset(offset);
     request.set_size(length);
 
-    if (!sourceInfo.cloneFileSource.empty()) {
+    if (sourceInfo.IsValid()) {
         request.set_clonefilesource(sourceInfo.cloneFileSource);
         request.set_clonefileoffset(sourceInfo.cloneFileOffset);
     }
@@ -94,7 +115,7 @@ int RequestSender::ReadChunk(ChunkIDInfo idinfo,
     return 0;
 }
 
-int RequestSender::WriteChunk(ChunkIDInfo idinfo,
+int RequestSender::WriteChunk(const ChunkIDInfo& idinfo,
                               uint64_t sn,
                               const char *buf,
                               off_t offset,
@@ -102,22 +123,14 @@ int RequestSender::WriteChunk(ChunkIDInfo idinfo,
                               const RequestSourceInfo& sourceInfo,
                               ClientClosure *done) {
     brpc::ClosureGuard doneGuard(done);
-
-    RequestClosure* rc = static_cast<RequestClosure*>(done->GetClosure());
-    MetricHelper::IncremRPCRPSCount(rc->GetMetric(), OpType::WRITE);
-    rc->SetStartTime(TimeUtility::GetTimeofDayUs());
+    brpc::Controller *cntl = new brpc::Controller();
+    ChunkResponse *response = new ChunkResponse();
 
     DVLOG(9) << "Sending request, buf header: "
              << " buf: " << *(unsigned int *)buf;
-    brpc::Controller *cntl = new brpc::Controller();
-    cntl->set_timeout_ms(
-    std::max(rc->GetNextTimeoutMS(),
-        iosenderopt_.failRequestOpt.chunkserverRPCTimeoutMS));
-    done->SetCntl(cntl);
-    ChunkResponse *response = new ChunkResponse();
-    done->SetResponse(response);
-    done->SetChunkServerID(chunkServerId_);
-    done->SetChunkServerEndPoint(serverEndPoint_);
+
+    UpdateRpcRPS(done, OpType::WRITE);
+    SetRpcStuff(done, cntl, response);
 
     ChunkRequest request;
     request.set_optype(curve::chunkserver::CHUNK_OP_TYPE::CHUNK_OP_WRITE);
@@ -128,7 +141,7 @@ int RequestSender::WriteChunk(ChunkIDInfo idinfo,
     request.set_offset(offset);
     request.set_size(length);
 
-    if (!sourceInfo.cloneFileSource.empty()) {
+    if (sourceInfo.IsValid()) {
         request.set_clonefilesource(sourceInfo.cloneFileSource);
         request.set_clonefileoffset(sourceInfo.cloneFileOffset);
     }
@@ -141,21 +154,17 @@ int RequestSender::WriteChunk(ChunkIDInfo idinfo,
     return 0;
 }
 
-int RequestSender::ReadChunkSnapshot(ChunkIDInfo idinfo,
+int RequestSender::ReadChunkSnapshot(const ChunkIDInfo& idinfo,
                                      uint64_t sn,
                                      off_t offset,
                                      size_t length,
                                      ClientClosure *done) {
     brpc::ClosureGuard doneGuard(done);
-
-    RequestClosure* rc = static_cast<RequestClosure*>(done->GetClosure());
     brpc::Controller *cntl = new brpc::Controller();
-    cntl->set_timeout_ms(
-    std::max(rc->GetNextTimeoutMS(),
-        iosenderopt_.failRequestOpt.chunkserverRPCTimeoutMS));
-    done->SetCntl(cntl);
     ChunkResponse *response = new ChunkResponse();
-    done->SetResponse(response);
+
+    UpdateRpcRPS(done, OpType::READ_SNAP);
+    SetRpcStuff(done, cntl, response);
 
     ChunkRequest request;
     request.set_optype(curve::chunkserver::CHUNK_OP_TYPE::CHUNK_OP_READ_SNAP);
@@ -171,19 +180,15 @@ int RequestSender::ReadChunkSnapshot(ChunkIDInfo idinfo,
     return 0;
 }
 
-int RequestSender::DeleteChunkSnapshotOrCorrectSn(ChunkIDInfo idinfo,
-                                       uint64_t correctedSn,
-                                       ClientClosure *done) {
+int RequestSender::DeleteChunkSnapshotOrCorrectSn(const ChunkIDInfo& idinfo,
+                                                  uint64_t correctedSn,
+                                                  ClientClosure* done) {
     brpc::ClosureGuard doneGuard(done);
-
-    RequestClosure* rc = static_cast<RequestClosure*>(done->GetClosure());
     brpc::Controller *cntl = new brpc::Controller();
-    cntl->set_timeout_ms(
-    std::max(rc->GetNextTimeoutMS(),
-        iosenderopt_.failRequestOpt.chunkserverRPCTimeoutMS));
-    done->SetCntl(cntl);
     ChunkResponse *response = new ChunkResponse();
-    done->SetResponse(response);
+
+    UpdateRpcRPS(done, OpType::DELETE_SNAP);
+    SetRpcStuff(done, cntl, response);
 
     ChunkRequest request;
     request.set_optype(curve::chunkserver::CHUNK_OP_TYPE::CHUNK_OP_DELETE_SNAP);
@@ -199,18 +204,14 @@ int RequestSender::DeleteChunkSnapshotOrCorrectSn(ChunkIDInfo idinfo,
     return 0;
 }
 
-int RequestSender::GetChunkInfo(ChunkIDInfo idinfo,
+int RequestSender::GetChunkInfo(const ChunkIDInfo& idinfo,
                                 ClientClosure *done) {
     brpc::ClosureGuard doneGuard(done);
-
-    RequestClosure* rc = static_cast<RequestClosure*>(done->GetClosure());
     brpc::Controller *cntl = new brpc::Controller();
-    cntl->set_timeout_ms(
-    std::max(rc->GetNextTimeoutMS(),
-        iosenderopt_.failRequestOpt.chunkserverRPCTimeoutMS));
-    done->SetCntl(cntl);
     GetChunkInfoResponse *response = new GetChunkInfoResponse();
-    done->SetResponse(response);
+
+    UpdateRpcRPS(done, OpType::GET_CHUNK_INFO);
+    SetRpcStuff(done, cntl, response);
 
     GetChunkInfoRequest request;
     request.set_logicpoolid(idinfo.lpid_);
@@ -221,22 +222,18 @@ int RequestSender::GetChunkInfo(ChunkIDInfo idinfo,
     return 0;
 }
 
-int RequestSender::CreateCloneChunk(ChunkIDInfo idinfo,
+int RequestSender::CreateCloneChunk(const ChunkIDInfo& idinfo,
                                 ClientClosure *done,
                                 const std::string &location,
                                 uint64_t correntSn,
                                 uint64_t sn,
                                 uint64_t chunkSize) {
     brpc::ClosureGuard doneGuard(done);
-
-    RequestClosure* rc = static_cast<RequestClosure*>(done->GetClosure());
     brpc::Controller *cntl = new brpc::Controller();
-    cntl->set_timeout_ms(
-    std::max(rc->GetNextTimeoutMS(),
-        iosenderopt_.failRequestOpt.chunkserverRPCTimeoutMS));
-    done->SetCntl(cntl);
     ChunkResponse *response = new ChunkResponse();
-    done->SetResponse(response);
+
+    UpdateRpcRPS(done, OpType::CREATE_CLONE);
+    SetRpcStuff(done, cntl, response);
 
     ChunkRequest request;
     request.set_optype(
@@ -258,15 +255,11 @@ int RequestSender::RecoverChunk(const ChunkIDInfo& idinfo,
                                 uint64_t offset,
                                 uint64_t len) {
     brpc::ClosureGuard doneGuard(done);
-
-    RequestClosure* rc = static_cast<RequestClosure*>(done->GetClosure());
     brpc::Controller *cntl = new brpc::Controller();
-    cntl->set_timeout_ms(
-    std::max(rc->GetNextTimeoutMS(),
-        iosenderopt_.failRequestOpt.chunkserverRPCTimeoutMS));
-    done->SetCntl(cntl);
     ChunkResponse *response = new ChunkResponse();
-    done->SetResponse(response);
+
+    UpdateRpcRPS(done, OpType::RECOVER_CHUNK);
+    SetRpcStuff(done, cntl, response);
 
     ChunkRequest request;
     request.set_optype(curve::chunkserver::CHUNK_OP_TYPE::CHUNK_OP_RECOVER);

src/client/request_sender.h

@@ -37,13 +37,6 @@
 namespace curve {
 namespace client {
 
-using curve::chunkserver::ChunkRequest;
-using curve::chunkserver::ChunkResponse;
-using curve::chunkserver::GetChunkInfoRequest;
-using curve::chunkserver::GetChunkInfoResponse;
-using curve::chunkserver::ChunkService_Stub;
-using ::google::protobuf::Closure;
-
 /**
  * 一个RequestSender负责管理一个ChunkServer的所有
  * connection，目前一个ChunkServer仅有一个connection
@@ -69,7 +62,7 @@ class RequestSender {
      * @param sourceInfo 数据源信息
      * @param done:上一层异步回调的closure
      */
-    int ReadChunk(ChunkIDInfo idinfo,
+    int ReadChunk(const ChunkIDInfo& idinfo,
                   uint64_t sn,
                   off_t offset,
                   size_t length,
@@ -87,7 +80,7 @@ class RequestSender {
    * @param sourceInfo 数据源信息
    * @param done:上一层异步回调的closure
    */
-    int WriteChunk(ChunkIDInfo idinfo,
+    int WriteChunk(const ChunkIDInfo& idinfo,
                    uint64_t sn,
                    const char *buf,
                    off_t offset,
@@ -103,7 +96,7 @@ class RequestSender {
      * @param length:读的长度
      * @param done:上一层异步回调的closure
      */
-    int ReadChunkSnapshot(ChunkIDInfo idinfo,
+    int ReadChunkSnapshot(const ChunkIDInfo& idinfo,
                           uint64_t sn,
                           off_t offset,
                           size_t length,
@@ -116,7 +109,7 @@ class RequestSender {
      * @param correctedSn:chunk需要修正的版本号
      * @param done:上一层异步回调的closure
      */
-    int DeleteChunkSnapshotOrCorrectSn(ChunkIDInfo idinfo,
+    int DeleteChunkSnapshotOrCorrectSn(const ChunkIDInfo& idinfo,
                             uint64_t correctedSn,
                             ClientClosure *done);
 
@@ -126,7 +119,7 @@ class RequestSender {
      * @param done:上一层异步回调的closure
      * @param retriedTimes:已经重试了几次
      */
-    int GetChunkInfo(ChunkIDInfo idinfo,
+    int GetChunkInfo(const ChunkIDInfo& idinfo,
                      ClientClosure *done);
 
     /**
@@ -146,7 +139,7 @@ class RequestSender {
     *
     * @return 错误码
     */
-    int CreateCloneChunk(ChunkIDInfo idinfo,
+    int CreateCloneChunk(const ChunkIDInfo& idinfo,
                   ClientClosure *done,
                   const std::string &location,
                   uint64_t sn,
@@ -178,6 +171,12 @@ class RequestSender {
        return channel_.CheckHealth() == 0;
     }
 
+ private:
+    void UpdateRpcRPS(ClientClosure* done, OpType type) const;
+
+    void SetRpcStuff(ClientClosure* done, brpc::Controller* cntl,
+                     google::protobuf::Message* rpcResponse) const;
+
  private:
     // Rpc stub配置
     IOSenderOption_t iosenderopt_;

src/client/service_helper.cpp

@@ -35,49 +35,59 @@
 namespace curve {
 namespace client {
 
-void ServiceHelper::ProtoFileInfo2Local(const curve::mds::FileInfo* finfo,
+void ServiceHelper::ProtoFileInfo2Local(const curve::mds::FileInfo& finfo,
                                         FInfo_t* fi) {
-    if (finfo->has_owner()) {
-        fi->owner = finfo->owner();
+    if (finfo.has_owner()) {
+        fi->owner = finfo.owner();
     }
-    if (finfo->has_filename()) {
-        fi->filename = finfo->filename();
+    if (finfo.has_filename()) {
+        fi->filename = finfo.filename();
     }
-    if (finfo->has_id()) {
-        fi->id = finfo->id();
+    if (finfo.has_id()) {
+        fi->id = finfo.id();
     }
-    if (finfo->has_parentid()) {
-        fi->parentid = finfo->parentid();
+    if (finfo.has_parentid()) {
+        fi->parentid = finfo.parentid();
     }
-    if (finfo->has_filetype()) {
-        fi->filetype = static_cast<FileType>(finfo->filetype());
+    if (finfo.has_filetype()) {
+        fi->filetype = static_cast<FileType>(finfo.filetype());
     }
-    if (finfo->has_chunksize()) {
-        fi->chunksize = finfo->chunksize();
+    if (finfo.has_chunksize()) {
+        fi->chunksize = finfo.chunksize();
     }
-    if (finfo->has_length()) {
-        fi->length = finfo->length();
+    if (finfo.has_length()) {
+        fi->length = finfo.length();
     }
-    if (finfo->has_ctime()) {
-        fi->ctime = finfo->ctime();
+    if (finfo.has_ctime()) {
+        fi->ctime = finfo.ctime();
     }
-    if (finfo->has_chunksize()) {
-        fi->chunksize = finfo->chunksize();
+    if (finfo.has_chunksize()) {
+        fi->chunksize = finfo.chunksize();
     }
-    if (finfo->has_segmentsize()) {
-        fi->segmentsize = finfo->segmentsize();
+    if (finfo.has_segmentsize()) {
+        fi->segmentsize = finfo.segmentsize();
     }
-    if (finfo->has_seqnum()) {
-        fi->seqnum = finfo->seqnum();
+    if (finfo.has_seqnum()) {
+        fi->seqnum = finfo.seqnum();
     }
-    if (finfo->has_filestatus()) {
-        fi->filestatus = (FileStatus)finfo->filestatus();
+    if (finfo.has_filestatus()) {
+        fi->filestatus = (FileStatus)finfo.filestatus();
     }
-    if (finfo->has_clonesource()) {
-        fi->cloneSource = finfo->clonesource();
-    }
-    if (finfo->has_clonelength()) {
-        fi->cloneLength = finfo->clonelength();
+}
+
+void ServiceHelper::ProtoCloneSourceInfo2Local(
+    const curve::mds::OpenFileResponse& openFileResponse,
+    CloneSourceInfo* info) {
+    const curve::mds::FileInfo& fileInfo = openFileResponse.fileinfo();
+    const curve::mds::CloneSourceSegment& sourceSegment =
+        openFileResponse.clonesourcesegment();
+
+    info->name = fileInfo.clonesource();
+    info->length = fileInfo.clonelength();
+    info->segmentSize = sourceSegment.segmentsize();
+    for (int i = 0; i < sourceSegment.allocatedsegmentoffset_size(); ++i) {
+        info->allocatedSegmentOffsets.insert(
+            sourceSegment.allocatedsegmentoffset(i));
     }
 }
 
@@ -338,7 +348,6 @@ bool ServiceHelper::GetUserInfoFromFilename(const std::string& filename,
     *realfilename = filename.substr(0, user_begin);
     *user = filename.substr(user_begin + 1, user_end - user_begin - 1);
 
-    LOG(INFO) << "user info [" << *user << "]";
     return true;
 }
 
@@ -375,4 +384,3 @@ int ServiceHelper::CheckChunkServerHealth(
 
 }   // namespace client
 }   // namespace curve
-

src/client/service_helper.h

@@ -95,8 +95,13 @@ class ServiceHelper {
      * @param: finfo为proto格式的文件信息
      * @param: fi为本地格式的文件信息
      */
-    static void ProtoFileInfo2Local(const curve::mds::FileInfo* finfo,
+    static void ProtoFileInfo2Local(const curve::mds::FileInfo& finfo,
                                     FInfo_t* fi);
+
+    static void ProtoCloneSourceInfo2Local(
+        const curve::mds::OpenFileResponse& openFileResponse,
+        CloneSourceInfo* info);
+
     /**
      * 从chunkserver端获取最新的leader信息
      * @param[in]: getLeaderInfo为对应copyset的信息

src/client/splitor.cpp

@@ -178,7 +178,7 @@ bool Splitor::AssignInternal(IOTracker* iotracker,
                                         (off_t)chunkidx * fileinfo->chunksize,
                                         fileinfo,
                                         &segInfo);
-        if (re == LIBCURVE_ERROR::FAILED || re == LIBCURVE_ERROR::AUTHFAIL) {
+        if (re != LIBCURVE_ERROR::OK) {
             LOG(ERROR) << "GetOrAllocateSegment failed! "
                        << "offset = " << chunkidx * fileinfo->chunksize;
             return false;
@@ -281,23 +281,21 @@ RequestContext* Splitor::GetInitedRequestContext() {
 RequestSourceInfo Splitor::CalcRequestSourceInfo(IOTracker* ioTracker,
                                                  MetaCache* metaCache,
                                                  ChunkIndex chunkIdx) {
-    const FInfo* fileInfo = metaCache->GetFileInfo();
-    if (fileInfo->cloneSource.empty()) {
-        return {};
-    }
-
     OpType type = ioTracker->Optype();
     if (type != OpType::READ && type != OpType::WRITE) {
         return {};
     }
 
-    uint64_t offset = static_cast<uint64_t>(chunkIdx) * fileInfo->chunksize;
-
-    if (offset >= fileInfo->cloneLength) {
-        return {};
+    const FInfo* fileInfo = metaCache->GetFileInfo();
+    if (fileInfo->filestatus == FileStatus::CloneMetaInstalled) {
+        const CloneSourceInfo& sourceInfo = fileInfo->sourceInfo;
+        uint64_t offset = static_cast<uint64_t>(chunkIdx) * fileInfo->chunksize;
+        if (sourceInfo.IsSegmentAllocated(offset)) {
+            return {sourceInfo.name, offset};
+        }
     }
 
-    return {fileInfo->cloneSource, offset};
+    return {};
 }
 
 }   // namespace client

src/common/namespace_define.h

@@ -39,7 +39,20 @@ const char CHUNKSTOREKEYEND[] = "06";
 const char LEADERCAMPAIGNNPFX[] = "07leader";
 const char SEGMENTALLOCSIZEKEY[] = "08";
 const char SEGMENTALLOCSIZEKEYEND[] = "09";
-const char TOPOLOGYITEMPRIFIX[] = "10";
+
+const char LOGICALPOOLKEYPREFIX[] = "1001";
+const char LOGICALPOOLKEYEND[] = "1002";
+const char PHYSICALPOOLKEYPREFIX[] = "1002";
+const char PHYSICALPOOLKEYEND[] = "1003";
+const char ZONEKEYPREFIX[] = "1003";
+const char ZONEKEYEND[] = "1004";
+const char SERVERKEYPREFIX[] = "1004";
+const char SERVERKEYEND[] = "1005";
+const char CHUNKSERVERKEYPREFIX[] = "1005";
+const char CHUNKSERVERKEYEND[] = "1006";
+const char CLUSTERINFOKEY[] = "1007";
+const char COPYSETKEYPREFIX[] = "1008";
+const char COPYSETKEYEND[] = "1009";
 
 const char SNAPINFOKEYPREFIX[] = "11";
 const char SNAPINFOKEYEND[] = "12";

src/kvstorageclient/etcd_client.cpp

@@ -129,6 +129,7 @@ int EtcdClientImp::List(const std::string &startKey, const std::string &endKey,
                 out->emplace_back(
                     std::string(objRes.r1, objRes.r1 + objRes.r2));
                 free(objRes.r1);
+                free(objRes.r3);
             }
             EtcdClientRemoveObject(res.r1);
         }

src/mds/common/mds_define.h

@@ -126,11 +126,7 @@ const uint64_t kMB = 1024*kKB;
 const uint64_t kGB = 1024*kMB;
 const uint64_t kTB = 1024*kGB;
 
-extern uint64_t DefaultSegmentSize;
-extern uint64_t kMiniFileLength;
-const uint64_t kMaxFileLength = 4 * kTB;
-
-// curve默认root目录&inodeid
+// curve default root path and inodeid
 const InodeID ROOTINODEID = 0;
 const char ROOTFILENAME[] = "/";
 

src/mds/main/main.cpp

@@ -35,12 +35,12 @@ DEFINE_uint32(dummyPort, 6667, "dummy server port");
 
 using ::curve::mds::kMB;
 using ::curve::mds::kGB;
-using ::curve::mds::DefaultSegmentSize;
-using ::curve::mds::kMiniFileLength;
+using ::curve::mds::kTB;
 
 DEFINE_uint64(chunkSize, 16 * kMB, "chunk size");
 DEFINE_uint64(segmentSize, 1 * kGB, "segment size");
-DEFINE_uint64(minFileLength, 10 * kGB, "min filglength");
+DEFINE_uint64(minFileLength, 10 * kGB, "min filelength");
+DEFINE_uint64(maxFileLength, 20 * kTB, "max filelength");
 
 void LoadConfigFromCmdline(Configuration *conf) {
     google::CommandLineFlagInfo info;
@@ -57,11 +57,16 @@ void LoadConfigFromCmdline(Configuration *conf) {
     }
 
     if (GetCommandLineFlagInfo("segmentSize", &info) && !info.is_default) {
-        DefaultSegmentSize = FLAGS_segmentSize;
+        conf->SetUInt64Value(
+                    "mds.curvefs.defaultSegmentSize", FLAGS_segmentSize);
     }
 
     if (GetCommandLineFlagInfo("minFileLength", &info) && !info.is_default) {
-        kMiniFileLength = FLAGS_minFileLength;
+        conf->SetUInt64Value("mds.curvefs.minFileLength", FLAGS_minFileLength);
+     }
+
+    if (GetCommandLineFlagInfo("maxFileLength", &info) && !info.is_default) {
+        conf->SetUInt64Value("mds.curvefs.maxFileLength", FLAGS_maxFileLength);
     }
 
     if (GetCommandLineFlagInfo("mdsDbName", &info) && !info.is_default) {

src/mds/nameserver2/clean_task.h

@@ -37,8 +37,13 @@ namespace mds {
 
 typedef  uint64_t TaskIDType;
 
+// default clean task retry times
+const uint32_t kDefaultTaskRetryTimes = 5;
+
 class Task {
  public:
+    Task() : taskID_(0), progress_(), retry_(1) {}
+
     virtual void Run(void) = 0;
 
     std::function<void()> Closure() {
@@ -67,9 +72,24 @@ class Task {
         return taskID_;
     }
 
+    void SetRetryTimes(uint32_t retry) {
+        retry_ = retry;
+    }
+
+    void Retry() {
+        retry_--;
+        progress_ = TaskProgress();
+    }
+
+    bool RetryTimesExceed() {
+        return retry_ == 0;
+    }
+
  protected:
     TaskIDType taskID_;
     TaskProgress progress_;
+    // 任务最大重试次数
+    uint32_t retry_;
 };
 
 class SnapShotCleanTask: public Task {
@@ -82,6 +102,7 @@ class SnapShotCleanTask: public Task {
         SetTaskProgress(TaskProgress());
         SetTaskID(taskID);
         asyncEntity_ = entity;
+        SetRetryTimes(kDefaultTaskRetryTimes);
     }
     void Run(void) override {
         StatusCode ret = cleanCore_->CleanSnapShotFile(fileInfo_,
@@ -126,6 +147,7 @@ class CommonFileCleanTask: public Task {
         fileInfo_ = fileInfo;
         SetTaskProgress(TaskProgress());
         SetTaskID(taskID);
+        SetRetryTimes(kDefaultTaskRetryTimes);
     }
 
     void Run(void) override {

src/mds/nameserver2/clean_task_manager.cpp

@@ -47,10 +47,22 @@ void CleanTaskManager::CheckCleanResult(void) {
                     iter = cleanTasks_.erase(iter);
                     continue;
                 } else if (taskProgress.GetStatus() == TaskStatus::FAILED) {
-                    LOG(WARNING) << "CleanTaskManager find Task Failed, retry,"
-                                 << " taskID = " << iter->second->GetTaskID();
-                    iter->second->SetTaskProgress(TaskProgress());
-                    cleanWorkers_->Enqueue(iter->second->Closure());
+                    iter->second->Retry();
+                    if (!iter->second->RetryTimesExceed()) {
+                        LOG(WARNING) << "CleanTaskManager find Task Failed,"
+                                     << " retry,"
+                                     << " taskID = "
+                                     << iter->second->GetTaskID();
+                        cleanWorkers_->Enqueue(iter->second->Closure());
+                    } else {
+                        LOG(ERROR) << "CleanTaskManager find Task Failed,"
+                                     << " retry times exceed,"
+                                     << " going to remove task,"
+                                     << " taskID = "
+                                     << iter->second->GetTaskID();
+                        iter = cleanTasks_.erase(iter);
+                        continue;
+                    }
                 }
                 ++iter;
             }

src/mds/nameserver2/curvefs.cpp

@@ -31,6 +31,7 @@
 #include "src/common/timeutility.h"
 #include "src/mds/nameserver2/namespace_storage.h"
 #include "src/mds/common/mds_define.h"
+#include "src/mds/nameserver2/helper/namespace_helper.h"
 
 using curve::common::TimeUtility;
 using ::std::chrono::steady_clock;
@@ -100,6 +101,9 @@ bool CurveFS::Init(std::shared_ptr<NameServerStorage> storage,
     rootAuthOptions_ = curveFSOptions.authOptions;
 
     defaultChunkSize_ = curveFSOptions.defaultChunkSize;
+    defaultSegmentSize_ = curveFSOptions.defaultSegmentSize;
+    minFileLength_ = curveFSOptions.minFileLength;
+    maxFileLength_ = curveFSOptions.maxFileLength;
     topology_ = topology;
 
     InitRootFile();
@@ -220,24 +224,24 @@ StatusCode CurveFS::CreateFile(const std::string & fileName,
 
     // check param
     if (filetype == FileType::INODE_PAGEFILE) {
-        if  (length < kMiniFileLength) {
-            LOG(ERROR) << "file Length < MinFileLength " << kMiniFileLength
+        if  (length < minFileLength_) {
+            LOG(ERROR) << "file Length < MinFileLength " << minFileLength_
                        << ", length = " << length;
             return StatusCode::kFileLengthNotSupported;
         }
 
-        if (length > kMaxFileLength) {
+        if (length > maxFileLength_) {
             LOG(ERROR) << "CreateFile file length > maxFileLength, fileName = "
                        << fileName << ", length = " << length
-                       << ", maxFileLength = " << kMaxFileLength;
+                       << ", maxFileLength = " << maxFileLength_;
             return StatusCode::kFileLengthNotSupported;
         }
 
-        if (length % DefaultSegmentSize != 0) {
+        if (length % defaultSegmentSize_ != 0) {
             LOG(ERROR) << "Create file length not align to segment size, "
                        << "fileName = " << fileName
                        << ", length = " << length
-                       << ", segment size = " << DefaultSegmentSize;
+                       << ", segment size = " << defaultSegmentSize_;
             return StatusCode::kFileLengthNotSupported;
         }
     }
@@ -271,7 +275,7 @@ StatusCode CurveFS::CreateFile(const std::string & fileName,
         fileInfo.set_filetype(filetype);
         fileInfo.set_owner(owner);
         fileInfo.set_chunksize(defaultChunkSize_);
-        fileInfo.set_segmentsize(DefaultSegmentSize);
+        fileInfo.set_segmentsize(defaultSegmentSize_);
         fileInfo.set_length(length);
         fileInfo.set_ctime(::curve::common::TimeUtility::GetTimeofDayUs());
         fileInfo.set_seqnum(kStartSeqNum);
@@ -848,10 +852,10 @@ StatusCode CurveFS::ExtendFile(const std::string &filename,
         return StatusCode::kNotSupported;
     }
 
-    if (newLength > kMaxFileLength) {
+    if (newLength > maxFileLength_) {
         LOG(ERROR) << "ExtendFile newLength > maxFileLength, fileName = "
                        << filename << ", newLength = " << newLength
-                       << ", maxFileLength = " << kMaxFileLength;
+                       << ", maxFileLength = " << maxFileLength_;
             return StatusCode::kFileLengthNotSupported;
     }
 
@@ -1201,7 +1205,7 @@ StatusCode CurveFS::CheckSnapShotFileStatus(const std::string &fileName,
     FileInfo snapShotFileInfo;
     StatusCode ret =  GetSnapShotFileInfo(fileName, seq, &snapShotFileInfo);
     if (ret != StatusCode::kOK) {
-        LOG(ERROR) << "GetSnapShotFileInfo file fail, fileName = "
+        LOG(WARNING) << "GetSnapShotFileInfo file fail, fileName = "
                    << fileName << ", seq = " << seq << ", ret = " << ret;
         return ret;
     }
@@ -1211,25 +1215,35 @@ StatusCode CurveFS::CheckSnapShotFileStatus(const std::string &fileName,
         TaskIDType taskID = static_cast<TaskIDType>(snapShotFileInfo.id());
         auto task = cleanManager_->GetTask(taskID);
         if (task == nullptr) {
-            *progress = 100;
-            return StatusCode::kOK;
+            // GetSnapShotFileInfo again
+            StatusCode ret2 =
+                GetSnapShotFileInfo(fileName, seq, &snapShotFileInfo);
+            // if not exist, means delete succeed.
+            if (StatusCode::kSnapshotFileNotExists == ret2) {
+                *progress = 100;
+                return StatusCode::kSnapshotFileNotExists;
+            // else the snapshotFile still exist,
+            // means delete failed and retry times exceed.
+            } else {
+                *progress = 0;
+                LOG(ERROR) << "snapshot file delete fail, fileName = "
+                           << fileName << ", seq = " << seq;
+                return StatusCode::kSnapshotFileDeleteError;
+            }
         }
 
         TaskStatus taskStatus = task->GetTaskProgress().GetStatus();
         switch (taskStatus) {
             case TaskStatus::PROGRESSING:
+            case TaskStatus::FAILED:  // FAILED task will retry
                 *progress = task->GetTaskProgress().GetProgress();
                 break;
-            case TaskStatus::FAILED:
-                *progress = 0;
-                LOG(ERROR) << "snapshot file delete fail, fileName = "
-                           << fileName << ", seq = " << seq;
-                return StatusCode::kSnapshotFileDeleteError;
             case TaskStatus::SUCCESS:
                 *progress = 100;
                 break;
         }
     } else {
+        // means delete haven't begin.
         *progress = 0;
     }
 
@@ -1246,7 +1260,7 @@ StatusCode CurveFS::GetSnapShotFileSegment(
     FileInfo snapShotFileInfo;
     StatusCode ret = GetSnapShotFileInfo(fileName, seq, &snapShotFileInfo);
     if (ret != StatusCode::kOK) {
-        LOG(ERROR) << "GetSnapShotFileInfo file fail, fileName = "
+        LOG(WARNING) << "GetSnapShotFileInfo file fail, fileName = "
                    << fileName << ", seq = " << seq << ", ret = " << ret;
         return ret;
     }
@@ -1308,8 +1322,9 @@ StatusCode CurveFS::GetSnapShotFileSegment(
 StatusCode CurveFS::OpenFile(const std::string &fileName,
                              const std::string &clientIP,
                              ProtoSession *protoSession,
-                             FileInfo  *fileInfo) {
-    // 检查文件是否存在
+                             FileInfo  *fileInfo,
+                             CloneSourceSegment* cloneSourceSegment) {
+    // check the existence of the file
     StatusCode ret;
     ret = GetFileInfo(fileName, fileInfo);
     if (ret == StatusCode::kFileNotExists) {
@@ -1326,6 +1341,8 @@ StatusCode CurveFS::OpenFile(const std::string &fileName,
         return ret;
     }
 
+    LOG(INFO) << "FileInfo, " << fileInfo->DebugString();
+
     if (fileInfo->filetype() != FileType::INODE_PAGEFILE) {
         LOG(ERROR) << "OpenFile file type not support, fileName = " << fileName
                    << ", clientIP = " << clientIP
@@ -1335,6 +1352,11 @@ StatusCode CurveFS::OpenFile(const std::string &fileName,
 
     fileRecordManager_->GetRecordParam(protoSession);
 
+    // clone file
+    if (fileInfo->has_clonesource() && isPathValid(fileInfo->clonesource())) {
+        return ListCloneSourceFileSegments(fileInfo, cloneSourceSegment);
+    }
+
     return StatusCode::kOK;
 }
 
@@ -1418,9 +1440,9 @@ StatusCode CurveFS::CreateCloneFile(const std::string &fileName,
         return StatusCode::kParaError;
     }
 
-    if  (length < kMiniFileLength || seq < kStartSeqNum) {
+    if  (length < minFileLength_ || seq < kStartSeqNum) {
         LOG(WARNING) << "CreateCloneFile err, filename = " << fileName
-                    << "file Length < MinFileLength " << kMiniFileLength
+                    << "file Length < MinFileLength " << minFileLength_
                     << ", length = " << length;
         return StatusCode::kParaError;
     }
@@ -1461,7 +1483,7 @@ StatusCode CurveFS::CreateCloneFile(const std::string &fileName,
         fileInfo.set_owner(owner);
 
         fileInfo.set_chunksize(chunksize);
-        fileInfo.set_segmentsize(DefaultSegmentSize);
+        fileInfo.set_segmentsize(defaultSegmentSize_);
         fileInfo.set_length(length);
         fileInfo.set_ctime(::curve::common::TimeUtility::GetTimeofDayUs());
 
@@ -1809,6 +1831,61 @@ StatusCode CurveFS::ListClient(bool listAllClient,
     return StatusCode::kOK;
 }
 
+StatusCode CurveFS::ListCloneSourceFileSegments(
+    const FileInfo* fileInfo, CloneSourceSegment* cloneSourceSegment) const {
+    if (fileInfo->filestatus() != FileStatus::kFileCloneMetaInstalled) {
+        LOG(INFO) << fileInfo->filename()
+                  << " hash clone source, but file status is "
+                  << FileStatus_Name(fileInfo->filestatus())
+                  << ", return empty CloneSourceSegment";
+        return StatusCode::kOK;
+    }
+
+    if (!cloneSourceSegment) {
+        LOG(ERROR) << "OpenFile failed, file has clone source, but "
+                      "cloneSourceSegments is nullptr, filename = "
+                   << fileInfo->filename();
+        return StatusCode::kParaError;
+    }
+
+    FileInfo cloneSourceFileInfo;
+    StatusCode ret = GetFileInfo(fileInfo->clonesource(), &cloneSourceFileInfo);
+    if (ret != StatusCode::kOK) {
+        LOG(ERROR)
+            << "OpenFile failed, Get clone source file info failed, ret = "
+            << StatusCode_Name(ret) << ", filename = " << fileInfo->filename()
+            << ", clone source = " << fileInfo->clonesource()
+            << ", file status = " << FileStatus_Name(fileInfo->filestatus());
+        return ret;
+    }
+
+    std::vector<PageFileSegment> segments;
+    StoreStatus status =
+        storage_->ListSegment(cloneSourceFileInfo.id(), &segments);
+    if (status != StoreStatus::OK) {
+        LOG(ERROR) << "OpenFile failed, list clone source segment failed, "
+                      "filename = "
+                   << fileInfo->filename()
+                   << ", source file name = " << fileInfo->clonesource()
+                   << ", ret = " << status;
+        return StatusCode::kStorageError;
+    }
+
+    cloneSourceSegment->set_segmentsize(fileInfo->segmentsize());
+
+    if (segments.empty()) {
+        LOG(WARNING) << "Clone source file has no segments, filename = "
+                     << fileInfo->clonesource();
+    } else {
+        for (const auto& segment : segments) {
+            cloneSourceSegment->add_allocatedsegmentoffset(
+                segment.startoffset());
+        }
+    }
+
+    return StatusCode::kOK;
+}
+
 StatusCode CurveFS::FindFileMountPoint(
     const std::string& fileName,
     ClientInfo* clientInfo) {
@@ -1836,6 +1913,18 @@ uint64_t CurveFS::GetDefaultChunkSize() {
     return defaultChunkSize_;
 }
 
+uint64_t CurveFS::GetDefaultSegmentSize() {
+    return defaultSegmentSize_;
+}
+
+uint64_t CurveFS::GetMinFileLength() {
+    return minFileLength_;
+}
+
+uint64_t CurveFS::GetMaxFileLength() {
+    return maxFileLength_;
+}
+
 CurveFS &kCurveFS = CurveFS::GetInstance();
 
 uint64_t GetOpenFileNum(void *varg) {

src/mds/nameserver2/curvefs.h

@@ -52,6 +52,9 @@ struct RootAuthOption {
 
 struct CurveFSOption {
     uint64_t defaultChunkSize;
+    uint64_t defaultSegmentSize;
+    uint64_t minFileLength;
+    uint64_t maxFileLength;
     RootAuthOption authOptions;
     FileRecordOptions fileRecordOptions;
 };
@@ -307,7 +310,8 @@ class CurveFS {
     StatusCode OpenFile(const std::string &fileName,
                         const std::string &clientIP,
                         ProtoSession *protoSession,
-                        FileInfo  *fileInfo);
+                        FileInfo  *fileInfo,
+                        CloneSourceSegment* cloneSourceSegment = nullptr);
 
     /**
      *  @brief 关闭文件
@@ -465,6 +469,27 @@ class CurveFS {
      */
     uint64_t GetDefaultChunkSize();
 
+    /**
+     *  @brief get the defaultSegmentSize info of curvefs
+     *  @param:
+     *  @return return defaultSegmentSize info obtained
+     */
+    uint64_t GetDefaultSegmentSize();
+
+    /**
+     *  @brief get the minFileLength info of curvefs
+     *  @param:
+     *  @return return minFileLength info obtained
+     */
+    uint64_t GetMinFileLength();
+
+    /**
+     *  @brief get the maxFileLength info of curvefs
+     *  @param:
+     *  @return return maxFileLength info obtained
+     */
+    uint64_t GetMaxFileLength();
+
  private:
     CurveFS() = default;
 
@@ -593,6 +618,28 @@ class CurveFS {
                            const FileInfo& fileInfo,
                            uint64_t* fileSize);
 
+    /**
+     * @brief check whether mds has started for enough time, based on the
+     *        file record expiration time(mds.file.expiredTimeUs)
+     * @param times multiple of file record expiration time
+     * @return return true if ok, otherwise return false
+     */
+    bool IsStartEnoughTime(int times) const {
+        std::chrono::steady_clock::duration timePass =
+            std::chrono::steady_clock::now() - startTime_;
+        uint32_t expiredUs = fileRecordManager_->GetFileRecordExpiredTimeUs();
+        return timePass >= times * std::chrono::microseconds(expiredUs);
+    }
+
+    /**
+     * @brief list clone source file's segment,
+     *        if current file status is in kFileCloneMetaInstalled
+     * @param fileInfo current file info
+     * @param[out] cloneSourceSegment source file allocated segments
+     */
+    StatusCode ListCloneSourceFileSegments(
+        const FileInfo* fileInfo, CloneSourceSegment* cloneSourceSegment) const;
+
  private:
     FileInfo rootFileInfo_;
     std::shared_ptr<NameServerStorage> storage_;
@@ -605,6 +652,9 @@ class CurveFS {
     struct RootAuthOption       rootAuthOptions_;
 
     uint64_t defaultChunkSize_;
+    uint64_t defaultSegmentSize_;
+    uint64_t minFileLength_;
+    uint64_t maxFileLength_;
     std::chrono::steady_clock::time_point startTime_;
 };
 extern CurveFS &kCurveFS;

src/mds/nameserver2/helper/namespace_helper.h

@@ -51,6 +51,32 @@ class NameSpaceStorageCodec {
     static bool DecodeSegmentAllocValue(
         const std::string &value, uint16_t *lid, uint64_t *alloc);
 };
+
+inline bool isPathValid(const std::string path) {
+    if (path.empty() || path[0] != '/') {
+        return false;
+    }
+
+    if (path.size() > 1U && path[path.size() - 1] == '/') {
+        return false;
+    }
+
+    bool slash = false;
+    for (uint32_t i = 0; i < path.size(); i++) {
+        if (path[i] == '/') {
+            if (slash) {
+                return false;
+            }
+            slash = true;
+        } else {
+            slash = false;
+        }
+    }
+
+    // if some other limits to path can add here in the future
+    return true;
+}
+
 }   // namespace mds
 }   // namespace curve
 

src/mds/nameserver2/namespace_service.cpp

@@ -28,6 +28,7 @@
 #include "src/mds/nameserver2/file_lock.h"
 #include "src/common/string_util.h"
 #include "src/common/timeutility.h"
+#include "src/mds/nameserver2/helper/namespace_helper.h"
 
 namespace curve {
 namespace mds {
@@ -1179,10 +1180,12 @@ void NameSpaceService::OpenFile(::google::protobuf::RpcController* controller,
 
     ProtoSession *protoSession = new ProtoSession();
     FileInfo *fileInfo = new FileInfo();
+    CloneSourceSegment* cloneSourceSegment = new CloneSourceSegment();
     retCode = kCurveFS.OpenFile(request->filename(),
                                 clientIP,
                                 protoSession,
-                                fileInfo);
+                                fileInfo,
+                                cloneSourceSegment);
     if (retCode != StatusCode::kOK)  {
         response->set_statuscode(retCode);
         if (google::ERROR != GetMdsLogLevel(retCode)) {
@@ -1206,11 +1209,19 @@ void NameSpaceService::OpenFile(::google::protobuf::RpcController* controller,
         }
         delete protoSession;
         delete fileInfo;
+        delete cloneSourceSegment;
         return;
     } else {
         response->set_allocated_protosession(protoSession);
         response->set_allocated_fileinfo(fileInfo);
         response->set_statuscode(StatusCode::kOK);
+
+        if (cloneSourceSegment->IsInitialized()) {
+            response->set_allocated_clonesourcesegment(cloneSourceSegment);
+        } else {
+            delete cloneSourceSegment;
+        }
+
         LOG(INFO) << "logid = " << cntl->log_id()
                   << ", OpenFile ok, filename = " << request->filename()
                   << ", clientip = " << clientIP
@@ -1430,32 +1441,6 @@ void NameSpaceService::RefreshSession(
     return;
 }
 
-bool isPathValid(const std::string path) {
-    if (path.empty() || path[0] != '/') {
-        return false;
-    }
-
-    if (path.size() > 1U && path[path.size() - 1] == '/') {
-        return false;
-    }
-
-    bool slash = false;
-    for (uint32_t i = 0; i < path.size(); i++) {
-        if (path[i] == '/') {
-            if (slash) {
-                return false;
-            }
-            slash = true;
-        } else {
-            slash = false;
-        }
-    }
-
-    // 将来如果有path有其他限制，可以在此处继续添加
-
-    return true;
-}
-
 bool IsRenamePathValid(const std::string& oldFileName,
                        const std::string& newFileName) {
     std::vector<std::string> oldFilePaths;
@@ -1815,6 +1800,7 @@ uint32_t GetMdsLogLevel(StatusCode code) {
         case StatusCode::kFileIdNotMatch:
         case StatusCode::kSessionNotExist:
         case StatusCode::kFileOccupied:
+        case StatusCode::kFileExists:
         case StatusCode::kFileNotExists:
         case StatusCode::kParaError:
             return google::WARNING;

src/mds/schedule/operatorController.cpp

@@ -129,7 +129,7 @@ bool OperatorController::ApplyOperator(const CopySetInfo &originInfo,
 
     // operator timeout or finish
     if (operators_[originInfo.id].IsTimeout()) {
-        LOG(ERROR) << "apply operator: "
+        LOG(WARNING) << "apply operator: "
                    << operators_[originInfo.id].OpToString()
                    << " on " << originInfo.CopySetInfoStr()
                    << " fail, operator is timeout";
@@ -139,10 +139,10 @@ bool OperatorController::ApplyOperator(const CopySetInfo &originInfo,
         auto res = operators_[originInfo.id].Apply(originInfo, newConf);
         switch (res) {
             case ApplyStatus::Failed:
-                LOG(ERROR) << "apply operator"
-                           << operators_[originInfo.id].OpToString()
-                           << " on " << originInfo.CopySetInfoStr()
-                           << " failed";
+                LOG(WARNING) << "apply operator"
+                             << operators_[originInfo.id].OpToString()
+                             << " on " << originInfo.CopySetInfoStr()
+                             << " failed";
                 RemoveOperatorLocked(originInfo.id);
                 return false;
             case ApplyStatus::Finished:

src/mds/schedule/recoverScheduler.cpp

@@ -221,7 +221,7 @@ void RecoverScheduler::CalculateExcludesChunkServer(
         if (item.second.size() < chunkserverFailureTolerance_) {
             continue;
         }
-        LOG(ERROR) << "server " << item.first << " has "
+        LOG(WARNING) << "server " << item.first << " has "
                     << item.second.size() << " offline chunkservers";
         for (auto cs : item.second) {
             excludes->emplace(cs);

src/mds/server/mds.cpp

@@ -458,6 +458,12 @@ void MDS::InitAuthOptions(RootAuthOption *authOptions) {
 void MDS::InitCurveFSOptions(CurveFSOption *curveFSOptions) {
     conf_->GetValueFatalIfFail(
         "mds.curvefs.defaultChunkSize", &curveFSOptions->defaultChunkSize);
+    conf_->GetValueFatalIfFail(
+        "mds.curvefs.defaultSegmentSize", &curveFSOptions->defaultSegmentSize);
+    conf_->GetValueFatalIfFail(
+        "mds.curvefs.minFileLength", &curveFSOptions->minFileLength);
+    conf_->GetValueFatalIfFail(
+        "mds.curvefs.maxFileLength", &curveFSOptions->maxFileLength);
     FileRecordOptions fileRecordOptions;
     InitFileRecordOptions(&curveFSOptions->fileRecordOptions);
 

src/mds/topology/topology_storage_codec.cpp

@@ -33,7 +33,7 @@ namespace topology {
 
 std::string TopologyStorageCodec::EncodeLogicalPoolKey(
     LogicalPoolIdType id) {
-    std::string key = TopologyStorageCodec::GetLogicalPoolKeyPrefix();
+    std::string key = LOGICALPOOLKEYPREFIX;
     size_t prefixLen = key.size();
     key.resize(prefixLen + sizeof(uint64_t));
     ::curve::common::EncodeBigEndian(&(key[prefixLen]), id);
@@ -52,7 +52,7 @@ bool TopologyStorageCodec::DecodeLogicalPoolData(
 
 std::string TopologyStorageCodec::EncodePhysicalPoolKey(
     PhysicalPoolIdType id) {
-    std::string key = TopologyStorageCodec::GetPhysicalPoolKeyPrefix();
+    std::string key = PHYSICALPOOLKEYPREFIX;
     size_t prefixLen = key.size();
     key.resize(prefixLen + sizeof(uint64_t));
     ::curve::common::EncodeBigEndian(&(key[prefixLen]), id);
@@ -71,7 +71,7 @@ bool TopologyStorageCodec::DecodePhysicalPoolData(
 
 std::string TopologyStorageCodec::EncodeZoneKey(
     ZoneIdType id) {
-    std::string key = TopologyStorageCodec::GetZoneKeyPrefix();
+    std::string key = ZONEKEYPREFIX;
     size_t prefixLen = key.size();
     key.resize(prefixLen + sizeof(uint64_t));
     ::curve::common::EncodeBigEndian(&(key[prefixLen]), id);
@@ -90,7 +90,7 @@ bool TopologyStorageCodec::DecodeZoneData(
 
 std::string TopologyStorageCodec::EncodeServerKey(
     ServerIdType id) {
-    std::string key = TopologyStorageCodec::GetServerKeyPrefix();
+    std::string key = SERVERKEYPREFIX;
     size_t prefixLen = key.size();
     key.resize(prefixLen + sizeof(uint64_t));
     ::curve::common::EncodeBigEndian(&(key[prefixLen]), id);
@@ -109,7 +109,7 @@ bool TopologyStorageCodec::DecodeServerData(
 
 std::string TopologyStorageCodec::EncodeChunkServerKey(
     ChunkServerIdType id) {
-    std::string key = TopologyStorageCodec::GetChunkServerKeyPrefix();
+    std::string key = CHUNKSERVERKEYPREFIX;
     size_t prefixLen = key.size();
     key.resize(prefixLen + sizeof(uint64_t));
     ::curve::common::EncodeBigEndian(&(key[prefixLen]), id);
@@ -128,11 +128,11 @@ bool TopologyStorageCodec::DecodeChunkServerData(
 
 std::string TopologyStorageCodec::EncodeCopySetKey(
     const CopySetKey &id) {
-    std::string key = TopologyStorageCodec::GetCopysetKeyPrefix();
+    std::string key = COPYSETKEYPREFIX;
     size_t prefixLen = key.size();
     key.resize(prefixLen + sizeof(uint64_t) + sizeof(uint64_t));
     ::curve::common::EncodeBigEndian(&(key[prefixLen]), id.first);
-    ::curve::common::EncodeBigEndian(&(key[prefixLen + sizeof(id.first)]),
+    ::curve::common::EncodeBigEndian(&(key[prefixLen + sizeof(uint64_t)]),
         id.second);
     return key;
 }

src/mds/topology/topology_storage_codec.h

@@ -28,7 +28,19 @@
 #include "src/common/namespace_define.h"
 #include "src/mds/topology/topology_item.h"
 
-using ::curve::common::TOPOLOGYITEMPRIFIX;
+using ::curve::common::LOGICALPOOLKEYPREFIX;
+using ::curve::common::LOGICALPOOLKEYEND;
+using ::curve::common::PHYSICALPOOLKEYPREFIX;
+using ::curve::common::PHYSICALPOOLKEYEND;
+using ::curve::common::ZONEKEYPREFIX;
+using ::curve::common::ZONEKEYEND;
+using ::curve::common::SERVERKEYPREFIX;
+using ::curve::common::SERVERKEYEND;
+using ::curve::common::CHUNKSERVERKEYPREFIX;
+using ::curve::common::CHUNKSERVERKEYEND;
+using ::curve::common::CLUSTERINFOKEY;
+using ::curve::common::COPYSETKEYPREFIX;
+using ::curve::common::COPYSETKEYEND;
 
 namespace curve {
 namespace mds {
@@ -76,66 +88,6 @@ class TopologyStorageCodec {
         const ClusterInformation &data, std::string *value);
     bool DecodeCluserInfoData(const std::string &value,
         ClusterInformation *data);
-
- public:
-    static std::string GetLogicalPoolKeyPrefix() {
-        static std::string key = std::string(TOPOLOGYITEMPRIFIX) + "01";
-        return key;
-    }
-
-    static std::string GetPhysicalPoolKeyPrefix() {
-        static std::string key = std::string(TOPOLOGYITEMPRIFIX) + "02";
-        return key;
-    }
-
-    static std::string GetZoneKeyPrefix() {
-        static std::string key = std::string(TOPOLOGYITEMPRIFIX) + "03";
-         return key;
-    }
-
-    static std::string GetServerKeyPrefix() {
-        static std::string key = std::string(TOPOLOGYITEMPRIFIX) + "04";
-        return key;
-    }
-
-    static std::string GetChunkServerKeyPrefix() {
-        static std::string key = std::string(TOPOLOGYITEMPRIFIX) + "05";
-        return key;
-    }
-
-    static std::string GetCopysetKeyPrefix() {
-        static std::string key = std::string(TOPOLOGYITEMPRIFIX) + "06";
-        return key;
-    }
-
-    static std::string GetClusterInfoKey() {
-        static std::string key = std::string(TOPOLOGYITEMPRIFIX) + "07";
-        return key;
-    }
-
-    static std::string GetLogicalPoolKeyEnd() {
-        return GetPhysicalPoolKeyPrefix();
-    }
-
-    static std::string GetPhysicalPoolKeyEnd() {
-        return GetZoneKeyPrefix();
-    }
-
-    static std::string GetZoneKeyEnd() {
-        return GetServerKeyPrefix();
-    }
-
-    static std::string GetServerKeyEnd() {
-        return GetChunkServerKeyPrefix();
-    }
-
-    static std::string GetChunkServerKeyEnd() {
-        return GetCopysetKeyPrefix();
-    }
-
-    static std::string GetCopysetKeyEnd() {
-        return GetClusterInfoKey();
-    }
 };
 
 

src/mds/topology/topology_storge_etcd.cpp

@@ -34,12 +34,10 @@ namespace topology {
 bool TopologyStorageEtcd::LoadLogicalPool(
     std::unordered_map<PoolIdType, LogicalPool> *logicalPoolMap,
     PoolIdType *maxLogicalPoolId) {
-    std::string startKey = TopologyStorageCodec::GetLogicalPoolKeyPrefix();
-    std::string endKey = TopologyStorageCodec::GetLogicalPoolKeyEnd();
     std::vector<std::string> out;
     logicalPoolMap->clear();
     *maxLogicalPoolId = 0;
-    int errCode = client_->List(startKey, endKey, &out);
+    int errCode = client_->List(LOGICALPOOLKEYPREFIX, LOGICALPOOLKEYEND, &out);
     if (errCode == EtcdErrCode::EtcdKeyNotExist) {
         return true;
     }
@@ -72,12 +70,11 @@ bool TopologyStorageEtcd::LoadLogicalPool(
 bool TopologyStorageEtcd::LoadPhysicalPool(
     std::unordered_map<PoolIdType, PhysicalPool> *physicalPoolMap,
     PoolIdType *maxPhysicalPoolId) {
-    std::string startKey = TopologyStorageCodec::GetPhysicalPoolKeyPrefix();
-    std::string endKey = TopologyStorageCodec::GetPhysicalPoolKeyEnd();
     std::vector<std::string> out;
     physicalPoolMap->clear();
     *maxPhysicalPoolId = 0;
-    int errCode = client_->List(startKey, endKey, &out);
+    int errCode = client_->List(
+        PHYSICALPOOLKEYPREFIX, PHYSICALPOOLKEYEND, &out);
     if (errCode == EtcdErrCode::EtcdKeyNotExist) {
         return true;
     }
@@ -110,12 +107,10 @@ bool TopologyStorageEtcd::LoadPhysicalPool(
 bool TopologyStorageEtcd::LoadZone(
     std::unordered_map<ZoneIdType, Zone> *zoneMap,
     ZoneIdType *maxZoneId) {
-    std::string startKey = TopologyStorageCodec::GetZoneKeyPrefix();
-    std::string endKey = TopologyStorageCodec::GetZoneKeyEnd();
     std::vector<std::string> out;
     zoneMap->clear();
     *maxZoneId = 0;
-    int errCode = client_->List(startKey, endKey, &out);
+    int errCode = client_->List(ZONEKEYPREFIX, ZONEKEYEND, &out);
     if (errCode == EtcdErrCode::EtcdKeyNotExist) {
         return true;
     }
@@ -148,12 +143,10 @@ bool TopologyStorageEtcd::LoadZone(
 bool TopologyStorageEtcd::LoadServer(
     std::unordered_map<ServerIdType, Server> *serverMap,
     ServerIdType *maxServerId) {
-    std::string startKey = TopologyStorageCodec::GetServerKeyPrefix();
-    std::string endKey = TopologyStorageCodec::GetServerKeyEnd();
     std::vector<std::string> out;
     serverMap->clear();
     *maxServerId = 0;
-    int errCode = client_->List(startKey, endKey, &out);
+    int errCode = client_->List(SERVERKEYPREFIX, SERVERKEYEND, &out);
     if (errCode == EtcdErrCode::EtcdKeyNotExist) {
         return true;
     }
@@ -186,12 +179,10 @@ bool TopologyStorageEtcd::LoadServer(
 bool TopologyStorageEtcd::LoadChunkServer(
     std::unordered_map<ChunkServerIdType, ChunkServer> *chunkServerMap,
     ChunkServerIdType *maxChunkServerId) {
-    std::string startKey = TopologyStorageCodec::GetChunkServerKeyPrefix();
-    std::string endKey = TopologyStorageCodec::GetChunkServerKeyEnd();
     std::vector<std::string> out;
     chunkServerMap->clear();
     *maxChunkServerId = 0;
-    int errCode = client_->List(startKey, endKey, &out);
+    int errCode = client_->List(CHUNKSERVERKEYPREFIX, CHUNKSERVERKEYEND, &out);
     if (errCode == EtcdErrCode::EtcdKeyNotExist) {
         return true;
     }
@@ -226,12 +217,10 @@ bool TopologyStorageEtcd::LoadChunkServer(
 bool TopologyStorageEtcd::LoadCopySet(
     std::map<CopySetKey, CopySetInfo> *copySetMap,
     std::map<PoolIdType, CopySetIdType> *copySetIdMaxMap) {
-    std::string startKey = TopologyStorageCodec::GetCopysetKeyPrefix();
-    std::string endKey = TopologyStorageCodec::GetCopysetKeyEnd();
     std::vector<std::string> out;
     copySetMap->clear();
     copySetIdMaxMap->clear();
-    int errCode = client_->List(startKey, endKey, &out);
+    int errCode = client_->List(COPYSETKEYPREFIX, COPYSETKEYEND, &out);
     if (errCode == EtcdErrCode::EtcdKeyNotExist) {
         return true;
     }
@@ -481,9 +470,8 @@ bool TopologyStorageEtcd::UpdateCopySet(const CopySetInfo &data) {
 
 bool TopologyStorageEtcd::LoadClusterInfo(
     std::vector<ClusterInformation> *info) {
-    std::string key = TopologyStorageCodec::GetClusterInfoKey();
     std::string value;
-    int errCode = client_->Get(key, &value);
+    int errCode = client_->Get(CLUSTERINFOKEY, &value);
     if (errCode == EtcdErrCode::EtcdKeyNotExist) {
         return true;
     }
@@ -503,7 +491,7 @@ bool TopologyStorageEtcd::LoadClusterInfo(
 }
 
 bool TopologyStorageEtcd::StorageClusterInfo(const ClusterInformation &info) {
-    std::string key = TopologyStorageCodec::GetClusterInfoKey();
+    std::string key = CLUSTERINFOKEY;
     std::string value;
     if (codec_->EncodeClusterInfoData(info, &value)
         != true) {

src/snapshotcloneserver/clone/clone_core.cpp

@@ -60,19 +60,43 @@ int CloneCoreImpl::CloneOrRecoverPre(const UUID &source,
     // 查询数据库中是否有任务正在执行
     std::vector<CloneInfo> cloneInfoList;
     metaStore_->GetCloneInfoByFileName(destination, &cloneInfoList);
+    bool needJudgeFileExist = false;
+    std::vector<CloneInfo> existCloneInfos;
     for (auto &info : cloneInfoList) {
-        if ((taskType == CloneTaskType::kClone) &&
-            (info.GetTaskType() == CloneTaskType::kClone)) {
-            if (info.GetStatus() == CloneStatus::done ||
-                info.GetStatus() == CloneStatus::error ||
-                info.GetStatus() == CloneStatus::metaInstalled ||
-                info.GetStatus() == CloneStatus::cloning ||
+        LOG(INFO) << "CloneOrRecoverPre find same clone task"
+                  << ", source = " << source
+                  << ", user = " << user
+                  << ", destination = " << destination
+                  << ", Exist CloneInfo : " << info;
+        // is clone
+        if (taskType == CloneTaskType::kClone) {
+            if (info.GetStatus() == CloneStatus::cloning ||
+                info.GetStatus() == CloneStatus::retrying) {
+                if ((info.GetUser() == user) &&
+                    (info.GetSrc() == source) &&
+                    (info.GetIsLazy() == lazyFlag) &&
+                    (info.GetTaskType() == taskType)) {
+                    // 视为同一个clone
+                    *cloneInfo = info;
+                    return kErrCodeTaskExist;
+                } else {
+                    // 视为不同的克隆，那么文件实际上已被占用，返回文件已存在
+                    return kErrCodeFileExist;
+                }
+            } else if (info.GetStatus() == CloneStatus::done ||
+                       info.GetStatus() == CloneStatus::error ||
+                       info.GetStatus() == CloneStatus::metaInstalled) {
+                // 可能已经删除，需要再判断文件存不存在，
+                // 在已删除的条件下，允许再克隆
+                existCloneInfos.push_back(info);
+                needJudgeFileExist = true;
+            } else {
+                // 此时，有个相同的克隆任务正在删除中, 返回文件被占用
+                return kErrCodeFileExist;
+            }
+        } else {  // is recover
+            if (info.GetStatus() == CloneStatus::recovering ||
                 info.GetStatus() == CloneStatus::retrying) {
-                LOG(INFO) << "CloneOrRecoverPre find same clone task"
-                          << ", source = " << source
-                          << ", user = " << user
-                          << ", destination = " << destination
-                          << ", Exist CloneInfo : " << info;
                 if ((info.GetUser() == user) &&
                     (info.GetSrc() == source) &&
                     (info.GetIsLazy() == lazyFlag) &&
@@ -84,30 +108,14 @@ int CloneCoreImpl::CloneOrRecoverPre(const UUID &source,
                     // 视为不同的克隆，那么文件实际上已被占用，返回文件已存在
                     return kErrCodeFileExist;
                 }
+            } else if (info.GetStatus() == CloneStatus::done ||
+                       info.GetStatus() == CloneStatus::error ||
+                       info.GetStatus() == CloneStatus::metaInstalled) {
+                // nothing
             } else {
-                // 此时，有个相同的克隆任务正在删除中, 返回文件被占用
+                // 此时，有个相同的任务正在删除中, 返回文件被占用
                 return kErrCodeFileExist;
             }
-        } else {
-            if (info.GetStatus() == CloneStatus::cloning ||
-                info.GetStatus() == CloneStatus::recovering ||
-                info.GetStatus() == CloneStatus::retrying) {
-                LOG(INFO) << "CloneOrRecoverPre find some task in progress"
-                          << ", source = " << source
-                          << ", user = " << user
-                          << ", destination = " << destination
-                          << ", Exist CloneInfo : " << info;
-                if ((info.GetUser() == user) &&
-                    (info.GetSrc() == source) &&
-                    (info.GetIsLazy() == lazyFlag) &&
-                    (info.GetTaskType() == taskType)) {
-                    // 视为同一个clone，返回任务已存在
-                    return kErrCodeTaskExist;
-                } else {
-                    // 视为不同的克隆，那么文件实际上已被占用，返回文件已存在
-                    return kErrCodeFileExist;
-                }
-            }
         }
     }
 
@@ -117,11 +125,36 @@ int CloneCoreImpl::CloneOrRecoverPre(const UUID &source,
     switch (ret) {
         case LIBCURVE_ERROR::OK:
             if (CloneTaskType::kClone == taskType) {
-                LOG(ERROR) << "Clone dest file must not exist"
-                           << ", source = " << source
-                           << ", user = " << user
-                           << ", destination = " << destination;
-                return kErrCodeFileExist;
+                if (needJudgeFileExist) {
+                    bool match = false;
+                    //  找出inodeid匹配的cloneInfo
+                    for (auto& existInfo : existCloneInfos) {
+                        if (destFInfo.id == existInfo.GetDestId()) {
+                            *cloneInfo = existInfo;
+                            match = true;
+                            break;
+                        }
+                    }
+                    if (match) {
+                        return kErrCodeTaskExist;
+                    } else {
+                        // 如果没找到，那么dest file都不是这些clone任务创建的，
+                        // 意味着文件重名了
+                        LOG(ERROR) << "Clone dest file exist, "
+                                   << "but task not match! "
+                                   << "source = " << source
+                                   << ", user = " << user
+                                   << ", destination = " << destination;
+                        return kErrCodeFileExist;
+                    }
+                } else {
+                    // 没有对应的cloneInfo，意味着文件重名了
+                    LOG(ERROR) << "Clone dest file must not exist"
+                               << ", source = " << source
+                               << ", user = " << user
+                               << ", destination = " << destination;
+                    return kErrCodeFileExist;
+                }
             }
             break;
         case -LIBCURVE_ERROR::NOTEXIST:
@@ -577,7 +610,11 @@ int CloneCoreImpl::CreateCloneFile(
     uint64_t fileLength = fInfo.length;
     uint64_t seqNum = fInfo.seqnum;
     uint32_t chunkSize = fInfo.chunksize;
-    std::string source = task->GetCloneInfo().GetSrc();
+    std::string source = "";
+    // 只有从文件克隆才带clone source
+    if (CloneFileType::kFile == task->GetCloneInfo().GetFileType()) {
+        source = task->GetCloneInfo().GetSrc();
+    }
 
     FInfo fInfoOut;
     int ret = client_->CreateCloneFile(source, fileName,

src/snapshotcloneserver/snapshot/snapshot_core.cpp

@@ -616,15 +616,12 @@ int SnapshotCoreImpl::DeleteSnapshotOnCurvefs(const SnapshotInfo &info) {
                   << ", seqNum = " << seqNum
                   << ", status = " << static_cast<int>(status)
                   << ", uuid = " << info.GetUuid();
+        // NOTEXIST means delete succeed.
         if (-LIBCURVE_ERROR::NOTEXIST == ret) {
             LOG(INFO) << "Check snapShot delete success"
                       << ", uuid = " << info.GetUuid();
             break;
-        // 目前mds删除快照失败会重试, 失败返回错误码DELETE_ERROR,
-        // 返回其他错误码一律InternalError
-        } else if (LIBCURVE_ERROR::OK == ret ||
-            -LIBCURVE_ERROR::DELETE_ERROR == ret) {
-            // nothing
+        } else if (LIBCURVE_ERROR::OK == ret) {
             if (status != FileStatus::Deleting) {
                 LOG(ERROR) << "CheckSnapShotStatus fail"
                            << ", ret = " << ret
@@ -640,8 +637,7 @@ int SnapshotCoreImpl::DeleteSnapshotOnCurvefs(const SnapshotInfo &info) {
         }
         std::this_thread::sleep_for(
             std::chrono::milliseconds(checkSnapshotStatusIntervalMs_));
-    } while (LIBCURVE_ERROR::OK == ret ||
-            -LIBCURVE_ERROR::DELETE_ERROR == ret);
+    } while (LIBCURVE_ERROR::OK == ret);
     return kErrCodeSuccess;
 }
 

test/client/BUILD

@@ -49,6 +49,7 @@ cc_test(
                 "libcurve_client_unittest.cpp",
                 "lease_executor_test.cpp",
                 "request_sender_test.cpp",
+                "mds_client_test.cpp"
                 ]
     ),
     copts = COPTS,

test/client/client_mdsclient_metacache_unittest.cpp

@@ -1464,8 +1464,8 @@ TEST_F(MDSClientTest, CreateCloneFile) {
                                          10 * 1024 * 1024, 0, 4 * 1024 * 1024,
                                          &finfo));
     ASSERT_EQ(5, finfo.id);
-    ASSERT_EQ(cloneSource, finfo.cloneSource);
-    ASSERT_EQ(cloneLength, finfo.cloneLength);
+    ASSERT_EQ(cloneSource, finfo.sourceInfo.name);
+    ASSERT_EQ(cloneLength, finfo.sourceInfo.length);
 }
 
 TEST_F(MDSClientTest, CompleteCloneMeta) {