curvefs/client: fix timer bug

and guarantee consistent of s3chunkinfo in inode. (#1304) (#1308)

.gitignore

@@ -97,8 +97,10 @@ curvefs/BUILD_MODE
 *.pyc
 .facts/
 *retry
+# monitor
+curvefs/monitor/prometheus/target.json
 
-curvefs/docker/curvefs
+curvefs/docker/*/curvefs
 curvefs/docker/base/*
 !curvefs/docker/base/Dockerfile
 !curvefs/docker/base/Makefile
@@ -109,3 +111,16 @@ __not_found__
 thirdparties/rocksdb/lib/
 thirdparties/rocksdb/include/
 thirdparties/rocksdb/rocksdb/
+thirdparties/rocksdb/*.tar.gz
+
+/external
+/bazel-*
+/compile_commands.json
+/.cache/
+
+docker/curvebs
+docker/*/curvebs
+
+curvefs/docker/curvefs
+curvefs/docker/*/curvefs
+storage_*

conf/client.conf

@@ -129,6 +129,9 @@ global.fileIOSplitMaxSizeKB=64
 #
 ################# log相关配置 ###############
 #
+# enable logging or not
+global.logging.enable=True
+#
 # log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
 global.logLevel=0
 # 设置log的路径

conf/s3.conf

@@ -27,3 +27,4 @@ s3.throttle.iopsWriteLimit=5000
 s3.throttle.bpsTotalMB=1280
 s3.throttle.bpsReadMB=1280
 s3.throttle.bpsWriteMB=1280
+s3.useVirtualAddressing=false

curvefs/Makefile

@@ -1,100 +1,23 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: build install deploy core config topo start stop reload status clean mount umount
+.PHONY: build dep install image
 
 prefix?= "$(PWD)/devops/projects"
-release?= 0
-build_rocksdb?= 0
 only?= "*"
-hosts?= "*"
-tag?= "curvefs:unknown"
+dep?= 0
+release?= 0
 os?= "debian9"
-
-define deploy_begin
-	@bash util/deploy.sh begin
-endef
-
-define deploy_end
-	@bash util/deploy.sh end
-endef
-
-define only_service
-	$(call deploy_begin)
-	@bash util/deploy.sh --hosts=$(hosts) --only="etcd" --tags=$(1)
-	@bash util/deploy.sh --hosts=$(hosts) --only="mds" --tags=$(1)
-	@bash util/deploy.sh --hosts=$(hosts) --only="metaserver" --tags=$(1)
-	$(call deploy_end)
-endef
-
-define only_specify
-	$(call deploy_begin)
-	@bash util/deploy.sh --hosts=$(hosts) --only=$(1) --tags=$(2)
-	$(call deploy_end)
-endef
+build_rocksdb?= 0
+tag?= "curvefs:unknown"
 
 build:
-	@bash util/build.sh --only=$(only) --release=$(release) --os=$(os) --build_rocksdb=$(build_rocksdb)
+	@bash util/build.sh --only=$(only) --dep=$(dep) --release=$(release) --os=$(os) --build_rocksdb=$(build_rocksdb)
+
+dep:
+	@bash util/build.sh --only="" --dep=1 --build_rocksdb=$(build_rocksdb)
 
 install:
 	@bash util/install.sh --prefix=$(prefix) --only=$(only)
 
 image:
 	@bash util/image.sh $(tag) $(os)
-
-deploy:
-	$(call deploy_begin)
-	@bash util/deploy.sh --hosts="etcd" --only="etcd" --tags="core,config,start"
-	@bash util/deploy.sh --hosts="mds" --only="mds" --tags="core,config,start"
-	@bash util/deploy.sh --hosts="metaserver" --only="metaserver" --tags="core,config,start"
-	@bash util/deploy.sh --hosts="tools" --only="tools" --tags="core,config,topo"
-	@bash util/deploy.sh --hosts="client" --only="client" --tags="core,config"
-	$(call deploy_end)
-
-core:
-	$(call only_specify,$(only),core)
-
-config:
-	$(call only_specify,$(only),config)
-
-topo:
-	$(call only_specify,tools,topo)
-
-debug:
-	$(call only_specify,$(only),debug)
-
-start:
-ifeq ($(only), "*")
-	$(call only_service,start)
-else
-	$(call only_specify,$(only),start)
-endif
-
-stop:
-ifeq ($(only), "*")
-	$(call only_service,stop)
-else
-	$(call only_specify,$(only),stop)
-endif
-
-reload:
-ifeq ($(only), "*")
-	$(call only_service,restart)
-else
-	$(call only_specify,$(only),restart)
-endif
-
-status:
-ifeq ($(only), "*")
-	$(call only_service,status)
-else
-	$(call only_specify,$(only),status)
-endif
-
-clean:
-	$(call only_specify,$(only),clean)
-
-mount:
-	$(call only_specify,client,mount)
-
-umount:
-	$(call only_specify,client,umount)

curvefs/README.md

@@ -3,127 +3,16 @@ CURVEFS
 
 Curve FileSystem
 
-Table of Contents
-===
-* [Requirement](#requirement)
-* [Quick Start](#quick-start)
-* [Devops](#devops)
-* [Hint](#hint)
-
-Requirement
-===
-
-* bazel
-* ansible
-* linux*
-
-[Back to Toc](#table-of-contents)
-
 Quick Start
 ===
 
-NOTE: If you are using CentOS-8, you can refer to this issue to compile CurveFS: https://github.com/opencurve/curve/issues/807
+```
+$ make build dep=1
+```
 
-step 1: clone repository, run build and install:
+OR
 
 ```
-$ git clone https://github.com/opencurve/curve.git
-$ cd curve/curvefs
+$ make dep
 $ make build
-$ make install
-$ make install only=etcd
 ```
-
-step 2: edit ansible config file, inventory file and client config file:
-
-* devops/ansible.cfg:
-    * `remoter_user`: we use this user to log in to the remote machine, create file and start service
-    * `private_key_file`: paste `remote_user`'s private key to `devops/ssh/pub_rsa` (you can also save the private key anywhere locally and point the `private_key_file` to it)
-    * Please make sure that the mode of private key file which `private_key_file` specfied is `600` (`-rw-------`)
-    * Please make sure that the public key already exists in `remote_user`'s ssh file (`~/.ssh/authorized_keys`)
-* inventory/server.ini:
-    * You can specify which hosts to deploy which services
-    * Please specify the client mount path (`client_mount_path`), the path will create automatic if it not exist
-    * Please specify the filesystem name which client mount (`client_mount_fsname`), the filesystem will create automatic if it not exist
-    * The `tools` only used to create topology, you can select one of `mds` hosts
-* conf/{client.conf, metaserver.conf}
-    * `s3.ak`
-    * `s3.sk`
-    * `s3.endpoint`
-    * `s3.bucket_name`
-
-step 3: deploy all and mount curve filesystem:
-
-```
-$ make deploy
-$ make mount
-```
-
-once this is done, you can enter the mount path and do anything like local filesystem.
-
-[Back to Toc](#table-of-contents)
-
-Devops
-===
-
-| command                                  | description                                   |
-| :---                                     | :---                                          |
-| make build [only=ONLY] [release=1]       | compile                                       |
-| make install [only=ONLY] [prefix=PREFIX] | install                                       |
-| [sudo] make image tag=TAG                | make docker image (maybe need sudo privilege) |
-| make deploy                              | deploy all                                    |
-| make core [only=ONLY] [hosts=HOSTS]      | sync binary file                              |
-| make config [only=ONLY] [hosts=HOSTS]    | sync config file                              |
-| make start [only=ONLY] [hosts=HOSTS]     | start service                                 |
-| make stop [only=ONLY] [hosts=HOSTS]      | stop service                                  |
-| make reload [only=ONLY] [hosts=HOSTS]    | restart service                               |
-| make status [only=ONLY] [hosts=HOSTS]    | show service status                           |
-| make clean [only=ONLY] [hosts=HOSTS]     | clean environment (include all created files) |
-| make topo                                | create topology                               |
-| make mount                               | mount curve filesystem                        |
-| make umount                              | umount curve filesystem                       |
-
-
-* If you want to execute action for specfied service, you can use `only` option, e.g: `make start only=mds`
-* If you want to execute action in specfied host, you can use `hosts` option, e.g: `make start hosts=machine1`
-* You can also specify both `only` and `hosts` option, e.g: `make start only=mds hosts=machine1:machine2`
-* The `only` option can be one of the following values: `etcd`、`mds`、`metaserver`、`space`、`client`、 `tools`
-
-[Back to Toc](#table-of-contents)
-
-HINT
-===
-
-* The default install prefix of projects is `devops/projects`
-* If you want to modify the service's config, you can modify the config under `conf` directory, then run `make config`
-* You can use `make status` to show service status, include active status, listen address, memory usage and etc:
-
-```
-$ make status only=mds
-
-curve-vm1:
-mds.service - CurveFS Mds
-    Active: [RUNNING] since 2021-09-26 19:27:39; 52:35 ago
-  Main PID: 326373 (curvefs_mds)
-    Daemon: True
-    Listen: 0.0.0.0:26700, 0.0.0.0:27700
-       Mem: 51688 KB
-
-curve-vm2:
-mds.service - CurveFS Mds
-    Active: [RUNNING] since 2021-09-26 19:27:39; 52:35 ago
-  Main PID: 326381 (curvefs_mds)
-    Daemon: True
-    Listen: 0.0.0.0:37700
-       Mem: 40688 KB
-
-curve-vm3:
-mds.service - CurveFS Mds
-    Active: [RUNNING] since 2021-09-26 19:27:39; 52:35 ago
-  Main PID: 326401 (curvefs_mds)
-    Daemon: True
-    Listen: 0.0.0.0:17700
-       Mem: 55520 KB
-```
-
-[Back to Toc](#table-of-contents)

curvefs/conf/client.conf

@@ -35,9 +35,9 @@ metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000
 #### excutorOpt
 # excutorOpt rpc with metaserver
 # rpc retry times with metaserver
-excutorOpt.maxRetry=1000000
+excutorOpt.maxRetry=4294967295
 # Retry sleep time between failed RPCs
-excutorOpt.retryIntervalUS=500
+excutorOpt.retryIntervalUS=100000
 # RPC timeout for communicating with metaserver
 excutorOpt.rpcTimeoutMS=1000
 # RPC stream idle timeout
@@ -88,6 +88,11 @@ fuseClient.dCacheLruSize=65536
 fuseClient.enableICacheMetrics=true
 fuseClient.enableDCacheMetrics=true
 fuseClient.cto=true
+# splice will bring higher performance in some cases
+# but there're might be a kernel issue that will cause kernel panic when enabling it
+# see https://lore.kernel.org/all/CAAmZXrsGg2xsP1CK+cbuEMumtrqdvD-NKnWzhNcvn71RV3c1yw@mail.gmail.com/
+# until this issue has been fixed, splice should be disabled
+fuseClient.enableSplice=false
 
 #### volume
 volume.bigFileSize=1048576
@@ -139,6 +144,7 @@ s3.logPrefix=/data/logs/curvefs/aws_ # __CURVEADM_TEMPLATE__ /curvefs/client/log
 s3.async_thread_num=30
 # limit all inflight async requests' bytes, |0| means not limited
 s3.max_async_request_inflight_bytes=104857600
+s3.chunkFlushThreads=5
 # throttle
 s3.throttle.iopsTotalLimit=0
 s3.throttle.iopsReadLimit=0
@@ -146,6 +152,7 @@ s3.throttle.iopsWriteLimit=0
 s3.throttle.bpsTotalMB=0
 s3.throttle.bpsReadMB=0
 s3.throttle.bpsWriteMB=0
+s3.useVirtualAddressing=false
 
 # TODO(hongsong): limit bytes、iops/bps
 #### disk cache options
@@ -161,6 +168,7 @@ diskCache.asyncLoadPeriodMs=5
 # util less than safeRatio
 diskCache.fullRatio=90
 diskCache.safeRatio=70
+diskCache.threads=5
 # the max size disk cache can use
 diskCache.maxUsableSpaceBytes=107374182400
 # the max time system command can run
@@ -168,16 +176,16 @@ diskCache.cmdTimeoutSec=300
 # directory of disk cache
 diskCache.cacheDir=/mnt/curvefs_cache  # __CURVEADM_TEMPLATE__ /curvefs/client/data/cache __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /mnt/curvefs_disk_cache/{{ 99999999 | random | to_uuid | upper }} __ANSIBLE_TEMPLATE__
 
-# the write throttle bps of disk cache, default 80MB/s
-diskCache.avgFlushBytes=83886080
-# the write burst bps of disk cache, default 100MB/s
-diskCache.burstFlushBytes=104857600
+# the write throttle bps of disk cache, default no limit
+diskCache.avgFlushBytes=0
+# the write burst bps of disk cache, default no limit
+diskCache.burstFlushBytes=0
 # the times that write burst bps can continue, default 180s
 diskCache.burstSecs=180
 # the write throttle iops of disk cache, default no limit
 diskCache.avgFlushIops=0
-# the read throttle bps of disk cache, default 80MB/s
-diskCache.avgReadFileBytes=83886080
+# the read throttle bps of disk cache, default no limit
+diskCache.avgReadFileBytes=0
 # the read throttle iops of disk cache, default no limit
 diskCache.avgReadFileIops=0
 

curvefs/conf/mds.conf

@@ -106,6 +106,8 @@ mds.scheduler.metaserver.cooling.timeSec=1800
 # the backend thread check whether fs is able to delete,
 # check partition of deleting fs is deleting
 mds.fsmanager.backEndThreadRunInterSec=10
+# number of threads that load space info of volume
+mds.fsmanager.reloadSpaceConcurrency=10
 
 #### s3
 # TODO(huyao): use more meaningfull name
@@ -128,3 +130,4 @@ s3.throttle.iopsWriteLimit=0
 s3.throttle.bpsTotalMB=0
 s3.throttle.bpsReadMB=0
 s3.throttle.bpsWriteMB=0
+s3.useVirtualAddressing=false

curvefs/conf/metaserver.conf

@@ -26,6 +26,7 @@ s3.throttle.iopsWriteLimit=0
 s3.throttle.bpsTotalMB=0
 s3.throttle.bpsReadMB=0
 s3.throttle.bpsWriteMB=0
+s3.useVirtualAddressing=false
 # s3 workqueue
 s3compactwq.enable=True
 s3compactwq.thread_num=2
@@ -228,5 +229,11 @@ storage.rocksdb.ordered_write_buffer_size=134217728
 # rocksdb column family's max_write_buffer_number
 # for store dentry and inode's s3chunkinfo list (default: 15)
 storage.rocksdb.ordered_max_write_buffer_number=15
-# rocksdb block cache(LRU) capacity (default: 128MB)
-storage.rocksdb.block_cache_capacity=134217728
+# rocksdb block cache(LRU) capacity (default: 512MB)
+storage.rocksdb.block_cache_capacity=536870912
+# rocksdb memtable prefix bloom size ratio (size=write_buffer_size*memtable_prefix_bloom_size_ratio)
+storage.rocksdb.memtable_prefix_bloom_size_ratio=0.1
+# if the number of inode's s3chunkinfo exceed the limit_size,
+# we will sending its with rpc streaming instead of
+# padding its into inode (default: 25000, about 25000 * 41 (byte) = 1MB)
+storage.s3_meta_inside_inode.limit_size=25000

curvefs/conf/tools.conf

@@ -29,5 +29,6 @@ s3.endpoint=endpoint
 s3.bucket_name=bucket
 s3.blocksize=4194304
 s3.chunksize=67108864
+s3.useVirtualAddressing=false
 # statistic info in xattr, hardlink will not be supported when enable
 enableSumInDir=true

curvefs/docker/debian10/Dockerfile

@@ -1,6 +1,6 @@
 FROM opencurvedocker/curve-base:debian10
 ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core
+RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
 COPY entrypoint.sh /
 COPY curvefs/tools/sbin/curvefs_tool /usr/bin

curvefs/docker/debian10/entrypoint.sh

@@ -115,7 +115,11 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
-    exec $g_binary $g_start_args
+    if [ $g_role == "etcd" ]; then
+        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
+    else
+        exec $g_binary $g_start_args
+    fi
 }
 
 ############################  MAIN()

curvefs/docker/debian11/Dockerfile

@@ -1,6 +1,6 @@
 FROM opencurvedocker/curve-base:debian11
 ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core
+RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
 COPY entrypoint.sh /
 COPY curvefs/tools/sbin/curvefs_tool /usr/bin

curvefs/docker/debian11/entrypoint.sh

@@ -115,7 +115,11 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
-    exec $g_binary $g_start_args
+    if [ $g_role == "etcd" ]; then
+        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
+    else
+        exec $g_binary $g_start_args
+    fi
 }
 
 ############################  MAIN()

curvefs/docker/debian9/Dockerfile

@@ -1,6 +1,6 @@
 FROM opencurvedocker/curve-base:debian9
 ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core
+RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
 COPY entrypoint.sh /
 COPY curvefs/tools/sbin/curvefs_tool /usr/bin

curvefs/docker/debian9/entrypoint.sh

@@ -115,7 +115,12 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
-    exec $g_binary $g_start_args
+    if [ $g_role == "etcd" ]; then
+        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
+    else
+        exec $g_binary $g_start_args
+    fi
+
 }
 
 ############################  MAIN()

curvefs/monitor/README.md

@@ -0,0 +1,131 @@
+# 目录结构介绍
+
+```
+monitor
+├── curve-monitor.sh        # curve集群监控的控制脚本，用于启动、停止、重启监控功能。
+├── docker-compose.yml      # 编排监控系统相关容器的配置文件，包括prometheus容器、grafana容器。
+|                           # 修改该文件来配置各组件的配置参数。
+├── grafana                 # grafana相关目录
+│   ├── dashboards          # grafana所有dashboards的json文件存放目录，grafana将从该目录加载文件来创建dashboards；
+|   |   |                   # 通过update_dashboard.sh脚本来更新最新的dashboards。
+│   │   ├── etcd.json
+│   │   ├── mds.json
+│   │   ├── metaserver.json
+│   │   └── clinet.json
+│   ├── grafana.ini         # grafana的启动配置文件，将映射到容器的 `/etc/grafana/grafana.ini` 上
+│   ├── provisioning        # grafana预配置相关目录，将映射到容器的`/etc/grafana/provisioning`上
+│   │   ├── dashboards
+│   │   │   └── all.yml
+│   │   └── datasources     # grafana的datasources的json文件存放目录，grafana将从该目录加载文件来创建datasources。
+│   │       └── all.yml
+│   └── report              # grafana日报临时目录，将映射到reporter容器的`/tmp/report`目录上
+│       └── README
+├── grafana-report.py
+├── prometheus              # prometheus相关目录
+│   ├── prometheus.yml      # prometheus的配置文件
+│   └── target.json
+├── README.md
+├── target.ini              # target_json.py脚本依赖的一些配置
+├── target_json.py          # 用于生成prometheus监控对象的python脚本，每隔一段时间用curvefs_tool拉取监控目标并更新。
+└── update_dashboard.sh     # 从grafana界面配置环境当中拉取最新的dashboard，用于更新该环境上grafana的界面。
+```
+
+## 使用说明
+
+以下步骤为不使用puppet进行部署的过程。
+
+### 环境初始化
+
+1.部署监控系统的机器需要安装如下组件：
+
+node_exporter、docker、docker-compose、jq
+
+* docker安装
+
+```
+$ curl -fsSL get.docker.com -o get-docker.sh
+$ sudo sh get-docker.sh --mirror Aliyun
+```
+
+或者直接安装
+
+```
+apt-get install docker-ce
+apt-get install docker-ce-cli
+```
+
+* docker-compose
+
+* ```
+  curl -L https://github.com/docker/compose/releases/download/1.18.0/docker-compose-`uname -s`-`uname -m` -o /usr/local/bin/docker-compose
+  chmod +x /usr/local/bin/docker-compose
+  ```
+
+  或者直接安装
+
+  ```
+  apt-get install docker-compose
+  ```
+
+* node_exporter
+
+  可能很多节点都要安装，可以用脚本来一起装，如下面的方式：
+
+  ```
+  for i in {1..4};
+  do
+  scp -P 1046 ~/Downloads/node_exporter-0.18.1.linux-amd64.tar.gz yangyaokai@pubt1-curve$i.yq.163.org:~/
+  ssh -p 1046 yangyaokai@pubt1-curve$i.yq.163.org "tar zxvf node_exporter-0.18.1.linux-amd64.tar.gz ; cd node_exporter-0.18.1.linux-amd64 ; nohup ./node_exporter >/dev/null 2>log &"
+  echo $i
+  done
+  ```
+
+* jq
+
+  update_dashboard.sh脚本需要依赖jq命令，这个一般机器上都没装
+
+  ```
+  apt-get install jq
+  ```
+
+2.chunkserver上安装node_exporter（机器监控可以依赖哨兵，可以不装）
+
+
+### 部署监控系统
+
+* 修改相关配置
+
+1.修改target_json.py文件中相应的配置
+
+2.修改update_dashboard.sh，将 URL 和 LOGIN 改为对应的地址和用户名密码
+
+3.修改docker-compose.yml文件，主要是映射的目录路径
+
+* 启动docker-compose
+
+在当前目录下执行如下命令即可
+
+```curve-monitor.sh start ```
+
+* 部署grafana每日报表
+
+crontab配置定时任务，添加如下任务：
+30 8 * * * python /etc/curve/monitor/grafana-report.py >> /etc/curve/monitor/cron.log 2>&1
+如果机器上没有配置其他的定时任务，可直接用下面命令
+echo "30 8 * * * python /etc/curve/monitor/grafana-report.py >> /etc/curve/monitor/cron.log 2>&1" >> conf && crontab conf && rm -f conf
+
+#### 对接puppet
+
+如果对接puppet，配置相关文件都会放到puppet上，配置的变更都要上传到puppet上。
+
+puppet上管理的配置包括：docker-compose.yml、target.ini、grafana.ini、prometheus.yml
+
+通过安装包安装完curve-monitor以后，会将curve-monitor.sh拷贝到/usr/bin目录下，可以通过以下命令管理监控系统：
+
+启动：```curve-monitor.sh start```
+
+停止：```curve-monitor.sh stop```
+
+重启：```curve-monitor.sh restart```
+
+上面环境初始化中的依赖的包puppet基本都会帮忙安装，除了node_exporter需要自己安装。

curvefs/monitor/curve-monitor.sh

@@ -0,0 +1,84 @@
+#!/bin/sh
+
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+#sh update_dashboard.sh
+#echo "update dashboards success!"
+
+WORKDIR=/etc/curvefs/monitor
+
+if [ ! -d $WORKDIR ]; then
+  echo "${WORKDIR} not exists"
+  exit 1
+fi
+
+cd $WORKDIR
+chmod -R 777 prometheus
+chmod -R 777 grafana
+
+start() {
+    echo "==========start==========="
+
+    echo "" > monitor.log
+
+    stdbuf -oL python3 target_json.py >> monitor.log 2>&1 &
+    echo "start prometheus targets service success!"
+
+    docker-compose up >> monitor.log 2>&1 &
+    echo "start metric system success!"
+}
+
+stop() {
+    echo "===========stop============"
+
+    docker-compose down
+
+    ID=`(ps -ef | grep "target_json.py"| grep -v "grep") | awk '{print $2}'`
+    for id in $ID
+    do
+        kill -9 $id
+	echo "killed $id"
+    done
+}
+
+restart() {
+    stop
+    echo "sleeping........."
+    sleep 3
+    start
+}
+
+case "$1" in
+    'start')
+        start
+        ;;
+    'stop')
+        stop
+        ;;
+    'status')
+        status
+        ;;
+    'restart')
+        restart
+        ;;
+    *)
+    echo "usage: $0 {start|stop|restart}"
+    exit 1
+        ;;
+    esac

curvefs/monitor/docker-compose.yml

@@ -0,0 +1,57 @@
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+version: '2.0'
+
+services:
+
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - ./prometheus/:/etc/prometheus/:rw
+      - ./prometheus/data:/prometheus:rw
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--storage.tsdb.retention.time=7d'
+      - '--storage.tsdb.retention.size=256GB'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+      - '--web.listen-address=:9090'
+    network_mode: host
+
+  grafana:
+    image: grafana/grafana
+    depends_on:
+      - prometheus
+    network_mode: host
+    volumes:
+      - ./grafana/data:/var/lib/grafana:rw
+      - ./grafana/grafana.ini:/etc/grafana/grafana.ini:rw
+    environment:
+      - GF_INSTALL_PLUGINS=grafana-piechart-panel
+      - GF_SECURITY_ADMIN_USER=admin
+      - GF_SECURITY_ADMIN_PASSWORD=curve
+
+  reporter:
+    image: promoon/reporter:latest
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - /etc/timezone:/etc/timezone:ro
+      - ./grafana/report:/tmp/report:rw
+    network_mode: host

curvefs/monitor/grafana-report.py

@@ -0,0 +1,135 @@
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+# coding: utf8
+
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.image import MIMEImage
+from email.utils import parseaddr, formataddr
+import time as Time
+import re
+import shutil
+import os
+
+sender = 'Grafana<xxxxxxxxx@163.com>'
+to_address = ['xxxxxxxxx@163.com']
+username = 'xxxxxxxxx@163.com'
+password = 'xxxxxxxxx' # SMTP授权码
+smtpserver = 'xxxx.163.com:1234'
+sourcefile= '/etc/curve/monitor/grafana/report/report.tex'
+imagedir= '/etc/curve/monitor/grafana/report/images/'
+pdfpath= '/etc/curve/monitor/grafana/report/report.pdf'
+clustername = '【CURVE】xxxxxxxxx'
+grafanauri = '127.0.0.1:3000'
+reporteruri = '127.0.0.1:8686'
+dashboardid = 'xxxxxxxxx'
+apitoken = 'xxxxxxxxx'
+
+def get_images():
+    image_name_list = []
+    file = open(sourcefile, 'r')
+    line = file.readline()
+    while line:
+        # print (line)
+        prefix_image_name = re.findall(r'image\d+', line)
+        if prefix_image_name:
+            print (prefix_image_name)
+            image_name_list.append(prefix_image_name[0])
+        line = file.readline()
+    file.close()
+
+    return image_name_list
+
+def getMsgImage(image_name):
+    file_name = imagedir+image_name+'.png'
+    print (file_name)
+    fp = open(file_name, 'rb')
+    msgImage = MIMEImage(fp.read())
+    fp.close()
+    msgImage.add_header('Content-ID', image_name)
+    msgImage.add_header("Content-Disposition", "inline", filename=file_name)
+    return msgImage
+
+def attach_body(msgRoot):
+    image_list = get_images()
+
+    image_body = ""
+    for image in image_list:
+        image_body += ('<img src="cid:%s" alt="%s">' % (image, image))
+        msgRoot.attach(getMsgImage(image))
+
+    html_str = '<html><head><style>#string{text-align:center;font-size:25px;}</style></head><body>%s</body></html>' % (image_body)
+
+    mailMsg = """
+    <p>可点击如下链接在grafana面板中查看（若显示混乱，请在附件pdf中查看）</p>
+    <p><a href="http://%s">grafana链接</a></p>
+    """ % (grafanauri)
+    mailMsg += html_str
+    print(mailMsg)
+    content = MIMEText(mailMsg,'html','utf-8')
+    msgRoot.attach(content)
+
+# 发送dashboard日报邮件
+def send_mail():
+    time_now = int(Time.time())
+    time_local = Time.localtime(time_now)
+    dt = Time.strftime("%Y%m%d",time_local)
+
+    msgRoot = MIMEMultipart('related')
+    msgRoot['Subject'] = '%s集群监控日报-%s' % (clustername, dt)
+    msgRoot['From'] = sender
+    msgRoot['To'] = ",".join( to_address ) # 发给多人
+
+    # 添加pdf附件
+    pdf_attach = MIMEText(open(pdfpath, 'rb').read(), 'base64', 'utf-8')
+    pdf_attach["Content-Type"] = 'application/octet-stream'
+    # 这里的filename可以任意写，写什么名字，邮件中显示什么名字
+    pdf_attach["Content-Disposition"] = 'attachment; filename="reporter-{}.pdf"'.format(dt)
+    msgRoot.attach(pdf_attach)
+
+    # 添加正文
+    attach_body(msgRoot)
+
+    smtp = smtplib.SMTP_SSL(smtpserver)
+    smtp.login(username, password)
+    smtp.sendmail(sender, to_address, msgRoot.as_string())
+    smtp.quit()
+
+def clear():
+    shutil.rmtree(imagedir)
+    os.mkdir(imagedir)
+    os.chmod(imagedir, 0777)
+
+def generate_report():
+    downloadcmd = (
+        "wget -O %s "
+        "http://%s/api/v5/report/%s?apitoken=%s"
+        "\&from=now-24h\&to=now"
+    ) % (pdfpath, reporteruri, dashboardid, apitoken)
+    print(downloadcmd)
+    os.system(downloadcmd)
+
+def main():
+    generate_report()
+    send_mail()
+    clear()
+
+if __name__ == '__main__':
+    main()

curvefs/monitor/grafana/dashboards/mds.json

@@ -0,0 +1,443 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Curvefs mds server",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 7,
+  "iteration": 1649851028366,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": true,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 6,
+      "panels": [
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 0,
+            "y": 2
+          },
+          "hiddenSeries": false,
+          "id": 4,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": false,
+            "max": true,
+            "min": false,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.4.4",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "tJLDJxsnz"
+              },
+              "exemplar": true,
+              "expr": "process_memory_resident{instance=~\"$instance\"}",
+              "interval": "",
+              "legendFormat": "process_memory_resident {{instance}}",
+              "refId": "process_memory_resident"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "tJLDJxsnz"
+              },
+              "exemplar": true,
+              "expr": "process_memory_virtual{instance=~\"$instance\"}",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "process_memory_virtual {{instance}}",
+              "refId": "process_memory_virtual"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "tJLDJxsnz"
+              },
+              "exemplar": true,
+              "expr": "process_memory_shared{instance=~\"$instance\"}",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "process_memory_shared {{instance}}",
+              "refId": "A"
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "process memory usage",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "$$hashKey": "object:63",
+              "format": "decbytes",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "$$hashKey": "object:64",
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        },
+        {
+          "aliasColors": {},
+          "bars": false,
+          "dashLength": 10,
+          "dashes": false,
+          "fill": 1,
+          "fillGradient": 0,
+          "gridPos": {
+            "h": 8,
+            "w": 12,
+            "x": 12,
+            "y": 2
+          },
+          "hiddenSeries": false,
+          "id": 2,
+          "legend": {
+            "alignAsTable": true,
+            "avg": true,
+            "current": false,
+            "max": true,
+            "min": false,
+            "rightSide": false,
+            "show": true,
+            "total": false,
+            "values": true
+          },
+          "lines": true,
+          "linewidth": 1,
+          "nullPointMode": "null",
+          "options": {
+            "alertThreshold": true
+          },
+          "percentage": false,
+          "pluginVersion": "8.4.4",
+          "pointradius": 2,
+          "points": false,
+          "renderer": "flot",
+          "seriesOverrides": [],
+          "spaceLength": 10,
+          "stack": false,
+          "steppedLine": false,
+          "targets": [
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "tJLDJxsnz"
+              },
+              "exemplar": true,
+              "expr": "process_cpu_usage{instance=~\"$instance\"}",
+              "interval": "",
+              "legendFormat": "process_cpu_usage {{instance}}",
+              "refId": "process_cpu_usage"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "tJLDJxsnz"
+              },
+              "exemplar": true,
+              "expr": "process_cpu_usage_system{instance=~\"$instance\"}",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "process_cpu_usage_system {{instance}}",
+              "refId": "process_cpu_usage_system"
+            },
+            {
+              "datasource": {
+                "type": "prometheus",
+                "uid": "tJLDJxsnz"
+              },
+              "exemplar": true,
+              "expr": "process_cpu_usage_user{instance=~\"$instance\"}",
+              "hide": false,
+              "interval": "",
+              "legendFormat": "process_cpu_usage_user {{instance}}",
+              "refId": "process_cpu_usage_user"
+            }
+          ],
+          "thresholds": [],
+          "timeRegions": [],
+          "title": "process cpu usage",
+          "tooltip": {
+            "shared": true,
+            "sort": 0,
+            "value_type": "individual"
+          },
+          "type": "graph",
+          "xaxis": {
+            "mode": "time",
+            "show": true,
+            "values": []
+          },
+          "yaxes": [
+            {
+              "$$hashKey": "object:495",
+              "format": "percentunit",
+              "logBase": 1,
+              "show": true
+            },
+            {
+              "$$hashKey": "object:496",
+              "format": "short",
+              "logBase": 1,
+              "show": true
+            }
+          ],
+          "yaxis": {
+            "align": false
+          }
+        }
+      ],
+      "title": "process usage",
+      "type": "row"
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 1
+      },
+      "id": 8,
+      "panels": [],
+      "title": "metaserver",
+      "type": "row"
+    },
+    {
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "auto",
+            "displayMode": "auto",
+            "filterable": true
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 11,
+        "w": 10,
+        "x": 0,
+        "y": 2
+      },
+      "id": 10,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "8.4.4",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "tJLDJxsnz"
+          },
+          "exemplar": true,
+          "expr": "{__name__=~\"topology_metric_metaserver_id_[[metaserver:regex]].*\",job=\"mds\"}",
+          "format": "table",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "topology",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "instance": false,
+              "job": true
+            },
+            "indexByName": {
+              "Time": 1,
+              "Value": 4,
+              "__name__": 0,
+              "instance": 2,
+              "job": 3
+            },
+            "renameByName": {
+              "__name__": "name"
+            }
+          }
+        },
+        {
+          "id": "groupBy",
+          "options": {
+            "fields": {
+              "Value": {
+                "aggregations": [
+                  "last"
+                ],
+                "operation": "aggregate"
+              },
+              "name": {
+                "aggregations": [],
+                "operation": "groupby"
+              }
+            }
+          }
+        }
+      ],
+      "type": "table"
+    }
+  ],
+  "refresh": false,
+  "schemaVersion": 35,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "definition": "{__name__=~\"bthread_count\", job=\"mds\"}",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Addr",
+        "multi": false,
+        "name": "instance",
+        "options": [],
+        "query": {
+          "query": "{__name__=~\"bthread_count\", job=\"mds\"}",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "/.*instance=\"([^\"]*).*/",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "definition": "{__name__=~\"topology_metric_metaserver_id_.*_copyset_num\",job=\"mds\"}",
+        "hide": 0,
+        "includeAll": true,
+        "label": "metaserverId",
+        "multi": false,
+        "name": "metaserver",
+        "options": [],
+        "query": {
+          "query": "{__name__=~\"topology_metric_metaserver_id_.*_copyset_num\",job=\"mds\"}",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "/topology_metric_metaserver_id_(.*)_copyset_num.*/",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "mds",
+  "uid": "dsgsdfg",
+  "version": 16,
+  "weekStart": ""
+}

curvefs/monitor/grafana/dashboards/metaserver.json

@@ -0,0 +1,423 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "Prometheus",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "target": {
+          "limit": 100,
+          "matchAny": false,
+          "tags": [],
+          "type": "dashboard"
+        },
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Curvefs mds server",
+  "editable": true,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 0,
+  "id": 5,
+  "iteration": 1649851101054,
+  "links": [],
+  "liveNow": false,
+  "panels": [
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 6,
+      "panels": [],
+      "title": "process usage",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "hiddenSeries": false,
+      "id": 4,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": false,
+        "max": true,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.4.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "tJLDJxsnz"
+          },
+          "exemplar": true,
+          "expr": "process_memory_resident{instance=~\"$instance\"}",
+          "interval": "",
+          "legendFormat": "process_memory_resident {{hostname}}",
+          "refId": "process_memory_resident"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "tJLDJxsnz"
+          },
+          "exemplar": true,
+          "expr": "process_memory_virtual{instance=~\"$instance\"}",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "process_memory_virtual {{hostname}}",
+          "refId": "process_memory_virtual"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "tJLDJxsnz"
+          },
+          "exemplar": true,
+          "expr": "process_memory_shared{instance=~\"$instance\"}",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "process_memory_shared {{hostname}}",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "process memory usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:63",
+          "format": "decbytes",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:64",
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 8,
+        "w": 12,
+        "x": 12,
+        "y": 1
+      },
+      "hiddenSeries": false,
+      "id": 2,
+      "legend": {
+        "alignAsTable": true,
+        "avg": true,
+        "current": false,
+        "max": true,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": true
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.4.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "tJLDJxsnz"
+          },
+          "exemplar": true,
+          "expr": "process_cpu_usage{instance=~\"$instance\"}",
+          "interval": "",
+          "legendFormat": "process_cpu_usage {{hostname}}",
+          "refId": "process_cpu_usage"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "tJLDJxsnz"
+          },
+          "exemplar": true,
+          "expr": "process_cpu_usage_system{instance=~\"$instance\"}",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "process_cpu_usage_system {{hostname}}",
+          "refId": "process_cpu_usage_system"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "tJLDJxsnz"
+          },
+          "exemplar": true,
+          "expr": "process_cpu_usage_user{instance=~\"$instance\"}",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "process_cpu_usage_user {{hostname}}",
+          "refId": "process_cpu_usage_user"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "process cpu usage",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:495",
+          "format": "percentunit",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:496",
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    },
+    {
+      "collapsed": false,
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 9
+      },
+      "id": 8,
+      "panels": [],
+      "title": "latency",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "fill": 1,
+      "fillGradient": 0,
+      "gridPos": {
+        "h": 7,
+        "w": 24,
+        "x": 0,
+        "y": 10
+      },
+      "hiddenSeries": false,
+      "id": 10,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "nullPointMode": "null",
+      "options": {
+        "alertThreshold": true
+      },
+      "percentage": false,
+      "pluginVersion": "8.4.4",
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "tJLDJxsnz"
+          },
+          "exemplar": true,
+          "expr": "{__name__=~\".*latency.*\",job=\"metaserver\"}",
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeRegions": [],
+      "title": "latency",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "mode": "time",
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "$$hashKey": "object:12821",
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        },
+        {
+          "$$hashKey": "object:12822",
+          "format": "short",
+          "logBase": 1,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false
+      }
+    }
+  ],
+  "schemaVersion": 35,
+  "style": "dark",
+  "tags": [],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "text": "10.219.192.50:6800",
+          "value": "10.219.192.50:6800"
+        },
+        "definition": "{__name__=~\"bthread_count\", job=\"metaserver\"}",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Addr",
+        "multi": false,
+        "name": "instance",
+        "options": [],
+        "query": {
+          "query": "{__name__=~\"bthread_count\", job=\"metaserver\"}",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "/.*instance=\"([^\"]*).*/",
+        "skipUrlSync": false,
+        "sort": 1,
+        "type": "query"
+      },
+      {
+        "current": {
+          "selected": false,
+          "text": "All",
+          "value": "$__all"
+        },
+        "definition": "{__name__=~\"bthread_count\", job=\"metaserver\"}",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Hostname",
+        "multi": false,
+        "name": "hostname",
+        "options": [],
+        "query": {
+          "query": "{__name__=~\"bthread_count\", job=\"metaserver\"}",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "/.*hostname=\"([^\"]*).*/",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
+      }
+    ]
+  },
+  "time": {
+    "from": "now-30m",
+    "to": "now"
+  },
+  "timepicker": {},
+  "timezone": "",
+  "title": "metaserver",
+  "uid": "t-YkFxy7z",
+  "version": 9,
+  "weekStart": ""
+}

curvefs/monitor/grafana/grafana.ini

@@ -0,0 +1,597 @@
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+##################### Grafana Configuration Example #####################
+#
+# Everything has defaults so you only need to uncomment things you want to
+# change
+
+# possible values : production, development
+;app_mode = production
+
+# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty
+;instance_name = ${HOSTNAME}
+
+#################################### Paths ####################################
+[paths]
+# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used)
+;data = /var/lib/grafana
+
+# Temporary files in `data` directory older than given duration will be removed
+;temp_data_lifetime = 24h
+
+# Directory where grafana can store logs
+;logs = /var/log/grafana
+
+# Directory where grafana will automatically scan and look for plugins
+;plugins = /var/lib/grafana/plugins
+
+# folder that contains provisioning config files that grafana will apply on startup and while running.
+;provisioning = conf/provisioning
+
+#################################### Server ####################################
+[server]
+# Protocol (http, https, socket)
+;protocol = http
+
+# The ip address to bind to, empty will bind to all interfaces
+;http_addr =
+
+# The http port  to use
+;http_port = 3000
+
+# The public facing domain name used to access grafana from a browser
+;domain = localhost
+
+# Redirect to correct domain if host header does not match domain
+# Prevents DNS rebinding attacks
+;enforce_domain = false
+
+# The full public facing url you use in browser, used for redirects and emails
+# If you use reverse proxy and sub path specify full url (with sub path)
+;root_url = http://localhost:3000
+
+# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons.
+;serve_from_sub_path = false
+
+# Log web requests
+;router_logging = false
+
+# the path relative working path
+;static_root_path = public
+
+# enable gzip
+;enable_gzip = false
+
+# https certs & key file
+;cert_file =
+;cert_key =
+
+# Unix socket path
+;socket =
+
+#################################### Database ####################################
+[database]
+# You can configure the database connection by specifying type, host, name, user and password
+# as separate properties or as on string using the url properties.
+
+# Either "mysql", "postgres" or "sqlite3", it's your choice
+;type = sqlite3
+;host = 127.0.0.1:3306
+;name = grafana
+;user = root
+# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;"""
+;password =
+
+# Use either URL or the previous fields to configure the database
+# Example: mysql://user:secret@host:port/database
+;url =
+
+# For "postgres" only, either "disable", "require" or "verify-full"
+;ssl_mode = disable
+
+# For "sqlite3" only, path relative to data_path setting
+;path = grafana.db
+
+# Max idle conn setting default is 2
+;max_idle_conn = 2
+
+# Max conn setting default is 0 (mean not set)
+;max_open_conn =
+
+# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours)
+;conn_max_lifetime = 14400
+
+# Set to true to log the sql calls and execution times.
+;log_queries =
+
+# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared)
+;cache_mode = private
+
+#################################### Cache server #############################
+[remote_cache]
+# Either "redis", "memcached" or "database" default is "database"
+;type = database
+
+# cache connectionstring options
+# database: will use Grafana primary database.
+# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0`. Only addr is required.
+# memcache: 127.0.0.1:11211
+;connstr =
+
+#################################### Data proxy ###########################
+[dataproxy]
+
+# This enables data proxy logging, default is false
+;logging = false
+
+# How long the data proxy should wait before timing out default is 30 (seconds)
+;timeout = 30
+
+# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false.
+;send_user_header = false
+
+#################################### Analytics ####################################
+[analytics]
+# Server reporting, sends usage counters to stats.grafana.org every 24 hours.
+# No ip addresses are being tracked, only simple counters to track
+# running instances, dashboard and error counts. It is very helpful to us.
+# Change this option to false to disable reporting.
+;reporting_enabled = true
+
+# Set to false to disable all checks to https://grafana.net
+# for new vesions (grafana itself and plugins), check is used
+# in some UI views to notify that grafana or plugin update exists
+# This option does not cause any auto updates, nor send any information
+# only a GET request to http://grafana.com to get latest versions
+;check_for_updates = true
+
+# Google Analytics universal tracking code, only enabled if you specify an id here
+;google_analytics_ua_id =
+
+# Google Tag Manager ID, only enabled if you specify an id here
+;google_tag_manager_id =
+
+#################################### Security ####################################
+[security]
+# default admin user, created on startup
+;admin_user = admin
+
+# default admin password, can be changed before first start of grafana,  or in profile settings
+;admin_password = admin
+
+# used for signing
+;secret_key = SW2YcwTIb9zpOOhoPsMm
+
+# disable gravatar profile images
+;disable_gravatar = false
+
+# data source proxy whitelist (ip_or_domain:port separated by spaces)
+;data_source_proxy_whitelist =
+
+# disable protection against brute force login attempts
+;disable_brute_force_login_protection = false
+
+# set to true if you host Grafana behind HTTPS. default is false.
+;cookie_secure = false
+
+# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict" and "none"
+;cookie_samesite = lax
+
+# set to true if you want to allow browsers to render Grafana in a <frame>, <iframe>, <embed> or <object>. default is false.
+;allow_embedding = false
+
+# Set to true if you want to enable http strict transport security (HSTS) response header.
+# This is only sent when HTTPS is enabled in this configuration.
+# HSTS tells browsers that the site should only be accessed using HTTPS.
+# The default version will change to true in the next minor release, 6.3.
+;strict_transport_security = false
+
+# Sets how long a browser should cache HSTS. Only applied if strict_transport_security is enabled.
+;strict_transport_security_max_age_seconds = 86400
+
+# Set to true if to enable HSTS preloading option. Only applied if strict_transport_security is enabled.
+;strict_transport_security_preload = false
+
+# Set to true if to enable the HSTS includeSubDomains option. Only applied if strict_transport_security is enabled.
+;strict_transport_security_subdomains = false
+
+# Set to true to enable the X-Content-Type-Options response header.
+# The X-Content-Type-Options response HTTP header is a marker used by the server to indicate that the MIME types advertised
+# in the Content-Type headers should not be changed and be followed. The default will change to true in the next minor release, 6.3.
+;x_content_type_options = false
+
+# Set to true to enable the X-XSS-Protection header, which tells browsers to stop pages from loading
+# when they detect reflected cross-site scripting (XSS) attacks. The default will change to true in the next minor release, 6.3.
+;x_xss_protection = false
+
+#################################### Snapshots ###########################
+[snapshots]
+# snapshot sharing options
+;external_enabled = true
+;external_snapshot_url = https://snapshots-origin.raintank.io
+;external_snapshot_name = Publish to snapshot.raintank.io
+
+# remove expired snapshot
+;snapshot_remove_expired = true
+
+#################################### Dashboards History ##################
+[dashboards]
+# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1
+;versions_to_keep = 20
+
+#################################### Users ###############################
+[users]
+# disable user signup / registration
+;allow_sign_up = true
+
+# Allow non admin users to create organizations
+;allow_org_create = true
+
+# Set to true to automatically assign new users to the default organization (id 1)
+;auto_assign_org = true
+
+# Default role new users will be automatically assigned (if disabled above is set to true)
+;auto_assign_org_role = Viewer
+
+# Background text for the user field on the login page
+;login_hint = email or username
+;password_hint = password
+
+# Default UI theme ("dark" or "light")
+;default_theme = dark
+
+# External user management, these options affect the organization users view
+;external_manage_link_url =
+;external_manage_link_name =
+;external_manage_info =
+
+# Viewers can edit/inspect dashboard settings in the browser. But not save the dashboard.
+;viewers_can_edit = false
+
+# Editors can administrate dashboard, folders and teams they create
+;editors_can_admin = false
+
+[auth]
+# Login cookie name
+;login_cookie_name = grafana_session
+
+# The lifetime (days) an authenticated user can be inactive before being required to login at next visit. Default is 7 days,
+;login_maximum_inactive_lifetime_days = 7
+
+# The maximum lifetime (days) an authenticated user can be logged in since login time before being required to login. Default is 30 days.
+;login_maximum_lifetime_days = 30
+
+# How often should auth tokens be rotated for authenticated users when being active. The default is each 10 minutes.
+;token_rotation_interval_minutes = 10
+
+# Set to true to disable (hide) the login form, useful if you use OAuth, defaults to false
+;disable_login_form = false
+
+# Set to true to disable the signout link in the side menu. useful if you use auth.proxy, defaults to false
+;disable_signout_menu = false
+
+# URL to redirect the user to after sign out
+;signout_redirect_url =
+
+# Set to true to attempt login with OAuth automatically, skipping the login screen.
+# This setting is ignored if multiple OAuth providers are configured.
+;oauth_auto_login = false
+
+#################################### Anonymous Auth ######################
+[auth.anonymous]
+# enable anonymous access
+;enabled = false
+
+# specify organization name that should be used for unauthenticated users
+;org_name = Main Org.
+
+# specify role for unauthenticated users
+;org_role = Viewer
+
+#################################### Github Auth ##########################
+[auth.github]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email,read:org
+;auth_url = https://github.com/login/oauth/authorize
+;token_url = https://github.com/login/oauth/access_token
+;api_url = https://api.github.com/user
+;team_ids =
+;allowed_organizations =
+
+#################################### Google Auth ##########################
+[auth.google]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_client_id
+;client_secret = some_client_secret
+;scopes = https://www.googleapis.com/auth/userinfo.profile https://www.googleapis.com/auth/userinfo.email
+;auth_url = https://accounts.google.com/o/oauth2/auth
+;token_url = https://accounts.google.com/o/oauth2/token
+;api_url = https://www.googleapis.com/oauth2/v1/userinfo
+;allowed_domains =
+
+#################################### Generic OAuth ##########################
+[auth.generic_oauth]
+;enabled = false
+;name = OAuth
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email,read:org
+;auth_url = https://foo.bar/login/oauth/authorize
+;token_url = https://foo.bar/login/oauth/access_token
+;api_url = https://foo.bar/user
+;team_ids =
+;allowed_organizations =
+;tls_skip_verify_insecure = false
+;tls_client_cert =
+;tls_client_key =
+;tls_client_ca =
+
+; Set to true to enable sending client_id and client_secret via POST body instead of Basic authentication HTTP header
+; This might be required if the OAuth provider is not RFC6749 compliant, only supporting credentials passed via POST payload
+;send_client_credentials_via_post = false
+
+#################################### SAML Auth ###########################
+;[auth.saml] # Enterprise only
+;enabled = false
+;private_key =
+;private_key_path =
+;certificate =
+;certificate_path =
+;idp_metadata =
+;idp_metadata_path =
+;idp_metadata_url =
+;max_issue_delay = 90s
+;metadata_valid_duration = 48h
+
+#################################### Grafana.com Auth ####################
+[auth.grafana_com]
+;enabled = false
+;allow_sign_up = true
+;client_id = some_id
+;client_secret = some_secret
+;scopes = user:email
+;allowed_organizations =
+
+#################################### Auth Proxy ##########################
+[auth.proxy]
+;enabled = false
+;header_name = X-WEBAUTH-USER
+;header_property = username
+;auto_sign_up = true
+;ldap_sync_ttl = 60
+;whitelist = 192.168.1.1, 192.168.2.1
+;headers = Email:X-User-Email, Name:X-User-Name
+
+#################################### Basic Auth ##########################
+[auth.basic]
+;enabled = true
+
+#################################### Auth LDAP ##########################
+[auth.ldap]
+;enabled = false
+;config_file = /etc/grafana/ldap.toml
+;allow_sign_up = true
+
+# LDAP backround sync (Enterprise only)
+# At 1 am every day
+;sync_cron = "0 0 1 * * *"
+;active_sync_enabled = true
+
+#################################### SMTP / Emailing ##########################
+[smtp]
+;enabled = false
+;host = localhost:25
+;user =
+# If the password contains # or ; you have to wrap it with trippel quotes. Ex """#password;"""
+;password =
+;cert_file =
+;key_file =
+;skip_verify = false
+;from_address = admin@grafana.localhost
+;from_name = Grafana
+# EHLO identity in SMTP dialog (defaults to instance_name)
+;ehlo_identity = dashboard.example.com
+
+[emails]
+;welcome_email_on_sign_up = false
+
+#################################### Logging ##########################
+[log]
+# Either "console", "file", "syslog". Default is console and  file
+# Use space to separate multiple modes, e.g. "console file"
+;mode = console file
+
+# Either "debug", "info", "warn", "error", "critical", default is "info"
+;level = info
+
+# optional settings to set different levels for specific loggers. Ex filters = sqlstore:debug
+;filters =
+
+# For "console" mode only
+[log.console]
+;level =
+
+# log line format, valid options are text, console and json
+;format = console
+
+# For "file" mode only
+[log.file]
+;level =
+
+# log line format, valid options are text, console and json
+;format = text
+
+# This enables automated log rotate(switch of following options), default is true
+;log_rotate = true
+
+# Max line number of single file, default is 1000000
+;max_lines = 1000000
+
+# Max size shift of single file, default is 28 means 1 << 28, 256MB
+;max_size_shift = 28
+
+# Segment log daily, default is true
+;daily_rotate = true
+
+# Expired days of log file(delete after max days), default is 7
+;max_days = 7
+
+[log.syslog]
+;level =
+
+# log line format, valid options are text, console and json
+;format = text
+
+# Syslog network type and address. This can be udp, tcp, or unix. If left blank, the default unix endpoints will be used.
+;network =
+;address =
+
+# Syslog facility. user, daemon and local0 through local7 are valid.
+;facility =
+
+# Syslog tag. By default, the process' argv[0] is used.
+;tag =
+
+#################################### Alerting ############################
+[alerting]
+# Disable alerting engine & UI features
+;enabled = true
+# Makes it possible to turn off alert rule execution but alerting UI is visible
+;execute_alerts = true
+
+# Default setting for new alert rules. Defaults to categorize error and timeouts as alerting. (alerting, keep_state)
+;error_or_timeout = alerting
+
+# Default setting for how Grafana handles nodata or null values in alerting. (alerting, no_data, keep_state, ok)
+;nodata_or_nullvalues = no_data
+
+# Alert notifications can include images, but rendering many images at the same time can overload the server
+# This limit will protect the server from render overloading and make sure notifications are sent out quickly
+;concurrent_render_limit = 5
+
+
+# Default setting for alert calculation timeout. Default value is 30
+;evaluation_timeout_seconds = 30
+
+# Default setting for alert notification timeout. Default value is 30
+;notification_timeout_seconds = 30
+
+# Default setting for max attempts to sending alert notifications. Default value is 3
+;max_attempts = 3
+
+#################################### Explore #############################
+[explore]
+# Enable the Explore section
+;enabled = true
+
+#################################### Internal Grafana Metrics ##########################
+# Metrics available at HTTP API Url /metrics
+[metrics]
+# Disable / Enable internal metrics
+;enabled           = true
+
+# Publish interval
+;interval_seconds  = 10
+
+# Send internal metrics to Graphite
+[metrics.graphite]
+# Enable by setting the address setting (ex localhost:2003)
+;address =
+;prefix = prod.grafana.%(instance_name)s.
+
+#################################### Distributed tracing ############
+[tracing.jaeger]
+# Enable by setting the address sending traces to jaeger (ex localhost:6831)
+;address = localhost:6831
+# Tag that will always be included in when creating new spans. ex (tag1:value1,tag2:value2)
+;always_included_tag = tag1:value1
+# Type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote
+;sampler_type = const
+# jaeger samplerconfig param
+# for "const" sampler, 0 or 1 for always false/true respectively
+# for "probabilistic" sampler, a probability between 0 and 1
+# for "rateLimiting" sampler, the number of spans per second
+# for "remote" sampler, param is the same as for "probabilistic"
+# and indicates the initial sampling rate before the actual one
+# is received from the mothership
+;sampler_param = 1
+# Whether or not to use Zipkin propagation (x-b3- HTTP headers).
+;zipkin_propagation = false
+# Setting this to true disables shared RPC spans.
+# Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
+;disable_shared_zipkin_spans = false
+
+#################################### Grafana.com integration  ##########################
+# Url used to import dashboards directly from Grafana.com
+[grafana_com]
+;url = https://grafana.com
+
+#################################### External image storage ##########################
+[external_image_storage]
+# Used for uploading images to public servers so they can be included in slack/email messages.
+# you can choose between (s3, webdav, gcs, azure_blob, local)
+;provider =
+
+[external_image_storage.s3]
+;bucket =
+;region =
+;path =
+;access_key =
+;secret_key =
+
+[external_image_storage.webdav]
+;url =
+;public_url =
+;username =
+;password =
+
+[external_image_storage.gcs]
+;key_file =
+;bucket =
+;path =
+
+[external_image_storage.azure_blob]
+;account_name =
+;account_key =
+;container_name =
+
+[external_image_storage.local]
+# does not require any configuration
+
+[rendering]
+# Options to configure external image rendering server like https://github.com/grafana/grafana-image-renderer
+;server_url =
+;callback_url =
+
+[enterprise]
+# Path to a valid Grafana Enterprise license.jwt file
+;license_path =
+
+[panels]
+# If set to true Grafana will allow script tags in text panels. Not recommended as it enable XSS vulnerabilities.
+;disable_sanitize_html = false
+
+[plugins]
+;enable_alpha = false
+;app_tls_skip_verify_insecure = false

curvefs/monitor/grafana/provisioning/dashboards/all.yml

@@ -0,0 +1,24 @@
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+- name: 'default'
+  org_id: 1
+  folder: ''
+  type: 'file'
+  options:
+    folder: '/etc/grafana/provisioning/dashboards'

curvefs/monitor/grafana/provisioning/datasources/all.yml

@@ -0,0 +1,27 @@
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+datasources:
+- name: 'Prometheus'
+  type: 'prometheus'
+  access: 'proxy'
+  org_id: 1
+  url: 'http://localhost:9090'
+  is_default: true
+  version: 1
+  editable: true

curvefs/monitor/prometheus/prometheus.yml

@@ -0,0 +1,53 @@
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+# my global config
+global:
+  scrape_interval:     3s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets:
+      # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: 'prometheus'
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+    - targets: ['localhost:9090']
+
+  - job_name: 'curve_metrics'
+
+    file_sd_configs:
+    - files: ['*.json']
+

curvefs/monitor/target.ini

@@ -0,0 +1,20 @@
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+[path]
+target_path=prometheus/target.json

curvefs/monitor/target_json.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+# coding=utf-8
+
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+from cProfile import label
+import os
+import time
+import json
+import configparser
+import subprocess
+import re
+
+CURVEFS_TOOL = "curvefs_tool"
+JSON_PATH = "/tmp/topology.json"
+HOSTNAME_PORT_REGEX = r"[^\"\ ]\S*:\d+"
+IP_PORT_REGEX = r"[0-9]+(?:\.[0-9]+){3}:\d+"
+
+targetPath=None
+
+def loadConf():
+    global targetPath
+    conf=configparser.ConfigParser()
+    conf.read("target.ini")
+    targetPath=conf.get("path", "target_path")
+
+def runCurvefsToolCommand(command):
+    cmd = [CURVEFS_TOOL]+command
+    try:
+        output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, timeout=5)
+    except subprocess.TimeoutExpired as e:
+        return -1, str(output)
+    except subprocess.CalledProcessError as e:
+        return 0, str(e.output)
+    return 0, str(output)
+    
+
+def loadServer():
+    ret, _ = runCurvefsToolCommand(["list-topology", "-jsonType=tree", "-jsonPath=%s"%JSON_PATH])
+    data = None
+    if ret == 0:
+        with open(JSON_PATH) as load_f:
+            data = json.load(load_f)
+    servers = []
+    if data is not None:
+        for pool in data["poollist"]:
+            for zone in pool["zonelist"]:
+                for server in zone["serverlist"]:
+                    servers.append(server)
+    return servers
+
+def loadClient():
+    ret, output = runCurvefsToolCommand(["list-fs"])
+    clients = []
+    if ret == 0 :
+        for line in output.split('\\n'):
+            if line.startswith("mountpoints:"):
+                targets = re.findall(HOSTNAME_PORT_REGEX, line)
+                clients.extend(targets)
+    label = lablesValue(None, "client")
+    return unitValue(label, clients)
+
+def loadType(hostType):
+    ret, output = runCurvefsToolCommand(["status-%s"%hostType])
+    targets = []
+    if ret == 0:
+        targets = re.findall(IP_PORT_REGEX, output)
+    labels = lablesValue(None, hostType)
+    return unitValue(labels, targets)
+
+def ipPort2Addr(ip, port):
+    return str(ip) + ":" + str(port)
+
+def server2Target(server):
+    labels = lablesValue(server["hostname"], "metaserver")
+    serverAddr = []
+    serverAddr.append(ipPort2Addr(server["internalip"], server["internalport"]))
+    targets = list(set(serverAddr))
+    return unitValue(labels, targets)
+
+def lablesValue(hostname, job):
+    labels = {}
+    if hostname is not None:
+        labels["hostname"] = hostname
+    if job is not None:
+        labels["job"] = job
+    return labels
+
+def unitValue(lables, targets):
+    unit = {}
+    if lables is not None:
+        unit["labels"] = lables
+    if targets is not None:
+        unit["targets"] = targets
+    return unit
+
+
+def refresh():
+    targets = []
+    # load metaserver
+    servers = loadServer()
+    for server in servers:
+        targets.append(server2Target(server))
+    # load etcd
+    etcd = loadType("etcd")
+    targets.append(etcd)
+    # load mds
+    mds = loadType("mds")
+    targets.append(mds)
+    # load client 
+    client = loadClient()
+    targets.append(client)
+    
+    with open(targetPath+'.new', 'w', 0o777) as fd:
+        json.dump(targets, fd, indent=4)
+        fd.flush()
+        os.fsync(fd.fileno())
+
+    os.rename(targetPath+'.new', targetPath)
+    os.chmod(targetPath, 0o777)
+
+
+if __name__ == '__main__':
+    while True:
+        loadConf()
+        refresh()
+        # refresh every 30s
+        time.sleep(30)

curvefs/monitor/update_dashboard.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+
+#
+#     Copyright (c) 2022 NetEase Inc.
+#
+#  This program is free software; you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation; either version 2 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License along
+#  with this program; if not, write to the Free Software Foundation, Inc.,
+#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+
+# Updates local dashboard configurations by retrieving
+# the new version from a Grafana instance.
+#
+# The script assumes that basic authentication is configured
+# (change the login credentials with `LOGIN`).
+#
+# DASHBOARD_DIRECTORY represents the path to the directory
+# where the JSON files corresponding to the dashboards exist.
+# The default location is relative to the execution of the
+# script.
+#
+# URL specifies the URL of the Grafana instance.
+#
+
+set -o errexit
+
+readonly URL=${URL:-"http://127.0.0.1:3000"}
+readonly LOGIN=${LOGIN:-"admin:curve"}
+readonly DASHBOARDS_DIRECTORY=${DASHBOARDS_DIRECTORY:-"./grafana/provisioning/dashboards"}
+
+
+main() {
+  local dashboards=$(list_dashboards)
+  local dashboard_json
+
+  show_config
+
+  for dashboard in $dashboards; do
+    dashboard_json=$(get_dashboard "$dashboard")
+
+    if [[ -z "$dashboard_json" ]]; then
+      echo "ERROR:
+  Couldn't retrieve dashboard $dashboard.
+      "
+      exit 1
+    fi
+
+    echo "$dashboard_json" >$DASHBOARDS_DIRECTORY/$dashboard.json
+  done
+}
+
+
+# Shows the global environment variables that have been configured
+# for this run.
+show_config() {
+  echo "INFO:
+  Starting dashboard extraction.
+
+  URL:                  $URL
+  LOGIN:                $LOGIN
+  DASHBOARDS_DIRECTORY: $DASHBOARDS_DIRECTORY
+  "
+}
+
+
+# Retrieves a dashboard ($1) from the database of dashboards.
+#
+# As we're getting it right from the database, it'll contain an `id`.
+#
+# Given that the ID is potentially different when we import it
+# later, to be make this dashboard importable we make the `id`
+# field NULL.
+get_dashboard() {
+  local dashboard=$1
+
+  if [[ -z "$dashboard" ]]; then
+    echo "ERROR:
+  A dashboard must be specified.
+  "
+    exit 1
+  fi
+
+  curl \
+    --silent \
+    --user "$LOGIN" \
+    $URL/api/dashboards/db/$dashboard |
+    jq '.dashboard | .id = null'
+}
+
+
+# lists all the dashboards available.
+#
+# `/api/search` lists all the dashboards and folders
+# that exist under our organization.
+#
+# Here we filter the response (that also contain folders)
+# to gather only the name of the dashboards.
+list_dashboards() {
+  curl \
+    --silent \
+    --user "$LOGIN" \
+    $URL/api/search |
+    jq -r '.[] | select(.type == "dash-db") | .uri' |
+    cut -d '/' -f2
+}
+
+main "$@"

curvefs/proto/metaserver.proto

@@ -46,6 +46,7 @@ enum MetaStatusCode {
     PARSE_FROM_STRING_FAILED = 23;
     STORAGE_INTERNAL_ERROR = 24;
     RPC_STREAM_ERROR = 25;
+    INODE_S3_META_TOO_LARGE = 26;
 }
 
 // dentry interface
@@ -147,14 +148,14 @@ message GetInodeRequest {
     required uint32 fsId = 4;
     required uint64 inodeId = 5;
     optional uint64 appliedIndex = 6;
+    optional bool supportStreaming = 7;  // for backward compatibility
 }
 
 enum FsFileType {
-    TYPE_DIRECTORY = 1;   // 1 << 0
-    TYPE_FILE = 2;        // 1 << 1
-    TYPE_SYM_LINK = 4;    // 1 << 2
-    TYPE_S3 = 18;         // (1 << 4) | TYPE_FILE
-    TYPE_VOLUME = 34;     // (1 << 5) | TYPE_FILE
+    TYPE_DIRECTORY = 1;
+    TYPE_FILE = 2;
+    TYPE_SYM_LINK = 3;
+    TYPE_S3 = 4;
 };
 
 message VolumeExtent {
@@ -211,6 +212,7 @@ message GetInodeResponse {
     required MetaStatusCode statusCode = 1;
     optional Inode inode = 2;
     optional uint64 appliedIndex = 3;
+    optional bool streaming = 4;
 }
 
 message CreateInodeRequest {
@@ -334,6 +336,7 @@ message GetOrModifyS3ChunkInfoRequest {
     required bool returnS3ChunkInfoMap = 8;
     optional bool fromS3Compaction = 9;
     // todo: we only need a bit flag to indicate a lot of bool
+    optional bool supportStreaming = 10;  // for backward compatibility
 }
 
 message GetOrModifyS3ChunkInfoResponse {

curvefs/src/client/BUILD

@@ -28,6 +28,7 @@ cc_binary(
     ],
     deps = [
         ":fuse_client_lib",
+        "@com_google_absl//absl/memory",
     ],
 )
 

curvefs/src/client/common/common.cpp

@@ -40,9 +40,18 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) {
     case MetaServerOpType::DeleteDentry:
         os << "DeleteDentry";
         break;
+    case MetaServerOpType::PrepareRenameTx:
+        os << "PrepareRenameTx";
+        break;
     case MetaServerOpType::GetInode:
         os << "GetInode";
         break;
+    case MetaServerOpType::BatchGetInodeAttr:
+        os << "BatchGetInodeAttr";
+        break;
+    case MetaServerOpType::BatchGetXAttr:
+        os << "BatchGetXAttr";
+        break;
     case MetaServerOpType::UpdateInode:
         os << "UpdateInode";
         break;

curvefs/src/client/common/config.cpp

@@ -118,6 +118,8 @@ void InitDiskCacheOption(Configuration *conf,
                               &diskCacheOption->maxUsableSpaceBytes);
     conf->GetValueFatalIfFail("diskCache.cmdTimeoutSec",
                               &diskCacheOption->cmdTimeoutSec);
+    conf->GetValueFatalIfFail("diskCache.threads",
+                              &diskCacheOption->threads);
     conf->GetValueFatalIfFail("diskCache.avgFlushBytes",
                               &diskCacheOption->avgFlushBytes);
     conf->GetValueFatalIfFail("diskCache.burstFlushBytes",
@@ -145,6 +147,8 @@ void InitS3Option(Configuration *conf, S3Option *s3Opt) {
                               &s3Opt->s3ClientAdaptorOpt.intervalSec);
     conf->GetValueFatalIfFail("s3.flushIntervalSec",
                               &s3Opt->s3ClientAdaptorOpt.flushIntervalSec);
+    conf->GetValueFatalIfFail("s3.chunkFlushThreads",
+                              &s3Opt->s3ClientAdaptorOpt.chunkFlushThreads);
     conf->GetValueFatalIfFail("s3.writeCacheMaxByte",
                               &s3Opt->s3ClientAdaptorOpt.writeCacheMaxByte);
     conf->GetValueFatalIfFail("s3.readCacheMaxByte",
@@ -234,6 +238,11 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
     conf->GetValueFatalIfFail("client.dummyserver.startport",
                               &clientOption->dummyServerStartPort);
 
+    LOG_IF(WARNING, conf->GetBoolValue("fuseClient.enableSplice",
+                                       &clientOption->enableFuseSplice))
+        << "Not found `fuseClient.enableSplice` in conf, use default value `"
+        << std::boolalpha << clientOption->enableFuseSplice << '`';
+
     SetBrpcOpt(conf);
 }
 

curvefs/src/client/common/config.h

@@ -97,6 +97,8 @@ struct DiskCacheOption {
     uint64_t maxUsableSpaceBytes;
     // the max time system command can run
     uint32_t cmdTimeoutSec;
+    // threads for disk cache
+    uint32_t threads;
     // the write throttle bps of disk cache
     uint64_t avgFlushBytes;
     // the write burst bps of disk cache
@@ -119,6 +121,7 @@ struct S3ClientAdaptorOption {
     uint32_t prefetchBlocks;
     uint32_t prefetchExecQueueNum;
     uint32_t intervalSec;
+    uint32_t chunkFlushThreads;
     uint32_t flushIntervalSec;
     uint64_t writeCacheMaxByte;
     uint64_t readCacheMaxByte;
@@ -180,6 +183,8 @@ struct FuseClientOption {
     bool enableDCacheMetrics;
 
     uint32_t dummyServerStartPort;
+
+    bool enableFuseSplice = false;
 };
 
 void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption);

curvefs/src/client/curve_fuse_op.cpp

@@ -52,9 +52,29 @@ static FuseClientOption *g_fuseClientOption = nullptr;
 
 DECLARE_int32(v);
 
+/**
+ * use vlog_level to set vlog level on the fly
+ * When vlog_level is set, CheckVLogLevel is called to check the validity of the
+ * value. Dynamically modify the vlog level by setting FLAG_v in CheckVLogLevel.
+ *
+ * You can modify the vlog level to 0 using:
+ * curl -s http://127.0.0.1:9000/flags/vlog_level?setvalue=0
+ */
+DEFINE_int32(vlog_level, 0, "set vlog level");
+static bool CheckVLogLevel(const char*, int32_t value) {
+    FLAGS_v = value;
+    return true;
+}
+DEFINE_validator(vlog_level, CheckVLogLevel);
+
 namespace {
 
 void EnableSplice(struct fuse_conn_info* conn) {
+    if (!g_fuseClientOption->enableFuseSplice) {
+        LOG(INFO) << "Fuse splice is disabled";
+        return;
+    }
+
     if (conn->capable & FUSE_CAP_SPLICE_MOVE) {
         conn->want |= FUSE_CAP_SPLICE_MOVE;
         LOG(INFO) << "FUSE_CAP_SPLICE_MOVE enabled";
@@ -110,6 +130,7 @@ int InitGlog(const char *confPath, const char *argv0) {
 
     curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy;
     dummy.Load(&conf, "v", "client.loglevel", &FLAGS_v);
+    FLAGS_vlog_level = FLAGS_v;
 
     // initialize logging module
     google::InitGoogleLogging(argv0);
@@ -478,3 +499,13 @@ void FuseOpBmap(fuse_req_t req,
     // TODO(wuhanqing): implement for volume storage
     FuseReplyErrByErrCode(req, CURVEFS_ERROR::NOTSUPPORT);
 }
+
+void FuseOpStatFs(fuse_req_t req, fuse_ino_t ino) {
+    struct statvfs stbuf;
+    CURVEFS_ERROR ret = g_ClientInstance->FuseOpStatFs(req, ino, &stbuf);
+    if (ret != CURVEFS_ERROR::OK) {
+        FuseReplyErrByErrCode(req, ret);
+        return;
+    }
+    fuse_reply_statfs(req, &stbuf);
+}

curvefs/src/client/fuse_client.cpp

@@ -35,7 +35,7 @@
 #include "curvefs/proto/mds.pb.h"
 #include "curvefs/src/client/fuse_common.h"
 #include "curvefs/src/client/client_operator.h"
-#include "src/common/timeutility.h"
+#include "src/common/net_common.h"
 #include "src/common/dummyserver.h"
 #include "src/client/client_common.h"
 #include "src/common/string_util.h"
@@ -167,9 +167,9 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
         (mOpts->mountPoint == nullptr) ? "" : mOpts->mountPoint;
     std::string fsName = (mOpts->fsName == nullptr) ? "" : mOpts->fsName;
 
-    int retVal = AddHostNameToMountPointStr(mountPointStr, &mountpoint_);
+    int retVal = AddHostPortToMountPointStr(mountPointStr, &mountpoint_);
     if (retVal < 0) {
-        LOG(ERROR) << "AddHostNameToMountPointStr failed, ret = " << retVal;
+        LOG(ERROR) << "AddHostPortToMountPointStr failed, ret = " << retVal;
         return CURVEFS_ERROR::INTERNAL;
     }
 
@@ -217,7 +217,7 @@ void FuseClient::FuseOpDestroy(void *userdata) {
         (mOpts->mountPoint == nullptr) ? "" : mOpts->mountPoint;
 
     std::string mountPointWithHost;
-    int retVal = AddHostNameToMountPointStr(mountPointStr, &mountPointWithHost);
+    int retVal = AddHostPortToMountPointStr(mountPointStr, &mountPointWithHost);
     if (retVal < 0) {
         return;
     }
@@ -1115,8 +1115,8 @@ CURVEFS_ERROR FuseClient::FastCalAllLayerSumInfo(Inode *inode) {
 CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, void* value,
                                          size_t size) {
-    LOG(INFO) << "FuseOpGetXattr, ino: " << ino
-              << ", name: " << name << ", size = " << size;
+    VLOG(6) << "FuseOpGetXattr, ino: " << ino
+            << ", name: " << name << ", size = " << size;
     std::shared_ptr<InodeWrapper> inodeWrapper;
     CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {

curvefs/src/client/fuse_client.h

@@ -204,6 +204,23 @@ class FuseClient {
         return CURVEFS_ERROR::OK;
     }
 
+    virtual CURVEFS_ERROR FuseOpStatFs(fuse_req_t req, fuse_ino_t ino,
+                                       struct statvfs* stbuf) {
+        // TODO(chengyi01,wuhanqing): implement in s3 and volume client
+        stbuf->f_frsize = stbuf->f_bsize = fsInfo_->blocksize();
+        stbuf->f_blocks = 10UL << 30;
+        stbuf->f_bavail = stbuf->f_bfree = stbuf->f_blocks - 1;
+
+        stbuf->f_files = 1UL << 30;
+        stbuf->f_ffree = stbuf->f_favail = stbuf->f_files - 1;
+
+        stbuf->f_fsid = fsInfo_->fsid();
+
+        stbuf->f_flag = 0;
+        stbuf->f_namemax = option_.maxNameLength;
+        return CURVEFS_ERROR::OK;
+    }
+
     void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
         fsInfo_ = fsInfo;
         init_ = true;
@@ -236,7 +253,7 @@ class FuseClient {
         const std::shared_ptr<InodeWrapper> &inodeWrapper_,
         fuse_entry_param *param);
 
-    int AddHostNameToMountPointStr(const std::string& mountPointStr,
+    int AddHostPortToMountPointStr(const std::string& mountPointStr,
                                    std::string* out) {
         char hostname[kMaxHostNameLength];
         int ret = gethostname(hostname, kMaxHostNameLength);
@@ -244,7 +261,11 @@ class FuseClient {
             LOG(ERROR) << "GetHostName failed, ret = " << ret;
             return ret;
         }
-        *out = std::string(hostname) + ":" + mountPointStr;
+        *out =
+            std::string(hostname) + ":" +
+            std::to_string(
+                curve::client::ClientDummyServerInfo::GetInstance().GetPort()) +
+            ":" + mountPointStr;
         return 0;
     }
 

curvefs/src/client/fuse_volume_client.cpp

@@ -224,7 +224,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
               << ", name: " << name
               << ", mode: " << mode;
     CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_VOLUME, 0, e);
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, e);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
@@ -236,7 +236,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
                                             dev_t rdev, fuse_entry_param *e) {
     VLOG(3) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_VOLUME, rdev, e);
+    return MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, rdev, e);
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,

curvefs/src/client/inode_cache_manager.cpp

@@ -61,8 +61,10 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInode(uint64_t inodeid,
     }
 
     Inode inode;
+    bool streaming;
 
-    MetaStatusCode ret2 = metaClient_->GetInode(fsId_, inodeid, &inode);
+    MetaStatusCode ret2 = metaClient_->GetInode(
+        fsId_, inodeid, &inode, &streaming);
     if (ret2 != MetaStatusCode::OK) {
         LOG_IF(ERROR, ret2 != MetaStatusCode::NOT_FOUND)
             << "metaClient_ GetInode failed, MetaStatusCode = " << ret2
@@ -74,18 +76,22 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInode(uint64_t inodeid,
     out = std::make_shared<InodeWrapper>(
         std::move(inode), metaClient_);
 
-    // NOTE: now the s3chunkinfo in inode is empty for
-    // we had store it with alone, so we should invoke
-    // RefreshS3ChunkInfo() to padding inode's s3chunkinfo.
-    CURVEFS_ERROR rc = out->RefreshS3ChunkInfo();
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "RefreshS3ChunkInfo() failed, retCode = " << rc;
-        return rc;
+    // NOTE: if the s3chunkinfo inside inode is too large,
+    // we should invoke RefreshS3ChunkInfo() to receive s3chunkinfo
+    // by streaming and padding its into inode.
+    if (streaming) {
+        CURVEFS_ERROR rc = out->RefreshS3ChunkInfo();
+        if (rc != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "RefreshS3ChunkInfo() failed, retCode = " << rc;
+            return rc;
+        }
     }
 
     std::shared_ptr<InodeWrapper> eliminatedOne;
     bool eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
     if (eliminated) {
+        VLOG(3) << "GetInode eliminate one inode, ino: "
+                << eliminatedOne->GetInodeId();
         eliminatedOne->FlushAsync();
     }
     return CURVEFS_ERROR::OK;
@@ -109,7 +115,11 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
         }
     }
 
-    MetaStatusCode ret = metaClient_->BatchGetInodeAttr(fsId_, inodeIds, attr);
+    if (inodeIds->empty()) {
+        return CURVEFS_ERROR::OK;
+    }
+
+    MetaStatusCode ret = metaClient_->BatchGetInodeAttr(fsId_, *inodeIds, attr);
     if (MetaStatusCode::OK != ret) {
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed, MetaStatusCode = "
                    << ret << ", MetaStatusCode_Name = "
@@ -136,7 +146,11 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
         }
     }
 
-    MetaStatusCode ret = metaClient_->BatchGetXAttr(fsId_, inodeIds, xattr);
+    if (inodeIds->empty()) {
+        return CURVEFS_ERROR::OK;
+    }
+
+    MetaStatusCode ret = metaClient_->BatchGetXAttr(fsId_, *inodeIds, xattr);
     if (MetaStatusCode::OK != ret) {
         LOG(ERROR) << "metaClient BatchGetXAttr failed, MetaStatusCode = "
                    << ret << ", MetaStatusCode_Name = "

curvefs/src/client/inode_wrapper.cpp

@@ -21,6 +21,7 @@
  */
 
 #include "curvefs/src/client/inode_wrapper.h"
+#include <sstream>
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 
@@ -144,9 +145,7 @@ CURVEFS_ERROR InodeWrapper::SyncFullInode() {
         return CURVEFS_ERROR::OK;
     }
 
-    auto tmp = extentCache_.ToInodePb();
-    inode_.mutable_volumeextentmap()->swap(tmp);
-    VLOG(9) << "Update inode: " << inode_.ShortDebugString();
+    AddVolumeExtentMapToInode();
     auto ret = metaClient_->UpdateInode(inode_);
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "update inode failed, error: " << MetaStatusCode_Name(ret)
@@ -161,6 +160,7 @@ CURVEFS_ERROR InodeWrapper::SyncFullInode() {
 CURVEFS_ERROR InodeWrapper::SyncAttr() {
     curve::common::UniqueLock lock = GetSyncingInodeUniqueLock();
     if (dirty_) {
+        AddVolumeExtentMapToInode();
         MetaStatusCode ret = metaClient_->UpdateInode(inode_);
 
         if (ret != MetaStatusCode::OK) {
@@ -170,6 +170,7 @@ CURVEFS_ERROR InodeWrapper::SyncAttr() {
                        << ", inodeid: " << inode_.inodeid();
             return MetaStatusCodeToCurvefsErrCode(ret);
         }
+
         dirty_ = false;
     }
     return CURVEFS_ERROR::OK;
@@ -196,7 +197,7 @@ void InodeWrapper::FlushAttrAsync() {
     if (dirty_) {
         LockSyncingInode();
 
-        if (inode_.type() == FsFileType::TYPE_VOLUME) {
+        if (inode_.type() == FsFileType::TYPE_FILE) {
             auto tmp = extentCache_.ToInodePb();
             inode_.mutable_volumeextentmap()->swap(tmp);
         }
@@ -258,6 +259,7 @@ CURVEFS_ERROR InodeWrapper::LinkLocked(uint64_t parent) {
     if (inode_.type() != FsFileType::TYPE_DIRECTORY && parent != 0) {
         inode_.add_parent(parent);
     }
+    AddVolumeExtentMapToInode();
     MetaStatusCode ret = metaClient_->UpdateInode(inode_);
     if (ret != MetaStatusCode::OK) {
         inode_.set_nlink(old);
@@ -329,10 +331,7 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) {
         //                      2. write "hello, world"
         //                      3. unlink this file
         //                      4. pread from 0 to 13, expected "hello, world"
-        auto extents = extentCache_.ToInodePb();
-        if (!extents.empty()) {
-            inode_.mutable_volumeextentmap()->swap(extents);
-        }
+        AddVolumeExtentMapToInode();
         MetaStatusCode ret = metaClient_->UpdateInode(inode_);
         VLOG(6) << "UnLinkInode, inodeid = " << inode_.inodeid()
                 << ", nlink = " << inode_.nlink();
@@ -401,6 +400,7 @@ CURVEFS_ERROR InodeWrapper::Release() {
 
 CURVEFS_ERROR
 InodeWrapper::UpdateInodeStatus(InodeOpenStatusChange statusChange) {
+    AddVolumeExtentMapToInode();
     MetaStatusCode ret = metaClient_->UpdateInode(inode_, statusChange);
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ UpdateInode failed, MetaStatusCode = " << ret
@@ -424,6 +424,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParentLocked(
     }
     inode_.add_parent(newParent);
 
+    AddVolumeExtentMapToInode();
     MetaStatusCode ret = metaClient_->UpdateInode(inode_);
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ UpdateInode failed, MetaStatusCode = " << ret
@@ -435,14 +436,48 @@ CURVEFS_ERROR InodeWrapper::UpdateParentLocked(
     return CURVEFS_ERROR::OK;
 }
 
+static std::ostream &operator<<(
+    std::ostream &os,
+    const google::protobuf::Map<uint64_t, curvefs::metaserver::VolumeExtentList>
+        &exts) {
+    if (exts.empty()) {
+        os << "[empty]";
+        return os;
+    }
+
+    for (const auto &range : exts) {
+        for (const auto &ext : range.second.volumeextents()) {
+            os << ext.ShortDebugString();
+        }
+    }
+
+    return os;
+}
+
 void InodeWrapper::BuildExtentCache() {
-    if (inode_.type() != FsFileType::TYPE_VOLUME) {
+    if (inode_.type() != FsFileType::TYPE_FILE) {
         return;
     }
 
-    VLOG(9) << "Build extent for inode: " << inode_.ShortDebugString();
+    VLOG(9) << "Build extent for inode: " << inode_.ShortDebugString()
+            << ", extents: " << inode_.volumeextentmap().size();
     extentCache_.Build(inode_.volumeextentmap());
 }
 
+void InodeWrapper::AddVolumeExtentMapToInode() {
+    if (inode_.type() != FsFileType::TYPE_FILE) {
+        return;
+    }
+
+    auto tmp = extentCache_.ToInodePb();
+    if (tmp.empty()) {
+        return;
+    }
+
+    VLOG(9) << "Volume extent map, ino: " << inode_.inodeid()
+            << ", extents: " << tmp;
+    inode_.mutable_volumeextentmap()->swap(tmp);
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_wrapper.h

@@ -271,7 +271,7 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
                 return SyncS3ChunkInfo();
             }
 
-            case FsFileType::TYPE_VOLUME: {
+            case FsFileType::TYPE_FILE: {
                 return SyncFullInode();
             }
 
@@ -373,6 +373,9 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
 
     void BuildExtentCache();
 
+    // TODO(wuhanqing): separate `volumeextentmap` from `Inode`
+    void AddVolumeExtentMapToInode();
+
  private:
     Inode inode_;
     uint32_t openCount_;

curvefs/src/client/main.c

@@ -51,6 +51,7 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     .releasedir = FuseOpReleaseDir,
     .flush      = FuseOpFlush,
     .bmap       = FuseOpBmap,
+    .statfs     = FuseOpStatFs,
 };
 
 int main(int argc, char *argv[]) {

curvefs/src/client/metric/client_metric.h

@@ -128,6 +128,8 @@ struct S3Metric {
     InterfaceMetric adaptorWriteDiskCache;
     InterfaceMetric adaptorReadS3;
     InterfaceMetric adaptorReadDiskCache;
+    bvar::LatencyRecorder readSize;
+    bvar::LatencyRecorder writeSize;
 
     explicit S3Metric(const std::string &name = "")
         : fsName(!name.empty() ? name
@@ -137,7 +139,9 @@ struct S3Metric {
           adaptorWriteS3(prefix, fsName + "_adaptor_write_s3"),
           adaptorWriteDiskCache(prefix, fsName + "_adaptor_write_disk_cache"),
           adaptorReadS3(prefix, fsName + "_adaptor_read_s3"),
-          adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache") {}
+          adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache"),
+          readSize(prefix, fsName + "_adaptor_read_size"),
+          writeSize(prefix, fsName + "_adaptor_write_size") {}
 };
 
 struct DiskCacheMetric {

curvefs/src/client/rpcclient/metacache.cpp

@@ -75,9 +75,6 @@ bool MetaCache::GetTxId(uint32_t fsId, uint64_t inodeId, uint32_t *partitionId,
 bool MetaCache::GetTarget(uint32_t fsID, uint64_t inodeID,
                           CopysetTarget *target, uint64_t *applyIndex,
                           bool refresh) {
-    VLOG(3) << "Get target, fsid: " << fsID << ", inode id " << inodeID
-            << ", target: " << *target;
-
     // list infos from mds
     if (!ListPartitions(fsID)) {
         LOG(ERROR) << "get target for {fsid:" << fsID
@@ -507,7 +504,7 @@ bool MetaCache::SelectPartition(CopysetTarget *target) {
     return true;
 }
 
-bool MetaCache::GetCopysetIDwithInodeID(uint32_t inodeID,
+bool MetaCache::GetCopysetIDwithInodeID(uint64_t inodeID,
                                         CopysetGroupID *groupID,
                                         PartitionID *partitionID,
                                         uint64_t *txId) {

curvefs/src/client/rpcclient/metacache.h

@@ -176,7 +176,7 @@ class MetaCache {
     bool SelectPartition(CopysetTarget *target);
 
     // get info from partitionMap or copysetMap
-    bool GetCopysetIDwithInodeID(uint32_t inodeID, CopysetGroupID *groupID,
+    bool GetCopysetIDwithInodeID(uint64_t inodeID, CopysetGroupID *groupID,
                                  PartitionID *patitionID, uint64_t *txId);
 
     bool GetCopysetInfowithCopySetID(const CopysetGroupID &groupID,

curvefs/src/client/rpcclient/metaserver_client.cpp

@@ -402,7 +402,7 @@ MetaServerClientImpl::PrepareRenameTx(const std::vector<Dentry> &dentrys) {
 }
 
 MetaStatusCode MetaServerClientImpl::GetInode(uint32_t fsId, uint64_t inodeid,
-                                              Inode *out) {
+                                              Inode *out, bool* streaming) {
     auto task = RPCTask {
         metaserverClientMetric_->getInode.qps.count << 1;
         GetInodeRequest request;
@@ -412,6 +412,7 @@ MetaStatusCode MetaServerClientImpl::GetInode(uint32_t fsId, uint64_t inodeid,
         request.set_partitionid(partitionID);
         request.set_fsid(fsId);
         request.set_inodeid(inodeid);
+        request.set_supportstreaming(true);
 
         curvefs::metaserver::MetaServerService_Stub stub(channel);
         stub.GetInode(cntl, &request, &response, nullptr);
@@ -441,6 +442,7 @@ MetaStatusCode MetaServerClientImpl::GetInode(uint32_t fsId, uint64_t inodeid,
             return -1;
         }
 
+        *streaming = response.has_streaming() ? response.streaming() : false;
         auto &s3chunkinfoMap = response.inode().s3chunkinfomap();
         for (auto &item : s3chunkinfoMap) {
             VLOG(9) << "inodeInfo, inodeId:" << inodeid
@@ -459,14 +461,14 @@ MetaStatusCode MetaServerClientImpl::GetInode(uint32_t fsId, uint64_t inodeid,
 bool GroupInodeIdByPartition(
     uint32_t fsId,
     std::shared_ptr<MetaCache> metaCache,
-    std::set<uint64_t> *inodeIds,
-    std::unordered_map<uint32_t, std::list<uint64_t>> *inodeGroups) {
-    for (const auto &it : *inodeIds) {
+    const std::set<uint64_t> &inodeIds,
+    std::unordered_map<uint32_t, std::vector<uint64_t>> *inodeGroups) {
+    for (const auto &it : inodeIds) {
         uint32_t pId = 0;
         if (metaCache->GetPartitionIdByInodeId(fsId, it, &pId)) {
             auto iter = inodeGroups->find(pId);
             if (iter == inodeGroups->end()) {
-                inodeGroups->emplace(pId, std::list<uint64_t>({it}));
+                inodeGroups->emplace(pId, std::vector<uint64_t>({it}));
             } else {
                 iter->second.push_back(it);
             }
@@ -479,159 +481,178 @@ bool GroupInodeIdByPartition(
     return true;
 }
 
+bool MetaServerClientImpl::SplitRequestInodes(
+    uint32_t fsId,
+    const std::set<uint64_t> &inodeIds,
+    std::vector<std::vector<uint64_t>> *inodeGroups) {
+    std::unordered_map<uint32_t, std::vector<uint64_t>> groups;
+    bool ret = GroupInodeIdByPartition(fsId, metaCache_, inodeIds, &groups);
+    if (!ret) {
+        return false;
+    }
+    for (const auto &it : groups) {
+        auto iter = it.second.begin();
+        while (iter != it.second.end()) {
+            std::vector<uint64_t> tmp;
+            uint32_t batchLimit = opt_.batchLimit;
+            while (iter != it.second.end() && batchLimit > 0) {
+                tmp.emplace_back(*iter);
+                iter++;
+                batchLimit--;
+            }
+            inodeGroups->emplace_back(std::move(tmp));
+        }
+    }
+    return true;
+}
+
+
 MetaStatusCode MetaServerClientImpl::BatchGetInodeAttr(uint32_t fsId,
-    std::set<uint64_t> *inodeIds,
+    const std::set<uint64_t> &inodeIds,
     std::list<InodeAttr> *attr) {
-    uint32_t limit = opt_.batchLimit;
-    // group inodeid by partition
-    std::unordered_map<uint32_t, std::list<uint64_t>> inodeGroups;
-    if (!GroupInodeIdByPartition(fsId, metaCache_, inodeIds, &inodeGroups)) {
+    // group inodeid by partition and batchlimit
+    std::vector<std::vector<uint64_t>> inodeGroups;
+    if (!SplitRequestInodes(fsId, inodeIds, &inodeGroups)) {
         return MetaStatusCode::NOT_FOUND;
     }
 
-    // send rpc
+    // TDOD(wanghai): send rpc parallelly
     for (const auto &it : inodeGroups) {
-        auto iter = it.second.begin();
-        while (iter != it.second.end()) {
-            uint64_t inodeId = *iter;
-            auto task = RPCTask {
-                metaserverClientMetric_->batchGetInodeAttr.qps.count << 1;
-                BatchGetInodeAttrRequest request;
-                BatchGetInodeAttrResponse response;
-                request.set_poolid(poolID);
-                request.set_copysetid(copysetID);
-                request.set_partitionid(partitionID);
-                request.set_fsid(fsId);
-                request.set_appliedindex(
-                    metaCache_->GetApplyIndex(CopysetGroupID(poolID,
-                                              copysetID)));
-                uint32_t batchLimit = limit;
-                while (iter != it.second.end() && batchLimit > 0) {
-                    request.add_inodeid(*iter);
-                    iter++;
-                    batchLimit--;
-                }
-                curvefs::metaserver::MetaServerService_Stub stub(channel);
-                stub.BatchGetInodeAttr(cntl, &request, &response, nullptr);
+        if (it.empty()) {
+            LOG(WARNING) << "BatchGetInodeAttr request empty.";
+            return MetaStatusCode::PARAM_ERROR;
+        }
+        uint64_t inodeId = *it.begin();
+        auto task = RPCTask {
+            metaserverClientMetric_->batchGetInodeAttr.qps.count << 1;
+            BatchGetInodeAttrRequest request;
+            BatchGetInodeAttrResponse response;
+            request.set_poolid(poolID);
+            request.set_copysetid(copysetID);
+            request.set_partitionid(partitionID);
+            request.set_fsid(fsId);
+            request.set_appliedindex(
+                metaCache_->GetApplyIndex(CopysetGroupID(poolID,
+                                            copysetID)));
+            *request.mutable_inodeid() = { it.begin(), it.end() };
 
-                if (cntl->Failed()) {
-                    metaserverClientMetric_->batchGetInodeAttr.eps.count << 1;
-                    LOG(WARNING) << "BatchGetInodeAttr Failed, errorcode = "
-                                 << cntl->ErrorCode()
-                                 << ", error content:" << cntl->ErrorText()
-                                 << ", log id = " << cntl->log_id();
-                    return -cntl->ErrorCode();
-                }
+            curvefs::metaserver::MetaServerService_Stub stub(channel);
+            stub.BatchGetInodeAttr(cntl, &request, &response, nullptr);
 
-                MetaStatusCode ret = response.statuscode();
-                if (ret != MetaStatusCode::OK) {
-                    LOG_IF(WARNING, ret != MetaStatusCode::NOT_FOUND)
-                        << "BatchGetInodeAttr failed, errcode = " << ret
-                        << ", errmsg = " << MetaStatusCode_Name(ret);
-                } else if (response.attr_size() > 0 &&
-                           response.has_appliedindex()) {
-                    auto retAttr = response.attr();
-                    for_each(retAttr.begin(), retAttr.end(),
-                        [&](InodeAttr &a) { attr->push_back(a); });
-                    metaCache_->UpdateApplyIndex(
-                        CopysetGroupID(poolID, copysetID),
-                        response.appliedindex());
-                } else {
-                    LOG(WARNING) << "BatchGetInodeAttr ok, but"
-                                 << " applyIndex or attr not set in response: "
-                                 << response.DebugString();
-                    return -1;
-                }
-                return ret;
-            };
-            auto taskCtx = std::make_shared<TaskContext>(
-                MetaServerOpType::BatchGetInodeAttr, task, fsId, inodeId);
-            BatchGetInodeAttrExcutor excutor(
-                opt_, metaCache_, channelManager_, taskCtx);
-            auto ret = ConvertToMetaStatusCode(excutor.DoRPCTask());
-            if (ret != MetaStatusCode::OK) {
-                attr->clear();
-                return ret;
+            if (cntl->Failed()) {
+                metaserverClientMetric_->batchGetInodeAttr.eps.count << 1;
+                LOG(WARNING) << "BatchGetInodeAttr Failed, errorcode = "
+                                << cntl->ErrorCode()
+                                << ", error content:" << cntl->ErrorText()
+                                << ", log id = " << cntl->log_id();
+                return -cntl->ErrorCode();
             }
+
+            MetaStatusCode ret = response.statuscode();
+            if (ret != MetaStatusCode::OK) {
+                LOG(ERROR) << "BatchGetInodeAttr failed, errcode = " << ret
+                           << ", errmsg = " << MetaStatusCode_Name(ret);
+            } else if (response.attr_size() > 0 &&
+                        response.has_appliedindex()) {
+                auto *attrs = response.mutable_attr();
+                attr->insert(attr->end(),
+                             std::make_move_iterator(attrs->begin()),
+                             std::make_move_iterator(attrs->end()));
+                metaCache_->UpdateApplyIndex(
+                    CopysetGroupID(poolID, copysetID),
+                    response.appliedindex());
+            } else {
+                LOG(WARNING) << "BatchGetInodeAttr ok, but"
+                                << " applyIndex or attr not set in response: "
+                                << response.DebugString();
+                return -1;
+            }
+            return ret;
+        };
+        auto taskCtx = std::make_shared<TaskContext>(
+            MetaServerOpType::BatchGetInodeAttr, task, fsId, inodeId);
+        BatchGetInodeAttrExcutor excutor(
+            opt_, metaCache_, channelManager_, taskCtx);
+        auto ret = ConvertToMetaStatusCode(excutor.DoRPCTask());
+        if (ret != MetaStatusCode::OK) {
+            attr->clear();
+            return ret;
         }
     }
     return MetaStatusCode::OK;
 }
 
 MetaStatusCode MetaServerClientImpl::BatchGetXAttr(uint32_t fsId,
-    std::set<uint64_t> *inodeIds,
+    const std::set<uint64_t> &inodeIds,
     std::list<XAttr> *xattr) {
-    uint32_t limit = opt_.batchLimit;
-    // group inodeid by partition
-    std::unordered_map<uint32_t, std::list<uint64_t>> inodeGroups;
-    if (!GroupInodeIdByPartition(fsId, metaCache_, inodeIds, &inodeGroups)) {
+    // group inodeid by partition and batchlimit
+    std::vector<std::vector<uint64_t>> inodeGroups;
+    if (!SplitRequestInodes(fsId, inodeIds, &inodeGroups)) {
         return MetaStatusCode::NOT_FOUND;
     }
 
-    // send rpc
+    // TDOD(wanghai): send rpc parallelly
     for (const auto &it : inodeGroups) {
-        auto iter = it.second.begin();
-        while (iter != it.second.end()) {
-            uint64_t inodeId = *iter;
-            auto task = RPCTask {
-                metaserverClientMetric_->batchGetXattr.qps.count << 1;
-                BatchGetXAttrRequest request;
-                BatchGetXAttrResponse response;
-                request.set_poolid(poolID);
-                request.set_copysetid(copysetID);
-                request.set_partitionid(partitionID);
-                request.set_fsid(fsId);
-                request.set_appliedindex(
-                    metaCache_->GetApplyIndex(
-                        CopysetGroupID(poolID, copysetID)));
-                uint32_t batchLimit = limit;
-                while (iter != it.second.end() && batchLimit > 0) {
-                    request.add_inodeid(*iter);
-                    iter++;
-                    batchLimit--;
-                }
-                curvefs::metaserver::MetaServerService_Stub stub(channel);
-                stub.BatchGetXAttr(cntl, &request, &response, nullptr);
+        if (it.empty()) {
+            LOG(WARNING) << "BatchGetInodeXAttr request empty.";
+            return MetaStatusCode::PARAM_ERROR;
+        }
 
-                if (cntl->Failed()) {
-                    metaserverClientMetric_->batchGetXattr.eps.count << 1;
-                    LOG(WARNING) << "BatchGetXAttr Failed, errorcode = "
-                                 << cntl->ErrorCode()
-                                 << ", error content:" << cntl->ErrorText()
-                                 << ", log id = " << cntl->log_id();
-                    return -cntl->ErrorCode();
-                }
+        uint64_t inodeId = *it.begin();
+        auto task = RPCTask {
+            metaserverClientMetric_->batchGetXattr.qps.count << 1;
+            BatchGetXAttrRequest request;
+            BatchGetXAttrResponse response;
+            request.set_poolid(poolID);
+            request.set_copysetid(copysetID);
+            request.set_partitionid(partitionID);
+            request.set_fsid(fsId);
+            request.set_appliedindex(
+                metaCache_->GetApplyIndex(
+                    CopysetGroupID(poolID, copysetID)));
+            *request.mutable_inodeid() = { it.begin(), it.end() };
 
-                MetaStatusCode ret = response.statuscode();
-                if (ret != MetaStatusCode::OK) {
-                    LOG_IF(WARNING, ret != MetaStatusCode::NOT_FOUND)
-                        << "BatchGetXAttr failed, errcode = " << ret
-                        << ", errmsg = " << MetaStatusCode_Name(ret);
-                } else if (response.xattr_size() > 0 &&
-                           response.has_appliedindex()) {
-                    auto retXattr = response.xattr();
-                    for_each(retXattr.begin(), retXattr.end(),
-                        [&](XAttr &a) { xattr->push_back(a); });
-                    metaCache_->UpdateApplyIndex(
-                        CopysetGroupID(poolID, copysetID),
-                            response.appliedindex());
-                } else {
-                    LOG(WARNING) << "BatchGetXAttr ok, but"
-                                 << " applyIndex or attr not set in response: "
-                                 << response.DebugString();
-                    return -1;
-                }
-                return ret;
-            };
-            auto taskCtx = std::make_shared<TaskContext>(
-                MetaServerOpType::BatchGetInodeAttr, task, fsId, inodeId);
-            BatchGetInodeAttrExcutor excutor(
-                opt_, metaCache_, channelManager_, taskCtx);
-            auto ret = ConvertToMetaStatusCode(excutor.DoRPCTask());
-            if (ret != MetaStatusCode::OK) {
-                xattr->clear();
-                return ret;
+            curvefs::metaserver::MetaServerService_Stub stub(channel);
+            stub.BatchGetXAttr(cntl, &request, &response, nullptr);
+
+            if (cntl->Failed()) {
+                metaserverClientMetric_->batchGetXattr.eps.count << 1;
+                LOG(WARNING) << "BatchGetXAttr Failed, errorcode = "
+                                << cntl->ErrorCode()
+                                << ", error content:" << cntl->ErrorText()
+                                << ", log id = " << cntl->log_id();
+                return -cntl->ErrorCode();
             }
+
+            MetaStatusCode ret = response.statuscode();
+            if (ret != MetaStatusCode::OK) {
+                LOG(ERROR) << "BatchGetXAttr failed, errcode = " << ret
+                           << ", errmsg = " << MetaStatusCode_Name(ret);
+            } else if (response.xattr_size() > 0 &&
+                        response.has_appliedindex()) {
+                auto *xattrs = response.mutable_xattr();
+                xattr->insert(xattr->end(),
+                              std::make_move_iterator(xattrs->begin()),
+                              std::make_move_iterator(xattrs->end()));
+                metaCache_->UpdateApplyIndex(
+                    CopysetGroupID(poolID, copysetID),
+                        response.appliedindex());
+            } else {
+                LOG(WARNING) << "BatchGetXAttr ok, but"
+                                << " applyIndex or attr not set in response: "
+                                << response.DebugString();
+                return -1;
+            }
+            return ret;
+        };
+        auto taskCtx = std::make_shared<TaskContext>(
+            MetaServerOpType::BatchGetInodeAttr, task, fsId, inodeId);
+        BatchGetInodeAttrExcutor excutor(
+            opt_, metaCache_, channelManager_, taskCtx);
+        auto ret = ConvertToMetaStatusCode(excutor.DoRPCTask());
+        if (ret != MetaStatusCode::OK) {
+            xattr->clear();
+            return ret;
         }
     }
     return MetaStatusCode::OK;
@@ -892,6 +913,7 @@ MetaStatusCode MetaServerClientImpl::GetOrModifyS3ChunkInfo(
         request.set_inodeid(inodeId);
         request.set_returns3chunkinfomap(returnS3ChunkInfoMap);
         *(request.mutable_s3chunkinfoadd()) = s3ChunkInfos;
+        request.set_supportstreaming(true);
 
         curvefs::metaserver::MetaServerService_Stub stub(channel);
 

curvefs/src/client/rpcclient/metaserver_client.h

@@ -88,14 +88,14 @@ class MetaServerClient {
     PrepareRenameTx(const std::vector<Dentry> &dentrys) = 0;
 
     virtual MetaStatusCode GetInode(uint32_t fsId, uint64_t inodeid,
-                                    Inode *out) = 0;
+                                    Inode *out, bool* streaming) = 0;
 
     virtual MetaStatusCode BatchGetInodeAttr(uint32_t fsId,
-        std::set<uint64_t> *inodeIds,
+        const std::set<uint64_t> &inodeIds,
         std::list<InodeAttr> *attr) = 0;
 
     virtual MetaStatusCode BatchGetXAttr(uint32_t fsId,
-        std::set<uint64_t> *inodeIds,
+        const std::set<uint64_t> &inodeIds,
         std::list<XAttr> *xattr) = 0;
 
     virtual MetaStatusCode UpdateInode(const Inode &inode,
@@ -127,6 +127,10 @@ class MetaServerClient {
     virtual MetaStatusCode CreateInode(const InodeParam &param, Inode *out) = 0;
 
     virtual MetaStatusCode DeleteInode(uint32_t fsId, uint64_t inodeid) = 0;
+
+    virtual bool SplitRequestInodes(uint32_t fsId,
+        const std::set<uint64_t> &inodeIds,
+        std::vector<std::vector<uint64_t>> *inodeGroups) = 0;
 };
 
 class MetaServerClientImpl : public MetaServerClient {
@@ -161,14 +165,14 @@ class MetaServerClientImpl : public MetaServerClient {
     MetaStatusCode PrepareRenameTx(const std::vector<Dentry> &dentrys) override;
 
     MetaStatusCode GetInode(uint32_t fsId, uint64_t inodeid,
-                            Inode *out) override;
+                            Inode *out, bool* streaming) override;
 
     MetaStatusCode BatchGetInodeAttr(uint32_t fsId,
-        std::set<uint64_t> *inodeIds,
+        const std::set<uint64_t> &inodeIds,
         std::list<InodeAttr> *attr) override;
 
     MetaStatusCode BatchGetXAttr(uint32_t fsId,
-        std::set<uint64_t> *inodeIds,
+        const std::set<uint64_t> &inodeIds,
         std::list<XAttr> *xattr) override;
 
     MetaStatusCode UpdateInode(const Inode &inode,
@@ -200,6 +204,10 @@ class MetaServerClientImpl : public MetaServerClient {
 
     MetaStatusCode DeleteInode(uint32_t fsId, uint64_t inodeid) override;
 
+    bool SplitRequestInodes(uint32_t fsId,
+        const std::set<uint64_t> &inodeIds,
+        std::vector<std::vector<uint64_t>> *inodeGroups) override;
+
  private:
     bool ParseS3MetaStreamBuffer(butil::IOBuf* buffer,
                                  uint64_t* chunkIndex,

curvefs/src/client/rpcclient/task_excutor.h

@@ -63,7 +63,7 @@ class TaskContext {
 
     TaskContext() = default;
     TaskContext(MetaServerOpType type, RpcFunc func, uint32_t fsid = 0,
-                uint32_t inodeid = 0, bool streaming = false)
+                uint64_t inodeid = 0, bool streaming = false)
         : optype(type), rpctask(func), fsID(fsid), inodeID(inodeid),
           streaming(streaming) {}
 

curvefs/src/client/s3/client_s3_adaptor.cpp

@@ -20,12 +20,12 @@
  * Author: huyao
  */
 
-
-
 #include <brpc/channel.h>
 #include <brpc/controller.h>
 #include <algorithm>
 #include <list>
+
+#include "absl/memory/memory.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
 #include "curvefs/src/common/s3util.h"
 
@@ -57,6 +57,7 @@ S3ClientAdaptorImpl::Init(
     memCacheNearfullRatio_ = option.nearfullRatio;
     throttleBaseSleepUs_ = option.baseSleepUs;
     flushIntervalSec_ = option.flushIntervalSec;
+    chunkFlushThreads_ = option.chunkFlushThreads;
     client_ = client;
     inodeManager_ = inodeManager;
     mdsClient_ = mdsClient;
@@ -95,6 +96,8 @@ S3ClientAdaptorImpl::Init(
               << ", readCacheMaxByte: " << option.readCacheMaxByte
               << ", nearfullRatio: " << option.nearfullRatio
               << ", baseSleepUs: " << option.baseSleepUs;
+    // start chunk flush threads
+    taskPool_.Start(chunkFlushThreads_);
     return CURVEFS_ERROR::OK;
 }
 
@@ -116,12 +119,16 @@ int S3ClientAdaptorImpl::Write(uint64_t inodeId, uint64_t offset,
         if ((size + pendingReq * fuseMaxSize_) >= maxSize) {
             LOG(INFO) << "write cache is full, wait flush. size:" << size
                       << ", maxSize:" << maxSize;
+            // offer to do flush
+            waitInterval_.StopWait();
             fsCacheManager_->WaitFlush();
         }
     }
     uint64_t memCacheRatio = fsCacheManager_->MemCacheRatio();
     int64_t exceedRatio = memCacheRatio - memCacheNearfullRatio_;
     if (exceedRatio > 0) {
+        // offer to do flush
+        waitInterval_.StopWait();
         // upload to s3 derectly or cache disk full
         bool needSleep =
             (DisableDiskCache() || IsReadCache()) ||
@@ -138,6 +145,7 @@ int S3ClientAdaptorImpl::Write(uint64_t inodeId, uint64_t offset,
     fsCacheManager_->DataCacheByteDec(length);
     if (s3Metric_.get() != nullptr) {
         CollectMetrics(&s3Metric_->adaptorWrite, ret, start);
+        s3Metric_->writeSize << length;
     }
     VLOG(6) << "write end inodeId:" << inodeId << ",ret:" << ret
             << ", pendingReq_ is: " << pendingReq_;
@@ -159,8 +167,10 @@ int S3ClientAdaptorImpl::Read(uint64_t inodeId, uint64_t offset,
     }
     if (s3Metric_.get() != nullptr) {
         CollectMetrics(&s3Metric_->adaptorRead, ret, start);
+        s3Metric_->readSize << length;
     }
-
+    VLOG(6) << "read end offset:" << offset << ", len:" << length
+            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
     return ret;
 }
 
@@ -300,8 +310,8 @@ int S3ClientAdaptorImpl::Stop() {
         }
         diskCacheManagerImpl_->UmountDiskCache();
     }
+    taskPool_.Stop();
     client_->Deinit();
-    LOG(INFO) << "Stopping S3ClientAdaptor success";
     return 0;
 }
 
@@ -369,5 +379,26 @@ int S3ClientAdaptorImpl::ClearDiskCache(int64_t inodeId) {
     return ret;
 }
 
+void S3ClientAdaptorImpl::Enqueue(
+  std::shared_ptr<FlushChunkCacheContext> context) {
+    auto task = [this, context]() {
+        this->FlushChunkClosure(context);
+    };
+    taskPool_.Enqueue(task);
+}
+
+int S3ClientAdaptorImpl::FlushChunkClosure(
+  std::shared_ptr<FlushChunkCacheContext> context) {
+    VLOG(9) << "FlushChunkCacheClosure start: " << context->inode;
+    CURVEFS_ERROR ret = context->chunkCacheManptr->Flush(
+      context->inode, context->force);
+    // set the returned value
+    // it is need in FlushChunkCacheCallBack
+    context->retCode = ret;
+    context->cb(context);
+    VLOG(9) << "FlushChunkCacheClosure end: " << context->inode;
+    return 0;
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/s3/client_s3_adaptor.h

@@ -44,6 +44,8 @@ namespace curvefs {
 namespace client {
 
 using ::curve::common::Thread;
+using ::curve::common::TaskThreadPool;
+using curvefs::client::common::S3ClientAdaptorOption;
 using curvefs::client::common::DiskCacheType;
 using curvefs::metaserver::Inode;
 using curvefs::metaserver::S3ChunkInfo;
@@ -52,6 +54,8 @@ using rpcclient::MdsClient;
 using curvefs::client::metric::S3Metric;
 
 class DiskCacheManagerImpl;
+class FlushChunkCacheContext;
+class ChunkCacheManager;
 
 class S3ClientAdaptor {
  public:
@@ -89,6 +93,17 @@ class S3ClientAdaptor {
                                 uint64_t start) = 0;
 };
 
+using FlushChunkCacheCallBack = std::function<
+  void(const std::shared_ptr<FlushChunkCacheContext>&)>;
+
+struct FlushChunkCacheContext {
+    uint64_t inode;
+    ChunkCacheManagerPtr chunkCacheManptr;
+    bool force;
+    FlushChunkCacheCallBack cb;
+    CURVEFS_ERROR retCode;
+};
+
 // client use s3 internal interface
 class S3ClientAdaptorImpl : public S3ClientAdaptor {
  public:
@@ -205,6 +220,8 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
     }
     std::shared_ptr<S3Metric> s3Metric_;
 
+    void Enqueue(std::shared_ptr<FlushChunkCacheContext> context);
+
  private:
     std::shared_ptr<S3Client> client_;
     uint64_t blockSize_;
@@ -214,6 +231,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
     uint32_t prefetchExecQueueNum_;
     std::string allocateServerEps_;
     uint32_t flushIntervalSec_;
+    uint32_t chunkFlushThreads_;
     uint32_t memCacheNearfullRatio_;
     uint32_t throttleBaseSleepUs_;
     Thread bgFlushThread_;
@@ -233,6 +251,11 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
     std::vector<bthread::ExecutionQueueId<AsyncDownloadTask>>
       downloadTaskQueues_;
     uint32_t pageSize_;
+
+    int FlushChunkClosure(std::shared_ptr<FlushChunkCacheContext> context);
+
+    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
+        taskPool_;
 };
 
 }  // namespace client

curvefs/src/client/s3/client_s3_cache_manager.cpp

@@ -85,7 +85,6 @@ void FsCacheManager::ReleaseFileCacheManager(uint64_t inodeId) {
 bool FsCacheManager::Set(DataCachePtr dataCache,
                          std::list<DataCachePtr>::iterator *outIter) {
     std::lock_guard<std::mutex> lk(lruMtx_);
-
     VLOG(3) << "lru current byte:" << lruByte_
             << ",lru max byte:" << readCacheMaxByte_
             << ", dataCache len:" << dataCache->GetLen();
@@ -154,8 +153,8 @@ CURVEFS_ERROR FsCacheManager::FsSync(bool force) {
         WriteLockGuard writeLockGuard(rwLock_);
         tmp = fileCacheManagerMap_;
     }
+
     auto iter = tmp.begin();
-    VLOG(3) << "FsSync force: " << force;
     for (; iter != tmp.end(); iter++) {
         ret = iter->second->Flush(force);
         if (ret == CURVEFS_ERROR::OK) {
@@ -932,45 +931,84 @@ void FileCacheManager::TruncateCache(uint64_t offset, uint64_t fileSize) {
 }
 
 CURVEFS_ERROR FileCacheManager::Flush(bool force, bool toS3) {
-    CURVEFS_ERROR ret;
+    // Todo: concurrent flushes within one file
+    // instead of multiple file flushes may be better
+    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
     std::map<uint64_t, ChunkCacheManagerPtr> tmp;
     {
         WriteLockGuard writeLockGuard(rwLock_);
         tmp = chunkCacheMap_;
     }
-    auto iter = tmp.begin();
 
-    for (; iter != tmp.end(); iter++) {
-        ret = iter->second->Flush(inode_, force, toS3);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "fileCacheManager Flush error, ret:" << ret
-                       << ",chunkIndex:" << iter->second->GetIndex();
-            return ret;
-        }
-
-        {
+    std::atomic<uint64_t> pendingReq(0);
+    curve::common::CountDownEvent cond(1);
+    FlushChunkCacheCallBack cb =
+        [&](const std::shared_ptr<FlushChunkCacheContext> &context) {
             WriteLockGuard writeLockGuard(rwLock_);
-            auto iter1 = chunkCacheMap_.find(iter->first);
-            if (iter1 == chunkCacheMap_.end()) {
-                VLOG(1) << "Flush, chunk cache for inodeid: " << inode_
+            if (ret != CURVEFS_ERROR::OK) {
+                return;
+            }
+            ret = context->retCode;
+            if (context->retCode != CURVEFS_ERROR::OK) {
+                LOG(ERROR) << "fileCacheManager Flush error, ret:" << ret
+                       << ", inode: " << context->inode
+                       << ", chunkIndex: "
+                       << context->chunkCacheManptr->GetIndex();
+                cond.Signal();
+                return;
+            }
+            {
+                auto iter1 = chunkCacheMap_.find(
+                  context->chunkCacheManptr->GetIndex());
+                if (iter1 == chunkCacheMap_.end()) {
+                    VLOG(9) << "Flush, chunk cache for inodeid: " << inode_
                         << " is removed";
-                continue;
-            } else {
-                // tmp and chunkCacheMap_ has this ChunkCacheManagerPtr, so
-                // count is 2 if count more than 2, this mean someone thread has
-                // this ChunkCacheManagerPtr
-                VLOG(9) << "Flush, ChunkCacheManagerPtr count:"
-                        << iter1->second.use_count();
-
+                    if (pendingReq.fetch_sub(1,
+                      std::memory_order_seq_cst) == 1) {
+                        VLOG(9) << "pendingReq is over";
+                        cond.Signal();
+                    }
+                    return;
+                }
+                VLOG(6) << "ChunkCacheManagerPtr count:"
+                        << context->chunkCacheManptr.use_count();
+                // tmp and chunkCacheMap_ has this ChunkCacheManagerPtr,
+                // so count is 2 if count more than 2,
+                // this mean someone thread has this ChunkCacheManagerPtr
                 if (iter1->second->IsEmpty() &&
-                    (2 == iter1->second.use_count())) {
-                    VLOG(9) << "chunkCacheMap_ erase.";
+                  (2 == iter1->second.use_count())) {
+                    VLOG(9) << "erase iter: " << iter1->first
+                            << ", inode: " << context->inode;
                     chunkCacheMap_.erase(iter1);
                 }
             }
+            if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
+                VLOG(9) << "pendingReq is over";
+                cond.Signal();
+            }
+        };
+        std::vector<std::shared_ptr<FlushChunkCacheContext>> flushTasks;
+        auto iter = tmp.begin();
+        VLOG(6) << "flush size is: " << tmp.size();
+        for (; iter != tmp.end(); iter++) {
+            auto context = std::make_shared<FlushChunkCacheContext>();
+            context->inode = inode_;
+            context->cb = cb;
+            context->force = force;
+            context->chunkCacheManptr = iter->second;
+            flushTasks.emplace_back(context);
         }
-    }
-    return CURVEFS_ERROR::OK;
+        pendingReq.fetch_add(flushTasks.size(), std::memory_order_seq_cst);
+        if (pendingReq.load(std::memory_order_seq_cst)) {
+            VLOG(6) << "wait for pendingReq";
+            for (auto iter = flushTasks.begin();
+              iter != flushTasks.end(); ++iter) {
+                s3ClientAdaptor_->Enqueue(*iter);
+            }
+            cond.Wait();
+        }
+        VLOG(6) << "file cache flush over";
+        return ret;
 }
 
 void ChunkCacheManager::ReadByWriteCache(uint64_t chunkPos, uint64_t readLen,
@@ -1261,13 +1299,11 @@ void ChunkCacheManager::AddReadDataCache(DataCachePtr dataCache) {
     uint64_t len = dataCache->GetLen();
     WriteLockGuard writeLockGuard(rwLockRead_);
     std::vector<uint64_t> deleteKeyVec;
-
     auto iter = dataRCacheMap_.begin();
     for (; iter != dataRCacheMap_.end(); iter++) {
         if (chunkPos + len <= iter->first) {
             break;
         }
-
         std::list<DataCachePtr>::iterator dcpIter = iter->second;
         uint64_t dcChunkPos = (*dcpIter)->GetChunkPos();
         uint64_t dcLen = (*dcpIter)->GetLen();
@@ -1278,7 +1314,6 @@ void ChunkCacheManager::AddReadDataCache(DataCachePtr dataCache) {
             deleteKeyVec.emplace_back(dcChunkPos);
         }
     }
-
     for (auto key : deleteKeyVec) {
         auto iter = dataRCacheMap_.find(key);
         std::list<DataCachePtr>::iterator dcpIter = iter->second;
@@ -1400,12 +1435,13 @@ CURVEFS_ERROR ChunkCacheManager::Flush(uint64_t inodeId, bool force,
                                        bool toS3) {
     std::map<uint64_t, DataCachePtr> tmp;
     curve::common::LockGuard lg(flushMtx_);
-    CURVEFS_ERROR ret;
+    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
     {
         WriteLockGuard writeLockGuard(rwLockChunk_);
         tmp = dataWCacheMap_;
     }
     auto iter = tmp.begin();
+    VLOG(6) << "ChunkCacheManager start , size: " << tmp.size();
     for (; iter != tmp.end(); iter++) {
         VLOG(9) << "Flush datacache chunkPos:" << iter->second->GetChunkPos()
                 << ",len:" << iter->second->GetLen() << ",inodeId:" << inodeId
@@ -1434,7 +1470,6 @@ CURVEFS_ERROR ChunkCacheManager::Flush(uint64_t inodeId, bool force,
             }
         }
     }
-
     return CURVEFS_ERROR::OK;
 }
 
@@ -1580,7 +1615,6 @@ void DataCache::CopyBufToDataCache(uint64_t dataCachePos, uint64_t len,
         len -= n;
         blockPos = (blockPos + n) % blockSize;
     }
-
     actualLen_ += addLen;
     VLOG(9) << "chunkPos:" << chunkPos_ << ", len:" << len_
             << ",actualChunkPos_:" << actualChunkPos_
@@ -1925,6 +1959,7 @@ void DataCache::CopyDataCacheToBuf(uint64_t offset, uint64_t len, char *data) {
 
     VLOG(9) << "CopyDataCacheToBuf start Offset:" << offset
             << ", newChunkPos:" << newChunkPos << ",len:" << len;
+
     while (len > 0) {
         if (blockPos + len > blockSize) {
             n = blockSize - blockPos;
@@ -1989,7 +2024,8 @@ CURVEFS_ERROR DataCache::Flush(uint64_t inodeId, bool force, bool toS3) {
             isFlush = false;
         }
     }
-    VLOG(9) << "now:" << now << ",createTime:" << createTime_
+    VLOG(9) << "DataCache::Flush : now:"
+            << now << ",createTime:" << createTime_
             << ",flushIntervalSec:" << flushIntervalSec
             << ",isFlush:" << isFlush << ",chunkPos:" << chunkPos_
             << ", len:" << len_ << ", inodeId:" << inodeId
@@ -2001,7 +2037,6 @@ CURVEFS_ERROR DataCache::Flush(uint64_t inodeId, bool force, bool toS3) {
         mtx_.unlock();
         return CURVEFS_ERROR::NOFLUSH;
     }
-
     if (isFlush) {
         tmpLen = len_;
         blockPos = chunkPos_ % blockSize;
@@ -2036,7 +2071,11 @@ CURVEFS_ERROR DataCache::Flush(uint64_t inodeId, bool force, bool toS3) {
                             &s3ClientAdaptor_->s3Metric_->adaptorWriteS3,
                             context->bufferSize, context->startTime);
                     }
-                    if (pendingReq.fetch_sub(1) == 1) {
+                    // Don't move the if sentence to the front
+                    // it will cause core dumped because s3Metric_
+                    // will be destructed before being accessed
+                    if (pendingReq.fetch_sub(
+                      1, std::memory_order_seq_cst) == 1) {
                         VLOG(9) << "pendingReq is over";
                         cond.Signal();
                     }
@@ -2044,7 +2083,6 @@ CURVEFS_ERROR DataCache::Flush(uint64_t inodeId, bool force, bool toS3) {
                             << " pendingReq is: " << pendingReq;
                     return;
                 }
-
                 LOG(WARNING) << "Put object failed, key: " << context->key;
                 s3ClientAdaptor_->GetS3Client()->UploadAsync(context);
             };
@@ -2065,44 +2103,35 @@ CURVEFS_ERROR DataCache::Flush(uint64_t inodeId, bool force, bool toS3) {
                 chunkId, blockIndex, 0, fsId, inodeId);
             int ret = 0;
             uint64_t start = butil::cpuwide_time_us();
-            if (useDiskCache) {
-                ret = s3ClientAdaptor_->GetDiskCacheManager()->Write(
-                    objectName, data + writeOffset, n);
-            } else {
-                auto context = std::make_shared<PutObjectAsyncContext>();
-                context->key = objectName;
-                context->buffer = data + writeOffset;
-                context->bufferSize = n;
-                context->cb = cb;
-                context->startTime = butil::cpuwide_time_us();
-                uploadTasks.emplace_back(context);
-            }
-            if (ret < 0) {
-                LOG(ERROR) << "write object fail. object: " << objectName;
-                delete[] data;
-                dirty_.store(true, std::memory_order_release);
-                return CURVEFS_ERROR::INTERNAL;
-            }
-            if (useDiskCache) {
-                if (s3ClientAdaptor_->s3Metric_.get() != nullptr) {
-                    s3ClientAdaptor_->CollectMetrics(
-                        &s3ClientAdaptor_->s3Metric_->adaptorWriteDiskCache, n,
-                        start);
-                }
-            }
+            auto context = std::make_shared<PutObjectAsyncContext>();
+            context->key = objectName;
+            context->buffer = data + writeOffset;
+            context->bufferSize = n;
+            context->cb = cb;
+            context->startTime = butil::cpuwide_time_us();
+            uploadTasks.emplace_back(context);
             tmpLen -= n;
             blockIndex++;
             writeOffset += n;
             blockPos = (blockPos + n) % blockSize;
         }
-        if (!useDiskCache) {
-            pendingReq.fetch_add(uploadTasks.size(), std::memory_order_seq_cst);
-            VLOG(9) << "pendingReq init: " << pendingReq;
-            for (auto iter = uploadTasks.begin(); iter != uploadTasks.end();
-                 ++iter) {
-                VLOG(9) << "upload start: " << (*iter)->key
-                        << " len : " << (*iter)->bufferSize;
+
+        pendingReq.fetch_add(uploadTasks.size(), std::memory_order_seq_cst);
+        VLOG(9) << "DataCache::Flush data cache flush pendingReq init: "
+          << pendingReq.load(std::memory_order_seq_cst);
+        if (pendingReq.load(std::memory_order_seq_cst) == 0) {
+            VLOG(3) << "upload task is empty.";
+            delete[] data;
+            return CURVEFS_ERROR::NOFLUSH;
+        }
+        for (auto iter = uploadTasks.begin(); iter != uploadTasks.end();
+            ++iter) {
+            VLOG(9) << "upload start: " << (*iter)->key
+                    << " len : " << (*iter)->bufferSize;
+            if (!useDiskCache) {
                 s3ClientAdaptor_->GetS3Client()->UploadAsync(*iter);
+            } else {
+                s3ClientAdaptor_->GetDiskCacheManager()->Enqueue(*iter);
             }
         }
 
@@ -2110,7 +2139,6 @@ CURVEFS_ERROR DataCache::Flush(uint64_t inodeId, bool force, bool toS3) {
             VLOG(9) << "wait for pendingReq";
             cond.Wait();
         }
-
         delete[] data;
         VLOG(9) << "update inode start, chunkId:" << chunkId
                 << ",offset:" << offset << ",len:" << writeOffset
@@ -2131,11 +2159,11 @@ CURVEFS_ERROR DataCache::Flush(uint64_t inodeId, bool force, bool toS3) {
             inodeWrapper->AppendS3ChunkInfo(chunkIndex, info);
             s3ClientAdaptor_->GetInodeCacheManager()->ShipToFlush(inodeWrapper);
         }
-
         VLOG(9) << "data flush end, inodeId: " << inodeId;
         return CURVEFS_ERROR::OK;
     }
     mtx_.unlock();
+    VLOG(9) << "Flush end " << inodeId;
     return CURVEFS_ERROR::NOFLUSH;
 }
 

curvefs/src/client/s3/disk_cache_manager.cpp

@@ -66,8 +66,9 @@ DiskCacheManager::DiskCacheManager(std::shared_ptr<PosixWrapper> posixWrapper,
     maxUsableSpaceBytes_ = 0;
     // cannot limit the size,
     // because cache is been delete must after upload to s3
-    cachedObjName_ = std::make_shared<SglLRUCache<std::string>>(
-        0, std::make_shared<CacheMetrics>("diskcache"));
+    cachedObjName_ = std::make_shared<
+      LRUCache<std::string, bool>>(0,
+        std::make_shared<CacheMetrics>("diskcache"));
 }
 
 int DiskCacheManager::Init(S3Client *client,
@@ -84,7 +85,7 @@ int DiskCacheManager::Init(S3Client *client,
     cmdTimeoutSec_ = option.diskCacheOpt.cmdTimeoutSec;
 
     cacheWrite_->Init(client_, posixWrapper_, cacheDir_,
-                      option.diskCacheOpt.asyncLoadPeriodMs);
+                      option.diskCacheOpt.asyncLoadPeriodMs, cachedObjName_);
     cacheRead_->Init(posixWrapper_, cacheDir_);
     int ret;
     ret = CreateDir();
@@ -92,19 +93,20 @@ int DiskCacheManager::Init(S3Client *client,
         LOG(ERROR) << "create cache dir error, ret = " << ret;
         return ret;
     }
-    // start aync upload thread
-    cacheWrite_->AsyncUploadRun();
-    std::thread uploadThread =
-        std::thread(&DiskCacheManager::UploadAllCacheWriteFile, this);
-    uploadThread.detach();
-
     // load all cache read file
+    // the all value of cachedObjName_ is set false
     ret = cacheRead_->LoadAllCacheReadFile(cachedObjName_);
     if (ret < 0) {
         LOG(ERROR) << "load all cache read file error. ret = " << ret;
         return ret;
     }
 
+    // start aync upload thread
+    cacheWrite_->AsyncUploadRun();
+    std::thread uploadThread =
+        std::thread(&DiskCacheManager::UploadAllCacheWriteFile, this);
+    uploadThread.detach();
+
     // start trim thread
     TrimRun();
 
@@ -153,13 +155,15 @@ int DiskCacheManager::ClearReadCache(const std::list<std::string> &files) {
     return cacheRead_->ClearReadCache(files);
 }
 
-void DiskCacheManager::AddCache(const std::string name) {
-    cachedObjName_->Put(name);
+void DiskCacheManager::AddCache(const std::string name,
+  bool cacheWriteExist) {
+    cachedObjName_->Put(name, cacheWriteExist);
     VLOG(9) << "cache size is: " << cachedObjName_->Size();
 }
 
 bool DiskCacheManager::IsCached(const std::string name) {
-    if (!cachedObjName_->IsCached(name)) {
+    bool exist;
+    if (!cachedObjName_->Get(name, &exist)) {
         VLOG(9) << "not cached, name = " << name;
         return false;
     }
@@ -268,8 +272,6 @@ int64_t DiskCacheManager::SetDiskFsUsedRatio() {
     int64_t usedBytes = totalBytes - freeBytes;
     int64_t usedPercent = 100 * usedBytes / (usedBytes + availableBytes) + 1;
 
-    VLOG(3) << "cache disk usage = " << usedPercent;
-
     diskFsUsedRatio_.store(usedPercent, std::memory_order_seq_cst);
     return usedPercent;
 }
@@ -289,7 +291,7 @@ void DiskCacheManager::SetDiskInitUsedBytes() {
         return;
     }
     usedBytes_.fetch_add(usedBytes, std::memory_order_seq_cst);
-    VLOG(3) << "cache disk used size is: " << result;
+    VLOG(9) << "cache disk used size is: " << result;
     return;
 }
 
@@ -297,11 +299,16 @@ bool DiskCacheManager::IsDiskCacheFull() {
     int64_t ratio = diskFsUsedRatio_.load(std::memory_order_seq_cst);
     uint64_t usedBytes = GetDiskUsedbytes();
     if (ratio >= fullRatio_ || usedBytes >= maxUsableSpaceBytes_) {
-        VLOG(3) << "disk cache is full"
-                << ", ratio is: " << ratio << ", fullRatio is: " << fullRatio_
-                << ", used bytes is: " << usedBytes;
+        VLOG(6) << "disk cache is full"
+                     << ", ratio is: " << ratio << ", fullRatio is: "
+                     << fullRatio_ << ", used bytes is: " << usedBytes;
+        waitIntervalSec_.StopWait();
         return true;
     }
+    if (!IsDiskCacheSafe()) {
+        VLOG(6) << "wake up trim thread.";
+        waitIntervalSec_.StopWait();
+    }
     return false;
 }
 
@@ -310,12 +317,12 @@ bool DiskCacheManager::IsDiskCacheSafe() {
     uint64_t usedBytes = GetDiskUsedbytes();
     if ((usedBytes < (safeRatio_ * maxUsableSpaceBytes_ / 100))
       && (ratio < safeRatio_)) {
-        VLOG(3) << "disk cache is safe"
+        VLOG(9) << "disk cache is safe"
                 << ", usedBytes is: " << usedBytes
                 << ", use ratio is: " << ratio;
         return true;
     }
-    VLOG(3) << "disk cache is not safe"
+    VLOG(6) << "disk cache is not safe"
                 << ", usedBytes is: " << usedBytes
                 << ", use ratio is: " << ratio;
     return false;
@@ -324,10 +331,16 @@ bool DiskCacheManager::IsDiskCacheSafe() {
 void DiskCacheManager::TrimCache() {
     const std::chrono::seconds sleepSec(trimCheckIntervalSec_);
     LOG(INFO) << "trim function start.";
+    waitIntervalSec_.Init(trimCheckIntervalSec_ * 1000);
     // 1. check cache disk usage every sleepSec seconds.
     // 2. if cache disk is full,
     //    then remove disk file until cache disk is lower than safeRatio_.
-    while (sleeper_.wait_for(sleepSec)) {
+    std::string cacheReadFullDir, cacheWriteFullDir,
+      cacheReadFile, cacheWriteFile, cacheKey;
+    cacheReadFullDir = GetCacheReadFullDir();
+    cacheWriteFullDir = GetCacheWriteFullDir();
+    while (true) {
+        waitIntervalSec_.WaitForNextExcution();
         if (!isRunning_) {
             LOG(INFO) << "trim thread end.";
             return;
@@ -335,29 +348,15 @@ void DiskCacheManager::TrimCache() {
         VLOG(9) << "trim thread wake up.";
         InitQosParam();
         SetDiskFsUsedRatio();
-        if (IsDiskCacheFull()) {
-            VLOG(3) << "disk cache full, begin trim.";
-            std::string cacheReadFullDir, cacheWriteFullDir,
-              cacheReadFile, cacheWriteFile;
-            cacheReadFullDir = GetCacheReadFullDir();
-            cacheWriteFullDir = GetCacheWriteFullDir();
-            std::string cacheKey, cacheKeyBfo, cacheKeyDel;
-            if (!cachedObjName_->GetBack(&cacheKey)) {
-                VLOG(3) << "remove disk file error"
-                        << ", cachedObjName is empty.";
-                continue;
-            }
             while (!IsDiskCacheSafe()) {
-                cacheKeyDel = cacheKey;
-                if (!cachedObjName_->GetBefore(cacheKey, &cacheKeyBfo)) {
-                    VLOG(3) << "obj is empty";
+                if (!cachedObjName_->GetLast(false, &cacheKey)) {
+                    VLOG(9) << "obj is empty";
                     break;
                 }
-                cacheKey = cacheKeyBfo;
 
-                VLOG(3) << "obj will be removed: " << cacheKeyDel;
-                cacheReadFile = cacheReadFullDir + "/" + cacheKeyDel;
-                cacheWriteFile = cacheWriteFullDir + "/" + cacheKeyDel;
+                VLOG(6) << "obj will be removed01: " << cacheKey;
+                cacheReadFile = cacheReadFullDir + "/" + cacheKey;
+                cacheWriteFile = cacheWriteFullDir + "/" + cacheKey;
                 struct stat statFile;
                 int ret;
                 ret = posixWrapper_->stat(cacheWriteFile.c_str(), &statFile);
@@ -367,17 +366,17 @@ void DiskCacheManager::TrimCache() {
                 // and then it cannot load the file from S3.
                 // so read is fail.
                 if (ret == 0) {
-                    VLOG(3) << "do not remove this disk file"
+                    VLOG(1) << "do not remove this disk file"
                             << ", file has not been uploaded to S3."
-                            << ", file is: " << cacheKeyDel;
+                            << ", file is: " << cacheKey;
                     continue;
                 }
-                cachedObjName_->Remove(cacheKeyDel);
+                cachedObjName_->Remove(cacheKey);
                 struct stat statReadFile;
                 ret = posixWrapper_->stat(cacheReadFile.c_str(), &statReadFile);
                 if (ret != 0) {
-                    VLOG(3) << "stat disk file error"
-                            << ", file is: " << cacheKeyDel;
+                    VLOG(0) << "stat disk file error"
+                            << ", file is: " << cacheKey;
                     continue;
                 }
                 // if remove disk file before delete cache,
@@ -387,14 +386,12 @@ void DiskCacheManager::TrimCache() {
                 ret = posixWrapper_->remove(toDelFile);
                 if (ret < 0) {
                     LOG(ERROR)
-                        << "remove disk file error, file is: " << cacheKeyDel;
+                        << "remove disk file error, file is: " << cacheKey;
                     continue;
                 }
                 DecDiskUsedBytes(statReadFile.st_size);
-                VLOG(3) << "remove disk file success, file is: " << cacheKeyDel;
+                VLOG(6) << "remove disk file success, file is: " << cacheKey;
             }
-            VLOG(3) << "trim over.";
-        }
     }
     LOG(INFO) << "trim function end.";
 }
@@ -413,7 +410,7 @@ int DiskCacheManager::TrimStop() {
     if (isRunning_.exchange(false)) {
         LOG(INFO) << "stop DiskCacheManager trim thread...";
         isRunning_ = false;
-        sleeper_.interrupt();
+        waitIntervalSec_.StopWait();
         backEndThread_.join();
         LOG(INFO) << "stop DiskCacheManager trim thread ok.";
         return -1;

curvefs/src/client/s3/disk_cache_manager.h

@@ -34,6 +34,7 @@
 #include "src/common/interruptible_sleeper.h"
 #include "src/common/lru_cache.h"
 #include "src/common/throttle.h"
+#include "src/common/wait_interval.h"
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/common/utils.h"
 #include "curvefs/src/client/s3/client_s3.h"
@@ -68,8 +69,11 @@ class DiskCacheManager {
     /**
      * @brief add obj to cachedObjName
      * @param[in] name obj name
+     * @param[in] cacheWriteExist whether the obj is
+     *                            exist in cache write
      */
-    void AddCache(const std::string name);
+    void AddCache(const std::string name,
+      bool cacheWriteExist = true);
 
     int CreateDir();
     std::string GetCacheReadFullDir();
@@ -111,8 +115,8 @@ class DiskCacheManager {
      */
     void AddDiskUsedBytes(uint64_t length) {
         usedBytes_.fetch_add(length, std::memory_order_seq_cst);
-        VLOG(9) << "add disk used size is: "
-                << usedBytes_.load(std::memory_order_seq_cst);
+        VLOG(9) << "add disk used size is: " << length
+                << ", now is: " << usedBytes_.load(std::memory_order_seq_cst);
         return;
     }
     /**
@@ -125,8 +129,8 @@ class DiskCacheManager {
         usedBytes = usedBytes_.fetch_sub(length, std::memory_order_seq_cst);
         assert(usedBytes >= 0);
         (void)usedBytes;
-        VLOG(9) << "dec disk used size is: "
-                << usedBytes_.load(std::memory_order_seq_cst);
+        VLOG(9) << "dec disk used size is: " << length
+                << ", now is: " << usedBytes_.load(std::memory_order_seq_cst);
         return;
     }
     void SetDiskInitUsedBytes();
@@ -143,6 +147,7 @@ class DiskCacheManager {
     curve::common::Thread backEndThread_;
     curve::common::Atomic<bool> isRunning_;
     curve::common::InterruptibleSleeper sleeper_;
+    curve::common::WaitInterval waitIntervalSec_;
     uint32_t trimCheckIntervalSec_;
     uint32_t fullRatio_;
     uint32_t safeRatio_;
@@ -155,7 +160,8 @@ class DiskCacheManager {
     std::string cacheDir_;
     std::shared_ptr<DiskCacheWrite> cacheWrite_;
     std::shared_ptr<DiskCacheRead> cacheRead_;
-    std::shared_ptr<SglLRUCache<std::string>> cachedObjName_;
+
+    std::shared_ptr<LRUCache<std::string, bool>> cachedObjName_;
 
     S3Client *client_;
     std::shared_ptr<PosixWrapper> posixWrapper_;

curvefs/src/client/s3/disk_cache_manager_impl.cpp

@@ -48,10 +48,32 @@ int DiskCacheManagerImpl::Init(const S3ClientAdaptorOption option) {
     }
 
     forceFlush_ = option.diskCacheOpt.forceFlush;
+    threads_ = option.diskCacheOpt.threads;
+    taskPool_.Start(threads_);
     LOG(INFO) << "DiskCacheManagerImpl init end.";
     return 0;
 }
 
+void DiskCacheManagerImpl::Enqueue(
+  std::shared_ptr<PutObjectAsyncContext> context) {
+    auto task = [this, context]() {
+        this->WriteClosure(context);
+    };
+    taskPool_.Enqueue(task);
+}
+
+int DiskCacheManagerImpl::WriteClosure(
+  std::shared_ptr<PutObjectAsyncContext> context) {
+    VLOG(9) << "WriteClosure start, name: " << context->key;
+    int ret = Write(context->key, context->buffer, context->bufferSize);
+     // set the returned value
+    // it is need in CallBack
+    context->retCode = ret;
+    context->cb(context);
+    VLOG(9) << "WriteClosure end, name: " << context->key;
+    return 0;
+}
+
 int DiskCacheManagerImpl::Write(const std::string name, const char *buf,
                                 uint64_t length) {
     VLOG(9) << "write name = " << name << ", length = " << length;
@@ -113,7 +135,7 @@ int DiskCacheManagerImpl::WriteReadDirect(const std::string fileName,
         return ret;
     }
     // add cache.
-    diskCacheManager_->AddCache(fileName);
+    diskCacheManager_->AddCache(fileName, false);
     return ret;
 }
 
@@ -154,6 +176,7 @@ int DiskCacheManagerImpl::UmountDiskCache() {
         LOG(ERROR) << "umount disk cache error.";
         return -1;
     }
+    taskPool_.Stop();
     return 0;
 }
 

curvefs/src/client/s3/disk_cache_manager_impl.h

@@ -43,6 +43,7 @@ namespace curvefs {
 namespace client {
 
 using curvefs::common::PosixWrapper;
+using curve::common::TaskThreadPool;
 
 struct DiskCacheOption {
     DiskCacheType diskCacheType;
@@ -54,6 +55,7 @@ struct DiskCacheOption {
     bool forceFlush;
     uint64_t maxUsableSpaceBytes;
     uint32_t cmdTimeoutSec;
+    uint32_t threads;
     uint64_t avgFlushBytes;
     uint64_t burstFlushBytes;
     uint64_t burstSecs;
@@ -114,12 +116,21 @@ class DiskCacheManagerImpl {
 
     virtual int ClearReadCache(const std::list<std::string> &files);
 
+    void Enqueue(std::shared_ptr<PutObjectAsyncContext> context);
+
  private:
     int WriteDiskFile(const std::string name, const char *buf, uint64_t length);
 
     std::shared_ptr<DiskCacheManager> diskCacheManager_;
+
     bool forceFlush_;
     S3Client *client_;
+
+    int WriteClosure(std::shared_ptr<PutObjectAsyncContext> context);
+    // threads for disk cache
+    uint32_t threads_;
+    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
+        taskPool_;
 };
 
 }  // namespace client

curvefs/src/client/s3/disk_cache_read.cpp

@@ -20,11 +20,11 @@
  * Author: hzwuhongsong
  */
 
-#include <sys/stat.h>
-#include <unistd.h>
 #include <errno.h>
 #include <fcntl.h>
+#include <sys/stat.h>
 #include <sys/types.h>
+#include <unistd.h>
 #include <dirent.h>
 #include <memory>
 #include <utility>
@@ -95,7 +95,8 @@ int DiskCacheRead::LinkWriteToRead(const std::string fileName,
         return -1;
     }
     ret = posixWrapper_->link(fullWritePath.c_str(), fullReadPath.c_str());
-    if (ret < 0) {
+    if (ret < 0 &&
+      errno != EEXIST ) {
         LOG(ERROR) << "link error. ret = " << ret << ", errno = " << errno
                    << ", write path = " << fullWritePath
                    << ", read path = " << fullReadPath;
@@ -106,7 +107,7 @@ int DiskCacheRead::LinkWriteToRead(const std::string fileName,
 }
 
 int DiskCacheRead::LoadAllCacheReadFile(
-    std::shared_ptr<SglLRUCache<std::string>> cachedObj) {
+    std::shared_ptr<LRUCache<std::string, bool>> cachedObj) {
     std::set<std::string> tmp;
     int ret = LoadAllCacheFile(&tmp);
     if (ret < 0) {
@@ -117,7 +118,7 @@ int DiskCacheRead::LoadAllCacheReadFile(
     }
 
     for (auto iter = tmp.begin(); iter != tmp.end(); iter++) {
-        cachedObj->Put(std::move(*iter));
+        cachedObj->Put(std::move(*iter), false);
     }
 
     return ret;

curvefs/src/client/s3/disk_cache_read.h

@@ -37,7 +37,7 @@
 namespace curvefs {
 namespace client {
 
-using curve::common::SglLRUCache;
+using curve::common::LRUCache;
 using curvefs::common::PosixWrapper;
 
 class DiskCacheRead : public DiskCacheBase {
@@ -58,7 +58,8 @@ class DiskCacheRead : public DiskCacheBase {
      * @brief after reboot，load all files that store in read cache.
      */
     virtual int
-    LoadAllCacheReadFile(std::shared_ptr<SglLRUCache<std::string>> cachedObj);
+    LoadAllCacheReadFile(std::shared_ptr<LRUCache<
+      std::string, bool>> cachedObj);
     virtual int ClearReadCache(const std::list<std::string> &files);
     virtual void InitMetrics(std::shared_ptr<DiskCacheMetric> metric) {
         metric_ = metric;

curvefs/src/client/s3/disk_cache_write.cpp

@@ -38,10 +38,13 @@ namespace client {
 void DiskCacheWrite::Init(S3Client *client,
                           std::shared_ptr<PosixWrapper> posixWrapper,
                           const std::string cacheDir,
-                          uint64_t asyncLoadPeriodMs) {
+                          uint64_t asyncLoadPeriodMs,
+                          std::shared_ptr<LRUCache<
+                            std::string, bool>> cachedObjName) {
     client_ = client;
     posixWrapper_ = posixWrapper;
     asyncLoadPeriodMs_ = asyncLoadPeriodMs;
+    cachedObjName_ = cachedObjName;
     DiskCacheBase::Init(posixWrapper, cacheDir);
 }
 
@@ -143,7 +146,7 @@ int DiskCacheWrite::UploadFile(const std::string &name,
                         << (butil::cpuwide_time_us() - context->startTime);
                 }
                 RemoveFile(context->key);
-                VLOG(9) << "PutObjectAsyncCallBack success, "
+                VLOG(9) << " PutObjectAsyncCallBack success, "
                         << "remove file: " << context->key;
                 posixWrapper_->free(buffer);
                 if (syncTask) {
@@ -187,12 +190,10 @@ int DiskCacheWrite::GetUploadFile(const std::string &inode,
     if (waitUpload_.empty()) {
         return 0;
     }
-
     if (inode.empty()) {
         toUpload->swap(waitUpload_);
         return toUpload->size();
     }
-
     waitUpload_.remove_if([&](const std::string &filename) {
         bool inodeFile =
             curvefs::common::s3util::ValidNameOfInode(inode, filename);
@@ -274,7 +275,7 @@ int DiskCacheWrite::AsyncUploadFunc() {
 
     std::list<std::string> toUpload;
 
-    VLOG(6) << "async upload function start.";
+    VLOG(3) << "async upload function start.";
     while (sleeper_.wait_for(std::chrono::milliseconds(asyncLoadPeriodMs_))) {
         if (!isRunning_) {
             LOG(INFO) << "async upload thread stop.";
@@ -285,8 +286,9 @@ int DiskCacheWrite::AsyncUploadFunc() {
             VLOG(9) << "no need to upload";
             continue;
         }
-        VLOG(3) << "async upload file size = " << toUpload.size();
+        VLOG(6) << "async upload file size = " << toUpload.size();
         UploadFile(toUpload, nullptr);
+        VLOG(6) << "async upload all files";
     }
     return 0;
 }
@@ -366,7 +368,7 @@ int DiskCacheWrite::UploadAllCacheWriteFile() {
         PutObjectAsyncCallBack cb =
         [&, buffer](const std::shared_ptr<PutObjectAsyncContext> &context) {
             if (context->retCode == 0) {
-                if (pendingReq.fetch_sub(1) == 1) {
+                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
                     VLOG(3) << "pendingReq is over";
                     cond.Signal();
                 }
@@ -391,6 +393,7 @@ int DiskCacheWrite::UploadAllCacheWriteFile() {
     }
     for (auto iter = uploadObjs.begin(); iter != uploadObjs.end(); iter++) {
         RemoveFile(*iter);
+        cachedObjName_->Put(*iter, false);
     }
     VLOG(3) << "upload all cached write file end.";
     return 0;
@@ -407,6 +410,7 @@ int DiskCacheWrite::RemoveFile(const std::string fileName) {
                    << ", errno = " << errno;
         return -1;
     }
+    cachedObjName_->Put(fileName, false);
     VLOG(9) << "remove file success, file = " << fileName;
     return 0;
 }
@@ -426,11 +430,13 @@ int DiskCacheWrite::WriteDiskFile(const std::string fileName, const char *buf,
     }
     ssize_t writeLen = posixWrapper_->write(fd, buf, length);
     if (writeLen < 0 || writeLen < length) {
-        LOG(ERROR) << "write disk file error. ret = " << writeLen
-                   << ", file = " << fileName;
+        LOG(ERROR) << "write disk file error. ret: " << writeLen
+                   << ", file: " << fileName
+                   << ", error: " << errno;
         posixWrapper_->close(fd);
         return -1;
     }
+
     // force to flush
     if (force) {
         ret = posixWrapper_->fdatasync(fd);
@@ -441,6 +447,7 @@ int DiskCacheWrite::WriteDiskFile(const std::string fileName, const char *buf,
             return -1;
         }
     }
+
     ret = posixWrapper_->close(fd);
     if (ret < 0) {
         LOG(ERROR) << "close disk file error. errno = " << errno

curvefs/src/client/s3/disk_cache_write.h

@@ -33,15 +33,23 @@
 
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
+#include "src/common/lru_cache.h"
+#include "src/common/throttle.h"
+#include "src/common/wait_interval.h"
 #include "curvefs/src/common/wrap_posix.h"
-#include "curvefs/src/client/s3/disk_cache_base.h"
+#include "curvefs/src/common/utils.h"
 #include "curvefs/src/client/s3/client_s3.h"
+#include "curvefs/src/client/s3/disk_cache_write.h"
+#include "curvefs/src/client/s3/disk_cache_read.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/s3/disk_cache_base.h"
 
 namespace curvefs {
 namespace client {
 
 using curvefs::common::PosixWrapper;
 using curve::common::InterruptibleSleeper;
+using ::curve::common::LRUCache;
 using curve::common::PutObjectAsyncCallBack;
 
 class DiskCacheWrite : public DiskCacheBase {
@@ -74,7 +82,8 @@ class DiskCacheWrite : public DiskCacheBase {
        AsyncUploadStop();
     }
     void Init(S3Client *client, std::shared_ptr<PosixWrapper> posixWrapper,
-              const std::string cacheDir, uint64_t asyncLoadPeriodMs);
+              const std::string cacheDir, uint64_t asyncLoadPeriodMs,
+              std::shared_ptr<LRUCache<std::string, bool>> cachedObjName);
     /**
      * @brief write obj to write cahce disk
      * @param[in] client S3Client
@@ -143,6 +152,8 @@ class DiskCacheWrite : public DiskCacheBase {
     // file system operation encapsulation
     std::shared_ptr<PosixWrapper> posixWrapper_;
     std::shared_ptr<DiskCacheMetric> metric_;
+
+    std::shared_ptr<LRUCache<std::string, bool>> cachedObjName_;
 };
 
 }  // namespace client

curvefs/src/mds/fs_manager.h

@@ -65,7 +65,7 @@ using ::curvefs::mds::topology::TopologyManager;
 
 struct FsManagerOption {
     uint32_t backEndThreadRunInterSec;
-    uint32_t spaceReloadConcurrency;
+    uint32_t spaceReloadConcurrency = 10;
     curve::common::S3AdapterOption s3AdapterOption;
 };
 

curvefs/src/mds/mds.cpp

@@ -133,6 +133,14 @@ void MDS::InitScheduleOption(ScheduleOption* scheduleOption) {
 void MDS::InitFsManagerOptions(FsManagerOption* fsManagerOption) {
     conf_->GetValueFatalIfFail("mds.fsmanager.backEndThreadRunInterSec",
                                &fsManagerOption->backEndThreadRunInterSec);
+
+    LOG_IF(ERROR,
+           conf_->GetUInt32Value("mds.fsmanager.reloadSpaceConcurrency",
+                                 &fsManagerOption->spaceReloadConcurrency))
+        << "Get `mds.fsmanager.reloadSpaceConcurrency` from conf error, use "
+           "default value: "
+        << fsManagerOption->spaceReloadConcurrency;
+
     ::curve::common::InitS3AdaptorOptionExceptS3InfoOption(
         conf_.get(), &fsManagerOption->s3AdapterOption);
 }

curvefs/src/mds/topology/topology_manager.cpp

@@ -632,6 +632,9 @@ void TopologyManager::CreatePartitions(const CreatePartitionRequest *request,
     auto partitionInfoList = response->mutable_partitioninfolist();
     response->set_statuscode(TopoStatusCode::TOPO_OK);
 
+    // get lock and avoid multiMountpoint create concurrently
+    NameLockGuard lock(createPartitionMutex_, std::to_string(fsId));
+
     while (partitionInfoList->size() < count) {
         if (topology_->GetAvailableCopysetNum()
                             < option_.minAvailableCopysetNum) {
@@ -693,7 +696,8 @@ void TopologyManager::CreatePartitions(const CreatePartitionRequest *request,
                 info->set_partitionid(partitionId);
                 info->set_start(idStart);
                 info->set_end(idEnd);
-                info->set_status(PartitionStatus::READONLY);
+                info->set_txid(0);
+                info->set_status(PartitionStatus::READWRITE);
             } else {
                 // TODO(wanghai): delete partition on metaserver
                 LOG(ERROR) << "Add partition failed after create partition."
@@ -853,7 +857,7 @@ TopoStatusCode TopologyManager::CreateCopyset(
         return TopoStatusCode::TOPO_CREATE_COPYSET_ON_METASERVER_FAIL;
     }
 
-    // add copyset record to topogy
+    // add copyset record to topology
     CopySetInfo copysetInfo(copyset.poolId, copyset.copysetId);
     copysetInfo.SetCopySetMembers(copyset.metaServerIds);
     auto ret = topology_->AddCopySet(copysetInfo);
@@ -866,6 +870,7 @@ TopoStatusCode TopologyManager::CreateCopyset(
         return ret;
     }
 
+    ClearCopysetCreating(copyset.poolId, copyset.copysetId);
     return TopoStatusCode::TOPO_OK;
 }
 

curvefs/src/mds/topology/topology_manager.h

@@ -184,6 +184,8 @@ class TopologyManager {
      */
     NameLock registMsMutex;
 
+    NameLock createPartitionMutex_;
+
     /**
      * @brief topology options
      */

curvefs/src/metaserver/copyset/meta_operator.cpp

@@ -176,7 +176,6 @@ void GetOrModifyS3ChunkInfoOperator::OnApply(int64_t index,
     MetaStatusCode rc;
     auto request = static_cast<const GetOrModifyS3ChunkInfoRequest*>(request_);
     auto response = static_cast<GetOrModifyS3ChunkInfoResponse*>(response_);
-    bool streaming = request->returns3chunkinfomap();
     auto metastore = node_->GetMetaStore();
     std::shared_ptr<StreamConnection> connection;
     std::shared_ptr<Iterator> iterator;
@@ -199,7 +198,9 @@ void GetOrModifyS3ChunkInfoOperator::OnApply(int64_t index,
         }
 
         brpc::Controller* cntl = static_cast<brpc::Controller*>(cntl_);
-        if (rc != MetaStatusCode::OK || !streaming) {
+        if (rc != MetaStatusCode::OK ||
+            !request->returns3chunkinfomap() ||
+            !request->supportstreaming()) {
             return;
         }
 

curvefs/src/metaserver/copyset/raft_log_codec.cpp

@@ -124,6 +124,12 @@ std::unique_ptr<MetaOperator> RaftLogCodec::Decode(CopysetNode* node,
         case OperatorType::GetInode:
             return ParseFromRaftLog<GetInodeOperator, GetInodeRequest>(
                 node, type, meta);
+        case OperatorType::BatchGetInodeAttr:
+            return ParseFromRaftLog<BatchGetInodeAttrOperator,
+                BatchGetInodeAttrRequest>(node, type, meta);
+        case OperatorType::BatchGetXAttr:
+            return ParseFromRaftLog<BatchGetXAttrOperator,
+                BatchGetXAttrRequest>(node, type, meta);
         case OperatorType::CreateInode:
             return ParseFromRaftLog<CreateInodeOperator, CreateInodeRequest>(
                 node, type, meta);

curvefs/src/metaserver/dentry_manager.cpp

@@ -42,8 +42,9 @@ void DentryManager::Log4Dentry(const std::string& request,
 
 void DentryManager::Log4Code(const std::string& request, MetaStatusCode rc) {
     auto succ = (rc == MetaStatusCode::OK ||
-                (rc == MetaStatusCode::NOT_FOUND &&
-                (request == "ListDentry" || request == "GetDentry")));
+                 rc == MetaStatusCode::IDEMPOTENCE_OK ||
+                 (rc == MetaStatusCode::NOT_FOUND &&
+                  (request == "ListDentry" || request == "GetDentry")));
     std::ostringstream message;
     message << request << " " << (succ ? "success" : "fail")
             << ", retCode = " << MetaStatusCode_Name(rc);
@@ -55,7 +56,8 @@ void DentryManager::Log4Code(const std::string& request, MetaStatusCode rc) {
     }
 }
 
-MetaStatusCode DentryManager::CreateDentry(const Dentry& dentry) {
+MetaStatusCode DentryManager::CreateDentry(const Dentry& dentry,
+                                           bool isLoadding) {
     Log4Dentry("CreateDentry", dentry);
     MetaStatusCode rc;
     // invoke only from snapshot loading
@@ -63,7 +65,7 @@ MetaStatusCode DentryManager::CreateDentry(const Dentry& dentry) {
         rc = dentryStorage_->HandleTx(DentryStorage::TX_OP_TYPE::PREPARE,
                                       dentry);
     } else {
-        rc = dentryStorage_->Insert(dentry);
+        rc = dentryStorage_->Insert(dentry, isLoadding);
     }
     Log4Code("CreateDentry", rc);
     return rc;

curvefs/src/metaserver/dentry_manager.h

@@ -40,7 +40,7 @@ class DentryManager {
     DentryManager(std::shared_ptr<DentryStorage> dentryStorage,
                   std::shared_ptr<TxManager> txManger);
 
-    MetaStatusCode CreateDentry(const Dentry& dentry);
+    MetaStatusCode CreateDentry(const Dentry& dentry, bool isLoadding = false);
 
     MetaStatusCode DeleteDentry(const Dentry& dentry);
 

curvefs/src/metaserver/dentry_storage.cpp

@@ -152,18 +152,20 @@ MetaStatusCode DentryStorage::Find(const Dentry& in,
     return rc;
 }
 
-MetaStatusCode DentryStorage::Insert(const Dentry& dentry) {
+MetaStatusCode DentryStorage::Insert(const Dentry& dentry, bool isLoadding) {
     WriteLockGuard w(rwLock_);
 
-    Dentry out;
-    MetaStatusCode rc = Find(dentry, &out, true);
-    if (rc == MetaStatusCode::OK) {
-        if (IsSameDentry(out, dentry)) {
-            return MetaStatusCode::IDEMPOTENCE_OK;
+    if (!isLoadding) {
+        Dentry out;
+        MetaStatusCode rc = Find(dentry, &out, true);
+        if (rc == MetaStatusCode::OK) {
+            if (IsSameDentry(out, dentry)) {
+                return MetaStatusCode::IDEMPOTENCE_OK;
+            }
+            return MetaStatusCode::DENTRY_EXIST;
+        } else if (rc != MetaStatusCode::NOT_FOUND) {
+            return MetaStatusCode::STORAGE_INTERNAL_ERROR;
         }
-        return MetaStatusCode::DENTRY_EXIST;
-    } else if (rc != MetaStatusCode::NOT_FOUND) {
-        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
     }
 
     // MetaStatusCode::NOT_FOUND

curvefs/src/metaserver/dentry_storage.h

@@ -65,7 +65,7 @@ class DentryStorage {
     DentryStorage(std::shared_ptr<KVStorage> kvStorage,
                   const std::string& tablename);
 
-    MetaStatusCode Insert(const Dentry& dentry);
+    MetaStatusCode Insert(const Dentry& dentry, bool isLoadding = false);
 
     MetaStatusCode Delete(const Dentry& dentry);
 

curvefs/src/metaserver/inode_manager.cpp

@@ -25,6 +25,7 @@
 #include <glog/logging.h>
 #include <google/protobuf/util/message_differencer.h>
 #include <list>
+#include <unordered_set>
 
 #include "curvefs/src/common/define.h"
 #include "src/common/timeutility.h"
@@ -130,21 +131,27 @@ void InodeManager::GenerateInodeInternal(uint64_t inodeId,
     return;
 }
 
-MetaStatusCode InodeManager::GetInode(uint32_t fsId, uint64_t inodeId,
-                                      Inode *inode) {
+MetaStatusCode InodeManager::GetInode(uint32_t fsId,
+                                      uint64_t inodeId,
+                                      Inode *inode,
+                                      bool paddingS3ChunkInfo) {
     VLOG(1) << "GetInode, fsId = " << fsId << ", inodeId = " << inodeId;
     NameLockGuard lg(inodeLock_, GetInodeLockName(fsId, inodeId));
-    MetaStatusCode ret = inodeStorage_->Get(Key4Inode(fsId, inodeId), inode);
-    if (ret != MetaStatusCode::OK) {
+    MetaStatusCode rc = inodeStorage_->Get(Key4Inode(fsId, inodeId), inode);
+    if (rc == MetaStatusCode::OK && paddingS3ChunkInfo) {
+        rc = PaddingInodeS3ChunkInfo(fsId, inodeId,
+                                     inode->mutable_s3chunkinfomap());
+    }
+
+    if (rc != MetaStatusCode::OK) {
         LOG(ERROR) << "GetInode fail, fsId = " << fsId
                    << ", inodeId = " << inodeId
-                   << ", ret = " << MetaStatusCode_Name(ret);
-        return ret;
+                   << ", retCode = " << MetaStatusCode_Name(rc);
+        return rc;
     }
 
     VLOG(1) << "GetInode success, fsId = " << fsId << ", inodeId = " << inodeId
             << ", " << inode->ShortDebugString();
-
     return MetaStatusCode::OK;
 }
 
@@ -308,30 +315,59 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest &request) {
 MetaStatusCode InodeManager::GetOrModifyS3ChunkInfo(
     uint32_t fsId, uint64_t inodeId,
     const S3ChunkInfoMap& map2add,
-    std::shared_ptr<Iterator>* iterator,
+    const S3ChunkInfoMap& map2del,
     bool returnS3ChunkInfoMap,
-    bool compaction) {
+    std::shared_ptr<Iterator>* iterator4InodeS3Meta) {
     VLOG(1) << "GetOrModifyS3ChunkInfo, fsId: " << fsId
             << ", inodeId: " << inodeId;
 
     NameLockGuard lg(inodeLock_, GetInodeLockName(fsId, inodeId));
 
-    if (!map2add.empty()) {
-        for (const auto& item : map2add) {
-            uint64_t chunkIndex = item.first;
-            auto list2add = item.second;
-            MetaStatusCode rc = inodeStorage_->AppendS3ChunkInfoList(
-                fsId, inodeId, chunkIndex, list2add, compaction);
-            if (rc != MetaStatusCode::OK) {
-                return rc;
-            }
+    const S3ChunkInfoList* list2add;
+    const S3ChunkInfoList* list2del;
+    std::unordered_set<uint64_t> deleted;
+    for (const auto& item : map2add) {
+        uint64_t chunkIndex = item.first;
+        list2add = &item.second;
+        auto iter = map2del.find(chunkIndex);
+        if (iter != map2del.end()) {
+            list2del = &iter->second;
+        } else {
+            list2del = nullptr;
+        }
+
+        MetaStatusCode rc = inodeStorage_->ModifyInodeS3ChunkInfoList(
+            fsId, inodeId, chunkIndex, list2add, list2del);
+        if (rc != MetaStatusCode::OK) {
+            LOG(ERROR) << "Modify inode s3chunkinfo list failed, fsId=" << fsId
+                       << ", inodeId=" << inodeId << ", retCode=" << rc;
+            return rc;
+        }
+        deleted.insert(chunkIndex);
+    }
+
+    for (const auto& item : map2del) {
+        uint64_t chunkIndex = item.first;
+        if (deleted.find(chunkIndex) != deleted.end()) {  // already deleted
+            continue;
+        }
+
+        list2add = nullptr;
+        list2del = &item.second;
+        MetaStatusCode rc = inodeStorage_->ModifyInodeS3ChunkInfoList(
+            fsId, inodeId, chunkIndex, list2add, list2del);
+        if (rc != MetaStatusCode::OK) {
+            LOG(ERROR) << "Modify inode s3chunkinfo list failed, fsId=" << fsId
+                       << ", inodeId=" << inodeId << ", retCode=" << rc;
+            return rc;
         }
     }
 
     // return if needed
     if (returnS3ChunkInfoMap) {
-        *iterator = inodeStorage_->GetInodeS3ChunkInfoList(fsId, inodeId);
-        if ((*iterator)->Status() != 0) {
+        *iterator4InodeS3Meta = inodeStorage_->GetInodeS3ChunkInfoList(
+            fsId, inodeId);
+        if ((*iterator4InodeS3Meta)->Status() != 0) {
             return MetaStatusCode::STORAGE_INTERNAL_ERROR;
         }
     }
@@ -343,10 +379,11 @@ MetaStatusCode InodeManager::GetOrModifyS3ChunkInfo(
 
 MetaStatusCode InodeManager::PaddingInodeS3ChunkInfo(int32_t fsId,
                                                      uint64_t inodeId,
-                                                     Inode* inode) {
+                                                     S3ChunkInfoMap* m,
+                                                     uint64_t limit) {
     VLOG(1) << "PaddingInodeS3ChunkInfo, fsId: " << fsId
             << ", inodeId: " << inodeId;
-    return inodeStorage_->PaddingInodeS3ChunkInfo(fsId, inodeId, inode);
+    return inodeStorage_->PaddingInodeS3ChunkInfo(fsId, inodeId, m, limit);
 }
 
 MetaStatusCode InodeManager::UpdateInodeWhenCreateOrRemoveSubNode(

curvefs/src/metaserver/inode_manager.h

@@ -34,7 +34,6 @@
 
 using ::curve::common::NameLock;
 using ::curvefs::metaserver::S3ChunkInfoList;
-using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 
 namespace curvefs {
 namespace metaserver {
@@ -61,7 +60,11 @@ class InodeManager {
     MetaStatusCode CreateInode(uint64_t inodeId, const InodeParam &param,
                                Inode *inode);
     MetaStatusCode CreateRootInode(const InodeParam &param);
-    MetaStatusCode GetInode(uint32_t fsId, uint64_t inodeId, Inode *inode);
+
+    MetaStatusCode GetInode(uint32_t fsId,
+                            uint64_t inodeId,
+                            Inode *inode,
+                            bool paddingS3ChunkInfo = false);
 
     MetaStatusCode GetInodeAttr(uint32_t fsId, uint64_t inodeId,
                                 InodeAttr *attr);
@@ -72,16 +75,18 @@ class InodeManager {
 
     MetaStatusCode UpdateInode(const UpdateInodeRequest &request);
 
-    MetaStatusCode GetOrModifyS3ChunkInfo(uint32_t fsId,
-                                          uint64_t inodeId,
-                                          const S3ChunkInfoMap& map2add,
-                                          std::shared_ptr<Iterator>* iterator,
-                                          bool returnS3ChunkInfoMap,
-                                          bool compaction);
+    MetaStatusCode GetOrModifyS3ChunkInfo(
+        uint32_t fsId,
+        uint64_t inodeId,
+        const S3ChunkInfoMap& map2add,
+        const S3ChunkInfoMap& map2del,
+        bool returnS3ChunkInfoMap,
+        std::shared_ptr<Iterator>* iterator4InodeS3Meta);
 
     MetaStatusCode PaddingInodeS3ChunkInfo(int32_t fsId,
                                            uint64_t inodeId,
-                                           Inode* inode);
+                                           S3ChunkInfoMap* m,
+                                           uint64_t limit = 0);
 
     MetaStatusCode UpdateInodeWhenCreateOrRemoveSubNode(uint32_t fsId,
         uint64_t inodeId, bool isCreate);

curvefs/src/metaserver/inode_storage.cpp

@@ -41,6 +41,8 @@ using ::curvefs::metaserver::storage::Prefix4InodeS3ChunkInfoList;
 using ::curvefs::metaserver::storage::Prefix4AllInode;
 using Transaction = std::shared_ptr<StorageTransaction>;
 
+using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+
 InodeStorage::InodeStorage(std::shared_ptr<KVStorage> kvStorage,
                            const std::string& tablename)
     : kvStorage_(kvStorage),
@@ -178,35 +180,46 @@ MetaStatusCode InodeStorage::AddS3ChunkInfoList(
     uint32_t fsId,
     uint64_t inodeId,
     uint64_t chunkIndex,
-    const S3ChunkInfoList& list2add) {
-    // key
-    size_t size = list2add.s3chunks_size();
-    uint64_t firstChunkId = list2add.s3chunks(0).chunkid();
-    uint64_t lastChunkId = list2add.s3chunks(size - 1).chunkid();
+    const S3ChunkInfoList* list2add) {
+    if (nullptr == list2add || list2add->s3chunks_size() == 0) {
+        return MetaStatusCode::OK;
+    }
+
+    size_t size = list2add->s3chunks_size();
+    uint64_t firstChunkId = list2add->s3chunks(0).chunkid();
+    uint64_t lastChunkId = list2add->s3chunks(size - 1).chunkid();
     Key4S3ChunkInfoList key(fsId, inodeId, chunkIndex,
-                            firstChunkId, lastChunkId);
+                            firstChunkId, lastChunkId, size);
     std::string skey = conv_->SerializeToString(key);
 
-    Status s = txn->SSet(table4s3chunkinfo_, skey, list2add);
+    Status s = txn->SSet(table4s3chunkinfo_, skey, *list2add);
     return s.ok() ? MetaStatusCode::OK :
                     MetaStatusCode::STORAGE_INTERNAL_ERROR;
 }
 
-// NOTE: s3chunkinfo which its chunkid equal or
-// less then min chunkid should be removed
-MetaStatusCode InodeStorage::RemoveS3ChunkInfoList(Transaction txn,
-                                                   uint32_t fsId,
-                                                   uint64_t inodeId,
-                                                   uint64_t chunkIndex,
-                                                   uint64_t minChunkId) {
+MetaStatusCode InodeStorage::DelS3ChunkInfoList(
+    Transaction txn,
+    uint32_t fsId,
+    uint64_t inodeId,
+    uint64_t chunkIndex,
+    const S3ChunkInfoList* list2del) {
+    if (nullptr == list2del || list2del->s3chunks_size() == 0) {
+        return MetaStatusCode::OK;
+    }
+
+    size_t size = list2del->s3chunks_size();
+    uint64_t delFirstChunkId = list2del->s3chunks(0).chunkid();
+    uint64_t delLastChunkId = list2del->s3chunks(size - 1).chunkid();
+
+    // prefix
     Prefix4ChunkIndexS3ChunkInfoList prefix(fsId, inodeId, chunkIndex);
     std::string sprefix = conv_->SerializeToString(prefix);
     auto iterator = txn->SSeek(table4s3chunkinfo_, sprefix);
     if (iterator->Status() != 0) {
+        LOG(ERROR) << "Get iterator failed, prefix=" << sprefix;
         return MetaStatusCode::STORAGE_INTERNAL_ERROR;
     }
 
-    uint64_t lastChunkId;
     Key4S3ChunkInfoList key;
     std::vector<std::string> key2del;
     for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
@@ -215,58 +228,83 @@ MetaStatusCode InodeStorage::RemoveS3ChunkInfoList(Transaction txn,
             break;
         } else if (!conv_->ParseFromString(skey, &key)) {
             return MetaStatusCode::PARSE_FROM_STRING_FAILED;
-        } else if (key.firstChunkId >= minChunkId) {
-            break;
         }
 
-        // firstChunkId < minChunkId
-        key2del.push_back(skey);
+        // current list range:    [  ]
+        // delete list range :  [      ]
+        if (delFirstChunkId <= key.firstChunkId &&
+            delLastChunkId >= key.lastChunkId) {
+            key2del.push_back(skey);
+        // current list range:       [  ]
+        // delete list range :  [  ]
+        } else if (delLastChunkId < key.firstChunkId) {
+            continue;
+        } else {
+            LOG(ERROR) << "wrong delete list range (" << delFirstChunkId
+                       << "," << delLastChunkId << "), skey=" << skey;
+            return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        }
     }
 
     for (const auto& skey : key2del) {
         if (!txn->SDel(table4s3chunkinfo_, skey).ok()) {
+            LOG(ERROR) << "Delete key failed, skey=" << skey;
             return MetaStatusCode::STORAGE_INTERNAL_ERROR;
         }
     }
     return MetaStatusCode::OK;
 }
 
-MetaStatusCode InodeStorage::AppendS3ChunkInfoList(
+MetaStatusCode InodeStorage::ModifyInodeS3ChunkInfoList(
     uint32_t fsId,
     uint64_t inodeId,
     uint64_t chunkIndex,
-    const S3ChunkInfoList& list2add,
-    bool compaction) {
+    const S3ChunkInfoList* list2add,
+    const S3ChunkInfoList* list2del) {
     WriteLockGuard writeLockGuard(rwLock_);
-    size_t size = list2add.s3chunks_size();
-    if (size == 0) {
-        return MetaStatusCode::OK;
-    }
-
     auto txn = kvStorage_->BeginTransaction();
     if (nullptr == txn) {
         return MetaStatusCode::STORAGE_INTERNAL_ERROR;
     }
 
-    MetaStatusCode rc;
-    rc = AddS3ChunkInfoList(txn, fsId, inodeId, chunkIndex, list2add);
-    if (rc == MetaStatusCode::OK && compaction) {
-        uint64_t minChunkId = list2add.s3chunks(0).chunkid();
-        rc = RemoveS3ChunkInfoList(txn, fsId, inodeId, chunkIndex, minChunkId);
+    auto rc = DelS3ChunkInfoList(txn, fsId, inodeId, chunkIndex, list2del);
+    if (rc == MetaStatusCode::OK) {
+        rc = AddS3ChunkInfoList(txn, fsId, inodeId, chunkIndex, list2add);
     }
 
     if (rc != MetaStatusCode::OK) {
-        txn->Rollback();
+        if (!txn->Rollback().ok()) {
+            LOG(ERROR) << "Rollback transaction failed";
+            rc = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        }
     } else if (!txn->Commit().ok()) {
+        LOG(ERROR) << "Commit transaction failed";
         rc = MetaStatusCode::STORAGE_INTERNAL_ERROR;
     }
+
+    if (rc != MetaStatusCode::OK) {
+        return rc;
+    }
+
+    // rc == MetaStatusCode::OK
+    uint64_t size4add = (nullptr == list2add) ? 0 : list2add->s3chunks_size();
+    uint64_t size4del = (nullptr == list2del) ? 0 : list2del->s3chunks_size();
+    if (!UpdateInodeS3MetaSize(fsId, inodeId, size4add, size4del)) {
+        LOG(ERROR) << "Update inode s3meta size failed";
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
     return rc;
 }
 
 MetaStatusCode InodeStorage::PaddingInodeS3ChunkInfo(int32_t fsId,
                                                      uint64_t inodeId,
-                                                     Inode* inode) {
+                                                     S3ChunkInfoMap* m,
+                                                     uint64_t limit) {
     ReadLockGuard readLockGuard(rwLock_);
+    if (limit != 0 && GetInodeS3MetaSize(fsId, inodeId) > limit) {
+        return MetaStatusCode::INODE_S3_META_TOO_LARGE;
+    }
+
     auto iterator = GetInodeS3ChunkInfoList(fsId, inodeId);
     if (iterator->Status() != 0) {
         LOG(ERROR) << "Get inode s3chunkinfo failed";
@@ -282,7 +320,6 @@ MetaStatusCode InodeStorage::PaddingInodeS3ChunkInfo(int32_t fsId,
 
     Key4S3ChunkInfoList key;
     S3ChunkInfoList list;
-    auto m = inode->mutable_s3chunkinfomap();
     for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
         std::string skey = iterator->Key();
         std::string svalue = iterator->Value();

curvefs/src/metaserver/inode_storage.h

@@ -24,14 +24,13 @@
 #define CURVEFS_SRC_METASERVER_INODE_STORAGE_H_
 
 #include <functional>
+#include <utility>
 #include <unordered_map>
 #include <unordered_set>
-#include <utility>
 #include <list>
 #include <string>
 #include <memory>
 
-#include "absl/container/btree_set.h"
 #include "src/common/concurrent/rw_lock.h"
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/metaserver/storage/converter.h"
@@ -45,12 +44,14 @@ using ::curvefs::metaserver::storage::Status;
 using ::curvefs::metaserver::storage::Iterator;
 using ::curvefs::metaserver::storage::KVStorage;
 using ::curvefs::metaserver::storage::StorageTransaction;
+using ::curvefs::metaserver::storage::Hash;
 
 namespace curvefs {
 namespace metaserver {
 
 using ::curvefs::metaserver::storage::Key4Inode;
 using ::curvefs::metaserver::storage::Converter;
+using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 
 enum TABLE_TYPE : unsigned char {
     kTypeInode = 1,
@@ -107,15 +108,16 @@ class InodeStorage {
      */
     MetaStatusCode Update(const Inode& inode);
 
-    MetaStatusCode AppendS3ChunkInfoList(uint32_t fsId,
-                                         uint64_t inodeId,
-                                         uint64_t chunkIndex,
-                                         const S3ChunkInfoList& list2add,
-                                         bool compaction);
+    MetaStatusCode ModifyInodeS3ChunkInfoList(uint32_t fsId,
+                                              uint64_t inodeId,
+                                              uint64_t chunkIndex,
+                                              const S3ChunkInfoList* list2add,
+                                              const S3ChunkInfoList* list2del);
 
     MetaStatusCode PaddingInodeS3ChunkInfo(int32_t fsId,
                                            uint64_t inodeId,
-                                           Inode* inode);
+                                           S3ChunkInfoMap* m,
+                                           uint64_t limit = 0);
 
     std::shared_ptr<Iterator> GetInodeS3ChunkInfoList(uint32_t fsId,
                                                       uint64_t inodeId);
@@ -136,14 +138,14 @@ class InodeStorage {
         uint32_t fsId,
         uint64_t inodeId,
         uint64_t chunkIndex,
-        const S3ChunkInfoList& list2add);
+        const S3ChunkInfoList* list2add);
 
-    MetaStatusCode RemoveS3ChunkInfoList(
+    MetaStatusCode DelS3ChunkInfoList(
         std::shared_ptr<StorageTransaction> txn,
         uint32_t fsId,
         uint64_t inodeId,
         uint64_t chunkIndex,
-        uint64_t minChunkId);
+        const S3ChunkInfoList* list2del);
 
     std::string RealTablename(TABLE_TYPE type, std::string tablename) {
         std::ostringstream oss;
@@ -151,6 +153,28 @@ class InodeStorage {
         return oss.str();
     }
 
+    static std::string InodeS3MetaSizeKey(uint32_t fsId, uint64_t inodeId) {
+        std::ostringstream oss;
+        oss << fsId << ":" << inodeId;
+        return oss.str();
+    }
+
+    bool UpdateInodeS3MetaSize(uint32_t fsId, uint64_t inodeId,
+                               uint64_t size4add, uint64_t size4del) {
+        std::string key = InodeS3MetaSizeKey(fsId, inodeId);
+        uint64_t size = inodeS3MetaSize_[key] + size4add;
+        if (size < size4del) {
+            return false;
+        }
+        inodeS3MetaSize_[key] = size - size4del;
+        return true;
+    }
+
+    uint64_t GetInodeS3MetaSize(uint32_t fsId, uint64_t inodeId) {
+        std::string key = InodeS3MetaSizeKey(fsId, inodeId);
+        return inodeS3MetaSize_[key];
+    }
+
     bool FindKey(const std::string& key) {
         return keySet_.find(key) != keySet_.end();
     }
@@ -170,6 +194,8 @@ class InodeStorage {
     std::string table4s3chunkinfo_;
     std::shared_ptr<Converter> conv_;
     std::unordered_set<std::string> keySet_;
+    // key: Hash(inode), value: the number of inode's chunkinfo size
+    std::unordered_map<std::string, uint64_t> inodeS3MetaSize_;
 };
 
 }  // namespace metaserver

curvefs/src/metaserver/metaserver.cpp

@@ -500,6 +500,12 @@ void Metaserver::InitStorage() {
     LOG_IF(FATAL, !conf_->GetUInt64Value(
         "storage.rocksdb.block_cache_capacity",
         &storageOptions_.blockCacheCapacity));
+    LOG_IF(FATAL, !conf_->GetDoubleValue(
+        "storage.rocksdb.memtable_prefix_bloom_size_ratio",
+        &storageOptions_.memtablePrefixBloomSizeRatio));
+    LOG_IF(FATAL, !conf_->GetUInt64Value(
+        "storage.s3_meta_inside_inode.limit_size",
+        &storageOptions_.s3MetaLimitSizeInsideInode));
 
     bool succ = ::curvefs::metaserver::storage::InitStorage(storageOptions_);
     LOG_IF(FATAL, !succ) << "Init storage failed";

curvefs/src/metaserver/metastore.cpp

@@ -397,13 +397,29 @@ MetaStatusCode MetaStoreImpl::GetInode(const GetInodeRequest* request,
         return status;
     }
 
-    MetaStatusCode status =
-        partition->GetInode(fsId, inodeId, response->mutable_inode());
-    if (status != MetaStatusCode::OK) {
+    Inode* inode = response->mutable_inode();
+    MetaStatusCode rc = partition->GetInode(fsId, inodeId, inode);
+    // NOTE: the following two cases we should padding inode's s3chunkinfo:
+    // (1): for RPC requests which unsupport streaming
+    // (2): inode's s3chunkinfo is small enough
+    if (rc == MetaStatusCode::OK) {
+        uint64_t limit = 0;
+        if (request->supportstreaming()) {
+            limit = kvStorage_->GetStorageOptions().s3MetaLimitSizeInsideInode;
+        }
+        rc = partition->PaddingInodeS3ChunkInfo(
+            fsId, inodeId, inode->mutable_s3chunkinfomap(), limit);
+        if (rc == MetaStatusCode::INODE_S3_META_TOO_LARGE) {
+            response->set_streaming(true);
+            rc = MetaStatusCode::OK;
+        }
+    }
+
+    if (rc != MetaStatusCode::OK) {
         response->clear_inode();
     }
-    response->set_statuscode(status);
-    return status;
+    response->set_statuscode(rc);
+    return rc;
 }
 
 MetaStatusCode MetaStoreImpl::BatchGetInodeAttr(
@@ -509,14 +525,24 @@ MetaStatusCode MetaStoreImpl::GetOrModifyS3ChunkInfo(
     auto partition = GetPartition(request->partitionid());
     if (nullptr == partition) {
         rc = MetaStatusCode::PARTITION_NOT_FOUND;
-    } else {
-        rc = partition->GetOrModifyS3ChunkInfo(request->fsid(),
-                                               request->inodeid(),
-                                               request->s3chunkinfoadd(),
-                                               iterator,
-                                               request->returns3chunkinfomap(),
-                                               request->froms3compaction());
+        response->set_statuscode(rc);
+        return rc;
     }
+
+    uint32_t fsId = request->fsid();
+    uint64_t inodeId = request->inodeid();
+    rc = partition->GetOrModifyS3ChunkInfo(fsId, inodeId,
+                                           request->s3chunkinfoadd(),
+                                           request->s3chunkinforemove(),
+                                           request->returns3chunkinfomap(),
+                                           iterator);
+    if (rc == MetaStatusCode::OK &&
+        !request->supportstreaming() &&
+        request->returns3chunkinfomap()) {
+        rc = partition->PaddingInodeS3ChunkInfo(
+            fsId, inodeId, response->mutable_s3chunkinfomap(), 0);
+    }
+
     response->set_statuscode(rc);
     return rc;
 }

curvefs/src/metaserver/metastore.h

@@ -77,6 +77,7 @@ using ::curvefs::metaserver::copyset::OnSnapshotSaveDoneClosure;
 using ::curvefs::metaserver::storage::Iterator;
 using ::curvefs::common::StreamServer;
 using ::curvefs::common::StreamConnection;
+using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 
 class MetaStore {
  public:

curvefs/src/metaserver/metastore_fstream.cpp

@@ -170,11 +170,12 @@ bool MetaStoreFStream::LoadInodeS3ChunkInfoList(uint32_t partitionId,
         return false;
     }
 
-    std::shared_ptr<Iterator> iterator;
     S3ChunkInfoMap map2add;
+    S3ChunkInfoMap map2del;
+    std::shared_ptr<Iterator> iterator;
     map2add.insert({key4list.chunkIndex, list});
     MetaStatusCode rc = partition->GetOrModifyS3ChunkInfo(
-        key4list.fsId, key4list.inodeId, map2add, &iterator, false, false);
+        key4list.fsId, key4list.inodeId, map2add, map2del, false, &iterator);
     if (rc != MetaStatusCode::OK) {
         LOG(ERROR) << "GetOrModifyS3ChunkInfo failed, retCode = "
                    << MetaStatusCode_Name(rc);

curvefs/src/metaserver/partition.cpp

@@ -67,7 +67,7 @@ MetaStatusCode Partition::CreateDentry(const Dentry& dentry, bool isLoadding) {
     if (GetStatus() == PartitionStatus::DELETING) {
         return MetaStatusCode::PARTITION_DELETING;
     }
-    MetaStatusCode ret = dentryManager_->CreateDentry(dentry);
+    MetaStatusCode ret = dentryManager_->CreateDentry(dentry, isLoadding);
     if (MetaStatusCode::OK == ret) {
         if (!isLoadding) {
             return inodeManager_->UpdateInodeWhenCreateOrRemoveSubNode(
@@ -262,20 +262,30 @@ MetaStatusCode Partition::UpdateInode(const UpdateInodeRequest& request) {
 MetaStatusCode Partition::GetOrModifyS3ChunkInfo(
     uint32_t fsId,
     uint64_t inodeId,
-    const S3ChunkInfoMap& list2add,
-    std::shared_ptr<Iterator>* iterator,
+    const S3ChunkInfoMap& map2add,
+    const S3ChunkInfoMap& map2del,
     bool returnS3ChunkInfoMap,
-    bool compaction) {
+    std::shared_ptr<Iterator>* iterator) {
     if (!IsInodeBelongs(fsId, inodeId)) {
         return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
-    if (GetStatus() == PartitionStatus::DELETING) {
+    } else if (GetStatus() == PartitionStatus::DELETING) {
         return MetaStatusCode::PARTITION_DELETING;
     }
 
     return inodeManager_->GetOrModifyS3ChunkInfo(
-        fsId, inodeId, list2add, iterator, returnS3ChunkInfoMap, compaction);
+        fsId, inodeId, map2add, map2del, returnS3ChunkInfoMap, iterator);
+}
+
+MetaStatusCode Partition::PaddingInodeS3ChunkInfo(int32_t fsId,
+                                                  uint64_t inodeId,
+                                                  S3ChunkInfoMap* m,
+                                                  uint64_t limit) {
+    if (!IsInodeBelongs(fsId, inodeId)) {
+        return MetaStatusCode::PARTITION_ID_MISSMATCH;
+    } else if (GetStatus() == PartitionStatus::DELETING) {
+        return MetaStatusCode::PARTITION_DELETING;
+    }
+    return inodeManager_->PaddingInodeS3ChunkInfo(fsId, inodeId, m, limit);
 }
 
 MetaStatusCode Partition::InsertInode(const Inode& inode) {

curvefs/src/metaserver/partition.h

@@ -92,10 +92,15 @@ class Partition {
 
     MetaStatusCode GetOrModifyS3ChunkInfo(uint32_t fsId,
                                           uint64_t inodeId,
-                                          const S3ChunkInfoMap& list2add,
-                                          std::shared_ptr<Iterator>* iterator,
+                                          const S3ChunkInfoMap& map2add,
+                                          const S3ChunkInfoMap& map2del,
                                           bool returnS3ChunkInfoMap,
-                                          bool compaction);
+                                          std::shared_ptr<Iterator>* iterator);
+
+    MetaStatusCode PaddingInodeS3ChunkInfo(int32_t fsId,
+                                           uint64_t inodeId,
+                                           S3ChunkInfoMap* m,
+                                           uint64_t limit = 0);
 
     MetaStatusCode InsertInode(const Inode& inode);
 

curvefs/src/metaserver/s3compact_wq_impl.cpp

@@ -458,7 +458,7 @@ bool S3CompactWorkQueueImpl::CompactPrecheck(
 
     // inode exist?
     MetaStatusCode ret = task.inodeManager->GetInode(
-        task.inodeKey.fsId, task.inodeKey.inodeId, inode);
+        task.inodeKey.fsId, task.inodeKey.inodeId, inode, true);
     if (ret != MetaStatusCode::OK) {
         LOG(WARNING) << "s3compact: GetInode fail, inodeKey = "
                      << task.inodeKey.fsId << "," << task.inodeKey.inodeId
@@ -472,22 +472,9 @@ bool S3CompactWorkQueueImpl::CompactPrecheck(
         return false;
     }
 
-    // pandding s3chunkinfomap for inode
-    {
-        auto inodeKey = task.inodeKey;
-        auto inodeManager = task.inodeManager;
-        MetaStatusCode rc = inodeManager->PaddingInodeS3ChunkInfo(
-            inodeKey.fsId, inodeKey.inodeId, inode);
-        if (rc != MetaStatusCode::OK) {
-            LOG(ERROR) << "Padding inode s3chunkinfo failed, "
-                       << "retCode = " << MetaStatusCode_Name(ret);
-            return false;
-        }
-
-        if (inode->s3chunkinfomap().size() == 0) {
-            VLOG(6) << "Inode s3chunkinfo is empty";
-            return false;
-        }
+    if (inode->s3chunkinfomap().size() == 0) {
+        VLOG(6) << "Inode s3chunkinfo is empty";
+        return false;
     }
 
     // need compact?

curvefs/src/metaserver/storage/config.h

@@ -53,6 +53,11 @@ struct StorageOptions {
     uint64_t orderedMaxWriteBufferNumber;
 
     uint64_t blockCacheCapacity;
+
+    double memtablePrefixBloomSizeRatio;
+
+    // misc config item
+    uint64_t s3MetaLimitSizeInsideInode;
 };
 
 }  // namespace storage

curvefs/src/metaserver/storage/converter.cpp

@@ -90,18 +90,21 @@ Key4S3ChunkInfoList::Key4S3ChunkInfoList()
       inodeId(0),
       chunkIndex(0),
       firstChunkId(0),
-      lastChunkId(0) {}
+      lastChunkId(0),
+      size(0) {}
 
 Key4S3ChunkInfoList::Key4S3ChunkInfoList(uint32_t fsId,
                                          uint64_t inodeId,
                                          uint64_t chunkIndex,
                                          uint64_t firstChunkId,
-                                         uint64_t lastChunkId)
+                                         uint64_t lastChunkId,
+                                         uint64_t size)
     : fsId(fsId),
       inodeId(inodeId),
       chunkIndex(chunkIndex),
       firstChunkId(firstChunkId),
-      lastChunkId(lastChunkId) {}
+      lastChunkId(lastChunkId),
+      size(size) {}
 
 std::string Key4S3ChunkInfoList::SerializeToString() const {
     std::ostringstream oss;
@@ -109,18 +112,20 @@ std::string Key4S3ChunkInfoList::SerializeToString() const {
         << inodeId << ":" << chunkIndex << ":"
         << std::setw(kMaxUint64Length_) << std::setfill('0') << firstChunkId
         << ":"
-        << std::setw(kMaxUint64Length_) << std::setfill('0') << lastChunkId;
+        << std::setw(kMaxUint64Length_) << std::setfill('0') << lastChunkId
+        << ":" << size;
     return oss.str();
 }
 
 bool Key4S3ChunkInfoList::ParseFromString(const std::string& value) {
     std::vector<std::string> items;
     SplitString(value, ":", &items);
-    return items.size() == 6 && CompareType(items[0], keyType_) &&
+    return items.size() == 7 && CompareType(items[0], keyType_) &&
         StringToUl(items[1], &fsId) && StringToUll(items[2], &inodeId) &&
         StringToUll(items[3], &chunkIndex) &&
         StringToUll(items[4], &firstChunkId) &&
-        StringToUll(items[5], &lastChunkId);
+        StringToUll(items[5], &lastChunkId) &&
+        StringToUll(items[6], &size);
 }
 
 Prefix4ChunkIndexS3ChunkInfoList::Prefix4ChunkIndexS3ChunkInfoList()

curvefs/src/metaserver/storage/converter.h

@@ -94,7 +94,8 @@ class Key4S3ChunkInfoList : public StorageKey {
                         uint64_t inodeId,
                         uint64_t chunkIndex,
                         uint64_t firstChunkId,
-                        uint64_t lastChunkId);
+                        uint64_t lastChunkId,
+                        uint64_t size);
 
     std::string SerializeToString() const override;
 
@@ -109,6 +110,7 @@ class Key4S3ChunkInfoList : public StorageKey {
      uint64_t chunkIndex;
      uint64_t firstChunkId;
      uint64_t lastChunkId;
+     uint64_t size;
 };
 
 class Prefix4ChunkIndexS3ChunkInfoList : public StorageKey {

curvefs/src/metaserver/storage/memory_storage.cpp

@@ -321,6 +321,10 @@ bool MemoryStorage::GetStatistics(StorageStatistics* statistics) {
     return true;
 }
 
+StorageOptions MemoryStorage::GetStorageOptions() const {
+    return options_;
+}
+
 }  // namespace storage
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/storage/memory_storage.h

@@ -97,6 +97,8 @@ class MemoryStorage : public KVStorage, public StorageTransaction {
 
     bool GetStatistics(StorageStatistics* Statistics) override;
 
+    StorageOptions GetStorageOptions() const override;
+
     Status HGet(const std::string& name,
                 const std::string& key,
                 ValueType* value) override;

curvefs/src/metaserver/storage/rocksdb_storage.cpp

@@ -42,38 +42,66 @@ const std::string RocksDBOptions::kOrderedColumnFamilyName_ =  // NOLINT
 
 RocksDBOptions::RocksDBOptions(StorageOptions options) {
     // db options
-    RocksDBStorageComparator cmp;
-    dbOptions_.comparator = &cmp;
+    // the database will be created if it is missing
     dbOptions_.create_if_missing = true;
+    // missing column families will be automatically created
     dbOptions_.create_missing_column_families = true;
-    dbOptions_.enable_blob_files = true;
+    // maximum number of concurrent background memtable flush jobs,
+    // submitted by default to the HIGH priority thread pool
     dbOptions_.max_background_flushes = 2;
+    // maximum number of concurrent background compaction jobs,
+    // submitted to the default LOW priority thread pool.
     dbOptions_.max_background_compactions = 4;
+    // allows OS to incrementally sync files to disk while they are being
+    // written, asynchronously, in the background.
     dbOptions_.bytes_per_sync = 1048576;
-    dbOptions_.compaction_pri = ROCKSDB_NAMESPACE::kMinOverlappingRatio;
-    dbOptions_.prefix_extractor.reset(NewFixedPrefixTransform(3));
 
     // table options
     std::shared_ptr<ROCKSDB_NAMESPACE::Cache> cache =
         NewLRUCache(options.blockCacheCapacity);
     BlockBasedTableOptions tableOptions;
+    tableOptions.block_size = 16 * 1024;  // 16KB
+    // default: an 8MB internal cache
     tableOptions.block_cache = cache;
-    tableOptions.block_size = 16 * 1024;  // 16MB
+    // whether to put index/filter blocks in the block cache
     tableOptions.cache_index_and_filter_blocks = true;
+    // only evicted from cache when the table reader is freed
     tableOptions.pin_l0_filter_and_index_blocks_in_cache = true;
-    dbOptions_.table_factory.reset(NewBlockBasedTableFactory(tableOptions));
+    // reset bloom filter
+    tableOptions.filter_policy.reset(NewBloomFilterPolicy(10, false));
 
     // column failmy options
-    auto unorderedCFOptions = ColumnFamilyOptions();
-    auto orderedCFOptions = ColumnFamilyOptions();
+    comparator_ = std::make_shared<RocksDBStorageComparator>();
+    ColumnFamilyOptions cfOptions = ColumnFamilyOptions();
+    // user-defined key comparator
+    cfOptions.comparator = comparator_.get();
+    // large values (blobs) are written to separate blob files, and
+    // only pointers to them are stored in SST files
+    cfOptions.enable_blob_files = true;
+    // RocksDB will pick target size of each level dynamically
+    cfOptions.level_compaction_dynamic_level_bytes = true;
+    cfOptions.compaction_pri = ROCKSDB_NAMESPACE::kMinOverlappingRatio;
+    // use the specified function to determine the prefixes for keys
+    cfOptions.prefix_extractor.reset(NewFixedPrefixTransform(sizeof(size_t)));
+    // The size in bytes of the filter for memtable is
+    // write_buffer_size * memtable_prefix_bloom_size_ratio
+    cfOptions.memtable_prefix_bloom_size_ratio =
+        options.memtablePrefixBloomSizeRatio;
+    // reset table options for column failmy
+    cfOptions.table_factory.reset(NewBlockBasedTableFactory(tableOptions));
+
+    ColumnFamilyOptions unorderedCFOptions = cfOptions;
+    // amount of data to build up in memory (backed by an unsorted log
+    // on disk) before converting to a sorted on-disk file.
     unorderedCFOptions.write_buffer_size = options.unorderedWriteBufferSize;
+    // the maximum number of write buffers that are built up in memory.
     unorderedCFOptions.max_write_buffer_number =
         options.unorderedMaxWriteBufferNumber;
-    unorderedCFOptions.level_compaction_dynamic_level_bytes = true;
+
+    ColumnFamilyOptions orderedCFOptions = cfOptions;
     orderedCFOptions.write_buffer_size = options.orderedWriteBufferSize;
     orderedCFOptions.max_write_buffer_number =
         options.orderedMaxWriteBufferNumber;
-    orderedCFOptions.level_compaction_dynamic_level_bytes = true;
 
     columnFamilies_.push_back(ColumnFamilyDescriptor(
         ROCKSDB_NAMESPACE::kDefaultColumnFamilyName, unorderedCFOptions));
@@ -209,6 +237,7 @@ Status RocksDBStorage::ToStorageStatus(const ROCKSDB_NAMESPACE::Status& s) {
     return Status::InternalError();
 }
 
+// "ordered:name"
 std::string RocksDBStorage::ToInternalName(const std::string& name,
                                            bool ordered) {
     std::ostringstream oss;
@@ -224,7 +253,7 @@ std::string RocksDBStorage::FormatInternalKey(size_t num4name,
 }
 
 // NOTE: we will convert name to number for compare prefix
-// eg: iname:key
+// eg: Hash(iname):key
 std::string RocksDBStorage::ToInternalKey(const std::string& iname,
                                           const std::string& key) {
     size_t num4name = Hash(iname);
@@ -269,6 +298,13 @@ inline void RocksDBStorage::CommitKeys() {
     for (const auto& pair : pending4del_) {
         counter_->Erase(pair.first, pair.second);
     }
+    pending4set_.clear();
+    pending4del_.clear();
+}
+
+inline void RocksDBStorage::RollbackKeys() {
+    pending4set_.clear();
+    pending4del_.clear();
 }
 
 Status RocksDBStorage::Get(const std::string& name,
@@ -281,7 +317,7 @@ Status RocksDBStorage::Get(const std::string& name,
 
     std::string iname = ToInternalName(name, ordered);
     std::string ikey = ToInternalKey(iname, key);
-    if (!FindKey(iname, ikey)) {
+    if (!InTransaction_ && !FindKey(iname, ikey)) {
         return Status::NotFound();
     }
 
@@ -332,7 +368,7 @@ Status RocksDBStorage::Del(const std::string& name,
 
     std::string iname = ToInternalName(name, ordered);
     std::string ikey = ToInternalKey(iname, key);
-    if (!counter_->Find(iname, ikey)) {
+    if (!InTransaction_ && !counter_->Find(iname, ikey)) {
         return Status::NotFound();
     }
 
@@ -351,7 +387,7 @@ Status RocksDBStorage::Del(const std::string& name,
 }
 
 std::shared_ptr<Iterator> RocksDBStorage::Seek(const std::string& name,
-                                                const std::string& prefix) {
+                                               const std::string& prefix) {
     size_t size = 0;
     int status = inited_ ? 0 : -1;
     std::string iname = ToInternalName(name, true);
@@ -378,11 +414,13 @@ size_t RocksDBStorage::Size(const std::string& name, bool ordered) {
 Status RocksDBStorage::Clear(const std::string& name, bool ordered) {
     if (!inited_) {
         return Status::DBClosed();
+    } else if (InTransaction_) {
+        return Status::NotSupported();
     }
 
     auto handle = GetColumnFamilyHandle(ordered);
-    std::string iname = ToInternalName(name, ordered);
-    std::string beginKey = ToInternalKey(iname, "");  // "name:"
+    std::string iname = ToInternalName(name, ordered);  // "1:name"
+    std::string beginKey = ToInternalKey(iname, "");  // "Hash(iname):"
     size_t beginNum = BinrayString2Number(beginKey);
     std::string endKey = FormatInternalKey(beginNum + 1, "");
     ROCKSDB_NAMESPACE::Status s = db_->DeleteRange(
@@ -399,28 +437,40 @@ std::shared_ptr<StorageTransaction> RocksDBStorage::BeginTransaction() {
     if (nullptr == txn) {
         return nullptr;
     }
+    pending4set_.clear();
+    pending4del_.clear();
     return std::make_shared<RocksDBStorage>(*this, txn);
 }
 
 Status RocksDBStorage::Commit() {
-    if (!InTransaction_) {
+    if (!InTransaction_ || nullptr == txn_) {
         return Status::NotSupported();
     }
 
-    Status s = ToStorageStatus(txn_->Commit());
-    if (s.ok()) {
+    ROCKSDB_NAMESPACE::Status s = txn_->Commit();
+    if (!s.ok()) {
+        LOG(ERROR) << "RocksDBStorage commit transaction failed"
+                   << ", status=" << s.ToString();
+    } else {
         CommitKeys();
     }
-    return s;
+    delete txn_;
+    return ToStorageStatus(s);
 }
 
 Status RocksDBStorage::Rollback()  {
-    if (!InTransaction_) {
+    if (!InTransaction_ || nullptr == txn_) {
         return Status::NotSupported();
     }
-    pending4set_.clear();
-    pending4del_.clear();
-    return ToStorageStatus(txn_->Rollback());
+    ROCKSDB_NAMESPACE::Status s = txn_->Rollback();
+    if (!s.ok()) {
+        LOG(ERROR) << "RocksDBStorage rollback transaction failed"
+                   << ", status=" << s.ToString();
+    } else {
+        RollbackKeys();
+    }
+    delete txn_;
+    return ToStorageStatus(s);
 }
 
 bool RocksDBStorage::GetStatistics(StorageStatistics* statistics) {
@@ -442,6 +492,10 @@ bool RocksDBStorage::GetStatistics(StorageStatistics* statistics) {
     return true;
 }
 
+StorageOptions RocksDBStorage::GetStorageOptions() const {
+    return options_;
+}
+
 }  // namespace storage
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/storage/rocksdb_storage.h

@@ -33,6 +33,7 @@
 #include "rocksdb/slice.h"
 #include "rocksdb/table.h"
 #include "rocksdb/options.h"
+#include "rocksdb/filter_policy.h"
 #include "rocksdb/slice_transform.h"
 #include "rocksdb/utilities/transaction.h"
 #include "rocksdb/utilities/transaction_db.h"
@@ -56,6 +57,7 @@ using ROCKSDB_NAMESPACE::BlockBasedTableOptions;
 using ROCKSDB_NAMESPACE::Transaction;
 using ROCKSDB_NAMESPACE::TransactionDB;
 using ROCKSDB_NAMESPACE::NewLRUCache;
+using ROCKSDB_NAMESPACE::NewBloomFilterPolicy;
 using ROCKSDB_NAMESPACE::NewFixedPrefixTransform;
 using ROCKSDB_NAMESPACE::NewBlockBasedTableFactory;
 using STORAGE_TYPE = KVStorage::STORAGE_TYPE;
@@ -84,12 +86,17 @@ class RocksDBOptions {
     std::vector<ColumnFamilyDescriptor> columnFamilies_;
 
     static const std::string kOrderedColumnFamilyName_;
+
+    std::shared_ptr<rocksdb::Comparator> comparator_;
 };
 
 class RocksDBStorageComparator : public rocksdb::Comparator {
  public:
+    // if slice1 < slice2, return -1
+    // if slice1 > slice2, return 1
+    // if slice1 == slice2, return 0
     int Compare(const rocksdb::Slice& slice1,
-                const rocksdb::Slice& slice2) const {
+                const rocksdb::Slice& slice2) const override {
         std::string key1 = std::string(slice1.data(), slice1.size());
         std::string key2 = std::string(slice2.data(), slice2.size());
         size_t num1 = BinrayString2Number(key1);
@@ -109,9 +116,12 @@ class RocksDBStorageComparator : public rocksdb::Comparator {
     }
 
     // Ignore the following methods for now
-    const char* Name() const { return "RocksDBStorageComparator"; }
-    void FindShortestSeparator(std::string*, const rocksdb::Slice&) const {}
-    void FindShortSuccessor(std::string*) const {}
+    const char* Name() const override { return "RocksDBStorageComparator"; }
+
+    void FindShortestSeparator(std::string*,
+                               const rocksdb::Slice&) const override {}
+
+    void FindShortSuccessor(std::string*) const override {}
 };
 
 class RocksDBStorage : public KVStorage, public StorageTransaction {
@@ -133,6 +143,8 @@ class RocksDBStorage : public KVStorage, public StorageTransaction {
 
     bool GetStatistics(StorageStatistics* Statistics) override;
 
+    StorageOptions GetStorageOptions() const override;
+
     // unordered
     Status HGet(const std::string& name,
                 const std::string& key,
@@ -211,6 +223,8 @@ class RocksDBStorage : public KVStorage, public StorageTransaction {
 
     void CommitKeys();
 
+    void RollbackKeys();
+
     Status Get(const std::string& name,
                const std::string& key,
                ValueType* value,
@@ -226,8 +240,11 @@ class RocksDBStorage : public KVStorage, public StorageTransaction {
                bool ordered);
 
     std::shared_ptr<Iterator> Seek(const std::string& name,
-                                    const std::string& prefix);
+                                   const std::string& prefix);
 
+    // TODO(@Wine93): We do not support transactions for the
+    // below 3 methods, maybe we should return Status::NotSupported
+    // when user invoke it in transaction.
     std::shared_ptr<Iterator> GetAll(const std::string& name, bool ordered);
 
     size_t Size(const std::string& name, bool ordered);
@@ -332,16 +349,25 @@ class RocksDBStorageIterator : public Iterator {
           size_(size),
           status_(status),
           prefixChecking_(true),
-          ordered_(ordered) {
+          ordered_(ordered),
+          iter_(nullptr) {
         if (status_ == 0) {
             readOptions_ = storage_->ReadOptions();
-            readOptions_.snapshot = storage_->db_->GetSnapshot();
+            if (storage_->InTransaction_) {
+                readOptions_.snapshot = storage_->txn_->GetSnapshot();
+            } else {
+                readOptions_.snapshot = storage_->db_->GetSnapshot();
+            }
         }
     }
 
     ~RocksDBStorageIterator() {
         if (status_ == 0) {
-            storage_->db_->ReleaseSnapshot(readOptions_.snapshot);
+            if (storage_->InTransaction_) {
+                storage_->txn_->ClearSnapshot();
+            } else {
+                storage_->db_->ReleaseSnapshot(readOptions_.snapshot);
+            }
         }
     }
 
@@ -362,7 +388,11 @@ class RocksDBStorageIterator : public Iterator {
 
     void SeekToFirst() {
         auto handler = storage_->GetColumnFamilyHandle(ordered_);
-        iter_ = storage_->db_->NewIterator(readOptions_, handler);
+        if (storage_->InTransaction_) {
+            iter_.reset(storage_->txn_->GetIterator(readOptions_, handler));
+        } else {
+            iter_.reset(storage_->db_->NewIterator(readOptions_, handler));
+        }
         iter_->Seek(prefix_);
     }
 
@@ -404,7 +434,7 @@ class RocksDBStorageIterator : public Iterator {
     int status_;
     bool ordered_;
     bool prefixChecking_;
-    rocksdb::Iterator* iter_;
+    std::unique_ptr<rocksdb::Iterator> iter_;
     RocksDBStorage* storage_;
     rocksdb::ReadOptions readOptions_;
 };

curvefs/src/metaserver/storage/storage.h

@@ -110,6 +110,8 @@ class KVStorage : public BaseStorage {
 
     virtual bool GetStatistics(StorageStatistics* Statistics) = 0;
 
+    virtual StorageOptions GetStorageOptions() const = 0;
+
     virtual std::shared_ptr<StorageTransaction> BeginTransaction() = 0;
 };
 

curvefs/src/metaserver/storage/storage_fstream.h

@@ -92,11 +92,17 @@ static std::pair<std::string, std::string> UserKey(const std::string& ikey) {
     if (items.size() >= 2) {
         prefix = items[0];
         ukey = ikey.substr(prefix.size() + 1);
+    } else if (items.size() == 1) {  // e.g: t3:
+        prefix = items[0];
     }
     return std::make_pair(prefix, ukey);
 }
 
 static std::pair<ENTRY_TYPE, uint32_t> Extract(const std::string& prefix) {
+    if (prefix.size() == 0) {
+        return std::make_pair(ENTRY_TYPE::UNKNOWN, 0);
+    }
+
     std::vector<std::string> items{
         prefix.substr(0, 1),  // eg: i
         prefix.substr(1),  // eg: 100

curvefs/src/metaserver/storage/utils.cpp

@@ -56,6 +56,8 @@ std::string Number2BinaryString(size_t num) {
 
 size_t BinrayString2Number(const std::string& str) {
     if (str.size() < sizeof(size_t)) {
+        LOG(ERROR) << "The length of binray string must equal or greater than "
+                   << sizeof(size_t) << ", but now is " << str.size();
         return 0;
     }
     return *reinterpret_cast<const size_t*>(str.c_str());

curvefs/src/tools/status/curvefs_mds_status.cpp

@@ -76,7 +76,7 @@ void MdsStatusTool::AfterGetMetric(const std::string hostAddr,
                                    const std::string& subUri,
                                    const std::string& value,
                                    const MetricStatusCode& statusCode) {
-    auto mainAddr = dummy2MainAddr_[hostAddr];
+    auto mainAddr = hostAddr;
     if (statusCode == MetricStatusCode::kOK) {
         onlineHosts_.insert(mainAddr);
         if (subUri == statusSubUri_) {
@@ -88,6 +88,7 @@ void MdsStatusTool::AfterGetMetric(const std::string hostAddr,
                     standbyHost_.insert(mainAddr);
                 } else if (keyValue == hostLeaderValue_) {
                     // leader host
+                    mainAddr = dummy2MainAddr_[hostAddr];
                     leaderHosts_.insert(mainAddr);
                 } else {
                     // error host

curvefs/src/tools/status/curvefs_status_base_tool.cpp

@@ -125,7 +125,7 @@ int StatusBaseTool::ProcessMetrics() {
     if (show_) {
         std::cout << "standby " << hostType_ << ": [ ";
         for (auto const& i : standbyHost_) {
-            std::cerr << i << " ";
+            std::cout << i << " ";
         }
         std::cout << "]." << std::endl;
     }

curvefs/test/client/client_s3_adaptor_test.cpp

@@ -67,6 +67,7 @@ class ClientS3AdaptorTest : public testing::Test {
         option.readCacheMaxByte = 104857600;
         option.writeCacheMaxByte = 10485760000;
         option.fuseMaxSize = 131072;
+        option.chunkFlushThreads = 5;
         option.diskCacheOpt.diskCacheType = (DiskCacheType)0;
         s3ClientAdaptor_->Init(option, mockS3Client_, mockInodeManager_,
                                mockMdsClient_, mockFsCacheManager_,
@@ -266,3 +267,4 @@ TEST_F(ClientS3AdaptorTest, FlushAllCache_with_cache) {
 
 }  // namespace client
 }  // namespace curvefs
+

curvefs/test/client/mock_metaserver_client.h

@@ -72,15 +72,15 @@ class MockMetaServerClient : public MetaServerClient {
     MOCK_METHOD1(PrepareRenameTx,
                  MetaStatusCode(const std::vector<Dentry>& dentrys));
 
-    MOCK_METHOD3(GetInode, MetaStatusCode(
-            uint32_t fsId, uint64_t inodeid, Inode *out));
+    MOCK_METHOD4(GetInode, MetaStatusCode(
+            uint32_t fsId, uint64_t inodeid, Inode *out, bool* streaming));
 
     MOCK_METHOD3(BatchGetInodeAttr, MetaStatusCode(
-        uint32_t fsId, std::set<uint64_t> *inodeIds,
+        uint32_t fsId, const std::set<uint64_t> &inodeIds,
         std::list<InodeAttr> *attr));
 
     MOCK_METHOD3(BatchGetXAttr, MetaStatusCode(
-        uint32_t fsId, std::set<uint64_t> *inodeIds,
+        uint32_t fsId, const std::set<uint64_t> &inodeIds,
         std::list<XAttr> *xattr));
 
     MOCK_METHOD2(UpdateInode,
@@ -112,6 +112,10 @@ class MockMetaServerClient : public MetaServerClient {
             const InodeParam &param, Inode *out));
 
     MOCK_METHOD2(DeleteInode, MetaStatusCode(uint32_t fsId, uint64_t inodeid));
+
+    MOCK_METHOD3(SplitRequestInodes, bool(uint32_t fsId,
+        const std::set<uint64_t> &inodeIds,
+        std::vector<std::vector<uint64_t>> *inodeGroups));
 };
 
 }  // namespace rpcclient

curvefs/test/client/rpcclient/metaserver_client_test.cpp

@@ -627,6 +627,8 @@ TEST_F(MetaServerClientImplTest, test_GetInode) {
     out.set_rdev(0);
     out.set_symlink("test9");
 
+    bool streaming;
+
     curvefs::metaserver::GetInodeResponse response;
 
     // test0: rpc error
@@ -637,7 +639,8 @@ TEST_F(MetaServerClientImplTest, test_GetInode) {
         .WillRepeatedly(DoAll(SetArgPointee<2>(target_),
                               SetArgPointee<3>(applyIndex), Return(true)));
 
-    MetaStatusCode status = metaserverCli_.GetInode(fsid, inodeid, &out);
+    MetaStatusCode status = metaserverCli_.GetInode(
+        fsid, inodeid, &out, &streaming);
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 
     // test1: get inode ok
@@ -652,7 +655,7 @@ TEST_F(MetaServerClientImplTest, test_GetInode) {
                   Invoke(SetRpcService<GetInodeRequest, GetInodeResponse>)));
     EXPECT_CALL(*mockMetacache_.get(), UpdateApplyIndex(_, _));
 
-    status = metaserverCli_.GetInode(fsid, inodeid, &out);
+    status = metaserverCli_.GetInode(fsid, inodeid, &out, &streaming);
     ASSERT_EQ(MetaStatusCode::OK, status);
 
     // test2: get inode with not found error
@@ -661,7 +664,7 @@ TEST_F(MetaServerClientImplTest, test_GetInode) {
         .WillOnce(
             DoAll(SetArgPointee<2>(response),
                   Invoke(SetRpcService<GetInodeRequest, GetInodeResponse>)));
-    status = metaserverCli_.GetInode(fsid, inodeid, &out);
+    status = metaserverCli_.GetInode(fsid, inodeid, &out, &streaming);
     ASSERT_EQ(MetaStatusCode::NOT_FOUND, status);
 
     // test3: test response do not have applyindex
@@ -672,20 +675,20 @@ TEST_F(MetaServerClientImplTest, test_GetInode) {
             DoAll(SetArgPointee<2>(response),
                   Invoke(SetRpcService<GetInodeRequest, GetInodeResponse>)));
 
-    status = metaserverCli_.GetInode(fsid, inodeid, &out);
+    status = metaserverCli_.GetInode(fsid, inodeid, &out, &streaming);
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 
     // test4: test response do not have inode
     response.set_appliedindex(10);
     response.clear_inode();
-    status = metaserverCli_.GetInode(fsid, inodeid, &out);
+    status = metaserverCli_.GetInode(fsid, inodeid, &out, &streaming);
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 
     // test5: do not have both dentrys and appliedindex
     response.clear_inode();
     response.clear_appliedindex();
 
-    status = metaserverCli_.GetInode(fsid, inodeid, &out);
+    status = metaserverCli_.GetInode(fsid, inodeid, &out, &streaming);
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 }
 
@@ -1151,7 +1154,7 @@ TEST_F(MetaServerClientImplTest, test_BatchGetInodeAttr) {
                               SetArgPointee<3>(applyIndex), Return(true)));
 
     MetaStatusCode status = metaserverCli_.BatchGetInodeAttr(
-        fsid, &inodeIds, &attr);
+        fsid, inodeIds, &attr);
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 
     // test1: batchGetInodeAttr ok
@@ -1173,7 +1176,7 @@ TEST_F(MetaServerClientImplTest, test_BatchGetInodeAttr) {
                   BatchGetInodeAttrResponse>)));
     EXPECT_CALL(*mockMetacache_.get(), UpdateApplyIndex(_, _));
 
-    status = metaserverCli_.BatchGetInodeAttr(fsid, &inodeIds, &attr);
+    status = metaserverCli_.BatchGetInodeAttr(fsid, inodeIds, &attr);
     ASSERT_EQ(MetaStatusCode::OK, status);
     ASSERT_EQ(attr.size(), 2);
     ASSERT_THAT(attr.begin()->inodeid(), AnyOf(inodeId1, inodeId2));
@@ -1189,7 +1192,7 @@ TEST_F(MetaServerClientImplTest, test_BatchGetInodeAttr) {
             DoAll(SetArgPointee<2>(response),
                   Invoke(SetRpcService<BatchGetInodeAttrRequest,
                   BatchGetInodeAttrResponse>)));
-    status = metaserverCli_.BatchGetInodeAttr(fsid, &inodeIds, &attr);
+    status = metaserverCli_.BatchGetInodeAttr(fsid, inodeIds, &attr);
     ASSERT_EQ(MetaStatusCode::NOT_FOUND, status);
 
     // test3: test response do not have applyindex
@@ -1205,7 +1208,7 @@ TEST_F(MetaServerClientImplTest, test_BatchGetInodeAttr) {
                   Invoke(SetRpcService<BatchGetInodeAttrRequest,
                   BatchGetInodeAttrResponse>)));
 
-    status = metaserverCli_.BatchGetInodeAttr(fsid, &inodeIds, &attr);
+    status = metaserverCli_.BatchGetInodeAttr(fsid, inodeIds, &attr);
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 }
 
@@ -1253,7 +1256,7 @@ TEST_F(MetaServerClientImplTest, test_BatchGetXAttr) {
                               SetArgPointee<3>(applyIndex), Return(true)));
 
     MetaStatusCode status = metaserverCli_.BatchGetXAttr(
-        fsid, &inodeIds, &xattr);
+        fsid, inodeIds, &xattr);
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 
     // test1: batchGetXAttr ok
@@ -1275,7 +1278,7 @@ TEST_F(MetaServerClientImplTest, test_BatchGetXAttr) {
                   BatchGetXAttrResponse>)));
     EXPECT_CALL(*mockMetacache_.get(), UpdateApplyIndex(_, _));
 
-    status = metaserverCli_.BatchGetXAttr(fsid, &inodeIds, &xattr);
+    status = metaserverCli_.BatchGetXAttr(fsid, inodeIds, &xattr);
     ASSERT_EQ(MetaStatusCode::OK, status);
     ASSERT_EQ(xattr.size(), 2);
     ASSERT_THAT(xattr.begin()->inodeid(), AnyOf(inodeId1, inodeId2));
@@ -1291,7 +1294,7 @@ TEST_F(MetaServerClientImplTest, test_BatchGetXAttr) {
             DoAll(SetArgPointee<2>(response),
                   Invoke(SetRpcService<BatchGetXAttrRequest,
                   BatchGetXAttrResponse>)));
-    status = metaserverCli_.BatchGetXAttr(fsid, &inodeIds, &xattr);
+    status = metaserverCli_.BatchGetXAttr(fsid, inodeIds, &xattr);
     ASSERT_EQ(MetaStatusCode::NOT_FOUND, status);
 
     // test3: test response do not have applyindex
@@ -1307,7 +1310,7 @@ TEST_F(MetaServerClientImplTest, test_BatchGetXAttr) {
                   Invoke(SetRpcService<BatchGetXAttrRequest,
                   BatchGetXAttrResponse>)));
 
-    status = metaserverCli_.BatchGetXAttr(fsid, &inodeIds, &xattr);
+    status = metaserverCli_.BatchGetXAttr(fsid, inodeIds, &xattr);
     ASSERT_EQ(MetaStatusCode::RPC_ERROR, status);
 }
 

curvefs/test/client/test_disk_cache_manager.cpp

@@ -64,7 +64,10 @@ class TestDiskCacheManager : public ::testing::Test {
         diskCacheManager_ = std::make_shared<DiskCacheManager>(
             wrapper, diskCacheWrite_, diskCacheRead_);
         diskCacheRead_->Init(wrapper, "/mnt/test");
-        diskCacheWrite_->Init(client, wrapper, "/mnt/test", 1);
+         std::shared_ptr<LRUCache<std::string, bool>> cachedObjName
+          = std::make_shared<LRUCache<std::string, bool>>
+              (0, std::make_shared<CacheMetrics>("diskcache"));
+        diskCacheWrite_->Init(client, wrapper, "/mnt/test", 1, cachedObjName);
     }
 
     virtual void TearDown() {
@@ -105,52 +108,6 @@ TEST_F(TestDiskCacheManager, Init) {
     EXPECT_CALL(*diskCacheRead_, CreateIoDir(_)).WillOnce(Return(-1));
     ret = diskCacheManager_->Init(client, s3AdaptorOption);
     ASSERT_EQ(-1, ret);
-    /*
-        EXPECT_CALL(*wrapper, stat(NotNull(), NotNull()))
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheWrite_, CreateIoDir(_))
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheRead_, CreateIoDir(_))
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheWrite_, AsyncUploadRun())
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
-              .WillOnce(Return(-1));
-        EXPECT_CALL(*diskCacheRead_, LoadAllCacheReadFile(_))
-              .WillOnce(Return(0));
-        ret = diskCacheManager_->Init(client, s3AdaptorOption);
-        ASSERT_EQ(0, ret);
-
-        EXPECT_CALL(*wrapper, stat(NotNull(), NotNull()))
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheWrite_, CreateIoDir(_))
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheRead_, CreateIoDir(_))
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheWrite_, AsyncUploadRun())
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheRead_, LoadAllCacheReadFile(_))
-              .WillOnce(Return(-1));
-        ret = diskCacheManager_->Init(client, s3AdaptorOption);
-        ASSERT_EQ(-1, ret);
-
-        EXPECT_CALL(*wrapper, stat(NotNull(), NotNull()))
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheWrite_, CreateIoDir(_))
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheRead_, CreateIoDir(_))
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheWrite_, AsyncUploadRun())
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheWrite_, UploadAllCacheWriteFile())
-              .WillOnce(Return(0));
-        EXPECT_CALL(*diskCacheRead_, LoadAllCacheReadFile(_))
-              .WillOnce(Return(0));
-        ret = diskCacheManager_->Init(client, s3AdaptorOption);
-        ASSERT_EQ(0, ret);
-    */
 }
 
 TEST_F(TestDiskCacheManager, CreateDir) {
@@ -237,12 +194,12 @@ TEST_F(TestDiskCacheManager, IsCached) {
     bool ret = diskCacheManager_->IsCached(fileName);
     ASSERT_EQ(false, ret);
 
-    diskCacheManager_->AddCache(fileName);
+    diskCacheManager_->AddCache(fileName, false);
     diskCacheManager_->AddCache(fileName2);
     ret = diskCacheManager_->IsCached(fileName2);
     ASSERT_EQ(true, ret);
 
-    diskCacheManager_->AddCache(fileName);
+    diskCacheManager_->AddCache(fileName, false);
     diskCacheManager_->AddCache(fileName2);
     ret = diskCacheManager_->IsCached(fileName);
     ASSERT_EQ(true, ret);

curvefs/test/client/test_disk_cache_manager_impl.cpp

@@ -59,6 +59,7 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
     ~TestDiskCacheManagerImpl() {}
 
     virtual void SetUp() {
+        Aws::InitAPI(awsOptions_);
         client_ = new MockS3Client();
         wrapper_ = std::make_shared<MockPosixWrapper>();
         diskCacheWrite_ = std::make_shared<MockDiskCacheWrite>();
@@ -66,7 +67,11 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
         diskCacheManager_ = std::make_shared<MockDiskCacheManager>(
             wrapper_, diskCacheWrite_, diskCacheRead_);
         diskCacheRead_->Init(wrapper_, "/mnt/test");
-        diskCacheWrite_->Init(client_, wrapper_, "/mnt/test", 1);
+
+        std::shared_ptr<LRUCache<std::string, bool>> cachedObjName
+          = std::make_shared<LRUCache<std::string, bool>>
+              (0, std::make_shared<CacheMetrics>("diskcache"));
+        diskCacheWrite_->Init(client_, wrapper_, "/mnt/test", 1, cachedObjName);
         diskCacheManagerImpl_ =
             std::make_shared<DiskCacheManagerImpl>(diskCacheManager_, client_);
     }
@@ -78,6 +83,7 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
         Mock::VerifyAndClear(diskCacheWrite_.get());
         Mock::VerifyAndClear(diskCacheRead_.get());
         Mock::VerifyAndClear(diskCacheManager_.get());
+        Aws::ShutdownAPI(awsOptions_);
     }
     std::shared_ptr<MockDiskCacheRead> diskCacheRead_;
     std::shared_ptr<MockDiskCacheWrite> diskCacheWrite_;
@@ -85,6 +91,7 @@ class TestDiskCacheManagerImpl : public ::testing::Test {
     std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl_;
     std::shared_ptr<MockPosixWrapper> wrapper_;
     MockS3Client *client_;
+    Aws::SDKOptions awsOptions_;
 };
 
 
@@ -99,6 +106,36 @@ TEST_F(TestDiskCacheManagerImpl, Init) {
     ASSERT_EQ(0, ret);
 }
 
+TEST_F(TestDiskCacheManagerImpl, WriteClosure) {
+    PutObjectAsyncCallBack cb =
+        [&](const std::shared_ptr<PutObjectAsyncContext> &context) {
+    };
+
+    auto context = std::make_shared<PutObjectAsyncContext>();
+    context->key = "objectName";
+    char data[5] = "gggg";
+    context->buffer = data + 0;
+    context->bufferSize = 2;
+    context->cb = cb;
+    context->startTime = butil::cpuwide_time_us();
+
+    S3ClientAdaptorOption s3AdaptorOption;
+    s3AdaptorOption.diskCacheOpt.threads = 5;
+    EXPECT_CALL(*diskCacheManager_, Init(_, _)).WillOnce(Return(0));
+    diskCacheManagerImpl_->Init(s3AdaptorOption);
+    std::string fileName = "test";
+    std::string buf = "test";
+    EXPECT_CALL(*diskCacheManager_, IsDiskCacheFull()).WillOnce(Return(false));
+    EXPECT_CALL(*diskCacheWrite_, WriteDiskFile(_, _, _, _))
+        .WillOnce(Return(0));
+    EXPECT_CALL(*diskCacheWrite_, GetCacheIoFullDir()).WillOnce(Return(buf));
+    EXPECT_CALL(*diskCacheRead_, GetCacheIoFullDir()).WillOnce(Return(buf));
+    EXPECT_CALL(*diskCacheRead_, LinkWriteToRead(_, _, _)).WillOnce(Return(0));
+    EXPECT_CALL(*diskCacheWrite_, AsyncUploadEnqueue(_)).WillOnce(Return());
+    diskCacheManagerImpl_->Enqueue(context);
+    sleep(5);
+}
+
 TEST_F(TestDiskCacheManagerImpl, Write) {
     std::string fileName = "test";
     std::string buf = "test";

curvefs/test/client/test_disk_cache_read.cpp

@@ -130,9 +130,9 @@ TEST_F(TestDiskCacheRead, LinkWriteToRead) {
 
 TEST_F(TestDiskCacheRead, LoadAllCacheFile) {
     EXPECT_CALL(*wrapper_, stat(NotNull(), NotNull())).WillOnce(Return(-1));
-    std::shared_ptr<SglLRUCache<std::string>> cachedObj;
-    cachedObj = std::make_shared<SglLRUCache<std::string>>(
-        0, std::make_shared<CacheMetrics>("diskcache"));
+    std::shared_ptr<LRUCache<std::string, bool>> cachedObj;
+    cachedObj =  std::make_shared<LRUCache<std::string, bool>>
+        (0, std::make_shared<CacheMetrics>("diskcache"));;
     int ret = diskCacheRead_->LoadAllCacheReadFile(cachedObj);
     ASSERT_EQ(-1, ret);
 
@@ -226,3 +226,4 @@ TEST_F(TestDiskCacheRead, ClearReadCache) {
 
 }  // namespace client
 }  // namespace curvefs
+