[feat] tools-v2: add snapshot copyset --all

Signed-off-by: montaguelhz <1443171175@qq.com>

.bazelrc

@@ -1,5 +1,10 @@
+build --verbose_failures
+
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
-build --incompatible_blacklisted_protos_requires_proto_info=false
+build --incompatible_blacklisted_protos_requires_proto_info=false
+build --copt=-fdiagnostics-color=always
+
+run --copt=-fdiagnostics-color=always

.clang-format

@@ -0,0 +1,147 @@
+---
+Language:        Cpp
+AccessModifierOffset: -3
+AlignAfterOpenBracket: Align
+AlignConsecutiveMacros: true
+AlignConsecutiveAssignments: false
+AlignConsecutiveBitFields: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   Align
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: true
+AllowAllConstructorInitializersOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortEnumsOnASingleLine: true
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortLambdasOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: MultiLine
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterCaseLabel:  false
+  AfterClass:      false
+  AfterControlStatement: Never
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  BeforeLambdaBody: false
+  BeforeWhile:     false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DeriveLineEnding: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority:        2
+    SortPriority:    0
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
+    SortPriority:    0
+  - Regex:           '.*'
+    Priority:        1
+    SortPriority:    0
+IncludeIsMainRegex: '(Test)?$'
+IncludeIsMainSourceRegex: ''
+IndentCaseLabels: false
+IndentCaseBlocks: false
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentExternBlock: AfterExternBlock
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+InsertTrailingCommas: None
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 2
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 2
+ObjCBreakBeforeNestedBlockParam: true
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    false
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInConditionalStatement: false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpaceBeforeSquareBrackets: false
+Standard:        Cpp11
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        4
+UseCRLF:         false
+UseTab:          Never
+WhitespaceSensitiveMacros:
+  - STRINGIZE
+  - PP_STRINGIZE
+  - BOOST_PP_STRINGIZE
+...

.clangd

@@ -0,0 +1,12 @@
+Diagnostics:
+  ClangTidy:
+    Add: [performance-*, modernize-*, readability-*,bugprone-*,]
+    Remove: [modernize-use-trailing-return-type]
+    CheckOptions:
+      readability-identifier-naming.VariableCase: camelBack
+  UnusedIncludes: Strict
+
+InlayHints:
+  Enabled: No
+  ParameterNames: No
+  DeducedTypes: No

.gitignore

@@ -78,6 +78,7 @@ runlog/
 .gdb_history
 
 .clang-format
+.clangd
 !curve-snapshotcloneserver-nginx/app/lib
 !nebd/nebd-package/usr/bin
 
@@ -90,6 +91,7 @@ test/client/configs/*
 projects/*
 docker/curvebs
 docker/base/*
+docker/debian*/*so*
 !docker/base/Dockerfile
 !docker/base/Makefile
 
@@ -107,6 +109,7 @@ curvefs/monitor/prometheus/target.json
 curvefs/docker/*/curvefs
 curvefs/docker/curvefs
 curvefs/docker/base/*
+curvefs/docker/debian*/*so*
 !curvefs/docker/base/Dockerfile
 !curvefs/docker/base/Makefile
 curvefs/BUILD_MODE
@@ -145,11 +148,21 @@ tools-v2/*/*.test
 tools-v2/__debug_bin
 tools-v2/vendor/
 
+.test
 .note
 .playground
+.dumpfile
+metastore_test.dat
 GPATH
 GRTAGS
 GTAGS
-curvefs/docker/debian9/libhashkit.so.2
-curvefs/docker/debian9/libmemcached.so.11
-curvefs/docker/debian9/libssl.so.3
+core.*
+
+test/integration/*.conf
+test/integration/client/config/client.conf*
+test/integration/snapshotcloneserver/config/*.conf
+
+.pre-commit-config.yaml
+
+*.deb
+*.whl

.obm.cfg

@@ -0,0 +1,2 @@
+container_name: curve-build-playground-master
+container_image: opencurvedocker/curve-base:build-debian9

CHANGELOG-2.5.md

@@ -0,0 +1,49 @@
+# CHANGELOG of v2.5
+
+Previous change logs can be found at [CHANGELOG-2.4](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.4.md)
+
+
+
+## New Features
+
+- Memcache support
+    - [curvefs/client: support memcached cluster](https://github.com/opencurve/curve/pull/2096) @ilixiaocui
+    - [curvefs/mds: support memcache cluster](https://github.com/opencurve/curve/pull/2108) @Cyber-SiKu
+    - [curvefs/client: add global cache client like memcached](https://github.com/opencurve/curve/pull/2102) @fansehep
+    - [curvefs/client: fix client core dump error](https://github.com/opencurve/curve/pull/2157) @ilixiaocui
+    - [add ut](https://github.com/opencurve/curve/pull/2164) @Cyber-SiKu
+
+- curvefs new tools 
+    - [add delete impl](https://github.com/opencurve/curve/pull/2088) @shentupenghui
+    - [feat: [tools-v2] bs list dir](https://github.com/opencurve/curve/pull/2082) @Sindweller
+    - [fix missing docs links](https://github.com/opencurve/curve/pull/2082) @zyb521
+    - [curve/toos-v2: add list client #2037](https://github.com/opencurve/curve/pull/2076) @tsonglew
+    - [support create volume directory by curve_ops_tool](https://github.com/opencurve/curve/pull/2078) @aspirer
+
+- [install: added playground.](https://github.com/opencurve/curve/pull/2053) @Wine93
+
+## Optimization
+
+- [update the braft to vesion v1.1.2](https://github.com/opencurve/curve/pull/2091) @tangwz
+
+- [curvefs/client: local cache policy optimization](https://github.com/opencurve/curve/pull/2064) @Tangruilin
+
+
+- [sync by threadpool](https://github.com/opencurve/curve/pull/1912) @fansehep
+
+- Merge block storage and file storage compilation scripts
+    - [Merge block storage and file storage compilation scripts
+](https://github.com/opencurve/curve/pull/2089) @linshiyx
+    - [Add help_msg in Makefile](https://github.com/opencurve/curve/pull/2133) @linshiyx
+    - [fix image.sh](https://github.com/opencurve/curve/pull/2124) @SeanHai
+
+- [fix bs list zone typo](https://github.com/opencurve/curve/pull/2066) @tsonglew 
+
+- [spell optimize](https://github.com/opencurve/curve/pull/2059) @shentupenghui
+
+
+## Bug Fix
+
+- aws s3 sdk revert
+    - [revert new s3 sdk due to critical bug](https://github.com/opencurve/curve/pull/2149) @h0hmj
+    - [curvefs/client: change the default value of s3](https://github.com/opencurve/curve/pull/2158) @wuhongsong

Makefile

@@ -1,6 +1,6 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build dep install image
+.PHONY: list build dep install image playground check test
 
 stor?=""
 prefix?= "$(PWD)/projects"
@@ -73,3 +73,9 @@ image:
 
 playground:
 	@bash util/playground.sh
+
+check:
+	@bash util/check.sh $(stor)
+
+test:
+	@bash util/test.sh $(stor) $(only)

README.md

@@ -4,7 +4,9 @@
 
 <div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
 
-**A could-native distributed storage system**
+**A cloud-native distributed storage system**
+
+**A sandbox project hosted by the CNCF Foundation**
 
 #### English | [简体中文](README_cn.md)
 ### 📄 [Documents](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [Official Website](https://www.opencurve.io/Curve/HOME) || 🏠 [Forum](https://ask.opencurve.io/t/topic/7)
@@ -42,11 +44,24 @@
   - [LICENSE](#license)
   - [Release Cycle](#release-cycle)
   - [Branch](#branch)
+  - [Contributors](#contributors)
+  - [Star History](#star-history)
   - [Feedback & Contact](#feedback--contact)
 
 ##  About Curve
 
-**Curve** is a mordern storage system developed by netease, currently supporting file storage(CurveFS) and block storage(CurveBS). Now it's hosted at [CNCF](https://www.cncf.io/) as a sandbox project.
+**Curve** is a modern storage system developed by netease, currently supporting file storage(CurveFS) and block storage(CurveBS). Now it's hosted at [CNCF](https://www.cncf.io/) as a sandbox project.
+
+The core application scenarios of CurveBS mainly include:
+- the performance, mixed, capacity cloud disk or persistent volume of virtual machine/container, and remote disks of physical machines
+- high-performance separation of storage and computation architecture: high-performance and low latency architecture based on RDMA+SPDK, supporting the separation deployment structure of various databases such as MySQL and Kafka
+
+
+The core application scenarios of CurveFS mainly include:
+- the cost-effective storage in AI training scene
+- the hot and cold data automation layered storage in big data scenarios
+- the cost-effective shared file storage on the public cloud: It can be used for business scenarios such as AI, big data, file sharing
+- Hybrid storage: Hot data is stored in the local IDC, cold data is stored in public cloud
 
 <details>
   <summary><b><font=5>High Performance | More stable | Easy Operation | Cloud Native</b></font></summary>
@@ -111,7 +126,7 @@
 
 </details>
 <details>
-  <summary><b><font=5>Docking Kubernates</b></font></summary>
+  <summary><b><font=5>Docking Kubernetes</b></font></summary>
 
 - Use [Curve CSI Driver](https://github.com/opencurve/curve-csi), The plugin implements the Container Storage Interface(CSI) between Container Orchestrator(CO) and Curve cluster. It allows dynamically provisioning curve volumes and attaching them to workloads.
 - For details of the documentation, see [CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md).
@@ -143,7 +158,7 @@ Curve supports deployment in private and public cloud environments, and can also
 <div align=center> <image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
 <div align=left>
 
-One of them, CurveFS shared file storage system, can be elasticly scaled to public cloud storage, which can provide users with greater capacity elasticity, lower cost, and better performance experience.
+One of them, CurveFS shared file storage system, can be elastically scaled to public cloud storage, which can provide users with greater capacity elasticity, lower cost, and better performance experience.
 
 </details>
 
@@ -199,6 +214,8 @@ Fio Curve engine is added, you can clone https://github.com/opencurve/fio and co
 $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
 ```
 
+If you have any questions during performance testing, please check the [Curve block storage performance tuning guide](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md).
+
 ##  CurveFS quick start
 Please use [CurveAdm](https://github.com/opencurve/curveadm/wiki) tool to deploy CurveFS，see [CurveFS Deployment Process](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), and the [CurveFS Command Instructions](curvefs/src/tools#readme).
 
@@ -208,12 +225,13 @@ Please refer to the [Test environment configuration](docs/cn/测试环境配置
 
 ## Practical
 - [CurveBS+NFS Build NFS Server](docs/practical/curvebs_nfs.md)
+- [CurveFS+MinIO S3 Gateway](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
 
 ## Governance
 See [Governance](https://github.com/opencurve/community/blob/master/GOVERNANCE.md).
 
 ##  Contribute us
-Participation in the Curve project is described in the [Curve Open Source Community Guidelines](Community_Guidelines.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
+Participation in the Curve project is described in the [Curve Developers Guidelines](developers_guide.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
 We welcome your contribution!
 
 ## Code of Conduct
@@ -230,6 +248,19 @@ Curve is distributed under the [**Apache 2.0 LICENSE**](LICENSE).
 ## Branch
 All the developments will be done under master branch. If there's any new version to establish, a new branch release-x.y will be pulled from the master, and the new version will be released from this branch.
 
+## Contributors
+This project exists thanks to all the people who contribute.
+
+<a href="https://github.com/opencurve/curve/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=opencurve/curve" />
+</a>
+
+Made with [contrib.rocks](https://contrib.rocks).
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=opencurve/curve&type=Date)](https://star-history.com/#opencurve/curve&Date)
+
 ## Feedback & Contact
 - [Github Issues](https://github.com/openCURVE/CURVE/issues)：You are sincerely welcomed to issue any bugs you came across or any suggestions through Github issues. If you have any question you can refer to our FAQ or join our user group for more details.
 - [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：Frequently asked question in our user group, and we'll keep working on it.

README_cn.md

@@ -4,7 +4,9 @@
 
 <div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
 
-**A could-native distributed storage system**
+**云原生高性能分布式存储系统**
+
+**CNCF基金会的沙箱托管项目**
 
 #### [English](README.md) | 简体中文
 ### 📄 [文档](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [官网](https://www.opencurve.io/Curve/HOME) || 🏠 [论坛](https://ask.opencurve.io/t/topic/7)
@@ -48,6 +50,18 @@
 ##  关于 Curve
 **Curve** 是网易主导自研的现代化存储系统, 目前支持文件存储(CurveFS)和块存储(CurveBS)。现作为沙箱项目托管于[CNCF](https://www.cncf.io/)。
 
+CurveBS的核心应用场景主要包括：
+- 虚拟机/容器的性能型、混合型、容量型云盘或持久化卷，以及物理机的远程存储盘
+- 高性能存算分离架构：基于RDMA+SPDK的高性能低时延架构，支撑MySQL、kafka等各类数据库、中间件的存算分离部署架构，提升实例交付效率和资源利用率
+
+
+CurveFS的核心应用场景主要包括：
+- AI训练（含机器学习等）场景下的高性价比存储
+- 大数据场景下的冷热数据自动化分层存储
+- 公有云上高性价比的共享文件存储：可用于AI、大数据、文件共享等业务场景
+- 混合云存储：热数据存储在本地IDC，冷数据存储在公有云
+
+
 <details>
   <summary><b><font=5>高性能 | 更稳定 | 易运维 | 云原生</b></font></summary>
 
@@ -112,7 +126,7 @@
 
 </details>
 <details>
-  <summary><b><font=5>对接 Kubernates</b></font></summary>
+  <summary><b><font=5>对接 Kubernetes</b></font></summary>
 
 - 使用 [Curve CSI Driver](https://github.com/opencurve/curve-csi) 插件在 Container Orchestrator (CO) 与 Curve 集群中实现了 Container Storage Interface(CSI)。
 - 文档详见[CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md)。
@@ -196,6 +210,8 @@ fio的Curve块存储引擎代码已经上传到 https://github.com/opencurve/fio
 $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
 ```
 
+在性能测试过程中有任何问题，请查看[Curve块存储性能调优指南](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md)
+
 ##  CurveFS 快速体验
 请使用 [CurveAdm](https://github.com/opencurve/curveadm/wiki) 工具进行 CurveFS 的部署，具体流程见：[CurveFS部署流程](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), 以及[CurveFS命令行工具说明](curvefs/src/tools#readme)。
 
@@ -207,10 +223,11 @@ $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest
 
 ##  贡献我们
 
-参与 Curve 项目开发详见[Curve 开源社区指南](Community_Guidelines_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
+参与 Curve 项目开发详见[Curve 开发者指南](developers_guide_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
 
 ## 最佳实践
 - [CurveBS+NFS搭建NFS存储](docs/practical/curvebs_nfs.md)
+- [CurveFS+S3网关部署实践](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
 
 ##  行为守则
 Curve 的行为守则遵循[CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md)。

WORKSPACE

@@ -75,11 +75,14 @@ bind(
 )
 
 #import the  gtest files.
-new_git_repository(
+http_archive(
     name = "com_google_googletest",
-    build_file = "//:thirdparties/gmock.BUILD",
-    remote = "https://github.com/google/googletest",
-    tag = "release-1.8.0",
+    urls = [
+        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
+        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
+    ],
+    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
+    strip_prefix = "googletest-release-1.12.1",
 )
 
 bind(

buildfs.sh

@@ -7,12 +7,22 @@ then
     exit
 fi
 
+if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
+then
+    bazelflags=''
+else
+    bazelflags='--copt -faligned-new'
+fi
+
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi
 
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
+--define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
+-DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
+
 if [ $? -ne 0 ]
 then
     echo "build curvefs failed"
@@ -34,4 +44,5 @@ then
     echo "mds_test failed"
     exit
 fi
-fi
+fi
+echo "end compile"

conf/chunkserver.conf

@@ -9,13 +9,20 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.block_size=4096
+
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +115,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/chunkserver.conf.example

@@ -13,9 +13,12 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +111,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/client.conf

@@ -168,13 +168,6 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
 
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
-
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@@ -188,4 +181,3 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
-

conf/cs_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/mds.conf

@@ -156,7 +156,7 @@ mds.topology.CreateCopysetRpcRetryTimes=20
 # 请求chunkserver上创建copyset重试间隔
 mds.topology.CreateCopysetRpcRetrySleepTimeMs=1000
 # Topology模块刷新metric时间间隔
-mds.topology.UpdateMetricIntervalSec=60
+mds.topology.UpdateMetricIntervalSec=10
 #和mds.chunkserver.failure.tolerance设置有关,一个zone 标准配置20台节点，如果允许3台节点failover,
 #那么剩余17台机器需要承载原先20台机器的空间,17/20=0.85,即使用量超过这个值即不再往这个池分配,
 #具体分为来两种情况, 当不使用chunkfilepool,物理池限制使用百分比,当使用 chunkfilepool 进行chunkfilepool分配时需预留failover空间,
@@ -197,6 +197,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -237,3 +239,16 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
+
+#
+## poolset rules
+#
+# for backward compatibility, rules are applied for select poolset when creating file
+#
+# for example
+#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
+#
+# when creating file reqeust doesn't have poolset, above rules are used to select poolset
+# - if filename is /dir1/file, then poolset1 is select
+# - if filename is /dir1/sub/file, then sub is select
+mds.poolset.rules=

conf/py_client.conf

@@ -153,10 +153,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/snap_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

curve-ansible/roles/generate_config/defaults/main.yml

@@ -164,7 +164,6 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
-chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -244,8 +243,6 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
-client_alignment_common: 512
-client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -14,8 +14,6 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
-# minimum alignment for io request
-global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -175,10 +175,3 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume={{ client_alignment_common }}
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume={{ client_alignment_clone }}

curvefs/conf/client.conf

@@ -104,23 +104,52 @@ fuseClient.maxDataSize=1024
 fuseClient.refreshDataIntervalSec=30
 fuseClient.warmupThreadsNum=10
 
+# the write throttle bps of fuseClient, default no limit
+fuseClient.throttle.avgWriteBytes=0
+# the write burst bps of fuseClient, default no limit
+fuseClient.throttle.burstWriteBytes=0
+# the times that write burst bps can continue, default 180s
+fuseClient.throttle.burstWriteBytesSecs=180
+
+# the write throttle iops of fuseClient, default no limit
+fuseClient.throttle.avgWriteIops=0
+# the write burst iops of fuseClient, default no limit
+fuseClient.throttle.burstWriteIops=0
+# the times that write burst Iops can continue, default 180s
+fuseClient.throttle.burstWriteIopsSecs=180
+
+# the read throttle bps of fuseClient, default no limit
+fuseClient.throttle.avgReadBytes=0
+# the read burst bps of fuseClient, default no limit
+fuseClient.throttle.burstReadBytes=0
+# the times that read burst bps can continue, default 180s
+fuseClient.throttle.burstReadBytesSecs=180
+
+# the read throttle iops of fuseClient, default no limit
+fuseClient.throttle.avgReadIops=0
+# the read burst Iops of fuseClient, default no limit
+fuseClient.throttle.burstReadIops=0
+# the times that read burst Iops can continue, default 180s
+fuseClient.throttle.burstReadIopsSecs=180
+
 #### filesystem metadata
 # {
 # fs.disableXattr:
-#   if you want to get curvefs specified xattr,
-#   you can mount another fs with |fs.disableXattr| is true
+#   if you want to get better metadata performance,
+#   you can mount fs with |fs.disableXattr| is true
 #
-# fs.lookupCache.negativeTimeout:
+# fs.lookupCache.negativeTimeoutSec:
 #   entry which not found will be cached if |timeout| > 0
 fs.cto=true
 fs.maxNameLength=255
-fs.disableXattr=true
+fs.disableXattr=false
 fs.accessLogging=true
-fs.kernelCache.attrTimeout=3600
-fs.kernelCache.dirAttrTimeout=3600
-fs.kernelCache.entryTimeout=3600
-fs.kernelCache.dirEntryTimeout=3600
-fs.lookupCache.negativeTimeout=0
+fs.kernelCache.attrTimeoutSec=3600
+fs.kernelCache.dirAttrTimeoutSec=3600
+fs.kernelCache.entryTimeoutSec=3600
+fs.kernelCache.dirEntryTimeoutSec=3600
+fs.lookupCache.negativeTimeoutSec=0
+fs.lookupCache.minUses=1
 fs.lookupCache.lruSize=100000
 fs.dirCache.lruSize=5000000
 fs.openFile.lruSize=65536
@@ -151,8 +180,6 @@ volume.blockGroup.allocateOnce=4
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false
-# the max size that fuse send
-s3.fuseMaxSize=131072
 s3.pageSize=65536
 # prefetch blocks that disk cache use
 s3.prefetchBlocks=1
@@ -167,8 +194,11 @@ s3.baseSleepUs=500
 s3.threadScheduleInterval=3
 # data cache flush wait time
 s3.cacheFlushIntervalSec=5
+# write cache < 8,388,608 (8MB) is not allowed
 s3.writeCacheMaxByte=838860800
 s3.readCacheMaxByte=209715200
+# file cache read thread num
+s3.readCacheThreads=5
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=False

curvefs/conf/mds.conf

@@ -6,6 +6,9 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
 mds.loglevel=0
+# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+mds.server.idleTimeoutSec=-1
 
 #
 # space options

curvefs/conf/metaserver.conf

@@ -128,16 +128,25 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000
 
-### apply queue options for each copyset
-### apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding worker queue by certain rules
-# number of apply queue workers for each, each worker will start a indepent thread
-applyqueue.worker_count=3
+#
+# Concurrent apply queue
+### concurrent apply queue options for each copyset
+### concurrent apply queue is used to isolate raft threads, each worker has its own queue
+### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
 
-# apply queue depth for each copyset
+# worker_count: number of apply queue workers for each, each worker will start a indepent thread
+# queue_depth: apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-applyqueue.queue_depth=1
+# write apply queue workers count
+applyqueue.write_worker_count=3
+# write apply queue depth
+applyqueue.write_queue_depth=1
+# read apply queue workers count
+applyqueue.read_worker_count=2
+# read apply queue depth
+applyqueue.read_queue_depth=1
+
 
 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@@ -145,6 +154,10 @@ applyqueue.queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto
 
+# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+server.idleTimeoutSec=-1
+
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need
@@ -267,3 +280,43 @@ storage.rocksdb.perf_sampling_ratio=0
 # we will sending its with rpc streaming instead of
 # padding its into inode (default: 25000, about 25000 * 41 (byte) = 1MB)
 storage.s3_meta_inside_inode.limit_size=25000
+
+# recycle options
+# metaserver scan recycle period, default 1h
+recycle.manager.scanPeriodSec=3600
+# metaserver recycle cleaner scan list dentry limit, default 1000
+recycle.cleaner.scanLimit=1000
+
+#### excutorOpt
+# excutorOpt rpc with metaserver
+# rpc retry times with metaserver
+excutorOpt.maxRetry=4294967295
+# internal rpc retry times with metaserver
+excutorOpt.maxInternalRetry = 3
+# Retry sleep time between failed RPCs
+excutorOpt.retryIntervalUS=100000
+# RPC timeout for communicating with metaserver
+excutorOpt.rpcTimeoutMS=1000
+# RPC stream idle timeout
+excutorOpt.rpcStreamIdleTimeoutMS=500
+# The maximum timeout RPC time of the retry request.
+# The timeout time will follow the exponential backoff policy.
+# Because the timeout occurs when the network is congested, the RPC timeout needs to be increased
+excutorOpt.maxRPCTimeoutMS=8000
+# Maximum sleep time between retry requests.
+# when the network is congested or the metaserver is overloaded,
+# it is necessary to increase the sleep time
+excutorOpt.maxRetrySleepIntervalUS=8000000
+excutorOpt.minRetryTimesForceTimeoutBackoff=5
+excutorOpt.maxRetryTimesBeforeConsiderSuspend=20
+# batch limit of get inode attr and xattr
+excutorOpt.batchInodeAttrLimit=10000
+excutorOpt.enableMultiMountPointRename=true
+
+#### metaCacheOpt
+# Gets the number of retries for the leader
+metaCacheOpt.metacacheGetLeaderRetry=3
+# Need to sleep for a period of time before each get leader retry
+metaCacheOpt.metacacheRPCRetryIntervalUS=100000
+# RPC timeout of get leader
+metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000

curvefs/conf/tools.conf

@@ -24,6 +24,7 @@ volumeName=volume
 volumeUser=user
 volumePassword=password
 volumeBlockGroupSize=134217728
+volumeCluster=127.0.0.1:6666,127.0.0.1:6667,127.0.0.1:6668
 # support |AtStart| and |AtEnd|
 volumeBitmapLocation=AtStart
 # s3
@@ -34,5 +35,12 @@ s3.bucket_name=bucket
 s3.blocksize=4194304
 s3.chunksize=67108864
 s3.useVirtualAddressing=false
+# s3 objectPrefix, if set 0, means no prefix, if set 1, means inode prefix
+# if set 2 and other values mean hash prefix
+s3.objectPrefix=0
 # statistic info in xattr, hardlink will not be supported when enable
 enableSumInDir=true
+
+# fs recycle, if set 0, disable fs recycle, delete files directly,
+# if set not 0, enable fs recycle, delete files after a period of time
+recycleTimeHour=0

curvefs/docker/debian10/Dockerfile

@@ -1,11 +0,0 @@
-FROM opencurvedocker/curve-base:debian10
-ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
-COPY curvefs /curvefs
-COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libmemcached.so /usr/lib/
-COPY libmemcached.so.11 /usr/lib/
-COPY libhashkit.so.2 /usr/lib
-RUN chmod a+x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian10/base/Dockerfile

@@ -1,39 +0,0 @@
-FROM opencurvedocker/curve-base:debian10
-
-RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        gcc \
-        gdb \
-        make \
-        openssl \
-        net-tools \
-        libcurl3-gnutls \
-        perl \
-        linux-perf \
-        vim \
-        curl \
-        cron \
-        procps \
-        lsof \
-        nginx \
-        less \
-        fuse3 \
-        libnl-3-200 \
-        libnl-genl-3-200 \
-        libjemalloc2 \
-        libsnappy1v5 \
-        liblz4-1 \
-        zlib1g \
-    && ln -sf /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
-
-COPY libetcdclient.so /usr/lib/

curvefs/docker/debian10/base/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:debian10 .

curvefs/docker/debian10/compile/Dockerfile

@@ -1,52 +0,0 @@
-FROM opencurvedocker/curve-base:build-debian10
-
-RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        wget \
-        git \
-        gcc \
-        g++ \
-        clang \
-        libssl-dev \
-        libnl-genl-3-dev \
-        libcurl4-gnutls-dev \
-        uuid-dev \
-        libfiu-dev \
-        libfuse3-dev \
-        zlib1g-dev \
-        make \
-        openjdk-11-jdk \
-        sudo \
-        zlib1g-dev \
-        libsnappy-dev \
-        liblz4-dev \
-        cmake \
-        ca-certificates \
-        curl \
-        gnupg \
-        lsb-release \
-        curl \
-        git \
-    && mkdir -p /etc/apt/keyrings \
-    && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
-    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
-  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
-    && apt-get update \
-    && apt-get install -y docker-ce docker-ce-cli containerd.io \
-    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
-    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
-    && ln -sf /usr/bin/bazelisk /usr/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
-    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
-    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk

curvefs/docker/debian10/compile/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:build-debian10 .

curvefs/docker/debian10/entrypoint.sh

@@ -1,126 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-############################  GLOBAL VARIABLES
-g_role=""
-g_args=""
-g_prefix=""
-g_binary=""
-g_start_args=""
-
-############################  BASIC FUNCTIONS
-function msg() {
-    printf '%b' "$1" >&2
-}
-
-function success() {
-    msg "\33[32m[✔]\33[0m ${1}${2}"
-}
-
-function die() {
-    msg "\33[31m[✘]\33[0m ${1}${2}"
-    exit 1
-}
-
-############################ FUNCTIONS
-function usage () {
-    cat << _EOC_
-Usage:
-    entrypoint.sh --role=ROLE
-    entrypoint.sh --role=ROLE --args=ARGS
-
-Examples:
-    entrypoint.sh --role=etcd
-    entrypoint.sh --role=client --args="-o default_permissions"
-_EOC_
-}
-
-function get_options() {
-    local long_opts="role:,args:,help"
-    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
-    eval set -- "${args}"
-    while true
-    do
-        case "$1" in
-            -r|--role)
-                g_role=$2
-                shift 2
-                ;;
-            -a|--args)
-                g_args=$2
-                shift 2
-                ;;
-            -h)
-                usage
-                exit 1
-                ;;
-            --)
-                shift
-                break
-                ;;
-            *)
-                exit 1
-                ;;
-        esac
-    done
-}
-
-function prepare() {
-    g_prefix="/curvefs/$g_role"
-    conf_path="$g_prefix/conf/$g_role.conf"
-
-    case $g_role in
-        etcd)
-            g_binary="$g_prefix/sbin/etcd"
-            g_start_args="--config-file $conf_path"
-            ;;
-        mds)
-            g_binary="$g_prefix/sbin/curvefs-mds"
-            g_start_args="--confPath $conf_path"
-            ;;
-        metaserver)
-            g_binary="$g_prefix/sbin/curvefs-metaserver"
-            g_start_args="--confPath $conf_path"
-            ;;
-        client)
-            g_binary="$g_prefix/sbin/curve-fuse"
-            g_start_args="--confPath $conf_path"
-            ;;
-        *)
-            usage
-            exit 1
-            ;;
-    esac
-
-    if [ "$g_args" != "" ]; then
-        g_start_args=$g_args
-    fi
-}
-
-function create_directory() {
-    chmod 700 "$g_prefix/data"
-    if [ "$g_role" == "etcd" ]; then
-        mkdir -p "$g_prefix/data/wal"
-    elif [ "$g_role" == "metaserver" ]; then
-        mkdir -p "$g_prefix/data/storage"
-    elif [ "$g_role" == "client" ]; then
-        mkdir -p "$g_prefix/mnt"
-    fi
-}
-
-function main() {
-    get_options "$@"
-
-    prepare
-    create_directory
-    [[ $(command -v crontab) ]] && cron
-    if [ $g_role == "etcd" ]; then
-        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
-    else
-        exec $g_binary $g_start_args
-    fi
-}
-
-############################  MAIN()
-main "$@"

curvefs/docker/debian11/Dockerfile

@@ -1,11 +0,0 @@
-FROM opencurvedocker/curve-base:debian11
-ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
-COPY curvefs /curvefs
-COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libmemcached.so /usr/lib/
-COPY libmemcached.so.11 /usr/lib/
-COPY libhashkit.so.2 /usr/lib
-RUN chmod a+x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian11/base/Dockerfile

@@ -1,39 +0,0 @@
-FROM opencurvedocker/curve-base:debian11
-
-RUN echo "deb http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        gcc \
-        gdb \
-        make \
-        openssl \
-        net-tools \
-        libcurl3-gnutls \
-        perl \
-        linux-perf \
-        vim \
-        curl \
-        cron \
-        procps \
-        lsof \
-        nginx \
-        less \
-        fuse3 \
-        libnl-3-200 \
-        libnl-genl-3-200 \
-        libjemalloc2 \
-        libsnappy1v5 \
-        liblz4-1 \
-        zlib1g
-    && ln -sf /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
-
-COPY libetcdclient.so /usr/lib/

curvefs/docker/debian11/base/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:debian11 .

curvefs/docker/debian11/compile/Dockerfile

@@ -1,51 +0,0 @@
-FROM opencurvedocker/curve-base:build-debian11 
-
-RUN echo "deb http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        wget \
-        git \
-        gcc \
-        g++ \
-        clang \
-        libssl-dev \
-        libnl-genl-3-dev \
-        libcurl4-gnutls-dev \
-        uuid-dev \
-        libfiu-dev \
-        libfuse3-dev \
-        zlib1g-dev \
-        make \
-        openjdk-11-jdk \
-        sudo \
-        libsnappy-dev \
-        liblz4-dev \
-        cmake \
-        ca-certificates \
-        curl \
-        gnupg \
-        lsb-release \
-        curl \
-        git \
-    && mkdir -p /etc/apt/keyrings \
-    # && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
-    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
-  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
-    && apt-get update \
-    && apt-get install -y docker-ce docker-ce-cli containerd.io \
-    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
-    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
-    && ln -sf /usr/bin/bazelisk /usr/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
-    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
-    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk

curvefs/docker/debian11/compile/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:build-debian11 .

curvefs/docker/debian11/entrypoint.sh

@@ -1,126 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-############################  GLOBAL VARIABLES
-g_role=""
-g_args=""
-g_prefix=""
-g_binary=""
-g_start_args=""
-
-############################  BASIC FUNCTIONS
-function msg() {
-    printf '%b' "$1" >&2
-}
-
-function success() {
-    msg "\33[32m[✔]\33[0m ${1}${2}"
-}
-
-function die() {
-    msg "\33[31m[✘]\33[0m ${1}${2}"
-    exit 1
-}
-
-############################ FUNCTIONS
-function usage () {
-    cat << _EOC_
-Usage:
-    entrypoint.sh --role=ROLE
-    entrypoint.sh --role=ROLE --args=ARGS
-
-Examples:
-    entrypoint.sh --role=etcd
-    entrypoint.sh --role=client --args="-o default_permissions"
-_EOC_
-}
-
-function get_options() {
-    local long_opts="role:,args:,help"
-    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
-    eval set -- "${args}"
-    while true
-    do
-        case "$1" in
-            -r|--role)
-                g_role=$2
-                shift 2
-                ;;
-            -a|--args)
-                g_args=$2
-                shift 2
-                ;;
-            -h)
-                usage
-                exit 1
-                ;;
-            --)
-                shift
-                break
-                ;;
-            *)
-                exit 1
-                ;;
-        esac
-    done
-}
-
-function prepare() {
-    g_prefix="/curvefs/$g_role"
-    conf_path="$g_prefix/conf/$g_role.conf"
-
-    case $g_role in
-        etcd)
-            g_binary="$g_prefix/sbin/etcd"
-            g_start_args="--config-file $conf_path"
-            ;;
-        mds)
-            g_binary="$g_prefix/sbin/curvefs-mds"
-            g_start_args="--confPath $conf_path"
-            ;;
-        metaserver)
-            g_binary="$g_prefix/sbin/curvefs-metaserver"
-            g_start_args="--confPath $conf_path"
-            ;;
-        client)
-            g_binary="$g_prefix/sbin/curve-fuse"
-            g_start_args="--confPath $conf_path"
-            ;;
-        *)
-            usage
-            exit 1
-            ;;
-    esac
-
-    if [ "$g_args" != "" ]; then
-        g_start_args=$g_args
-    fi
-}
-
-function create_directory() {
-    chmod 700 "$g_prefix/data"
-    if [ "$g_role" == "etcd" ]; then
-        mkdir -p "$g_prefix/data/wal"
-    elif [ "$g_role" == "metaserver" ]; then
-        mkdir -p "$g_prefix/data/storage"
-    elif [ "$g_role" == "client" ]; then
-        mkdir -p "$g_prefix/mnt"
-    fi
-}
-
-function main() {
-    get_options "$@"
-
-    prepare
-    create_directory
-    [[ $(command -v crontab) ]] && cron
-    if [ $g_role == "etcd" ]; then
-        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
-    else
-        exec $g_binary $g_start_args
-    fi
-}
-
-############################  MAIN()
-main "$@"

curvefs/docker/debian9/Dockerfile

@@ -1,9 +1,8 @@
 FROM opencurvedocker/curve-base:debian9
-ENV TZ=Asia/Shanghai
 COPY entrypoint.sh /
 COPY curvefs /curvefs
 COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 /usr/lib/
 RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \
     && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \
-    && cp curvefs/tools-v2/sbin/curve /usr/bin/
+    && cp /curvefs/tools-v2/sbin/curve /usr/bin/
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian9/base/Dockerfile

@@ -1,38 +0,0 @@
-FROM opencurvedocker/curve-base:debian9
-
-RUN echo "deb http://mirrors.163.com/debian/ stretch main\n" \
-         "deb http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ stretch/updates main non-free contrib\n" \
-         "deb http://httpredir.debian.org/debian stretch-backports main contrib non-free\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        gcc \
-        gdb \
-        make \
-        openssl \
-        net-tools \
-        libcurl3-gnutls \
-        podlators-perl \
-        linux-perf \
-        vim \
-        curl \
-        cron \
-        procps \
-        lsof \
-        nginx \
-        less \
-        zlib1g \
-        libsnappy1v5 \
-        liblz4-1 \
-        python \
-        wget \
-        python3 \
-    && wget https://curve-build.nos-eastchina1.126.net/curve-base.tar.gz \
-    && tar -xzvf curve-base.tar.gz
-COPY fusermount3 /usr/local/bin
-COPY libetcdclient.so /usr/lib/
-COPY libfuse3.so libfuse3.so.3 libfuse3.so.3.10.5 libnl-3.so.200 libnl-genl-3.so.200 /usr/lib/x86_64-linux-gnu/
-COPY libjemalloc.so libjemalloc.so.1 libjemalloc.so.2 /usr/local/lib/

curvefs/docker/debian9/base/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:debian9 .

curvefs/docker/debian9/compile/Dockerfile

@@ -1,50 +0,0 @@
-FROM opencurvedocker/curve-base:build-debian9 
-
-RUN apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        g++ \
-        wget \
-        clang \
-        git \
-        libnl-genl-3-dev \
-        libssl-dev \
-        uuid-dev \
-        libcurl4-gnutls-dev \
-        libfiu-dev \
-        openjdk-8-jdk \
-        sudo \
-        python-dev \
-        python-setuptools \
-        python-wheel \
-        python3-dev \
-        python3-setuptools \
-        python3-wheel \
-        zlib1g-dev \
-        libsnappy-dev \
-        liblz4-dev \
-        cmake \
-        ca-certificates \
-        curl \
-        gnupg \
-        lsb-release \
-        curl \
-        apt-transport-https \
-        git \
-    && mkdir -p /etc/apt/keyrings \
-    && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
-    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
-  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
-    && apt-get update \
-    && apt-get install -y docker-ce docker-ce-cli containerd.io \
-    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
-    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
-    && ln -sf /usr/bin/bazelisk /usr/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
-    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/0.17.2/bazel-0.17.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin \
-    && mv bazel-0.17.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel \
-    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk
-COPY fuse3/* /usr/include/fuse3/

curvefs/docker/debian9/compile/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:build-debian9 .

curvefs/docker/debian9/entrypoint.sh

@@ -8,6 +8,7 @@ g_args=""
 g_prefix=""
 g_binary=""
 g_start_args=""
+g_preexec="/curvefs/tools-v2/sbin/daemon"
 
 ############################  BASIC FUNCTIONS
 function msg() {
@@ -119,6 +120,7 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
+    [[ ! -z $g_preexec ]] && $g_preexec &
     if [ $g_role == "etcd" ]; then
         exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
     elif [ $g_role == "monitor" ]; then

curvefs/proto/common.proto

@@ -58,6 +58,7 @@ message S3Info {
     required string bucketname = 4;
     required uint64 blockSize = 5;
     required uint64 chunkSize = 6;
+    optional uint32 objectPrefix = 7;
 }
 
 enum PartitionStatus {
@@ -83,6 +84,7 @@ message PartitionInfo {
     optional uint64 inodeNum = 10;  // hearbeat upload this value to mds topo, topo/metaserver needn't persist this value
     optional uint64 dentryNum = 11;  // hearbeat upload this value to mds topo, topo/metaserver needn't persist this value
     map<int32, uint64> fileType2inodeNum = 12;
+    optional bool manageFlag = 13; // if a partition has recyclebin inode, set this flag true
 }
 
 message Peer {

curvefs/proto/mds.proto

@@ -56,6 +56,10 @@ enum FSStatusCode {
     VOLUME_INFO_ERROR = 31;
     MOUNT_POINT_CONFLICT = 32;
     FSNAME_INVALID = 33;
+    INSERT_DENTRY_FAIL = 34;
+    INSERT_MANAGE_INODE_FAIL = 35;
+    DELETE_DENTRY_FAIL = 36;
+    UPDATE_FS_FAIL = 37;
 }
 
 // fs interface
@@ -97,6 +101,7 @@ message FsInfo {
     required string owner = 12;
     optional uint64 txSequence = 13;
     optional string txOwner = 14;
+    optional uint32 recycleTimeHour =15;
 }
 
 message GetFsInfoResponse {
@@ -112,6 +117,7 @@ message CreateFsRequest {
     required bool enableSumInDir = 5;
     required string owner = 6;
     required uint64 capacity = 7;
+    optional uint32 recycleTimeHour = 8;
 }
 
 message CreateFsResponse {
@@ -185,6 +191,7 @@ message RefreshSessionRequest {
 message RefreshSessionResponse {
     required FSStatusCode statusCode = 1;
     repeated topology.PartitionTxId latestTxIdList = 2;
+    optional bool enableSumInDir = 3;
 }
 
 message DLockValue {

curvefs/proto/metaserver.proto

@@ -272,6 +272,28 @@ message CreateRootInodeResponse {
     optional uint64 appliedIndex = 2;
 }
 
+enum ManageInodeType {
+    TYPE_NOT_MANAGE = 0;
+    TYPE_RECYCLE = 1;
+};
+
+message CreateManageInodeRequest {
+    required uint32 poolId = 1;
+    required uint32 copysetId = 2;
+    required uint32 partitionId = 3;
+    required uint32 fsId = 4;
+    required uint32 uid = 5;
+    required uint32 gid = 6;
+    required uint32 mode = 7;
+    required ManageInodeType manageType = 8;
+}
+
+message CreateManageInodeResponse {
+    required MetaStatusCode statusCode = 1;
+    optional Inode inode = 2;
+    optional uint64 appliedIndex = 3;
+}
+
 message UpdateInodeRequest {
     required uint32 poolId = 1;
     required uint32 copysetId = 2;
@@ -474,6 +496,7 @@ service MetaServerService {
     rpc DeleteInode(DeleteInodeRequest) returns (DeleteInodeResponse);
     rpc CreateRootInode(CreateRootInodeRequest) returns
                                             (CreateRootInodeResponse);
+    rpc CreateManageInode(CreateManageInodeRequest) returns (CreateManageInodeResponse);
     rpc GetOrModifyS3ChunkInfo(GetOrModifyS3ChunkInfoRequest) returns (GetOrModifyS3ChunkInfoResponse);
     rpc BatchGetInodeAttr(BatchGetInodeAttrRequest) returns (BatchGetInodeAttrResponse);
     rpc BatchGetXAttr(BatchGetXAttrRequest) returns (BatchGetXAttrResponse);

curvefs/src/client/BUILD

@@ -76,6 +76,7 @@ cc_library(
         "//curvefs/src/common:threading",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/synchronization",
         "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",

curvefs/src/client/async_request_closure.cpp

@@ -101,7 +101,7 @@ void UpdateInodeAttrAndExtentClosure::Run() {
                    << ", inode: " << inode_->GetInodeId();
         inode_->MarkInodeError();
     }
-
+    inode_->ClearDirty();
     inode_->syncingVolumeExtentsMtx_.unlock();
     inode_->ReleaseSyncingInode();
 

curvefs/src/client/client_operator.cpp

@@ -397,6 +397,27 @@ CURVEFS_ERROR RenameOperator::UpdateInodeParent() {
     return rc;
 }
 
+CURVEFS_ERROR RenameOperator::UpdateInodeCtime() {
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    auto rc = inodeManager_->GetInode(srcDentry_.inodeid(), inodeWrapper);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG_ERROR("GetInode", rc);
+        return rc;
+    }
+
+    curve::common::UniqueLock lk = inodeWrapper->GetUniqueLock();
+    inodeWrapper->UpdateTimestampLocked(kChangeTime);
+
+    rc = inodeWrapper->SyncAttr();
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG_ERROR("UpdateInodeCtime", rc);
+        return rc;
+    }
+
+    LOG(INFO) << "UpdateInodeCtime inodeid = " << srcDentry_.inodeid();
+    return rc;
+}
+
 void RenameOperator::UpdateCache() {
     SetTxId(srcPartitionId_, srcTxId_ + 1);
     SetTxId(dstPartitionId_, dstTxId_ + 1);

curvefs/src/client/client_operator.h

@@ -59,6 +59,7 @@ class RenameOperator {
     CURVEFS_ERROR UnlinkSrcParentInode();
     void UnlinkOldInode();
     CURVEFS_ERROR UpdateInodeParent();
+    CURVEFS_ERROR UpdateInodeCtime();
     void UpdateCache();
 
     void GetOldInode(uint64_t *oldInodeId, int64_t *oldInodeSize,

curvefs/src/client/common/common.cpp

@@ -22,6 +22,8 @@
  */
 
 #include "curvefs/src/client/common/common.h"
+#include "src/common/string_util.h"
+
 namespace curvefs {
 namespace client {
 namespace common {
@@ -103,6 +105,59 @@ WarmupType GetWarmupType(const std::string& type) {
     return ret;
 }
 
+const char kCurveFsWarmupStorageDisk[] = "disk";
+const char kCurveFsWarmupStorageKvclient[] = "kvclient";
+
+WarmupStorageType GetWarmupStorageType(const std::string &type) {
+    auto ret = WarmupStorageType::kWarmupStorageTypeUnknown;
+    if (type == kCurveFsWarmupStorageDisk) {
+        ret = WarmupStorageType::kWarmupStorageTypeDisk;
+    } else if (type == kCurveFsWarmupStorageKvclient) {
+        ret = WarmupStorageType::kWarmupStorageTypeKvClient;
+    }
+    return ret;
+}
+
+using ::curve::common::StringToUll;
+
+// if direction is true means '+', false means '-'
+// is direction is true, add second to first
+// if direction is false, sub second from first
+bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction) {
+    uint64_t firstNum = 0;
+    uint64_t secondNum = second;
+    if (StringToUll(*first, &firstNum)) {
+        if (direction) {
+            *first = std::to_string(firstNum + secondNum);
+        } else {
+            if (firstNum < secondNum) {
+                *first = std::to_string(0);
+                LOG(WARNING) << "AddUllStringToFirst failed when minus,"
+                             << " first = " << firstNum
+                             << ", second = " << secondNum;
+                return false;
+            }
+            *first = std::to_string(firstNum - secondNum);
+        }
+    } else {
+        LOG(ERROR) << "StringToUll failed, first = " << *first
+                   << ", second = " << second;
+        return false;
+    }
+    return true;
+}
+
+bool AddUllStringToFirst(uint64_t *first, const std::string &second) {
+    uint64_t secondNum = 0;
+    if (StringToUll(second, &secondNum)) {
+        *first += secondNum;
+        return true;
+    }
+
+    LOG(ERROR) << "StringToUll failed, second = " << second;
+    return false;
+}
+
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/common/common.h

@@ -59,6 +59,7 @@ enum class MetaServerOpType {
     GetOrModifyS3ChunkInfo,
     GetVolumeExtent,
     UpdateVolumeExtent,
+    CreateManageInode,
 };
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);
@@ -68,6 +69,9 @@ const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024;
 
 const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op";
 
+
+constexpr int kWarmupOpNum = 4;
+
 enum class WarmupOpType {
     kWarmupOpUnknown = 0,
     kWarmupOpAdd = 1,
@@ -82,7 +86,15 @@ enum class WarmupType {
     kWarmupTypeSingle = 2,
 };
 
-WarmupType GetWarmupType(const std::string& type);
+WarmupType GetWarmupType(const std::string &type);
+
+enum class WarmupStorageType {
+    kWarmupStorageTypeUnknown = 0,
+    kWarmupStorageTypeDisk = 1,
+    kWarmupStorageTypeKvClient = 2,
+};
+
+WarmupStorageType GetWarmupStorageType(const std::string &type);
 
 enum class FileHandle : uint64_t {
     kDefaultValue = 0,
@@ -94,6 +106,11 @@ enum class NlinkChange : int32_t {
     kSubOne = -1,
 };
 
+// if direction is true means '+', false means '-'
+// is direction is true, add second to first
+// if direction is false, sub second from first
+bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction);
+bool AddUllStringToFirst(uint64_t *first, const std::string &second);
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/common/config.cpp

@@ -52,6 +52,55 @@ DEFINE_bool(useFakeS3, false,
 DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing");
 DEFINE_bool(access_logging, true, "enable access log");
 
+/**
+ * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true
+ * for dynamic parameter configuration
+ */
+static bool pass_uint64(const char *, uint64_t) { return true; }
+
+DEFINE_uint64(fuseClientAvgWriteBytes, 0,
+              "the write throttle bps of fuse client");
+DEFINE_validator(fuseClientAvgWriteBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteBytes, 0,
+              "the write burst bps of fuse client");
+DEFINE_validator(fuseClientBurstWriteBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteBytesSecs, 180,
+              "the times that write burst bps can continue");
+DEFINE_validator(fuseClientBurstWriteBytesSecs, &pass_uint64);
+
+
+DEFINE_uint64(fuseClientAvgWriteIops, 0,
+              "the write throttle iops of fuse client");
+DEFINE_validator(fuseClientAvgWriteIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteIops, 0,
+              "the write burst iops of fuse client");
+DEFINE_validator(fuseClientBurstWriteIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteIopsSecs, 180,
+              "the times that write burst iops can continue");
+DEFINE_validator(fuseClientBurstWriteIopsSecs, &pass_uint64);
+
+
+DEFINE_uint64(fuseClientAvgReadBytes, 0,
+              "the Read throttle bps of fuse client");
+DEFINE_validator(fuseClientAvgReadBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadBytes, 0,
+              "the Read burst bps of fuse client");
+DEFINE_validator(fuseClientBurstReadBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadBytesSecs, 180,
+              "the times that Read burst bps can continue");
+DEFINE_validator(fuseClientBurstReadBytesSecs, &pass_uint64);
+
+
+DEFINE_uint64(fuseClientAvgReadIops, 0,
+              "the Read throttle iops of fuse client");
+DEFINE_validator(fuseClientAvgReadIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadIops, 0,
+              "the Read burst iops of fuse client");
+DEFINE_validator(fuseClientBurstReadIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadIopsSecs, 180,
+              "the times that Read burst iops can continue");
+DEFINE_validator(fuseClientBurstReadIopsSecs, &pass_uint64);
+
 void InitMdsOption(Configuration *conf, MdsOption *mdsOpt) {
     conf->GetValueFatalIfFail("mdsOpt.mdsMaxRetryMS", &mdsOpt->mdsMaxRetryMS);
     conf->GetValueFatalIfFail("mdsOpt.rpcRetryOpt.maxRPCTimeoutMS",
@@ -158,8 +207,6 @@ void InitDiskCacheOption(Configuration *conf,
 
 void InitS3Option(Configuration *conf, S3Option *s3Opt) {
     conf->GetValueFatalIfFail("s3.fakeS3", &FLAGS_useFakeS3);
-    conf->GetValueFatalIfFail("s3.fuseMaxSize",
-                              &s3Opt->s3ClientAdaptorOpt.fuseMaxSize);
     conf->GetValueFatalIfFail("s3.pageSize",
                               &s3Opt->s3ClientAdaptorOpt.pageSize);
     conf->GetValueFatalIfFail("s3.prefetchBlocks",
@@ -176,6 +223,8 @@ void InitS3Option(Configuration *conf, S3Option *s3Opt) {
                               &s3Opt->s3ClientAdaptorOpt.writeCacheMaxByte);
     conf->GetValueFatalIfFail("s3.readCacheMaxByte",
                               &s3Opt->s3ClientAdaptorOpt.readCacheMaxByte);
+    conf->GetValueFatalIfFail("s3.readCacheThreads",
+                              &s3Opt->s3ClientAdaptorOpt.readCacheThreads);
     conf->GetValueFatalIfFail("s3.nearfullRatio",
                               &s3Opt->s3ClientAdaptorOpt.nearfullRatio);
     conf->GetValueFatalIfFail("s3.baseSleepUs",
@@ -250,36 +299,43 @@ void InitFileSystemOption(Configuration* c, FileSystemOption* option) {
     c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr);
     c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength);
     c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging);
-    { // kernel cache option
+    {  // kernel cache option
         auto o = &option->kernelCacheOption;
-        c->GetValueFatalIfFail("fs.kernelCache.attrTimeout", &o->attrTimeout);
-        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeout", &o->dirAttrTimeout);
-        c->GetValueFatalIfFail("fs.kernelCache.entryTimeout", &o->entryTimeout);
-        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeout", &o->dirEntryTimeout);
+        c->GetValueFatalIfFail("fs.kernelCache.attrTimeoutSec",
+                               &o->attrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeoutSec",
+                               &o->dirAttrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.entryTimeoutSec",
+                               &o->entryTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeoutSec",
+                               &o->dirEntryTimeoutSec);
     }
-    { // memory lookup cache option
+    {  // lookup cache option
         auto o = &option->lookupCacheOption;
-        c->GetValueFatalIfFail("fs.lookupCache.lruSize", &o->lruSize);
-        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeout",
-                               &o->negativeTimeout);
+        c->GetValueFatalIfFail("fs.lookupCache.lruSize",
+                               &o->lruSize);
+        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeoutSec",
+                               &o->negativeTimeoutSec);
+        c->GetValueFatalIfFail("fs.lookupCache.minUses",
+                               &o->minUses);
     }
-    { // dir cache option
+    {  // dir cache option
         auto o = &option->dirCacheOption;
         c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize);
     }
-    { // open file option
-        auto o = &option->openFileOption;
+    {  // open file option
+        auto o = &option->openFilesOption;
         c->GetValueFatalIfFail("fs.openFile.lruSize", &o->lruSize);
     }
-    { // attr watcher option
+    {  // attr watcher option
         auto o = &option->attrWatcherOption;
         c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize);
     }
-    { // rpc option
+    {  // rpc option
         auto o = &option->rpcOption;
         c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit);
     }
-    { // defer sync option
+    {  // defer sync option
         auto o = &option->deferSyncOption;
         c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay);
         c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime);
@@ -325,6 +381,33 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
         << "Not found `fuseClient.enableSplice` in conf, use default value `"
         << std::boolalpha << clientOption->enableFuseSplice << '`';
 
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteBytes",
+                              &FLAGS_fuseClientAvgWriteBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytes",
+                              &FLAGS_fuseClientBurstWriteBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytesSecs",
+                              &FLAGS_fuseClientBurstWriteBytesSecs);
+
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteIops",
+                              &FLAGS_fuseClientAvgWriteIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIops",
+                              &FLAGS_fuseClientBurstWriteIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIopsSecs",
+                              &FLAGS_fuseClientBurstWriteIopsSecs);
+
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadBytes",
+                              &FLAGS_fuseClientAvgReadBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytes",
+                              &FLAGS_fuseClientBurstReadBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytesSecs",
+                              &FLAGS_fuseClientBurstReadBytesSecs);
+
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadIops",
+                              &FLAGS_fuseClientAvgReadIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIops",
+                              &FLAGS_fuseClientBurstReadIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIopsSecs",
+                              &FLAGS_fuseClientBurstReadIopsSecs);
     SetBrpcOpt(conf);
 }
 
@@ -332,6 +415,7 @@ void SetFuseClientS3Option(FuseClientOption *clientOption,
     const S3InfoOption &fsS3Opt) {
     clientOption->s3Opt.s3ClientAdaptorOpt.blockSize = fsS3Opt.blockSize;
     clientOption->s3Opt.s3ClientAdaptorOpt.chunkSize = fsS3Opt.chunkSize;
+    clientOption->s3Opt.s3ClientAdaptorOpt.objectPrefix = fsS3Opt.objectPrefix;
     clientOption->s3Opt.s3AdaptrOpt.s3Address = fsS3Opt.s3Address;
     clientOption->s3Opt.s3AdaptrOpt.ak = fsS3Opt.ak;
     clientOption->s3Opt.s3AdaptrOpt.sk = fsS3Opt.sk;
@@ -346,6 +430,7 @@ void S3Info2FsS3Option(const curvefs::common::S3Info& s3,
     fsS3Opt->bucketName = s3.bucketname();
     fsS3Opt->blockSize = s3.blocksize();
     fsS3Opt->chunkSize = s3.chunksize();
+    fsS3Opt->objectPrefix = s3.has_objectprefix() ? s3.objectprefix() : 0;
 }
 
 }  // namespace common

curvefs/src/client/common/config.h

@@ -32,15 +32,13 @@
 #include "src/common/configuration.h"
 #include "src/common/s3_adapter.h"
 
+namespace curvefs {
+namespace client {
+namespace common {
 using ::curve::common::Configuration;
 using ::curve::common::S3AdapterOption;
 using ::curvefs::client::common::DiskCacheType;
 using ::curve::common::S3InfoOption;
-
-namespace curvefs {
-namespace client {
-namespace common {
-
 using MdsOption = ::curve::client::MetaServerOption;
 
 struct BlockDeviceClientOptions {
@@ -106,7 +104,7 @@ struct DiskCacheOption {
     // the max time system command can run
     uint32_t cmdTimeoutSec;
     // threads for disk cache
-    uint32_t threads;
+    uint32_t threads = 10;
     // the write throttle bps of disk cache
     uint64_t avgFlushBytes;
     // the write burst bps of disk cache
@@ -124,7 +122,6 @@ struct DiskCacheOption {
 struct S3ClientAdaptorOption {
     uint64_t blockSize;
     uint64_t chunkSize;
-    uint32_t fuseMaxSize;
     uint64_t pageSize;
     uint32_t prefetchBlocks;
     uint32_t prefetchExecQueueNum;
@@ -133,10 +130,12 @@ struct S3ClientAdaptorOption {
     uint32_t flushIntervalSec;
     uint64_t writeCacheMaxByte;
     uint64_t readCacheMaxByte;
+    uint32_t readCacheThreads;
     uint32_t nearfullRatio;
     uint32_t baseSleepUs;
     uint32_t maxReadRetryIntervalMs;
     uint32_t readRetryIntervalMs;
+    uint32_t objectPrefix;
     DiskCacheOption diskCacheOpt;
 };
 
@@ -178,27 +177,28 @@ struct RefreshDataOption {
 
 // { filesystem option
 struct KernelCacheOption {
-    uint32_t entryTimeout;
-    uint32_t dirEntryTimeout;
-    uint32_t attrTimeout;
-    uint32_t dirAttrTimeout;
+    uint32_t entryTimeoutSec;
+    uint32_t dirEntryTimeoutSec;
+    uint32_t attrTimeoutSec;
+    uint32_t dirAttrTimeoutSec;
 };
 
 struct LookupCacheOption {
     uint64_t lruSize;
-    uint32_t negativeTimeout;
+    uint32_t negativeTimeoutSec;
+    uint32_t minUses;
 };
 
 struct DirCacheOption {
     uint64_t lruSize;
-    uint32_t timeout;
+    uint32_t timeoutSec;
 };
 
 struct AttrWatcherOption {
     uint64_t lruSize;
 };
 
-struct OpenFileOption {
+struct OpenFilesOption {
     uint64_t lruSize;
     uint32_t deferSyncSecond;
 };
@@ -220,7 +220,7 @@ struct FileSystemOption {
     KernelCacheOption kernelCacheOption;
     LookupCacheOption lookupCacheOption;
     DirCacheOption dirCacheOption;
-    OpenFileOption openFileOption;
+    OpenFilesOption openFilesOption;
     AttrWatcherOption attrWatcherOption;
     RPCOption rpcOption;
     DeferSyncOption deferSyncOption;
@@ -248,7 +248,7 @@ struct FuseClientOption {
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;
     uint32_t downloadMaxRetryTimes;
-    uint32_t warmupThreadsNum;
+    uint32_t warmupThreadsNum = 10;
 };
 
 void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption);

curvefs/src/client/curve_fuse_op.cpp

@@ -59,6 +59,7 @@ using ::curvefs::common::LatencyUpdater;
 using ::curvefs::client::metric::InflightGuard;
 using ::curvefs::client::filesystem::EntryOut;
 using ::curvefs::client::filesystem::AttrOut;
+using ::curvefs::client::filesystem::FileOut;
 using ::curvefs::client::filesystem::AccessLogGuard;
 using ::curvefs::client::filesystem::StrFormat;
 using ::curvefs::client::filesystem::InitAccessLog;
@@ -148,18 +149,17 @@ int InitLog(const char *confPath, const char *argv0) {
     return 0;
 }
 
-int InitFuseClient(const char *confPath, const char* fsName,
-    const char *fsType, const char *mdsAddr) {
+int InitFuseClient(const struct MountOption *mountOption) {
     g_clientOpMetric = new ClientOpMetric();
 
     Configuration conf;
-    conf.SetConfigPath(confPath);
+    conf.SetConfigPath(mountOption->conf);
     if (!conf.LoadConfig()) {
-        LOG(ERROR) << "LoadConfig failed, confPath = " << confPath;
+        LOG(ERROR) << "LoadConfig failed, confPath = " << mountOption->conf;
         return -1;
     }
-    if (mdsAddr)
-        conf.SetStringValue("mdsOpt.rpcRetryOpt.addrs", mdsAddr);
+    if (mountOption->mdsAddr)
+        conf.SetStringValue("mdsOpt.rpcRetryOpt.addrs", mountOption->mdsAddr);
 
     conf.PrintConfig();
 
@@ -167,11 +167,12 @@ int InitFuseClient(const char *confPath, const char* fsName,
     curvefs::client::common::InitFuseClientOption(&conf, g_fuseClientOption);
 
     std::shared_ptr<FsInfo> fsInfo = std::make_shared<FsInfo>();
-    if (GetFsInfo(fsName, fsInfo.get()) != 0) {
+    if (GetFsInfo(mountOption->fsName, fsInfo.get()) != 0) {
         return -1;
     }
 
-    std::string fsTypeStr = (fsType == nullptr) ? "" : fsType;
+    std::string fsTypeStr =
+        (mountOption->fsType == nullptr) ? "" : mountOption->fsType;
     std::string fsTypeMds;
     if (fsInfo->fstype() == FSType::TYPE_S3) {
        fsTypeMds = "s3";
@@ -201,6 +202,11 @@ int InitFuseClient(const char *confPath, const char* fsName,
         return -1;
     }
 
+    ret = g_ClientInstance->SetMountStatus(mountOption);
+    if (ret != CURVEFS_ERROR::OK) {
+        return -1;
+    }
+
     return 0;
 }
 
@@ -215,15 +221,16 @@ void UnInitFuseClient() {
 }
 
 int AddWarmupTask(curvefs::client::common::WarmupType type, fuse_ino_t key,
-                  const std::string &path) {
+                  const std::string &path,
+                  curvefs::client::common::WarmupStorageType storageType) {
     int ret = 0;
     bool result = true;
     switch (type) {
     case curvefs::client::common::WarmupType::kWarmupTypeList:
-        result = g_ClientInstance->PutWarmFilelistTask(key);
+        result = g_ClientInstance->PutWarmFilelistTask(key, storageType);
         break;
     case curvefs::client::common::WarmupType::kWarmupTypeSingle:
-        result = g_ClientInstance->PutWarmFileTask(key, path);
+        result = g_ClientInstance->PutWarmFileTask(key, path, storageType);
         break;
     default:
         // not support add warmup type (warmup single file/dir or filelist)
@@ -257,16 +264,23 @@ int Warmup(fuse_ino_t key, const std::string& name, const std::string& value) {
 
     std::vector<std::string> opTypePath;
     curve::common::SplitString(value, "\n", &opTypePath);
-    if (opTypePath.size() != 3) {
+    if (opTypePath.size() != curvefs::client::common::kWarmupOpNum) {
         LOG(ERROR) << name << " has invalid xattr value " << value;
         return ERANGE;
     }
+    auto storageType =
+        curvefs::client::common::GetWarmupStorageType(opTypePath[3]);
+    if (storageType ==
+        curvefs::client::common::WarmupStorageType::kWarmupStorageTypeUnknown) {
+        LOG(ERROR) << name << " not support storage type: " << value;
+        return ERANGE;
+    }
     int ret = 0;
     switch (curvefs::client::common::GetWarmupOpType(opTypePath[0])) {
     case curvefs::client::common::WarmupOpType::kWarmupOpAdd:
         ret =
             AddWarmupTask(curvefs::client::common::GetWarmupType(opTypePath[1]),
-                          key, opTypePath[2]);
+                          key, opTypePath[2], storageType);
         if (ret != 0) {
             LOG(ERROR) << name << " has invalid xattr value " << value;
         }
@@ -280,13 +294,27 @@ int Warmup(fuse_ino_t key, const std::string& name, const std::string& value) {
 
 namespace {
 
+struct CodeGuard {
+    explicit CodeGuard(CURVEFS_ERROR* rc, bvar::Adder<uint64_t>* ecount)
+    : rc_(rc), ecount_(ecount) {}
+
+    ~CodeGuard() {
+        if (*rc_ != CURVEFS_ERROR::OK) {
+            (*ecount_) << 1;
+        }
+    }
+
+    CURVEFS_ERROR* rc_;
+    bvar::Adder<uint64_t>* ecount_;
+};
+
 FuseClient* Client() {
     return g_ClientInstance;
 }
 
 const char* warmupXAttr = ::curvefs::client::common::kCurveFsWarmupXAttr;
 
-bool IsSpecialReq(const char* name) {
+bool IsWamupReq(const char* name) {
     return strcmp(name, warmupXAttr) == 0;
 }
 
@@ -313,8 +341,12 @@ void QueryWarmup(fuse_req_t req, fuse_ino_t ino, size_t size) {
     return fs->ReplyBuffer(req, data.data(), data.length());
 }
 
+void ReadThrottleAdd(size_t size) { Client()->Add(true, size); }
+void WriteThrottleAdd(size_t size) { Client()->Add(false, size); }
+
 #define MetricGuard(REQUEST) \
-    InflightGuard guard(&g_clientOpMetric->op##REQUEST.inflightOpNum); \
+    InflightGuard iGuard(&g_clientOpMetric->op##REQUEST.inflightOpNum); \
+    CodeGuard cGuard(&rc, &g_clientOpMetric->op##REQUEST.ecount); \
     LatencyUpdater updater(&g_clientOpMetric->op##REQUEST.latency)
 
 }  // namespace
@@ -539,8 +571,9 @@ void FuseOpLink(fuse_req_t req,
     auto fs = client->GetFileSystem();
     MetricGuard(Link);
     AccessLogGuard log([&](){
-        return StrFormat("link (%d,%d,%s): %s%s",
-                         ino, newparent, newname, StrErr(rc), StrEntry(entryOut));
+        return StrFormat(
+            "link (%d,%d,%s): %s%s",
+            ino, newparent, newname, StrErr(rc), StrEntry(entryOut));
     });
 
     rc = client->FuseOpLink(req, ino, newparent, newname, &entryOut);
@@ -552,6 +585,7 @@ void FuseOpLink(fuse_req_t req,
 
 void FuseOpOpen(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) {
     CURVEFS_ERROR rc;
+    FileOut fileOut;
     auto client = Client();
     auto fs = client->GetFileSystem();
     MetricGuard(Open);
@@ -559,12 +593,12 @@ void FuseOpOpen(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) {
         return StrFormat("open (%d): %s [fh:%d]", ino, StrErr(rc), fi->fh);
     });
 
-    rc = client->FuseOpOpen(req, ino, fi);
+    rc = client->FuseOpOpen(req, ino, fi, &fileOut);
     if (rc != CURVEFS_ERROR::OK) {
         fs->ReplyError(req, rc);
         return;
     }
-    return fs->ReplyOpen(req, fi);
+    return fs->ReplyOpen(req, &fileOut);
 }
 
 void FuseOpRead(fuse_req_t req,
@@ -583,6 +617,7 @@ void FuseOpRead(fuse_req_t req,
                          ino, size, off, fi->fh, StrErr(rc), rSize);
     });
 
+    ReadThrottleAdd(size);
     rc = client->FuseOpRead(req, ino, size, off, fi, buffer.get(), &rSize);
     if (rc != CURVEFS_ERROR::OK) {
         return fs->ReplyError(req, rc);
@@ -599,20 +634,21 @@ void FuseOpWrite(fuse_req_t req,
                  off_t off,
                  struct fuse_file_info *fi) {
     CURVEFS_ERROR rc;
-    size_t wSize = 0;
+    FileOut fileOut;
     auto client = Client();
     auto fs = client->GetFileSystem();
     MetricGuard(Read);
     AccessLogGuard log([&](){
         return StrFormat("write (%d,%d,%d,%d): %s (%d)",
-                         ino, size, off, fi->fh, StrErr(rc), wSize);
+                         ino, size, off, fi->fh, StrErr(rc), fileOut.nwritten);
     });
 
-    rc = client->FuseOpWrite(req, ino, buf, size, off, fi, &wSize);
+    WriteThrottleAdd(size);
+    rc = client->FuseOpWrite(req, ino, buf, size, off, fi, &fileOut);
     if (rc != CURVEFS_ERROR::OK) {
         return fs->ReplyError(req, rc);
     }
-    return fs->ReplyWrite(req, wSize);
+    return fs->ReplyWrite(req, &fileOut);
 }
 
 void FuseOpFlush(fuse_req_t req,
@@ -767,7 +803,7 @@ void FuseOpSetXattr(fuse_req_t req,
                          ino, name, size, flags, StrErr(rc));
     });
 
-    if (IsSpecialReq(name)) {
+    if (IsWamupReq(name)) {
         return TriggerWarmup(req, ino, name, value, size);
     }
     rc = client->FuseOpSetXattr(req, ino, name, value, size, flags);
@@ -788,7 +824,7 @@ void FuseOpGetXattr(fuse_req_t req,
                          ino, name, size, StrErr(rc), value.size());
     });
 
-    if (IsSpecialReq(name)) {
+    if (IsWamupReq(name)) {
         return QueryWarmup(req, ino, size);
     }
 

curvefs/src/client/curve_fuse_op.h

@@ -39,8 +39,7 @@ extern "C" {
 
 int InitLog(const char *confPath, const char *argv0);
 
-int InitFuseClient(const char *confPath, const char *fsName,
-        const char* fsType, const char* mdsAddr);
+int InitFuseClient(const struct MountOption *mountOption);
 
 void UnInitFuseClient();
 

curvefs/src/client/filesystem/access_log.h

@@ -20,9 +20,14 @@
  * Author: Jingli Chen (Wine93)
  */
 
+#include <unistd.h>
 #include <butil/time.h>
 #include <spdlog/spdlog.h>
 #include <spdlog/sinks/basic_file_sink.h>
+#include <spdlog/sinks/daily_file_sink.h>
+
+#include <string>
+#include <memory>
 
 #include "absl/strings/str_format.h"
 #include "curvefs/src/client/common/config.h"
@@ -46,13 +51,14 @@ using MessageHandler = std::function<std::string()>;
 static std::shared_ptr<spdlog::logger> Logger;
 
 bool InitAccessLog(const std::string& prefix) {
-    Logger = spdlog::basic_logger_mt("fuse_access", prefix + "/access.log");
+    std::string filename = StrFormat("%s/access.%d.log", prefix, getpid());
+    Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0);
     spdlog::flush_every(std::chrono::seconds(1));
     return true;
 }
 
 struct AccessLogGuard {
-    AccessLogGuard(MessageHandler handler)
+    explicit AccessLogGuard(MessageHandler handler)
         : enable(FLAGS_access_logging),
           handler(handler) {
         if (!enable) {

curvefs/src/client/filesystem/attr_watcher.cpp

@@ -59,10 +59,26 @@ void AttrWatcher::UpdateDirEntryAttr(Ino ino, const InodeAttr& attr) {
     }
 }
 
+void AttrWatcher::UpdateDirEntryLength(Ino ino, const InodeAttr& open) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : open.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateLength(ino, open);
+
+        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
+                << ", attr = " << open.ShortDebugString();
+    }
+}
+
 AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
                                    InodeAttr* attr,
+                                   ReplyType type,
                                    bool writeBack)
-    : watcher(watcher), attr(attr), writeBack(writeBack) {
+    : watcher(watcher), attr(attr), type(type), writeBack(writeBack) {
     InodeAttr open;
     Ino ino = attr->inodeid();
     bool yes = watcher->openFiles_->GetFileAttr(ino, &open);
@@ -80,9 +96,19 @@ AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
 }
 
 AttrWatcherGuard::~AttrWatcherGuard() {
-    watcher->RemeberMtime(*attr);
-    if (writeBack) {
-        watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
+    switch (type) {
+        case ReplyType::ATTR:
+            watcher->RemeberMtime(*attr);
+            if (writeBack) {
+                watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
+            }
+            break;
+
+        case ReplyType::ONLY_LENGTH:
+            if (writeBack) {
+                watcher->UpdateDirEntryLength(attr->inodeid(), *attr);
+            }
+            break;
     }
 }
 

curvefs/src/client/filesystem/attr_watcher.h

@@ -23,6 +23,8 @@
 #ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
 #define CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
 
+#include <memory>
+
 #include "src/common/lru_cache.h"
 #include "curvefs/src/client/common/config.h"
 #include "curvefs/src/client/filesystem/meta.h"
@@ -53,6 +55,8 @@ class AttrWatcher {
 
     void UpdateDirEntryAttr(Ino ino, const InodeAttr& attr);
 
+    void UpdateDirEntryLength(Ino ino, const InodeAttr& open);
+
  private:
     friend class AttrWatcherGuard;
 
@@ -63,18 +67,26 @@ class AttrWatcher {
     std::shared_ptr<DirCache> dirCache_;
 };
 
+
+enum class ReplyType {
+    ATTR,
+    ONLY_LENGTH
+};
+
 /*
  * each attribute reply to kernel, the watcher will:
- *    before reply:
- *      1) set attibute length if the corresponding file is opened
- *    after reply:
- *     1) remeber attribute modified time
- *     2) write back attribute to dir entry cache if |writeBack| it true
+ *  before reply:
+ *    1) set attibute length if the corresponding file is opened
+ *  after reply:
+ *    1) remeber attribute modified time.
+ *    2) write back attribute to dir entry cache if |writeBack| is true,
+ *       because the dir-entry attribute maybe stale.
  */
 struct AttrWatcherGuard {
  public:
     AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
                      InodeAttr* attr,
+                     ReplyType type,
                      bool writeBack);
 
     ~AttrWatcherGuard();
@@ -82,6 +94,7 @@ struct AttrWatcherGuard {
  private:
     std::shared_ptr<AttrWatcher> watcher;
     InodeAttr* attr;
+    ReplyType type;
     bool writeBack;
 };
 

curvefs/src/client/filesystem/defer_sync.cpp

@@ -20,6 +20,9 @@
  * Author: Jingli Chen (Wine93)
  */
 
+#include <vector>
+#include <memory>
+
 #include "curvefs/src/client/filesystem/defer_sync.h"
 #include "curvefs/src/client/filesystem/utils.h"
 
@@ -62,7 +65,7 @@ void DeferSync::SyncTask() {
             inodes.swap(inodes_);
         }
         for (const auto& inode : inodes) {
-            UniqueLock lck(inode->GetUniqueLock());
+            UniqueLock lk(inode->GetUniqueLock());
             inode->Async(nullptr, true);
         }
         inodes.clear();

curvefs/src/client/filesystem/defer_sync.h

@@ -24,7 +24,8 @@
 #define CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
 
 #include <atomic>
-#include <thread>
+#include <vector>
+#include <memory>
 
 #include "src/common/interruptible_sleeper.h"
 #include "curvefs/src/client/common/config.h"

curvefs/src/client/filesystem/dir_cache.cpp

@@ -20,6 +20,8 @@
  * Author: Jingli Chen (Wine93)
  */
 
+#include <utility>
+
 #include "curvefs/src/client/filesystem/dir_cache.h"
 #include "curvefs/src/client/filesystem/utils.h"
 
@@ -117,7 +119,7 @@ DirCache::DirCache(DirCacheOption option)
         entries->Clear();
     });
 
-    LOG(INFO) << "Using directory cache, capacity = " << option_.lruSize;
+    LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize;
 }
 
 void DirCache::Start() {
@@ -134,14 +136,14 @@ void DirCache::Delete(Ino parent,
                       std::shared_ptr<DirEntryList> entries,
                       bool evit) {
     nentries_ -= entries->Size();
-    mq_->Publish(entries);
+    mq_->Publish(entries);  // clear entries in background
     lru_->Remove(parent);
 
-    LOG(INFO) << "Delete directory cache (evit=" << evit << "): "
-              << "parent = " << parent
-              << ", mtime = " << entries->GetMtime()
-              << ", size = " << entries->Size()
-              << ", nentries = " << nentries_;
+    VLOG(1) << "Delete directory cache (evit=" << evit << "): "
+            << "parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
 }
 
 void DirCache::Evit(size_t size) {
@@ -158,14 +160,14 @@ void DirCache::Evit(size_t size) {
 
 void DirCache::Put(Ino parent, std::shared_ptr<DirEntryList> entries) {
     WriteLockGuard lk(rwlock_);
-    Evit(entries->Size());
+    Evit(entries->Size());  // it guarantee put entries success
     lru_->Put(parent, entries);
     nentries_ += entries->Size();
 
-    LOG(INFO) << "Insert directory cache: parent = " << parent
-              << ", mtime = " << entries->GetMtime()
-              << ", size = " << entries->Size()
-              << ", nentries = " << nentries_;
+    VLOG(1) << "Insert directory cache: parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
 }
 
 bool DirCache::Get(Ino parent, std::shared_ptr<DirEntryList>* entries) {

curvefs/src/client/filesystem/dir_cache.h

@@ -25,6 +25,7 @@
 
 #include <map>
 #include <list>
+#include <memory>
 
 #include "src/common/lru_cache.h"
 #include "src/common/concurrent/concurrent.h"

curvefs/src/client/filesystem/error.cpp

@@ -21,6 +21,7 @@
  */
 
 #include <map>
+#include <utility>
 #include <ostream>
 
 #include "curvefs/src/client/filesystem/error.h"

curvefs/src/client/filesystem/error.h

@@ -21,6 +21,7 @@
  */
 
 #include <map>
+#include <string>
 #include <iostream>
 
 #include "curvefs/proto/metaserver.pb.h"
@@ -53,8 +54,9 @@ enum class CURVEFS_ERROR {
     OUT_OF_RANGE = -15,
     NODATA = -16,
     IO_ERROR = -17,
-    STALE = -18,
-    NOSYS = -19,
+    CACHETOOSMALL = -18,
+    STALE = -19,
+    NOSYS = -20,
 };
 
 std::string StrErr(CURVEFS_ERROR code);

curvefs/src/client/filesystem/filesystem.cpp

@@ -34,8 +34,8 @@ FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
     deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
     negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
     dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
-    openFiles_ = std::make_shared<OpenFiles>(option_.openFileOption,
-                                             deferSync_, dirCache_);
+    openFiles_ = std::make_shared<OpenFiles>(option_.openFilesOption,
+                                             deferSync_);
     attrWatcher_ = std::make_shared<AttrWatcher>(option_.attrWatcherOption,
                                                  openFiles_, dirCache_);
     handlerManager_ = std::make_shared<HandlerManager>();
@@ -55,22 +55,22 @@ void FileSystem::Destory() {
 
 void FileSystem::Attr2Stat(InodeAttr* attr, struct stat* stat) {
     std::memset(stat, 0, sizeof(struct stat));
-    stat->st_ino = attr->inodeid();
-    stat->st_mode = attr->mode();
-    stat->st_nlink = attr->nlink();
-    stat->st_uid = attr->uid();
-    stat->st_gid = attr->gid();
-    stat->st_size = attr->length();
-    stat->st_rdev = attr->rdev();
-    stat->st_atim.tv_sec = attr->atime(); // atime
+    stat->st_ino = attr->inodeid();  //  inode number
+    stat->st_mode = attr->mode();  // permission mode
+    stat->st_nlink = attr->nlink();  // number of links
+    stat->st_uid = attr->uid();  // user ID of owner
+    stat->st_gid = attr->gid();  // group ID of owner
+    stat->st_size = attr->length();  // total size, in bytes
+    stat->st_rdev = attr->rdev();  // device ID (if special file)
+    stat->st_atim.tv_sec = attr->atime();  // time of last access
     stat->st_atim.tv_nsec = attr->atime_ns();
-    stat->st_mtim.tv_sec = attr->mtime();
+    stat->st_mtim.tv_sec = attr->mtime();  // time of last modification
     stat->st_mtim.tv_nsec = attr->mtime_ns();
-    stat->st_ctim.tv_sec = attr->ctime();
+    stat->st_ctim.tv_sec = attr->ctime();  // time of last status change
     stat->st_ctim.tv_nsec = attr->ctime_ns();
-    stat->st_blksize = option_.blockSize;;
-    stat->st_blocks = 0;
-    if (IsS3File(*attr) || IsDir(*attr)) {
+    stat->st_blksize = option_.blockSize;  // blocksize for file system I/O
+    stat->st_blocks = 0;  // number of 512B blocks allocated
+    if (IsS3File(*attr)) {
         stat->st_blocks = (attr->length() + 511) / 512;
     }
 }
@@ -88,20 +88,20 @@ void FileSystem::Entry2Param(EntryOut* entryOut,
 void FileSystem::SetEntryTimeout(EntryOut* entryOut) {
     auto option = option_.kernelCacheOption;
     if (IsDir(entryOut->attr)) {
-        entryOut->entryTimeout = option.dirEntryTimeout;
-        entryOut->attrTimeout = option.dirAttrTimeout;
+        entryOut->entryTimeout = option.dirEntryTimeoutSec;
+        entryOut->attrTimeout = option.dirAttrTimeoutSec;
     } else {
-        entryOut->entryTimeout = option.entryTimeout;
-        entryOut->attrTimeout = option.attrTimeout;
+        entryOut->entryTimeout = option.entryTimeoutSec;
+        entryOut->attrTimeout = option.attrTimeoutSec;
     }
 }
 
 void FileSystem::SetAttrTimeout(AttrOut* attrOut) {
     auto option = option_.kernelCacheOption;
     if (IsDir(attrOut->attr)) {
-        attrOut->attrTimeout = option.dirAttrTimeout;
+        attrOut->attrTimeout = option.dirAttrTimeoutSec;
     } else {
-        attrOut->attrTimeout = option.attrTimeout;
+        attrOut->attrTimeout = option.attrTimeoutSec;
     }
 }
 
@@ -112,7 +112,8 @@ void FileSystem::ReplyError(Request req, CURVEFS_ERROR code) {
 
 void FileSystem::ReplyEntry(Request req,
                             EntryOut* entryOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr, true);
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
     fuse_entry_param e;
     SetEntryTimeout(entryOut);
     Entry2Param(entryOut, &e);
@@ -121,7 +122,8 @@ void FileSystem::ReplyEntry(Request req,
 
 void FileSystem::ReplyAttr(Request req,
                            AttrOut* attrOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr, true);
+    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr,
+                             ReplyType::ATTR, true);
     struct stat stat;
     SetAttrTimeout(attrOut);
     Attr2Stat(&attrOut->attr, &stat);
@@ -132,18 +134,26 @@ void FileSystem::ReplyReadlink(Request req, const std::string& link) {
     fuse_reply_readlink(req, link.c_str());
 }
 
-void FileSystem::ReplyOpen(Request req, FileInfo *fi) {
+void FileSystem::ReplyOpen(Request req, FileInfo* fi) {
     fuse_reply_open(req, fi);
 }
 
+void FileSystem::ReplyOpen(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_open(req, fileOut->fi);
+}
+
 void FileSystem::ReplyData(Request req,
                            struct fuse_bufvec *bufv,
-		                   enum fuse_buf_copy_flags flags) {
+                           enum fuse_buf_copy_flags flags) {
     fuse_reply_data(req, bufv, flags);
 }
 
-void FileSystem::ReplyWrite(Request req, size_t count) {
-    fuse_reply_write(req, count);
+void FileSystem::ReplyWrite(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_write(req, fileOut->nwritten);
 }
 
 void FileSystem::ReplyBuffer(Request req, const char *buf, size_t size) {
@@ -159,7 +169,8 @@ void FileSystem::ReplyXattr(Request req, size_t size) {
 }
 
 void FileSystem::ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi) {
-    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr, true);
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
     fuse_entry_param e;
     SetEntryTimeout(entryOut);
     Entry2Param(entryOut, &e);
@@ -187,7 +198,8 @@ void FileSystem::AddDirEntry(Request req,
 void FileSystem::AddDirEntryPlus(Request req,
                                  DirBufferHead* buffer,
                                  DirEntry* dirEntry) {
-    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr, false);
+    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr,
+                             ReplyType::ATTR, false);
     struct fuse_entry_param e;
     EntryOut entryOut(dirEntry->attr);
     SetEntryTimeout(&entryOut);
@@ -218,7 +230,7 @@ void FileSystem::ReleaseHandler(uint64_t fh) {
 }
 
 FileSystemMember FileSystem::BorrowMember() {
-    return FileSystemMember(deferSync_, openFiles_);
+    return FileSystemMember(deferSync_, openFiles_, attrWatcher_);
 }
 
 // fuse request*
@@ -236,7 +248,9 @@ CURVEFS_ERROR FileSystem::Lookup(Request req,
     }
 
     auto rc = rpc_->Lookup(parent, name, entryOut);
-    if (rc == CURVEFS_ERROR::NOTEXIST) {
+    if (rc == CURVEFS_ERROR::OK) {
+        negative_->Delete(parent, name);
+    } else if (rc == CURVEFS_ERROR::NOTEXIST) {
         negative_->Put(parent, name);
     }
     return rc;
@@ -318,7 +332,7 @@ CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) {
         LOG(WARNING) << "open(" << ino << "): stale file handler"
                      << ": attribute not found in wacther";
         return CURVEFS_ERROR::STALE;
-    } else if(mtime != InodeMtime(inode)) {
+    } else if (mtime != InodeMtime(inode)) {
         LOG(WARNING) << "open(" << ino << "): stale file handler"
                      << ", cache(" << mtime << ") vs remote("
                      << InodeMtime(inode) << ")";

curvefs/src/client/filesystem/filesystem.h

@@ -23,6 +23,11 @@
 #ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
 #define CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
 
+#include <gtest/gtest_prod.h>
+
+#include <memory>
+#include <string>
+
 #include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/common/config.h"
 #include "curvefs/src/client/filesystem/package.h"
@@ -42,12 +47,15 @@ using ::curvefs::client::common::FileSystemOption;
 
 struct FileSystemMember {
     FileSystemMember(std::shared_ptr<DeferSync> deferSync,
-                     std::shared_ptr<OpenFiles> openFiles)
+                     std::shared_ptr<OpenFiles> openFiles,
+                     std::shared_ptr<AttrWatcher> attrWatcher)
         : deferSync(deferSync),
-          openFiles(openFiles) {}
+          openFiles(openFiles),
+          attrWatcher(attrWatcher) {}
 
     std::shared_ptr<DeferSync> deferSync;
     std::shared_ptr<OpenFiles> openFiles;
+    std::shared_ptr<AttrWatcher> attrWatcher;
 };
 
 class FileSystem {
@@ -90,11 +98,13 @@ class FileSystem {
 
     void ReplyOpen(Request req, FileInfo *fi);
 
+    void ReplyOpen(Request req, FileOut* fileOut);
+
     void ReplyData(Request req,
                    struct fuse_bufvec *bufv,
-		           enum fuse_buf_copy_flags flags);
+                   enum fuse_buf_copy_flags flags);
 
-    void ReplyWrite(Request req, size_t count);
+    void ReplyWrite(Request req, FileOut* fileOut);
 
     void ReplyBuffer(Request req, const char *buf, size_t size);
 
@@ -123,6 +133,11 @@ class FileSystem {
     FileSystemMember BorrowMember();
 
  private:
+    FRIEND_TEST(FileSystemTest, Attr2Stat);
+    FRIEND_TEST(FileSystemTest, Entry2Param);
+    FRIEND_TEST(FileSystemTest, SetEntryTimeout);
+    FRIEND_TEST(FileSystemTest, SetAttrTimeout);
+
     // utility: convert to system type.
     void Attr2Stat(InodeAttr* attr, struct stat* stat);
 

curvefs/src/client/filesystem/lookup_cache.cpp

@@ -32,43 +32,71 @@ namespace curvefs {
 namespace client {
 namespace filesystem {
 
+#define RETURN_FALSE_IF_DISABLED() \
+    do {                           \
+        if (!enable_) {            \
+            return false;          \
+        }                          \
+    } while (0)
+
+
 LookupCache::LookupCache(LookupCacheOption option)
-    : enable_(option.negativeTimeout > 0),
+    : enable_(option.negativeTimeoutSec > 0),
       rwlock_(),
       option_(option) {
     lru_ = std::make_shared<LRUType>(option.lruSize);
+    if (enable_) {
+        LOG(INFO) << "Using lookup negative lru cache"
+                  << ", timeout = " << option.negativeTimeoutSec
+                  << ", capacity = " << option.lruSize;
+    }
 }
 
-std::string LookupCache::Entry2Key(Ino parent, const std::string& name) {
+std::string LookupCache::CacheKey(Ino parent, const std::string& name) {
     return absl::StrFormat("%d:%s", parent, name);
 }
 
 bool LookupCache::Get(Ino parent, const std::string& name) {
-    if (!enable_) {
-        return false;
-    }
-
+    RETURN_FALSE_IF_DISABLED();
     ReadLockGuard lk(rwlock_);
-    std::string key = Entry2Key(parent, name);
-    TimeSpec expireTime;
-    bool yes = lru_->Get(key, &expireTime);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
     if (!yes) {
+        VLOG(1) << absl::StrFormat("Lookup cache not found: key(%d,%s)",
+                                   parent, name);
         return false;
-    } else if (Now() > expireTime) {
+    } else if (entry.uses < option_.minUses) {
+        return false;
+    } else if (entry.expireTime < Now()) {
         return false;
     }
     return true;
 }
 
-void LookupCache::Put(Ino parent, const std::string& name) {
-    if (!enable_) {
-        return;
+bool LookupCache::Put(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    WriteLockGuard lk(rwlock_);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
+    if (yes) {
+        entry.uses++;
+    } else {
+        entry.uses = 0;
     }
 
+    entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0);
+    lru_->Put(key, entry);
+    return true;
+}
+
+bool LookupCache::Delete(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
     WriteLockGuard lk(rwlock_);
-    auto key = Entry2Key(parent, name);
-    TimeSpec expireTime = Now() + TimeSpec(option_.negativeTimeout, 0);
-    lru_->Put(key, expireTime);
+    auto key = CacheKey(parent, name);
+    lru_->Remove(key);
+    return true;
 }
 
 }  // namespace filesystem

curvefs/src/client/filesystem/lookup_cache.h

@@ -23,6 +23,9 @@
 #ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
 #define CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
 
+#include <memory>
+#include <string>
+
 #include "src/common/lru_cache.h"
 #include "curvefs/src/client/common/config.h"
 #include "curvefs/src/client/filesystem/meta.h"
@@ -38,20 +41,27 @@ using ::curve::common::WriteLockGuard;
 using ::curvefs::client::common::LookupCacheOption;
 
 // memory cache for lookup result, now we only support cache negative result,
-// other positive entry will be cached in kernel.
+// and other positive entry will be cached in kernel.
 class LookupCache {
  public:
-    using LRUType = LRUCache<std::string, TimeSpec>;
+    struct CacheEntry {
+        uint32_t uses;
+        TimeSpec expireTime;
+    };
+
+    using LRUType = LRUCache<std::string, CacheEntry>;
 
  public:
     explicit LookupCache(LookupCacheOption option);
 
     bool Get(Ino parent, const std::string& name);
 
-    void Put(Ino parent, const std::string& name);
+    bool Put(Ino parent, const std::string& name);
+
+    bool Delete(Ino parent, const std::string& name);
 
  private:
-    std::string Entry2Key(Ino parent, const std::string& name);
+    std::string CacheKey(Ino parent, const std::string& name);
 
  private:
     bool enable_;

curvefs/src/client/filesystem/message_queue.h

@@ -97,6 +97,10 @@ class MessageQueue {
         while (running_.load(std::memory_order_relaxed)) {
             queue_.Pop()();
         }
+
+        while (queue_.Size() > 0) {
+            queue_.Pop()();
+        }
     }
 
  private:

curvefs/src/client/filesystem/meta.cpp

@@ -65,37 +65,37 @@ void HandlerManager::ReleaseHandler(uint64_t fh) {
 
 std::string StrMode(uint16_t mode) {
     static std::map<uint16_t, char> type2char = {
-	    { S_IFSOCK, 's' },
-	    { S_IFLNK, 'l' },
-	    { S_IFREG, '-' },
-	    { S_IFBLK, 'b' },
-	    { S_IFDIR, 'd' },
-	    { S_IFCHR, 'c' },
-	    { S_IFIFO, 'f' },
-	    { 0, '?' },
+        { S_IFSOCK, 's' },
+        { S_IFLNK, 'l' },
+        { S_IFREG, '-' },
+        { S_IFBLK, 'b' },
+        { S_IFDIR, 'd' },
+        { S_IFCHR, 'c' },
+        { S_IFIFO, 'f' },
+        { 0, '?' },
     };
 
     std::string s("?rwxrwxrwx");
-	s[0] = type2char[mode & (S_IFMT & 0xffff)];
-	if (mode & S_ISUID) {
-		s[3] = 's';
-	}
-	if (mode & S_ISGID) {
-		s[6] = 's';
-	}
-	if (mode & S_ISVTX) {
-		s[9] = 't';
-	}
+    s[0] = type2char[mode & (S_IFMT & 0xffff)];
+    if (mode & S_ISUID) {
+        s[3] = 's';
+    }
+    if (mode & S_ISGID) {
+        s[6] = 's';
+    }
+    if (mode & S_ISVTX) {
+        s[9] = 't';
+    }
 
     for (auto i = 0; i < 9; i++) {
-		if ((mode & (1 << i)) == 0) {
-			if ((s[9-i] == 's') || (s[9-i] == 't')) {
-				s[9-i] &= 0xDF;
-			} else {
-				s[9-i] = '-';
-			}
-		}
-	}
+        if ((mode & (1 << i)) == 0) {
+            if ((s[9-i] == 's') || (s[9-i] == 't')) {
+                s[9-i] &= 0xDF;
+            } else {
+                s[9-i] = '-';
+            }
+        }
+    }
     return s;
 }
 

curvefs/src/client/filesystem/meta.h

@@ -23,6 +23,9 @@
 #ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
 #define CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
 
+#include <map>
+#include <memory>
+#include <string>
 #include <iostream>
 
 #include "src/common/concurrent/concurrent.h"
@@ -54,7 +57,7 @@ using FileInfo = struct fuse_file_info;
 struct EntryOut {
     EntryOut() = default;
 
-    EntryOut(InodeAttr attr) : attr(attr) {}
+    explicit EntryOut(InodeAttr attr) : attr(attr) {}
 
     InodeAttr attr;
     double entryTimeout;
@@ -64,7 +67,7 @@ struct EntryOut {
 struct AttrOut {
     AttrOut() = default;
 
-    AttrOut(InodeAttr attr) : attr(attr) {}
+    explicit AttrOut(InodeAttr attr) : attr(attr) {}
 
     InodeAttr attr;
     double attrTimeout;
@@ -81,6 +84,20 @@ struct DirEntry {
     InodeAttr attr;
 };
 
+struct FileOut {
+    FileOut() = default;
+
+    FileOut(FileInfo* fi, InodeAttr attr)
+        : fi(fi), attr(attr), nwritten(0) {}
+
+    FileOut(InodeAttr attr, size_t nwritten)
+        : fi(nullptr), attr(attr), nwritten(nwritten) {}
+
+    FileInfo* fi;
+    InodeAttr attr;
+    size_t nwritten;
+};
+
 struct TimeSpec {
     TimeSpec() : seconds(0), nanoSeconds(0) {}
 

curvefs/src/client/filesystem/openfile.cpp

@@ -27,46 +27,23 @@ namespace curvefs {
 namespace client {
 namespace filesystem {
 
-OpenFiles::OpenFiles(OpenFileOption option,
-                     std::shared_ptr<DeferSync> deferSync,
-                     std::shared_ptr<DirCache> dirCache)
+OpenFiles::OpenFiles(OpenFilesOption option,
+                     std::shared_ptr<DeferSync> deferSync)
     : rwlock_(),
       option_(option),
       deferSync_(deferSync),
-      dirCache_(dirCache),
       files_(std::make_shared<LRUType>(option.lruSize)) {
-}
-
-void OpenFiles::UpdateDirEntryLength(Ino ino,
-                                     const std::shared_ptr<OpenFile>& file) {
-    InodeAttr attr;
-    std::shared_ptr<DirEntryList> entries;
-    file->inode->GetInodeAttr(&attr);
-    for (const auto parent : attr.parent()) {
-        bool yes = dirCache_->Get(parent, &entries);
-        if (!yes) {
-            continue;
-        }
-
-        entries->UpdateLength(ino, attr);
-
-        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
-                << ", length = " << attr.length()
-                << ", mtime = " << AttrMtime(attr)
-                << ", ctime = " << AttrCtime(attr);
-    }
+    LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize;
 }
 
 /*
  * Delete(...) does:
- *   1) write back file length to dir entry cache
- *   2) publish to message queue which will flush file to server with async
- *   3) delete file from lru cache
+ *   1) publish to message queue which will flush file to server with async
+ *   2) delete file from lru cache
  */
 void OpenFiles::Delete(Ino ino,
                        const std::shared_ptr<OpenFile>& file,
                        bool flush) {
-    UpdateDirEntryLength(ino, file);  // write back file length to dir cache
     if (flush) {
         deferSync_->Push(file->inode);
     }
@@ -140,8 +117,7 @@ void OpenFiles::Close(Ino ino) {
 
 /*
  * CloseAll() does:
- *   1) flush all file to server and delete from LRU cache
- *   2) stop message queue
+ *   flush all file to server and delete from LRU cache
  */
 void OpenFiles::CloseAll() {
     WriteLockGuard lk(rwlock_);

curvefs/src/client/filesystem/openfile.h

@@ -23,6 +23,8 @@
 #ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
 #define CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
 
+#include <memory>
+
 #include "src/common/lru_cache.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/filesystem/meta.h"
@@ -37,10 +39,10 @@ using ::curve::common::LRUCache;
 using ::curve::common::ReadLockGuard;
 using ::curve::common::WriteLockGuard;
 using ::curvefs::client::InodeWrapper;
-using ::curvefs::client::common::OpenFileOption;
+using ::curvefs::client::common::OpenFilesOption;
 
 struct OpenFile {
-    OpenFile(std::shared_ptr<InodeWrapper> inode)
+    explicit OpenFile(std::shared_ptr<InodeWrapper> inode)
         : inode(inode), refs(0) {}
 
     std::shared_ptr<InodeWrapper> inode;
@@ -52,9 +54,8 @@ class OpenFiles {
     using LRUType = LRUCache<Ino, std::shared_ptr<OpenFile>>;
 
  public:
-    explicit OpenFiles(OpenFileOption option,
-                       std::shared_ptr<DeferSync> deferSync,
-                       std::shared_ptr<DirCache> dirCache);
+    explicit OpenFiles(OpenFilesOption option,
+                       std::shared_ptr<DeferSync> deferSync);
 
     void Open(Ino ino, std::shared_ptr<InodeWrapper> inode);
 
@@ -67,18 +68,14 @@ class OpenFiles {
     bool GetFileAttr(Ino ino, InodeAttr* inode);
 
  private:
-    void UpdateDirEntryLength(Ino ino,
-                              const std::shared_ptr<OpenFile>& file);
-
     void Delete(Ino ino, const std::shared_ptr<OpenFile>& file, bool flush);
 
     void Evit(size_t size);
 
  private:
     RWLock rwlock_;
-    OpenFileOption option_;
+    OpenFilesOption option_;
     std::shared_ptr<DeferSync> deferSync_;
-    std::shared_ptr<DirCache> dirCache_;
     std::shared_ptr<LRUType> files_;
 };
 

curvefs/src/client/filesystem/package.h

@@ -23,6 +23,8 @@
 #ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
 #define CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
 
+#include <memory>
+
 #include "curvefs/src/client/dentry_cache_manager.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/xattr_manager.h"
@@ -32,16 +34,14 @@ namespace client {
 namespace filesystem {
 
 struct ExternalMember {  // external member depended by FileSystem
-    ExternalMember(std::shared_ptr<InodeCacheManager> inodeManager,
-                   std::shared_ptr<DentryCacheManager> dentryManager,
-                   std::shared_ptr<XattrManager> xattrManager)
-        : inodeManager(inodeManager),
-          dentryManager(dentryManager),
-          xattrManager(xattrManager) {}
+    ExternalMember() = delete;
+    ExternalMember(std::shared_ptr<DentryCacheManager> dentryManager,
+                   std::shared_ptr<InodeCacheManager> inodeManager)
+        : dentryManager(dentryManager),
+          inodeManager(inodeManager) {}
 
-    std::shared_ptr<InodeCacheManager> inodeManager;
     std::shared_ptr<DentryCacheManager> dentryManager;
-    std::shared_ptr<XattrManager> xattrManager;
+    std::shared_ptr<InodeCacheManager> inodeManager;
 };
 
 }  // namespace filesystem

curvefs/src/client/filesystem/rpc_client.cpp

@@ -22,6 +22,10 @@
 
 #include <map>
 #include <set>
+#include <list>
+#include <string>
+#include <memory>
+#include <utility>
 
 #include "curvefs/src/client/filesystem/rpc_client.h"
 
@@ -51,8 +55,8 @@ CURVEFS_ERROR RPCClient::Lookup(Ino parent,
     CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry);
     if (rc != CURVEFS_ERROR::OK) {
         if (rc != CURVEFS_ERROR::NOTEXIST) {
-            LOG(WARNING) << "rpc(lookup::GetDentry) failed, retCode = " << rc
-                         << ", parent = " << parent << ", name = " << name;
+            LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc
+                       << ", parent = " << parent << ", name = " << name;
         }
         return rc;
     }
@@ -66,7 +70,8 @@ CURVEFS_ERROR RPCClient::Lookup(Ino parent,
     return rc;
 }
 
-CURVEFS_ERROR RPCClient::ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries) {
+CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
+                                 std::shared_ptr<DirEntryList>* entries) {
     uint32_t limit = option_.listDentryLimit;
 
     std::list<Dentry> dentries;
@@ -99,9 +104,11 @@ CURVEFS_ERROR RPCClient::ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries
             continue;
         }
 
+        // NOTE: we can't use std::move() for attribute for hard link
+        // which will sharing inode attribute.
         dirEntry.ino = ino;
         dirEntry.name = std::move(dentry.name());
-        dirEntry.attr = std::move(iter->second);
+        dirEntry.attr = iter->second;
         (*entries)->Add(dirEntry);
     }
     return CURVEFS_ERROR::OK;

curvefs/src/client/filesystem/rpc_client.h

@@ -23,6 +23,9 @@
 #ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
 #define CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
 
+#include <memory>
+#include <string>
+
 #include "curvefs/src/client/common/config.h"
 #include "curvefs/src/client/filesystem/meta.h"
 #include "curvefs/src/client/filesystem/package.h"
@@ -59,4 +62,4 @@ class RPCClient {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_

curvefs/src/client/filesystem/utils.h

@@ -23,6 +23,8 @@
 #ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
 #define CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
 
+#include <memory>
+
 #include "curvefs/src/client/filesystem/meta.h"
 #include "curvefs/src/client/filesystem/package.h"
 

curvefs/src/client/fuse_client.cpp

@@ -58,6 +58,7 @@ using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
 using ::curvefs::client::filesystem::ExternalMember;
 using ::curvefs::client::filesystem::DirEntry;
 using ::curvefs::client::filesystem::DirEntryList;
+using ::curvefs::client::filesystem::FileOut;
 
 #define RETURN_IF_UNSUCCESS(action)                                            \
     do {                                                                       \
@@ -72,6 +73,21 @@ namespace curvefs {
 namespace client {
 namespace common {
 DECLARE_bool(enableCto);
+DECLARE_uint64(fuseClientAvgWriteIops);
+DECLARE_uint64(fuseClientBurstWriteIops);
+DECLARE_uint64(fuseClientBurstWriteIopsSecs);
+
+DECLARE_uint64(fuseClientAvgWriteBytes);
+DECLARE_uint64(fuseClientBurstWriteBytes);
+DECLARE_uint64(fuseClientBurstWriteBytesSecs);
+
+DECLARE_uint64(fuseClientAvgReadIops);
+DECLARE_uint64(fuseClientBurstReadIops);
+DECLARE_uint64(fuseClientBurstReadIopsSecs);
+
+DECLARE_uint64(fuseClientAvgReadBytes);
+DECLARE_uint64(fuseClientBurstReadBytes);
+DECLARE_uint64(fuseClientBurstReadBytesSecs);
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs
@@ -85,6 +101,27 @@ using rpcclient::Cli2ClientImpl;
 using rpcclient::MetaCache;
 using common::FLAGS_enableCto;
 
+using common::FLAGS_fuseClientAvgWriteIops;
+using common::FLAGS_fuseClientBurstWriteIops;
+using common::FLAGS_fuseClientBurstWriteIopsSecs;
+
+using common::FLAGS_fuseClientAvgWriteBytes;
+using common::FLAGS_fuseClientBurstWriteBytes;
+using common::FLAGS_fuseClientBurstWriteBytesSecs;
+
+using common::FLAGS_fuseClientAvgReadIops;
+using common::FLAGS_fuseClientBurstReadIops;
+using common::FLAGS_fuseClientBurstReadIopsSecs;
+
+using common::FLAGS_fuseClientAvgReadBytes;
+using common::FLAGS_fuseClientBurstReadBytes;
+using common::FLAGS_fuseClientBurstReadBytesSecs;
+
+static void on_throttle_timer(void *arg) {
+    FuseClient *fuseClient = reinterpret_cast<FuseClient *>(arg);
+    fuseClient->InitQosParam();
+}
+
 CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     option_ = option;
 
@@ -100,7 +137,8 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     auto channelManager = std::make_shared<ChannelManager<MetaserverID>>();
 
     leaseExecutor_ = absl::make_unique<LeaseExecutor>(option.leaseOpt,
-                                                      metaCache, mdsClient_);
+                                                      metaCache, mdsClient_,
+                                                      &enableSumInDir_);
 
     xattrManager_ = std::make_shared<XattrManager>(inodeManager_,
         dentryManager_, option_.listDentryLimit, option_.listDentryThreads);
@@ -120,9 +158,7 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     curve::client::ClientDummyServerInfo::GetInstance().SetIP(localIp);
 
     {
-        ExternalMember member(inodeManager_,
-                              dentryManager_,
-                              xattrManager_);
+        ExternalMember member(dentryManager_, inodeManager_);
         fs_ = std::make_shared<FileSystem>(option_.fileSystemOption, member);
     }
 
@@ -148,6 +184,8 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
         warmupManager_->SetFsInfo(fsInfo_);
     }
 
+    InitQosParam();
+
     return CURVEFS_ERROR::OK;
 }
 
@@ -158,6 +196,10 @@ void FuseClient::UnInit() {
 
     delete mdsBase_;
     mdsBase_ = nullptr;
+
+    while (bthread_timer_del(throttleTimer_) == 1) {
+        bthread_usleep(1000);
+    }
 }
 
 CURVEFS_ERROR FuseClient::Run() {
@@ -175,51 +217,9 @@ void FuseClient::Fini() {
 
 CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                      struct fuse_conn_info *conn) {
-    struct MountOption* mOpts = (struct MountOption*)userdata;
-    // set path
-    mountpoint_.set_path((mOpts->mountPoint == nullptr) ? ""
-                                                       : mOpts->mountPoint);
-    std::string fsName = (mOpts->fsName == nullptr) ? "" : mOpts->fsName;
-
-    mountpoint_.set_cto(false);
-
-    int retVal = SetHostPortInMountPoint(&mountpoint_);
-    if (retVal < 0) {
-        LOG(ERROR) << "Set Host and Port in MountPoint failed, ret = "
-                   << retVal;
-        return CURVEFS_ERROR::INTERNAL;
-    }
-
-    auto ret = mdsClient_->MountFs(fsName, mountpoint_, fsInfo_.get());
-    if (ret != FSStatusCode::OK && ret != FSStatusCode::MOUNT_POINT_EXIST) {
-        LOG(ERROR) << "MountFs failed, FSStatusCode = " << ret
-                   << ", FSStatusCode_Name = "
-                   << FSStatusCode_Name(ret)
-                   << ", fsName = " << fsName
-                   << ", mountPoint = " << mountpoint_.ShortDebugString();
-        return CURVEFS_ERROR::MOUNT_FAILED;
-    }
-    inodeManager_->SetFsId(fsInfo_->fsid());
-    dentryManager_->SetFsId(fsInfo_->fsid());
-    enableSumInDir_ = fsInfo_->enablesumindir() && !FLAGS_enableCto;
-    LOG(INFO) << "Mount " << fsName << " on " << mountpoint_.ShortDebugString()
-              << " success!" << " enableSumInDir = " << enableSumInDir_;
-
-    fsMetric_ = std::make_shared<FSMetric>(fsName);
-
+    (void)userdata;
+    (void)conn;
     fs_->Run();
-
-    // init fsname and mountpoint
-    leaseExecutor_->SetFsName(fsName);
-    leaseExecutor_->SetMountPoint(mountpoint_);
-    if (!leaseExecutor_->Start()) {
-        return CURVEFS_ERROR::INTERNAL;
-    }
-
-    init_ = true;
-    if (warmupManager_ != nullptr) {
-        warmupManager_->SetMounted(true);
-    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -275,7 +275,8 @@ CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req,
 
 CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
                                           fuse_ino_t ino,
-                                          struct fuse_file_info* fi) {
+                                          struct fuse_file_info* fi,
+                                          FileOut* fileOut) {
     std::shared_ptr<InodeWrapper> inodeWrapper;
     // alredy opened
     CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
@@ -284,9 +285,14 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
                    << ", inodeid = " << ino;
         return ret;
     }
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+
+    fileOut->fi = fi;
+    inodeWrapper->GetInodeAttr(&fileOut->attr);
+
     if (fi->flags & O_TRUNC) {
         if (fi->flags & O_WRONLY || fi->flags & O_RDWR) {
+            ::curve::common::UniqueLock lgGuard =
+                inodeWrapper->GetUniqueLock();
             uint64_t length = inodeWrapper->GetLengthLocked();
             CURVEFS_ERROR tRet = Truncate(inodeWrapper.get(), 0);
             if (tRet != CURVEFS_ERROR::OK) {
@@ -305,7 +311,7 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
                 inodeWrapper->MarkDirty();
             }
 
-            if (enableSumInDir_ && length != 0) {
+            if (enableSumInDir_.load() && length != 0) {
                 // update parent summary info
                 const Inode *inode = inodeWrapper->GetInodeLocked();
                 XAttr xattr;
@@ -321,6 +327,7 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
                     }
                 }
             }
+            inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
         } else {
             return CURVEFS_ERROR::NOPERMISSION;
         }
@@ -330,13 +337,14 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
 
 CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req,
                                      fuse_ino_t ino,
-                                     struct fuse_file_info* fi) {
+                                     struct fuse_file_info* fi,
+                                     FileOut* fileOut) {
     CURVEFS_ERROR rc = fs_->Open(req, ino, fi);
     if (rc != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "open(" << ino << ") failed, retCode = " << rc;
         return rc;
     }
-    return HandleOpenFlags(req, ino, fi);
+    return HandleOpenFlags(req, ino, fi, fileOut);
 }
 
 CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
@@ -361,27 +369,38 @@ CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
         if (option_.fileSystemOption.deferSyncOption.deferDirMtime) {
             inodeManager_->ShipToFlush(parentInodeWrapper);
         } else {
-            ret = parentInodeWrapper->SyncAttr();
-            if (ret != CURVEFS_ERROR::OK) {
-                return CURVEFS_ERROR::OK;
-            }
+            return parentInodeWrapper->SyncAttr();
         }
     }
 
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req,
-                                   fuse_ino_t parent,
-                                   const char* name,
-                                   mode_t mode,
-                                   FsFileType type,
-                                   dev_t rdev,
-                                   bool internal,
-                                   std::shared_ptr<InodeWrapper>& inodeWrapper) {
+CURVEFS_ERROR FuseClient::MakeNode(
+    fuse_req_t req,
+    fuse_ino_t parent,
+    const char* name,
+    mode_t mode,
+    FsFileType type,
+    dev_t rdev,
+    bool internal,
+    std::shared_ptr<InodeWrapper>& inodeWrapper) {
     if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
+
+    // check if node is recycle or under recycle
+    if (!internal && strcmp(name, RECYCLENAME) == 0 && parent == ROOTINODEID) {
+        LOG(WARNING) << "Can not make node " << RECYCLENAME
+                     << " under root dir.";
+        return CURVEFS_ERROR::NOPERMISSION;
+    }
+
+    if (!internal && parent == RECYCLEINODEID) {
+        LOG(WARNING) << "Can not make node under recycle.";
+        return CURVEFS_ERROR::NOPERMISSION;
+    }
+
     const struct fuse_ctx *ctx = fuse_req_ctx(req);
     InodeParam param;
     param.fsId = fsInfo_->fsid();
@@ -448,7 +467,7 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req,
             << ", parent = " << parent << ", name = " << name
             << ", mode = " << mode;
 
-    if (enableSumInDir_) {
+    if (enableSumInDir_.load()) {
         // update parent summary info
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
@@ -495,38 +514,9 @@ CURVEFS_ERROR FuseClient::FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
     return RemoveNode(req, parent, name, FsFileType::TYPE_DIRECTORY);
 }
 
-CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
-                                     const char *name, FsFileType type) {
-    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
-        return CURVEFS_ERROR::NAMETOOLONG;
-    }
-    Dentry dentry;
-    CURVEFS_ERROR ret = dentryManager_->GetDentry(parent, name, &dentry);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(WARNING) << "dentryManager_ GetDentry fail, ret = " << ret
-                     << ", parent = " << parent << ", name = " << name;
-        return ret;
-    }
-
-    uint64_t ino = dentry.inodeid();
-
-    // judge dir empty
-    if (FsFileType::TYPE_DIRECTORY == type) {
-        std::list<Dentry> dentryList;
-        auto limit = option_.listDentryLimit;
-        ret = dentryManager_->ListDentry(ino, &dentryList, limit);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
-                       << ", parent = " << ino;
-            return ret;
-        }
-        if (!dentryList.empty()) {
-            LOG(ERROR) << "rmdir not empty";
-            return CURVEFS_ERROR::NOTEMPTY;
-        }
-    }
-
-    ret = dentryManager_->DeleteDentry(parent, name, type);
+CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent,
+                             const char* name, FsFileType type) {
+    CURVEFS_ERROR ret = dentryManager_->DeleteDentry(parent, name, type);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "dentryManager_ DeleteDentry fail, ret = " << ret
                    << ", parent = " << parent << ", name = " << name;
@@ -556,7 +546,7 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
                    << ", parent = " << parent << ", name = " << name;
     }
 
-    if (enableSumInDir_) {
+    if (enableSumInDir_.load()) {
         // update parent summary info
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
@@ -577,6 +567,272 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
     return ret;
 }
 
+std::string GetRecycleTimeDirName() {
+    time_t timeStamp;
+    time(&timeStamp);
+    struct tm p = *localtime_r(&timeStamp, &p);
+    char now[64];
+    strftime(now, 64, "%Y-%m-%d-%H", &p);
+    return now;
+}
+
+CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req,
+                                           uint64_t parent,
+                                           const char* name,
+                                           mode_t mode,
+                                           ManageInodeType manageType,
+                                           EntryOut* entryOut) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+        return CURVEFS_ERROR::NAMETOOLONG;
+    }
+
+    InodeParam param;
+    param.fsId = fsInfo_->fsid();
+    const struct fuse_ctx *ctx = fuse_req_ctx(req);
+    param.uid = ctx->uid;
+    param.gid = ctx->gid;
+    param.mode = mode;
+    param.manageType = manageType;
+
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->CreateManageInode(param, inodeWrapper);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager CreateManageNode fail, ret = " << ret
+                   << ", parent = " << parent << ", name = " << name
+                   << ", mode = " << mode;
+        return ret;
+    }
+
+    VLOG(6) << "inodeManager CreateManageNode success"
+            << ", parent = " << parent << ", name = " << name
+            << ", mode = " << mode
+            << ", inode id = " << inodeWrapper->GetInodeId();
+
+    Dentry dentry;
+    dentry.set_fsid(fsInfo_->fsid());
+    dentry.set_inodeid(inodeWrapper->GetInodeId());
+    dentry.set_parentinodeid(parent);
+    dentry.set_name(name);
+    dentry.set_type(inodeWrapper->GetType());
+    FsFileType type = inodeWrapper->GetType();
+    if (type == FsFileType::TYPE_FILE || type == FsFileType::TYPE_S3) {
+        dentry.set_flag(DentryFlag::TYPE_FILE_FLAG);
+    }
+
+    ret = dentryManager_->CreateDentry(dentry);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "dentryManager_ CreateDentry fail, ret = " << ret
+                   << ", parent = " << parent << ", name = " << name
+                   << ", mode = " << mode;
+
+        CURVEFS_ERROR ret2 =
+            inodeManager_->DeleteInode(inodeWrapper->GetInodeId());
+        if (ret2 != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "Also delete inode failed, ret = " << ret2
+                       << ", inodeid = " << inodeWrapper->GetInodeId();
+        }
+        return ret;
+    }
+
+    ret = UpdateParentMCTimeAndNlink(parent, type, NlinkChange::kAddOne);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "UpdateParentMCTimeAndNlink failed"
+                   << ", parent: " << parent
+                   << ", name: " << name
+                   << ", type: " << type;
+        return ret;
+    }
+
+    VLOG(6) << "dentryManager_ CreateDentry success"
+            << ", parent = " << parent << ", name = " << name
+            << ", mode = " << mode;
+
+    if (enableSumInDir_.load()) {
+        // update parent summary info
+        XAttr xattr;
+        xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
+        if (type == FsFileType::TYPE_DIRECTORY) {
+            xattr.mutable_xattrinfos()->insert({XATTRSUBDIRS, "1"});
+        } else {
+            xattr.mutable_xattrinfos()->insert({XATTRFILES, "1"});
+        }
+        xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
+            std::to_string(inodeWrapper->GetLength())});
+        auto tret = xattrManager_->UpdateParentInodeXattr(parent, xattr, true);
+        if (tret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "UpdateParentInodeXattr failed,"
+                       << " inodeId = " << parent
+                       << ", xattr = " << xattr.DebugString();
+        }
+    }
+
+    inodeWrapper->GetInodeAttrLocked(&entryOut->attr);
+    return ret;
+}
+
+CURVEFS_ERROR FuseClient::GetOrCreateRecycleDir(fuse_req_t req, Dentry *out) {
+    auto ret = dentryManager_->GetDentry(ROOTINODEID, RECYCLENAME, out);
+    if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOTEXIST) {
+        LOG(ERROR) << "dentryManager_ GetDentry fail, ret = " << ret
+                   << ", inode = " << ROOTINODEID
+                   << ", name = " << RECYCLENAME;
+        return ret;
+    } else if (ret == CURVEFS_ERROR::NOTEXIST) {
+        LOG(INFO) << "recycle dir is not exist, create " << RECYCLENAME
+                  << ", parentid = " << ROOTINODEID;
+        EntryOut entryOut;
+        ret = CreateManageNode(req, ROOTINODEID, RECYCLENAME,
+                               S_IFDIR | 0755, ManageInodeType::TYPE_RECYCLE,
+                               &entryOut);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "CreateManageNode failed, ret = " << ret
+                       << ", inode = " << ROOTINODEID
+                       << ", name = " << RECYCLENAME
+                       << ", type = TYPE_RECYCLE";
+            return ret;
+        }
+    }
+
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
+            fuse_ino_t parent, const char* name, FsFileType type) {
+    (void)type;
+    // 1. check recycle exist, if not exist, create recycle dir
+    Dentry recycleDir;
+    CURVEFS_ERROR ret = GetOrCreateRecycleDir(req, &recycleDir);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "GetOrCreateRecycleDir fail, move " << name
+                   << " to recycle fail, ret = " << ret;
+        return ret;
+    }
+
+    // 2. check recycle time dir is exist, if not exist, create time dir
+    std::string recycleTimeDirName = GetRecycleTimeDirName();
+    Dentry dentry;
+    uint64_t recycleTimeDirIno;
+    ret = dentryManager_->GetDentry(RECYCLEINODEID,
+                                        recycleTimeDirName.c_str(), &dentry);
+    if (ret != CURVEFS_ERROR::OK && ret != CURVEFS_ERROR::NOTEXIST) {
+        LOG(ERROR) << "dentryManager_ GetDentry fail, ret = " << ret
+                   << ", inode = " << RECYCLEINODEID
+                   << ", name = " << recycleTimeDirName;
+        return ret;
+    } else if (ret == CURVEFS_ERROR::NOTEXIST) {
+        std::shared_ptr<InodeWrapper> inode;
+        bool internal = true;
+        ret = MakeNode(req, RECYCLEINODEID, recycleTimeDirName.c_str(),
+              S_IFDIR | 0755, FsFileType::TYPE_DIRECTORY, 0, internal, inode);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "MakeNode failed, ret = " << ret
+                       << ", inode = " << RECYCLEINODEID
+                       << ", name = " << recycleTimeDirName;
+            return ret;
+        }
+        recycleTimeDirIno = inode->GetInodeId();
+    } else {
+        recycleTimeDirIno = dentry.inodeid();
+    }
+
+    // 3. generate new name, parentid_inodeid_name
+    std::string newName = GenerateNewRecycleName(ino, parent, name);
+
+    // 4. move inode to recycle time dir
+    ret = FuseOpRename(req, parent, name,
+                       recycleTimeDirIno, newName.c_str(), 0);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "MoveToRecycle failed, ret = " << ret
+                   << ", inodeid = " << ino
+                   << ", parent = " << parent << ", name = " << name;
+        return ret;
+    }
+
+    return CURVEFS_ERROR::OK;
+}
+
+bool FuseClient::ShouldMoveToRecycle(fuse_ino_t parent) {
+    // 1. check if recycle is open, if recycle not open, return false
+    if (!fsInfo_->has_recycletimehour() || fsInfo_->recycletimehour() ==  0) {
+        return false;
+    }
+
+    // 2. check if inode is in recycle, if node in recycle, return false
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(parent, inodeWrapper);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << parent;
+        return false;
+    }
+
+    InodeAttr attr;
+    inodeWrapper->GetInodeAttrLocked(&attr);
+    if (attr.parent_size() != 0 && attr.parent(0) == RECYCLEINODEID) {
+        return false;
+    }
+
+    return true;
+}
+
+CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
+                                     const char *name, FsFileType type) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+        return CURVEFS_ERROR::NAMETOOLONG;
+    }
+
+    // check if node is recycle or recycle time dir
+    if ((strcmp(name, RECYCLENAME) == 0 && parent == ROOTINODEID) ||
+         parent == RECYCLEINODEID) {
+        return CURVEFS_ERROR::NOPERMISSION;
+    }
+
+    Dentry dentry;
+    CURVEFS_ERROR ret = dentryManager_->GetDentry(parent, name, &dentry);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(WARNING) << "dentryManager_ GetDentry fail, ret = " << ret
+                     << ", parent = " << parent << ", name = " << name;
+        return ret;
+    }
+
+    uint64_t ino = dentry.inodeid();
+
+    // check dir empty
+    if (FsFileType::TYPE_DIRECTORY == type) {
+        std::list<Dentry> dentryList;
+        auto limit = option_.listDentryLimit;
+        ret = dentryManager_->ListDentry(ino, &dentryList, limit);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
+                       << ", parent = " << ino;
+            return ret;
+        }
+        if (!dentryList.empty()) {
+            LOG(ERROR) << "rmdir not empty";
+            return CURVEFS_ERROR::NOTEMPTY;
+        }
+    }
+
+    // check if inode should move to recycle
+    if (ShouldMoveToRecycle(parent)) {
+        ret = MoveToRecycle(req, ino, parent, name, type);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "MoveToRecycle failed, ret = " << ret
+                       << ", inodeid = " << ino;
+            return ret;
+        }
+    } else {
+        ret = DeleteNode(ino, parent, name, type);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "DeleteNode failed, ret = " << ret
+                       << ", inodeid = " << ino;
+            return ret;
+        }
+    }
+
+    return CURVEFS_ERROR::OK;
+}
+
 CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req,
                                         fuse_ino_t ino,
                                         struct fuse_file_info* fi) {
@@ -615,7 +871,6 @@ CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req,
             }
         });
         handler->padding = true;
-        //LOG(INFO) << "<<< padding";
     }
 
     if (off < buffer->size) {
@@ -686,9 +941,10 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
     if (parent != newparent) {
         renameOp.UpdateInodeParent();
     }
+    renameOp.UpdateInodeCtime();
     renameOp.UpdateCache();
 
-    if (enableSumInDir_) {
+    if (enableSumInDir_.load()) {
         xattrManager_->UpdateParentXattrAfterRename(
             parent, newparent, newname, &renameOp);
     }
@@ -772,7 +1028,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
         }
         inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
 
-        if (enableSumInDir_ && changeSize != 0) {
+        if (enableSumInDir_.load() && changeSize != 0) {
             // update parent summary info
             const Inode* inode = inodeWrapper->GetInodeLocked();
             XAttr xattr;
@@ -802,6 +1058,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
 CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, std::string* value,
                                          size_t size) {
+    (void)req;
     VLOG(9) << "FuseOpGetXattr, ino: " << ino
             << ", name: " << name << ", size = " << size;
     if (option_.fileSystemOption.disableXattr) {
@@ -816,7 +1073,8 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
         return ret;
     }
 
-    ret = xattrManager_->GetXattr(name, value, &inodeAttr, enableSumInDir_);
+    ret = xattrManager_->GetXattr(name, value,
+                    &inodeAttr, enableSumInDir_.load());
     if (CURVEFS_ERROR::OK != ret) {
         LOG(ERROR) << "xattrManager get xattr failed, name = " << name;
         return ret;
@@ -876,6 +1134,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
 
 CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize) {
+    (void)req;
     VLOG(1) << "FuseOpListXattr, ino: " << ino << ", size = " << size;
     InodeAttr inodeAttr;
     CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &inodeAttr);
@@ -981,7 +1240,7 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req,
         return ret;
     }
 
-    if (enableSumInDir_) {
+    if (enableSumInDir_.load()) {
         // update parent summary info
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
@@ -1051,7 +1310,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
         return ret;
     }
 
-    if (enableSumInDir_) {
+    if (enableSumInDir_.load()) {
         // update parent summary info
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
@@ -1073,6 +1332,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
 
 CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string *linkStr) {
+    (void)req;
     VLOG(1) << "FuseOpReadLink, ino: " << ino << ", linkStr: " << linkStr;
     InodeAttr attr;
     CURVEFS_ERROR ret = inodeManager_->GetInodeAttr(ino, &attr);
@@ -1099,5 +1359,84 @@ void FuseClient::FlushAll() {
     FlushData();
 }
 
+CURVEFS_ERROR
+FuseClient::SetMountStatus(const struct MountOption *mountOption) {
+    mountpoint_.set_path(
+        (mountOption->mountPoint == nullptr) ? "" : mountOption->mountPoint);
+    std::string fsName =
+        (mountOption->fsName == nullptr) ? "" : mountOption->fsName;
+
+    mountpoint_.set_cto(FLAGS_enableCto);
+
+    int retVal = SetHostPortInMountPoint(&mountpoint_);
+    if (retVal < 0) {
+        LOG(ERROR) << "Set Host and Port in MountPoint failed, ret = "
+                   << retVal;
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    auto ret = mdsClient_->MountFs(fsName, mountpoint_, fsInfo_.get());
+    if (ret != FSStatusCode::OK && ret != FSStatusCode::MOUNT_POINT_EXIST) {
+        LOG(ERROR) << "MountFs failed, FSStatusCode = " << ret
+                   << ", FSStatusCode_Name = " << FSStatusCode_Name(ret)
+                   << ", fsName = " << fsName
+                   << ", mountPoint = " << mountpoint_.ShortDebugString();
+        return CURVEFS_ERROR::MOUNT_FAILED;
+    }
+    inodeManager_->SetFsId(fsInfo_->fsid());
+    dentryManager_->SetFsId(fsInfo_->fsid());
+    enableSumInDir_.store(fsInfo_->enablesumindir());
+    if (fsInfo_->has_recycletimehour()) {
+        enableSumInDir_.store(enableSumInDir_.load() &&
+        (fsInfo_->recycletimehour() == 0));
+    }
+
+    LOG(INFO) << "Mount " << fsName << " on " << mountpoint_.ShortDebugString()
+              << " success!"
+              << " enableSumInDir = " << enableSumInDir_.load();
+
+    fsMetric_ = std::make_shared<FSMetric>(fsName);
+
+    // init fsname and mountpoint
+    leaseExecutor_->SetFsName(fsName);
+    leaseExecutor_->SetMountPoint(mountpoint_);
+    if (!leaseExecutor_->Start()) {
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    init_ = true;
+    if (warmupManager_ != nullptr) {
+        warmupManager_->SetMounted(true);
+    }
+    return CURVEFS_ERROR::OK;
+}
+
+void FuseClient::InitQosParam() {
+    ReadWriteThrottleParams params;
+    params.iopsWrite = ThrottleParams(FLAGS_fuseClientAvgWriteIops,
+                                      FLAGS_fuseClientBurstWriteIops,
+                                      FLAGS_fuseClientBurstWriteIopsSecs);
+
+    params.bpsWrite = ThrottleParams(FLAGS_fuseClientAvgWriteBytes,
+                                     FLAGS_fuseClientBurstWriteBytes,
+                                     FLAGS_fuseClientBurstWriteBytesSecs);
+
+    params.iopsRead = ThrottleParams(FLAGS_fuseClientAvgReadIops,
+                                     FLAGS_fuseClientBurstReadIops,
+                                     FLAGS_fuseClientBurstReadIopsSecs);
+
+    params.bpsRead = ThrottleParams(FLAGS_fuseClientAvgReadBytes,
+                                    FLAGS_fuseClientBurstReadBytes,
+                                    FLAGS_fuseClientBurstReadBytesSecs);
+
+    throttle_.UpdateThrottleParams(params);
+
+    int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1),
+                                on_throttle_timer, this);
+    if (ret != 0) {
+        LOG(ERROR) << "Create fuse client throttle timer failed!";
+    }
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/fuse_client.h

@@ -25,6 +25,7 @@
 
 #include <unistd.h>
 #include <sys/stat.h>
+#include <bthread/unstable.h>
 
 #include <map>
 #include <memory>
@@ -32,6 +33,7 @@
 #include <list>
 #include <utility>
 #include <vector>
+#include <atomic>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/mds.pb.h"
@@ -53,6 +55,7 @@
 #include "curvefs/src/client/lease/lease_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
+#include "src/common/throttle.h"
 #include "curvefs/src/client/filesystem/meta.h"
 #include "curvefs/src/client/filesystem/filesystem.h"
 
@@ -63,6 +66,7 @@ using ::curve::common::InterruptibleSleeper;
 using ::curve::common::Thread;
 using ::curvefs::common::FSType;
 using ::curvefs::metaserver::DentryFlag;
+using ::curvefs::metaserver::ManageInodeType;
 using ::curvefs::client::metric::FSMetric;
 
 namespace curvefs {
@@ -82,6 +86,7 @@ using rpcclient::MetaServerClientImpl;
 using ::curvefs::client::filesystem::FileSystem;
 using ::curvefs::client::filesystem::EntryOut;
 using ::curvefs::client::filesystem::AttrOut;
+using ::curvefs::client::filesystem::FileOut;
 
 using curvefs::common::is_aligned;
 
@@ -97,11 +102,11 @@ class FuseClient {
         inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
         dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
         fsInfo_(nullptr),
-        mdsBase_(nullptr),
-        isStop_(true),
         init_(false),
         enableSumInDir_(false),
-        warmupManager_(nullptr) {}
+        warmupManager_(nullptr),
+        mdsBase_(nullptr),
+        isStop_(true) {}
 
     virtual ~FuseClient() {}
 
@@ -115,11 +120,11 @@ class FuseClient {
             inodeManager_(inodeManager),
             dentryManager_(dentryManager),
             fsInfo_(nullptr),
-            mdsBase_(nullptr),
-            isStop_(true),
             init_(false),
             enableSumInDir_(false),
-            warmupManager_(warmupManager) {}
+            warmupManager_(warmupManager),
+            mdsBase_(nullptr),
+            isStop_(true) {}
 
     virtual CURVEFS_ERROR Init(const FuseClientOption &option);
 
@@ -137,7 +142,7 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                       const char* buf, size_t size, off_t off,
                                       struct fuse_file_info* fi,
-                                      size_t* wSize) = 0;
+                                      FileOut* fileOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                      size_t size, off_t off,
@@ -151,7 +156,8 @@ class FuseClient {
 
     virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req,
                                      fuse_ino_t ino,
-                                     struct fuse_file_info* fi);
+                                     struct fuse_file_info* fi,
+                                     FileOut* fileOut);
 
     virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
                                        fuse_ino_t parent,
@@ -246,11 +252,16 @@ class FuseClient {
                                       struct fuse_file_info* fi) = 0;
     virtual CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                       struct fuse_file_info *fi) {
+        (void)req;
+        (void)ino;
+        (void)fi;
         return CURVEFS_ERROR::OK;
     }
 
     virtual CURVEFS_ERROR FuseOpStatFs(fuse_req_t req, fuse_ino_t ino,
                                        struct statvfs* stbuf) {
+        (void)req;
+        (void)ino;
         // TODO(chengyi01,wuhanqing): implement in s3 and volume client
         stbuf->f_frsize = stbuf->f_bsize = fsInfo_->blocksize();
         stbuf->f_blocks = 10UL << 30;
@@ -294,20 +305,17 @@ class FuseClient {
         enableSumInDir_ = enable;
     }
 
-    bool EnableXattrSumming() {
-        return enableSumInDir_;
-    }
-
-    bool PutWarmFilelistTask(fuse_ino_t key) {
+    bool PutWarmFilelistTask(fuse_ino_t key, common::WarmupStorageType type) {
         if (fsInfo_->fstype() == FSType::TYPE_S3) {
-            return warmupManager_->AddWarmupFilelist(key);
+            return warmupManager_->AddWarmupFilelist(key, type);
         }  // only support s3
         return true;
     }
 
-    bool PutWarmFileTask(fuse_ino_t key, const std::string& path) {
+    bool PutWarmFileTask(fuse_ino_t key, const std::string &path,
+                         common::WarmupStorageType type) {
         if (fsInfo_->fstype() == FSType::TYPE_S3) {
-            return warmupManager_->AddWarmupFile(key, path);
+            return warmupManager_->AddWarmupFile(key, path, type);
         }  // only support s3
         return true;
     }
@@ -319,6 +327,12 @@ class FuseClient {
         return false;
     }
 
+    CURVEFS_ERROR SetMountStatus(const struct MountOption *mountOption);
+
+    void Add(bool isRead, size_t size) { throttle_.Add(isRead, size); }
+
+    void InitQosParam();
+
  protected:
     CURVEFS_ERROR MakeNode(fuse_req_t req,
                            fuse_ino_t parent,
@@ -327,11 +341,29 @@ class FuseClient {
                            FsFileType type,
                            dev_t rdev,
                            bool internal,
-                           std::shared_ptr<InodeWrapper>& InodeWrapper);
+                           std::shared_ptr<InodeWrapper>& InodeWrapper);  // NOLINT
 
     CURVEFS_ERROR RemoveNode(fuse_req_t req, fuse_ino_t parent,
                              const char* name, FsFileType type);
 
+    CURVEFS_ERROR CreateManageNode(fuse_req_t req,
+                                   uint64_t parent,
+                                   const char* name,
+                                   mode_t mode,
+                                   ManageInodeType manageType,
+                                   EntryOut* entryOut);
+
+    CURVEFS_ERROR GetOrCreateRecycleDir(fuse_req_t req, Dentry *out);
+
+    CURVEFS_ERROR DeleteNode(fuse_ino_t ino, fuse_ino_t parent,
+                             const char* name, FsFileType type);
+
+    CURVEFS_ERROR MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
+                                fuse_ino_t parent, const char* name,
+                                FsFileType type);
+
+    bool ShouldMoveToRecycle(fuse_ino_t parent);
+
     CURVEFS_ERROR FuseOpLink(fuse_req_t req,
                              fuse_ino_t ino,
                              fuse_ino_t newparent,
@@ -341,7 +373,8 @@ class FuseClient {
 
     CURVEFS_ERROR HandleOpenFlags(fuse_req_t req,
                                   fuse_ino_t ino,
-                                  struct fuse_file_info* fi);
+                                  struct fuse_file_info* fi,
+                                  FileOut* fileOut);
 
     int SetHostPortInMountPoint(Mountpoint* out) {
         char hostname[kMaxHostNameLength];
@@ -402,7 +435,7 @@ class FuseClient {
     bool init_;
 
     // enable record summary info in dir inode xattr
-    bool enableSumInDir_;
+    std::atomic<bool> enableSumInDir_;
 
     std::shared_ptr<FSMetric> fsMetric_;
 
@@ -419,6 +452,10 @@ class FuseClient {
     Atomic<bool> isStop_;
 
     curve::common::Mutex renameMutex_;
+
+    Throttle throttle_;
+
+    bthread_timer_t throttleTimer_;
 };
 
 }  // namespace client

curvefs/src/client/fuse_common.h

@@ -34,9 +34,9 @@ extern "C" {
 #endif
 
 struct MountOption {
-    char* mountPoint;
-    char* fsName;
-    char* fsType;
+    const char* mountPoint;
+    const char* fsName;
+    const char* fsType;
     char* conf;
     char* mdsAddr;
 };

curvefs/src/client/fuse_s3_client.cpp

@@ -67,10 +67,20 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
 
     auto s3Client = std::make_shared<S3ClientImpl>();
     s3Client->Init(opt.s3Opt.s3AdaptrOpt);
+
+    const uint64_t writeCacheMaxByte =
+        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
+    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
+        LOG(ERROR) << "writeCacheMaxByte is too small"
+                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
+                      ", writeCacheMaxByte = " << writeCacheMaxByte;
+        return CURVEFS_ERROR::CACHETOOSMALL;
+    }
+
     auto fsCacheManager = std::make_shared<FsCacheManager>(
         dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
-        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte, kvClientManager_);
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte, writeCacheMaxByte,
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheThreads, kvClientManager_);
     if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
         DiskCacheType::Disable) {
         auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
@@ -145,7 +155,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
 CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                         const char *buf, size_t size, off_t off,
                                         struct fuse_file_info *fi,
-                                        size_t *wSize) {
+                                        FileOut* fileOut) {
+    size_t *wSize = &fileOut->nwritten;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -207,6 +218,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
             }
         }
     }
+
+    inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
     return ret;
 }
 
@@ -214,6 +227,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                        size_t size, off_t off,
                                        struct fuse_file_info *fi, char *buffer,
                                        size_t *rSize) {
+    (void)req;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -232,7 +246,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
     uint64_t fileSize = inodeWrapper->GetLength();
 
     size_t len = 0;
-    if (fileSize <= off) {
+    if (static_cast<int64_t>(fileSize) <= off) {
         *rSize = 0;
         return CURVEFS_ERROR::OK;
     } else if (fileSize < off + size) {
@@ -329,6 +343,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
 CURVEFS_ERROR FuseS3Client::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                         int datasync,
                                         struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(1) << "FuseOpFsync, ino: " << ino << ", datasync: " << datasync;
 
     CURVEFS_ERROR ret = s3Adaptor_->Flush(ino);
@@ -357,6 +373,8 @@ CURVEFS_ERROR FuseS3Client::Truncate(InodeWrapper *inode, uint64_t length) {
 
 CURVEFS_ERROR FuseS3Client::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(1) << "FuseOpFlush, ino: " << ino;
     CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
 

curvefs/src/client/fuse_s3_client.h

@@ -33,6 +33,7 @@
 #include "curvefs/src/client/fuse_client.h"
 #include "curvefs/src/client/s3/client_s3_cache_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
+#include "curvefs/src/volume/common.h"
 #include "src/common/s3_adapter.h"
 
 namespace curvefs {
@@ -40,6 +41,7 @@ namespace client {
 
 using curve::common::GetObjectAsyncContext;
 using curve::common::GetObjectAsyncCallBack;
+using curvefs::volume::kMiB;
 namespace warmup {
 class WarmupManager;
 class WarmupManagerS3Impl;
@@ -79,7 +81,7 @@ class FuseS3Client : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, size_t *wSize) override;
+        struct fuse_file_info *fi, FileOut* fileOut) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
         fuse_ino_t ino, size_t size, off_t off,
@@ -127,6 +129,8 @@ class FuseS3Client : public FuseClient {
     // s3 adaptor
     std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
     std::shared_ptr<KVClientManager> kvClientManager_;
+
+    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };
 
 

curvefs/src/client/fuse_volume_client.cpp

@@ -133,10 +133,11 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
                                             size_t size,
                                             off_t off,
                                             struct fuse_file_info *fi,
-                                            size_t *wSize) {
+                                            FileOut* fileOut) {
     VLOG(9) << "write start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
+    size_t* wSize = &fileOut->nwritten;
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
               is_aligned(size, DirectIOAlignment))) {
@@ -148,7 +149,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
     butil::Timer timer;
     timer.start();
 
-    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf);
+    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf, fileOut);
     if (ret != CURVEFS_ERROR::OK) {
         if (fsMetric_) {
             fsMetric_->userWrite.eps.count << 1;
@@ -189,6 +190,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
                                            struct fuse_file_info *fi,
                                            char *buffer,
                                            size_t *rSize) {
+    (void)req;
     VLOG(3) << "read start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
@@ -239,8 +241,8 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
               << ", mode: " << mode;
 
     std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR ret = MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, false,
-                                 inode);
+    CURVEFS_ERROR ret = MakeNode(
+        req, parent, name, mode, FsFileType::TYPE_FILE, 0, false, inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
@@ -295,6 +297,8 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
 CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                             int datasync,
                                             struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(3) << "FuseOpFsync start, ino: " << ino << ", datasync: " << datasync;
 
     CURVEFS_ERROR ret = storage_->Flush(ino);
@@ -321,12 +325,16 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
 }
 
 CURVEFS_ERROR FuseVolumeClient::Truncate(InodeWrapper *inode, uint64_t length) {
+    (void)inode;
+    (void)length;
     // Todo: call volume truncate
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                             struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(9) << "FuseOpFlush, ino: " << ino;
 
     CURVEFS_ERROR ret = storage_->Flush(ino);

curvefs/src/client/fuse_volume_client.h

@@ -65,7 +65,7 @@ class FuseVolumeClient : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, size_t *wSize) override;
+        struct fuse_file_info *fi, FileOut* fileOut) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
             fuse_ino_t ino, size_t size, off_t off,

curvefs/src/client/inode_cache_manager.cpp

@@ -202,6 +202,23 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
     return CURVEFS_ERROR::OK;
 }
 
+CURVEFS_ERROR InodeCacheManagerImpl::CreateManageInode(
+    const InodeParam &param,
+    std::shared_ptr<InodeWrapper> &out) {
+    Inode inode;
+    MetaStatusCode ret = metaClient_->CreateManageInode(param, &inode);
+    if (ret != MetaStatusCode::OK) {
+        LOG(ERROR) << "metaClient_ CreateManageInode failed, MetaStatusCode = "
+                   << ret << ", MetaStatusCode_Name = "
+                   << MetaStatusCode_Name(ret);
+        return ToFSError(ret);
+    }
+    out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
+        s3ChunkInfoMetric_, option_.maxDataSize,
+        option_.refreshDataIntervalSec);
+    return CURVEFS_ERROR::OK;
+}
+
 CURVEFS_ERROR InodeCacheManagerImpl::DeleteInode(uint64_t inodeId) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
     MetaStatusCode ret = metaClient_->DeleteInode(fsId_, inodeId);

curvefs/src/client/inode_cache_manager.h

@@ -104,6 +104,9 @@ class InodeCacheManager {
     virtual CURVEFS_ERROR CreateInode(const InodeParam &param,
         std::shared_ptr<InodeWrapper> &out) = 0;   // NOLINT
 
+    virtual CURVEFS_ERROR CreateManageInode(const InodeParam &param,
+        std::shared_ptr<InodeWrapper> &out) = 0;   // NOLINT
+
     virtual CURVEFS_ERROR DeleteInode(uint64_t inodeId) = 0;
 
     virtual void ShipToFlush(
@@ -115,7 +118,6 @@ class InodeCacheManager {
 
 class InodeCacheManagerImpl : public InodeCacheManager,
     public std::enable_shared_from_this<InodeCacheManagerImpl> {
-
  public:
     InodeCacheManagerImpl()
       : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
@@ -152,6 +154,9 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     CURVEFS_ERROR CreateInode(const InodeParam &param,
         std::shared_ptr<InodeWrapper> &out) override;
 
+    CURVEFS_ERROR CreateManageInode(const InodeParam &param,
+        std::shared_ptr<InodeWrapper> &out) override;
+
     CURVEFS_ERROR DeleteInode(uint64_t inodeId) override;
 
     void ShipToFlush(

curvefs/src/client/inode_wrapper.cpp

@@ -107,6 +107,7 @@ class UpdateInodeAsyncDone : public MetaServerClientDone {
                        << ", inodeid: " << inodeWrapper_->GetInodeId();
             inodeWrapper_->MarkInodeError();
         }
+        inodeWrapper_->ClearDirty();
         inodeWrapper_->ReleaseSyncingInode();
 
         if (parent_ != nullptr) {
@@ -226,7 +227,6 @@ void InodeWrapper::AsyncFlushAttr(MetaServerClientDone* done,
         metaClient_->UpdateInodeWithOutNlinkAsync(
             inode_.fsid(), inode_.inodeid(), dirtyAttr_,
             new UpdateInodeAsyncDone(shared_from_this(), done));
-        dirty_ = false;
         dirtyAttr_.Clear();
         return;
     }
@@ -301,7 +301,7 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
     UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize() - before);
     ClearS3ChunkInfoAdd();
     UpdateMaxS3ChunkInfoSize();
-    lastRefreshTime_ = ::curve::common::TimeUtility::GetTimeofDaySec();
+    lastRefreshTime_ = TimeUtility::GetTimeofDaySec();
     return CURVEFS_ERROR::OK;
 }
 
@@ -489,7 +489,6 @@ void InodeWrapper::AsyncFlushAttrAndExtents(MetaServerClientDone *done,
             new UpdateInodeAttrAndExtentClosure{shared_from_this(), done},
             std::move(indices));
 
-        dirty_ = false;
         dirtyAttr_.Clear();
         return;
     }
@@ -543,6 +542,7 @@ class UpdateInodeAsyncS3Done : public MetaServerClientDone {
             inodeWrapper_->MarkInodeError();
         }
         VLOG(9) << "inode " << inodeWrapper_->GetInodeId() << " async success.";
+        inodeWrapper_->ClearDirty();
         inodeWrapper_->ReleaseSyncingInode();
         inodeWrapper_->ReleaseSyncingS3ChunkInfo();
 
@@ -559,6 +559,7 @@ class UpdateInodeAsyncS3Done : public MetaServerClientDone {
 }  // namespace
 
 void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
+    (void)internal;
     if (dirty_ || !s3ChunkInfoAdd_.empty()) {
         LockSyncingInode();
         LockSyncingS3ChunkInfo();
@@ -570,7 +571,6 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
             inode_.fsid(), inode_.inodeid(), dirtyAttr_,
             new UpdateInodeAsyncS3Done{shared_from_this(), done},
             std::move(indices));
-        dirty_ = false;
         dirtyAttr_.Clear();
         ClearS3ChunkInfoAdd();
         return;
@@ -610,7 +610,8 @@ CURVEFS_ERROR InodeWrapper::RefreshNlink() {
         return CURVEFS_ERROR::INTERNAL;
     }
     inode_.set_nlink(attr.nlink());
-    VLOG(3) << "RefreshNlink from metaserver, newnlink: " << attr.nlink();
+    VLOG(3) << "RefreshNlink from metaserver, newnlink: " << attr.nlink()
+            << ", inodeid: " << inode_.inodeid();
     return CURVEFS_ERROR::OK;
 }
 

curvefs/src/client/inode_wrapper.h

@@ -65,6 +65,7 @@ using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;
 using metric::S3ChunkInfoMetric;
 using common::NlinkChange;
+using curve::common::TimeUtility;
 
 std::ostream &operator<<(std::ostream &os, const struct stat &attr);
 void AppendS3ChunkInfoToMap(uint64_t chunkIndex, const S3ChunkInfo &info,
@@ -77,18 +78,19 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     InodeWrapper(Inode inode,
                  std::shared_ptr<MetaServerClient> metaClient,
                  std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric = nullptr,
-                 uint64_t maxDataSize = ULONG_MAX,
+                 int64_t maxDataSize = LONG_MAX,
                  uint32_t refreshDataInterval = UINT_MAX)
         : inode_(std::move(inode)),
           status_(InodeStatus::kNormal),
           baseMaxDataSize_(maxDataSize),
           maxDataSize_(maxDataSize),
           refreshDataInterval_(refreshDataInterval),
-          lastRefreshTime_(::curve::common::TimeUtility::GetTimeofDaySec()),
+          lastRefreshTime_(TimeUtility::GetTimeofDaySec()),
           s3ChunkInfoAddSize_(0),
           metaClient_(std::move(metaClient)),
           s3ChunkInfoMetric_(std::move(s3ChunkInfoMetric)),
-          dirty_(false) {
+          dirty_(false),
+          time_(TimeUtility::GetTimeofDaySec()) {
         UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize());
         g_alive_inode_count << 1;
     }
@@ -337,6 +339,10 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         return false;
     }
 
+    uint32_t GetCachedTime() const {
+        return time_;
+    }
+
  private:
     CURVEFS_ERROR SyncS3ChunkInfo(bool internal = false);
 
@@ -392,8 +398,8 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     InodeAttr dirtyAttr_;
 
     InodeStatus status_;
-    uint64_t baseMaxDataSize_;
-    uint64_t maxDataSize_;
+    int64_t baseMaxDataSize_;
+    int64_t maxDataSize_;
     uint32_t refreshDataInterval_;
     uint64_t lastRefreshTime_;
 
@@ -411,6 +417,9 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
 
     mutable ::curve::common::Mutex syncingVolumeExtentsMtx_;
     ExtentCache extentCache_;
+
+    // timestamp when put in cache
+    uint64_t time_;
 };
 
 }  // namespace client

curvefs/src/client/kvclient/memcache_client.h

@@ -118,6 +118,8 @@ class MemCachedClient : public KVClient {
             return true;
         }
         *errorlog = ResError(res);
+        memcached_free(tcli);
+        tcli = nullptr;
         LOG(ERROR) << "Set key = " << key << " error = " << *errorlog;
         return false;
     }
@@ -144,9 +146,11 @@ class MemCachedClient : public KVClient {
 
         *errorlog = ResError(ue);
         if (ue != MEMCACHED_NOTFOUND) {
-            LOG(ERROR) << "Get key = " << key << " error = " << *errorlog
-                       << ", get_value_len = " << value_length
-                       << ", expect_value_len = " << length;
+          LOG(ERROR) << "Get key = " << key << " error = " << *errorlog
+                     << ", get_value_len = " << value_length
+                     << ", expect_value_len = " << length;
+          memcached_free(tcli);
+          tcli = nullptr;
         }
 
         return false;

curvefs/src/client/lease/lease_excutor.cpp

@@ -41,7 +41,7 @@ LeaseExecutor::~LeaseExecutor() {
 bool LeaseExecutor::Start() {
     if (opt_.leaseTimeUs <= 0 || opt_.refreshTimesPerLease <= 0) {
         LOG(ERROR) << "LeaseExecutor start fail. Invalid param in leaseopt, "
-                      "leasTimesUs = "
+                      "leaseTimeUs = "
                    << opt_.leaseTimeUs
                    << ", refreshTimePerLease = " << opt_.refreshTimesPerLease;
         return false;
@@ -78,7 +78,8 @@ bool LeaseExecutor::RefreshLease() {
     // refresh from mds
     std::vector<PartitionTxId> latestTxIdList;
     FSStatusCode ret = mdsCli_->RefreshSession(txIds, &latestTxIdList,
-                                               fsName_, mountpoint_);
+                                               fsName_, mountpoint_,
+                                               enableSumInDir_);
     if (ret != FSStatusCode::OK) {
         LOG(ERROR) << "LeaseExecutor refresh session fail, ret = " << ret
                    << ", errorName = " << FSStatusCode_Name(ret);

curvefs/src/client/lease/lease_excutor.h

@@ -25,6 +25,7 @@
 
 #include <memory>
 #include <string>
+#include <atomic>
 
 #include "curvefs/src/client/rpcclient/metacache.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
@@ -44,8 +45,10 @@ namespace client {
 class LeaseExecutor : public LeaseExecutorBase {
  public:
     LeaseExecutor(const LeaseOpt &opt, std::shared_ptr<MetaCache> metaCache,
-                  std::shared_ptr<MdsClient> mdsCli)
-        : opt_(opt), metaCache_(metaCache), mdsCli_(mdsCli) {}
+                  std::shared_ptr<MdsClient> mdsCli,
+                  std::atomic<bool>* enableSumInDir)
+         : opt_(opt), metaCache_(metaCache), mdsCli_(mdsCli),
+         enableSumInDir_(enableSumInDir) {}
 
     ~LeaseExecutor();
 
@@ -73,6 +76,7 @@ class LeaseExecutor : public LeaseExecutorBase {
     std::unique_ptr<RefreshSessionTask> task_;
     std::string fsName_;
     Mountpoint mountpoint_;
+    std::atomic<bool>* enableSumInDir_;
 };
 
 }  // namespace client

curvefs/src/client/main.cpp

@@ -21,7 +21,11 @@
  * Author: xuchaojie
  */
 
+#include <glog/logging.h>
+
+#include <string>
 #include <unordered_map>
+
 #include "curvefs/src/client/curve_fuse_op.h"
 #include "curvefs/src/client/fuse_common.h"
 
@@ -103,7 +107,7 @@ void extra_options_help() {
 
 std::string match_any_pattern(
     const std::unordered_map<std::string, char**>& patterns, const char* src) {
-    int src_len = strlen(src);
+    size_t src_len = strlen(src);
     for (const auto& pair : patterns) {
         const auto& pattern = pair.first;
         if (pattern.length() < src_len &&
@@ -220,14 +224,14 @@ int main(int argc, char *argv[]) {
         printf("Init log failed, confpath = %s\n", mOpts.conf);
     }
 
-    ret = InitFuseClient(mOpts.conf, mOpts.fsName, mOpts.fsType, mOpts.mdsAddr);
+    ret = InitFuseClient(&mOpts);
     if (ret < 0) {
-        printf("init fuse client fail, conf =%s\n", mOpts.conf);
+        LOG(ERROR) << "init fuse client fail, conf = " << mOpts.conf;
         goto err_out4;
     }
 
-    printf("fuse start loop, singlethread = %d, max_idle_threads = %d\n",
-        opts.singlethread, opts.max_idle_threads);
+    LOG(INFO) << "fuse start loop, singlethread = " << opts.singlethread
+              << ", max_idle_threads = " << opts.max_idle_threads;
 
     /* Block until ctrl+c or fusermount -u */
     if (opts.singlethread) {

curvefs/src/client/metric/BUILD

@@ -22,5 +22,9 @@ cc_library(
     hdrs = glob(["*.h"]),
     copts = CURVE_DEFAULT_COPTS,
     visibility = ["//visibility:public"],
-    deps = ["//external:gflags"],
+    deps = [
+        "//external:gflags",
+        "//external:bvar",
+        "//src/client:curve_client",
+    ],
 )

curvefs/src/client/metric/client_metric.cpp

@@ -0,0 +1,43 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Created Date: Fri Apr 21 2023
+ * Author: Xinlong-Chen
+ */
+
+#include "curvefs/src/client/metric/client_metric.h"
+
+namespace curvefs {
+namespace client {
+namespace metric {
+
+const std::string MDSClientMetric::prefix = "curvefs_mds_client";  // NOLINT
+const std::string MetaServerClientMetric::prefix = "curvefs_metaserver_client";  // NOLINT
+const std::string ClientOpMetric::prefix = "curvefs_client";  // NOLINT
+const std::string S3MultiManagerMetric::prefix = "curvefs_client_manager";  // NOLINT
+const std::string FSMetric::prefix = "curvefs_client";  // NOLINT
+const std::string S3Metric::prefix = "curvefs_s3";  // NOLINT
+const std::string DiskCacheMetric::prefix = "curvefs_disk_cache";  // NOLINT
+const std::string KVClientMetric::prefix = "curvefs_kvclient";  // NOLINT
+const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info";  // NOLINT
+const std::string WarmupManagerS3Metric::prefix = "curvefs_warmup";  // NOLINT
+
+}  // namespace metric
+}  // namespace client
+}  // namespace curvefs
+

curvefs/src/client/metric/client_metric.h

@@ -36,8 +36,7 @@ namespace client {
 namespace metric {
 
 struct MDSClientMetric {
-    std::string prefix;
-    std::string mdsAddrs;
+    static const std::string prefix;
 
     InterfaceMetric mountFs;
     InterfaceMetric umountFs;
@@ -53,11 +52,8 @@ struct MDSClientMetric {
     InterfaceMetric commitTx;
     InterfaceMetric allocOrGetMemcacheCluster;
 
-    explicit MDSClientMetric(const std::string& prefix_ = "")
-        : prefix(!prefix_.empty() ? prefix_
-                                  : "curvefs_mds_client_" +
-                                        curve::common::ToHexString(this)),
-          mountFs(prefix, "mountFs"),
+    MDSClientMetric()
+        : mountFs(prefix, "mountFs"),
           umountFs(prefix, "umountFs"),
           getFsInfo(prefix, "getFsInfo"),
           getMetaServerInfo(prefix, "getMetaServerInfo"),
@@ -73,7 +69,7 @@ struct MDSClientMetric {
 };
 
 struct MetaServerClientMetric {
-    std::string prefix;
+    static const std::string prefix;
 
     // dentry
     InterfaceMetric getDentry;
@@ -97,11 +93,8 @@ struct MetaServerClientMetric {
     InterfaceMetric updateVolumeExtent;
     InterfaceMetric getVolumeExtent;
 
-    explicit MetaServerClientMetric(const std::string &prefix_ = "")
-        : prefix(!prefix_.empty() ? prefix_
-                                  : "curvefs_metaserver_client_" +
-                                        curve::common::ToHexString(this)),
-          getDentry(prefix, "getDentry"),
+    MetaServerClientMetric()
+        : getDentry(prefix, "getDentry"),
           listDentry(prefix, "listDentry"),
           createDentry(prefix, "createDentry"),
           deleteDentry(prefix, "deleteDentry"),
@@ -142,7 +135,7 @@ struct OpMetric {
 };
 
 struct ClientOpMetric {
-    std::string prefix;
+    static const std::string prefix;
 
     OpMetric opLookup;
     OpMetric opOpen;
@@ -169,10 +162,8 @@ struct ClientOpMetric {
     OpMetric opWrite;
 
 
-    explicit ClientOpMetric(const std::string &prefix_ = "")
-        : prefix(!prefix_.empty() ? prefix_
-                                  : "curvefs_client"),
-          opLookup(prefix, "opLookup"),
+    ClientOpMetric()
+        : opLookup(prefix, "opLookup"),
           opOpen(prefix, "opOpen"),
           opCreate(prefix, "opCreate"),
           opMkNod(prefix, "opMkNod"),
@@ -198,7 +189,8 @@ struct ClientOpMetric {
 };
 
 struct S3MultiManagerMetric {
-    const std::string prefix;
+    static const std::string prefix;
+
     bvar::Adder<int64_t> fileManagerNum;
     bvar::Adder<int64_t> chunkManagerNum;
     bvar::Adder<int64_t> writeDataCacheNum;
@@ -206,9 +198,7 @@ struct S3MultiManagerMetric {
     bvar::Adder<int64_t> readDataCacheNum;
     bvar::Adder<int64_t> readDataCacheByte;
 
-    explicit S3MultiManagerMetric(
-        const std::string &prefix_ = "curvefs_client_manager")
-        : prefix(prefix_) {
+    S3MultiManagerMetric() {
         fileManagerNum.expose_as(prefix, "file_manager_num");
         chunkManagerNum.expose_as(prefix, "chunk_manager_num");
         writeDataCacheNum.expose_as(prefix, "write_data_cache_num");
@@ -219,7 +209,7 @@ struct S3MultiManagerMetric {
 };
 
 struct FSMetric {
-    const std::string prefix = "curvefs_client";
+    static const std::string prefix;
 
     std::string fsName;
 
@@ -238,7 +228,7 @@ struct FSMetric {
 };
 
 struct S3Metric {
-    const std::string prefix = "curvefs_s3";
+    static const std::string prefix;
 
     std::string fsName;
     InterfaceMetric adaptorWrite;
@@ -264,7 +254,7 @@ struct S3Metric {
 };
 
 struct DiskCacheMetric {
-    const std::string prefix = "curvefs_disk_cache";
+    static const std::string prefix;
 
     std::string fsName;
     InterfaceMetric writeS3;
@@ -278,16 +268,16 @@ struct DiskCacheMetric {
 };
 
 struct KVClientMetric {
-    const std::string prefix = "curvefs_kvclient";
-    InterfaceMetric kvClientSet;
+    static const std::string prefix;
     InterfaceMetric kvClientGet;
+    InterfaceMetric kvClientSet;
 
     KVClientMetric()
         : kvClientGet(prefix, "get"), kvClientSet(prefix, "set") {}
 };
 
 struct S3ChunkInfoMetric {
-    const std::string prefix = "inode_s3_chunk_info";
+    static const std::string prefix;
 
     bvar::Adder<int64_t> s3ChunkInfoSize;
 
@@ -295,7 +285,8 @@ struct S3ChunkInfoMetric {
 };
 
 struct WarmupManagerS3Metric {
-    const std::string prefix = "curvefs_warmup";
+    static const std::string prefix;
+
     InterfaceMetric warmupS3Cached;
     bvar::Adder<uint64_t> warmupS3CacheSize;
 

curvefs/src/client/rpcclient/base_client.h

@@ -42,6 +42,8 @@ using curvefs::metaserver::CreateDentryRequest;
 using curvefs::metaserver::CreateDentryResponse;
 using curvefs::metaserver::CreateInodeRequest;
 using curvefs::metaserver::CreateInodeResponse;
+using curvefs::metaserver::CreateManageInodeRequest;
+using curvefs::metaserver::CreateManageInodeResponse;
 using curvefs::metaserver::DeleteDentryRequest;
 using curvefs::metaserver::DeleteDentryResponse;
 using curvefs::metaserver::DeleteInodeRequest;
@@ -59,6 +61,7 @@ using curvefs::metaserver::PrepareRenameTxRequest;
 using curvefs::metaserver::PrepareRenameTxResponse;
 using curvefs::metaserver::UpdateInodeRequest;
 using curvefs::metaserver::UpdateInodeResponse;
+using curvefs::metaserver::ManageInodeType;
 
 using curvefs::common::FSType;
 using curvefs::common::PartitionInfo;
@@ -122,6 +125,7 @@ struct InodeParam {
     uint64_t rdev;
     std::string symlink;
     uint64_t parent;
+    ManageInodeType manageType = ManageInodeType::TYPE_NOT_MANAGE;
 };
 
 inline std::ostream& operator<<(std::ostream& os, const InodeParam& p) {

curvefs/src/client/rpcclient/mds_client.cpp

@@ -59,6 +59,8 @@ MdsClientImpl::Init(const ::curve::client::MetaServerOption &mdsOpt,
 FSStatusCode MdsClientImpl::MountFs(const std::string& fsName,
                                     const Mountpoint& mountPt, FsInfo* fsInfo) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.mountFs.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.mountFs.latency);
         MountFsResponse response;
@@ -88,6 +90,8 @@ FSStatusCode MdsClientImpl::MountFs(const std::string& fsName,
 FSStatusCode MdsClientImpl::UmountFs(const std::string& fsName,
                                      const Mountpoint& mountPt) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.umountFs.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.umountFs.latency);
         UmountFsResponse response;
@@ -113,6 +117,8 @@ FSStatusCode MdsClientImpl::UmountFs(const std::string& fsName,
 FSStatusCode MdsClientImpl::GetFsInfo(const std::string &fsName,
                                       FsInfo *fsInfo) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.getFsInfo.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.getFsInfo.latency);
         GetFsInfoResponse response;
@@ -143,6 +149,8 @@ FSStatusCode MdsClientImpl::GetFsInfo(const std::string &fsName,
 
 FSStatusCode MdsClientImpl::GetFsInfo(uint32_t fsId, FsInfo *fsInfo) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.getFsInfo.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.getFsInfo.latency);
         GetFsInfoResponse response;
@@ -203,6 +211,9 @@ bool MdsClientImpl::GetMetaServerInfo(
     ::curve::common::StringToUll(strs[1], &port);
 
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
+        (void)addrindex;
         mdsClientMetric_.getMetaServerInfo.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.getMetaServerInfo.latency);
         GetMetaServerInfoResponse response;
@@ -239,6 +250,8 @@ bool MdsClientImpl::GetMetaServerListInCopysets(
     const LogicPoolID &logicalpooid, const std::vector<CopysetID> &copysetidvec,
     std::vector<CopysetInfo<MetaserverID>> *cpinfoVec) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.getMetaServerListInCopysets.qps.count << 1;
         LatencyUpdater updater(
             &mdsClientMetric_.getMetaServerListInCopysets.latency);
@@ -290,6 +303,8 @@ bool MdsClientImpl::GetMetaServerListInCopysets(
 bool MdsClientImpl::CreatePartition(
     uint32_t fsID, uint32_t count, std::vector<PartitionInfo> *partitionInfos) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.createPartition.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.createPartition.latency);
         CreatePartitionResponse response;
@@ -335,6 +350,8 @@ bool MdsClientImpl::GetCopysetOfPartitions(
     const std::vector<uint32_t> &partitionIDList,
     std::map<uint32_t, Copyset> *copysetMap) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.getCopysetOfPartitions.qps.count << 1;
         LatencyUpdater updater(
             &mdsClientMetric_.getCopysetOfPartitions.latency);
@@ -379,6 +396,8 @@ bool MdsClientImpl::GetCopysetOfPartitions(
 bool MdsClientImpl::ListPartition(uint32_t fsID,
                                   std::vector<PartitionInfo> *partitionInfos) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.listPartition.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.listPartition.latency);
         ListPartitionResponse response;
@@ -416,6 +435,8 @@ bool MdsClientImpl::ListPartition(uint32_t fsID,
 bool MdsClientImpl::AllocOrGetMemcacheCluster(uint32_t fsId,
                                               MemcacheClusterInfo* cluster) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.allocOrGetMemcacheCluster.qps.count << 1;
         LatencyUpdater updater(
             &mdsClientMetric_.allocOrGetMemcacheCluster.latency);
@@ -447,6 +468,8 @@ bool MdsClientImpl::AllocOrGetMemcacheCluster(uint32_t fsId,
 FSStatusCode MdsClientImpl::AllocS3ChunkId(uint32_t fsId, uint32_t idNum,
                                            uint64_t *chunkId) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.allocS3ChunkId.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.allocS3ChunkId.latency);
         AllocateS3ChunkResponse response;
@@ -478,8 +501,11 @@ FSStatusCode
 MdsClientImpl::RefreshSession(const std::vector<PartitionTxId> &txIds,
                               std::vector<PartitionTxId> *latestTxIdList,
                               const std::string& fsName,
-                              const Mountpoint& mountpoint) {
+                              const Mountpoint& mountpoint,
+                              std::atomic<bool>* enableSumInDir) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         mdsClientMetric_.refreshSession.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.refreshSession.latency);
         RefreshSessionRequest request;
@@ -507,6 +533,11 @@ MdsClientImpl::RefreshSession(const std::vector<PartitionTxId> &txIds,
             LOG(INFO) << "RefreshSession need update partition txid list: "
                       << response.DebugString();
         }
+        if (enableSumInDir->load() && !response.enablesumindir()) {
+            enableSumInDir->store(response.enablesumindir());
+            LOG(INFO) << "update enableSumInDir to "
+                      << response.enablesumindir();
+        }
 
         return ret;
     };
@@ -517,6 +548,8 @@ MdsClientImpl::RefreshSession(const std::vector<PartitionTxId> &txIds,
 FSStatusCode MdsClientImpl::GetLatestTxId(const GetLatestTxIdRequest& request,
                                           GetLatestTxIdResponse* response) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         VLOG(3) << "GetLatestTxId [request]: " << request.DebugString();
         mdsClientMetric_.getLatestTxId.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.getLatestTxId.latency);
@@ -552,6 +585,8 @@ FSStatusCode MdsClientImpl::GetLatestTxId(const GetLatestTxIdRequest& request,
 
 FSStatusCode MdsClientImpl::CommitTx(const CommitTxRequest& request) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         VLOG(3) << "CommitTx [request]: " << request.DebugString();
         mdsClientMetric_.commitTx.qps.count << 1;
         LatencyUpdater updater(&mdsClientMetric_.commitTx.latency);
@@ -670,6 +705,8 @@ SpaceErrCode MdsClientImpl::AllocateVolumeBlockGroup(
     const std::string &owner,
     std::vector<curvefs::mds::space::BlockGroup> *groups) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         AllocateBlockGroupResponse response;
         mdsbasecli_->AllocateVolumeBlockGroup(fsId, count, owner, &response,
                                               cntl, channel);
@@ -705,6 +742,8 @@ SpaceErrCode MdsClientImpl::AcquireVolumeBlockGroup(
     const std::string &owner,
     curvefs::mds::space::BlockGroup *groups) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         AcquireBlockGroupResponse response;
         mdsbasecli_->AcquireVolumeBlockGroup(fsId, blockGroupOffset, owner,
                                              &response, cntl, channel);
@@ -730,6 +769,8 @@ SpaceErrCode MdsClientImpl::ReleaseVolumeBlockGroup(
     const std::string &owner,
     const std::vector<curvefs::mds::space::BlockGroup> &blockGroups) {
     auto task = RPCTask {
+        (void)addrindex;
+        (void)rpctimeoutMS;
         ReleaseBlockGroupResponse response;
         mdsbasecli_->ReleaseVolumeBlockGroup(fsId, owner, blockGroups,
                                              &response, cntl, channel);

curvefs/src/client/rpcclient/mds_client.h

@@ -26,6 +26,7 @@
 #include <memory>
 #include <string>
 #include <vector>
+#include <atomic>
 
 #include "curvefs/proto/mds.pb.h"
 #include "curvefs/proto/topology.pb.h"
@@ -115,7 +116,8 @@ class MdsClient {
     RefreshSession(const std::vector<PartitionTxId> &txIds,
                    std::vector<PartitionTxId> *latestTxIdList,
                    const std::string& fsName,
-                   const Mountpoint& mountpoint) = 0;
+                   const Mountpoint& mountpoint,
+                   std::atomic<bool>* enableSumInDir) = 0;
 
     virtual FSStatusCode GetLatestTxId(uint32_t fsId,
                                        std::vector<PartitionTxId>* txIds) = 0;
@@ -159,8 +161,7 @@ class MdsClient {
 
 class MdsClientImpl : public MdsClient {
  public:
-    explicit MdsClientImpl(const std::string &metricPrefix = "")
-        : mdsClientMetric_(metricPrefix) {}
+    MdsClientImpl() = default;
 
     FSStatusCode Init(const ::curve::client::MetaServerOption &mdsOpt,
                       MDSBaseClient *baseclient) override;
@@ -203,7 +204,8 @@ class MdsClientImpl : public MdsClient {
     FSStatusCode RefreshSession(const std::vector<PartitionTxId> &txIds,
                                 std::vector<PartitionTxId> *latestTxIdList,
                                 const std::string& fsName,
-                                const Mountpoint& mountpoint) override;
+                                const Mountpoint& mountpoint,
+                                std::atomic<bool>* enableSumInDir) override;
 
     FSStatusCode GetLatestTxId(uint32_t fsId,
                                std::vector<PartitionTxId>* txIds) override;

curvefs/src/client/rpcclient/metacache.cpp

@@ -50,13 +50,11 @@ void MetaCache::GetTxId(uint32_t partitionId, uint64_t *txId) {
     }
 }
 
-bool MetaCache::GetTxId(uint32_t fsId,
-                        uint64_t inodeId,
-                        uint32_t *partitionId,
+bool MetaCache::GetTxId(uint32_t fsId, uint64_t inodeId, uint32_t *partitionId,
                         uint64_t *txId) {
     for (const auto &partition : partitionInfos_) {
-        if (fsId == partition.fsid() &&
-            inodeId >= partition.start() && inodeId <= partition.end()) {
+        if (fsId == partition.fsid() && inodeId >= partition.start() &&
+            inodeId <= partition.end()) {
             *partitionId = partition.partitionid();
             *txId = partition.txid();
             GetTxId(*partitionId, txId);
@@ -85,7 +83,7 @@ bool MetaCache::RefreshTxId() {
         return false;
     }
 
-    for (const auto& item : txIds) {
+    for (const auto &item : txIds) {
         SetTxId(item.partitionid(), item.txid());
     }
     return true;
@@ -215,7 +213,7 @@ bool MetaCache::GetTargetLeader(CopysetTarget *target, uint64_t *applyindex,
     // if cacahe do not have invalid leader, refresh leader
     VLOG(3) << "refresh leader for " << target->groupID.ToString();
     bool ret = true;
-    uint32_t retry = 0;
+    int retry = 0;
     while (retry++ < metacacheopt_.metacacheGetLeaderRetry) {
         // refresh from metaserver
         ret = UpdateLeaderInternal(target->groupID, &copysetInfo);
@@ -272,7 +270,7 @@ bool MetaCache::CreatePartitions(int currentNum,
     // already create
     {
         ReadLockGuard rl(rwlock4Partitions_);
-        if (partitionInfos_.size() > currentNum) {
+        if (static_cast<int>(partitionInfos_.size()) > currentNum) {
             newPartitions->reserve(partitionInfos_.size() - currentNum);
             newPartitions->insert(newPartitions->end(),
                                   partitionInfos_.begin() + currentNum,
@@ -290,8 +288,8 @@ bool MetaCache::CreatePartitions(int currentNum,
     // add partition and copyset info
     WriteLockGuard wl4PartitionMap(rwlock4Partitions_);
     WriteLockGuard wl4CopysetMap(rwlock4copysetInfoMap_);
-    DoAddOrResetPartitionAndCopyset(*newPartitions,
-                                    std::move(copysetMap), false);
+    DoAddOrResetPartitionAndCopyset(*newPartitions, std::move(copysetMap),
+                                    false);
 
     return true;
 }
@@ -476,6 +474,7 @@ void MetaCache::UpdateCopysetInfoIfMatchCurrentLeader(
     std::vector<CopysetInfo<MetaserverID>> metaServerInfos;
     bool ret = mdsClient_->GetMetaServerListInCopysets(
         groupID.poolID, {groupID.copysetID}, &metaServerInfos);
+    (void)ret;
 
     bool needUpdate = (!metaServerInfos.empty()) &&
                       (metaServerInfos[0].HasPeerInCopyset(leaderAddr));
@@ -562,7 +561,8 @@ bool MetaCache::GetCopysetInfowithCopySetID(
 }
 
 bool TryGetPartitionIdByInodeId(const std::vector<PartitionInfo> &plist,
-    RWLock *lock, uint64_t inodeID, PartitionID *pid) {
+                                RWLock *lock, uint64_t inodeID,
+                                PartitionID *pid) {
     ReadLockGuard rl(*lock);
     for (const auto &it : plist) {
         if (it.start() <= inodeID && it.end() >= inodeID) {
@@ -574,17 +574,17 @@ bool TryGetPartitionIdByInodeId(const std::vector<PartitionInfo> &plist,
 }
 
 bool MetaCache::GetPartitionIdByInodeId(uint32_t fsID, uint64_t inodeID,
-    PartitionID *pid) {
-    if (!TryGetPartitionIdByInodeId(partitionInfos_,
-                                    &rwlock4Partitions_, inodeID, pid)) {
+                                        PartitionID *pid) {
+    if (!TryGetPartitionIdByInodeId(partitionInfos_, &rwlock4Partitions_,
+                                    inodeID, pid)) {
         // list form mds
         if (!ListPartitions(fsID)) {
             LOG(ERROR) << "ListPartitions for {fsid:" << fsID
                        << "} fail, partition list not exist";
             return false;
         }
-        return TryGetPartitionIdByInodeId(partitionInfos_,
-            &rwlock4Partitions_, inodeID, pid);
+        return TryGetPartitionIdByInodeId(partitionInfos_, &rwlock4Partitions_,
+                                          inodeID, pid);
     }
     return true;
 }