[feat] tools-v2: add snapshot copyset --all

Signed-off-by: montaguelhz <1443171175@qq.com>

.bazelrc

@@ -1,5 +1,10 @@
+build --verbose_failures
+
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
-build --incompatible_blacklisted_protos_requires_proto_info=false
+build --incompatible_blacklisted_protos_requires_proto_info=false
+build --copt=-fdiagnostics-color=always
+
+run --copt=-fdiagnostics-color=always

.clang-format

@@ -0,0 +1,147 @@
+---
+Language:        Cpp
+AccessModifierOffset: -3
+AlignAfterOpenBracket: Align
+AlignConsecutiveMacros: true
+AlignConsecutiveAssignments: false
+AlignConsecutiveBitFields: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   Align
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: true
+AllowAllConstructorInitializersOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortEnumsOnASingleLine: true
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortLambdasOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: MultiLine
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterCaseLabel:  false
+  AfterClass:      false
+  AfterControlStatement: Never
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  BeforeLambdaBody: false
+  BeforeWhile:     false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DeriveLineEnding: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority:        2
+    SortPriority:    0
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
+    SortPriority:    0
+  - Regex:           '.*'
+    Priority:        1
+    SortPriority:    0
+IncludeIsMainRegex: '(Test)?$'
+IncludeIsMainSourceRegex: ''
+IndentCaseLabels: false
+IndentCaseBlocks: false
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentExternBlock: AfterExternBlock
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+InsertTrailingCommas: None
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 2
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 2
+ObjCBreakBeforeNestedBlockParam: true
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    false
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInConditionalStatement: false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpaceBeforeSquareBrackets: false
+Standard:        Cpp11
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        4
+UseCRLF:         false
+UseTab:          Never
+WhitespaceSensitiveMacros:
+  - STRINGIZE
+  - PP_STRINGIZE
+  - BOOST_PP_STRINGIZE
+...

.clangd

@@ -0,0 +1,12 @@
+Diagnostics:
+  ClangTidy:
+    Add: [performance-*, modernize-*, readability-*,bugprone-*,]
+    Remove: [modernize-use-trailing-return-type]
+    CheckOptions:
+      readability-identifier-naming.VariableCase: camelBack
+  UnusedIncludes: Strict
+
+InlayHints:
+  Enabled: No
+  ParameterNames: No
+  DeducedTypes: No

.gitignore

@@ -78,6 +78,7 @@ runlog/
 .gdb_history
 
 .clang-format
+.clangd
 !curve-snapshotcloneserver-nginx/app/lib
 !nebd/nebd-package/usr/bin
 
@@ -90,6 +91,7 @@ test/client/configs/*
 projects/*
 docker/curvebs
 docker/base/*
+docker/debian*/*so*
 !docker/base/Dockerfile
 !docker/base/Makefile
 
@@ -107,6 +109,7 @@ curvefs/monitor/prometheus/target.json
 curvefs/docker/*/curvefs
 curvefs/docker/curvefs
 curvefs/docker/base/*
+curvefs/docker/debian*/*so*
 !curvefs/docker/base/Dockerfile
 !curvefs/docker/base/Makefile
 curvefs/BUILD_MODE
@@ -144,3 +147,22 @@ tools-v2/proto/curvefs/*
 tools-v2/*/*.test
 tools-v2/__debug_bin
 tools-v2/vendor/
+
+.test
+.note
+.playground
+.dumpfile
+metastore_test.dat
+GPATH
+GRTAGS
+GTAGS
+core.*
+
+test/integration/*.conf
+test/integration/client/config/client.conf*
+test/integration/snapshotcloneserver/config/*.conf
+
+.pre-commit-config.yaml
+
+*.deb
+*.whl

.obm.cfg

@@ -0,0 +1,2 @@
+container_name: curve-build-playground-master
+container_image: opencurvedocker/curve-base:build-debian9

CHANGELOG-2.5.md

@@ -0,0 +1,49 @@
+# CHANGELOG of v2.5
+
+Previous change logs can be found at [CHANGELOG-2.4](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.4.md)
+
+
+
+## New Features
+
+- Memcache support
+    - [curvefs/client: support memcached cluster](https://github.com/opencurve/curve/pull/2096) @ilixiaocui
+    - [curvefs/mds: support memcache cluster](https://github.com/opencurve/curve/pull/2108) @Cyber-SiKu
+    - [curvefs/client: add global cache client like memcached](https://github.com/opencurve/curve/pull/2102) @fansehep
+    - [curvefs/client: fix client core dump error](https://github.com/opencurve/curve/pull/2157) @ilixiaocui
+    - [add ut](https://github.com/opencurve/curve/pull/2164) @Cyber-SiKu
+
+- curvefs new tools 
+    - [add delete impl](https://github.com/opencurve/curve/pull/2088) @shentupenghui
+    - [feat: [tools-v2] bs list dir](https://github.com/opencurve/curve/pull/2082) @Sindweller
+    - [fix missing docs links](https://github.com/opencurve/curve/pull/2082) @zyb521
+    - [curve/toos-v2: add list client #2037](https://github.com/opencurve/curve/pull/2076) @tsonglew
+    - [support create volume directory by curve_ops_tool](https://github.com/opencurve/curve/pull/2078) @aspirer
+
+- [install: added playground.](https://github.com/opencurve/curve/pull/2053) @Wine93
+
+## Optimization
+
+- [update the braft to vesion v1.1.2](https://github.com/opencurve/curve/pull/2091) @tangwz
+
+- [curvefs/client: local cache policy optimization](https://github.com/opencurve/curve/pull/2064) @Tangruilin
+
+
+- [sync by threadpool](https://github.com/opencurve/curve/pull/1912) @fansehep
+
+- Merge block storage and file storage compilation scripts
+    - [Merge block storage and file storage compilation scripts
+](https://github.com/opencurve/curve/pull/2089) @linshiyx
+    - [Add help_msg in Makefile](https://github.com/opencurve/curve/pull/2133) @linshiyx
+    - [fix image.sh](https://github.com/opencurve/curve/pull/2124) @SeanHai
+
+- [fix bs list zone typo](https://github.com/opencurve/curve/pull/2066) @tsonglew 
+
+- [spell optimize](https://github.com/opencurve/curve/pull/2059) @shentupenghui
+
+
+## Bug Fix
+
+- aws s3 sdk revert
+    - [revert new s3 sdk due to critical bug](https://github.com/opencurve/curve/pull/2149) @h0hmj
+    - [curvefs/client: change the default value of s3](https://github.com/opencurve/curve/pull/2158) @wuhongsong

Makefile

@@ -1,6 +1,6 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build dep install image
+.PHONY: list build dep install image playground check test
 
 stor?=""
 prefix?= "$(PWD)/projects"
@@ -70,3 +70,12 @@ install:
 
 image:
 	@bash util/image.sh $(stor) $(tag) $(os)
+
+playground:
+	@bash util/playground.sh
+
+check:
+	@bash util/check.sh $(stor)
+
+test:
+	@bash util/test.sh $(stor) $(only)

README.md

@@ -4,7 +4,9 @@
 
 <div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
 
-**A could-native distributed storage system**
+**A cloud-native distributed storage system**
+
+**A sandbox project hosted by the CNCF Foundation**
 
 #### English | [简体中文](README_cn.md)
 ### 📄 [Documents](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [Official Website](https://www.opencurve.io/Curve/HOME) || 🏠 [Forum](https://ask.opencurve.io/t/topic/7)
@@ -42,11 +44,24 @@
   - [LICENSE](#license)
   - [Release Cycle](#release-cycle)
   - [Branch](#branch)
+  - [Contributors](#contributors)
+  - [Star History](#star-history)
   - [Feedback & Contact](#feedback--contact)
 
 ##  About Curve
 
-**Curve** is a mordern storage system developed by netease, currently supporting file storage(CurveFS) and block storage(CurveBS). Now it's hosted at [CNCF](https://www.cncf.io/) as a sandbox project.
+**Curve** is a modern storage system developed by netease, currently supporting file storage(CurveFS) and block storage(CurveBS). Now it's hosted at [CNCF](https://www.cncf.io/) as a sandbox project.
+
+The core application scenarios of CurveBS mainly include:
+- the performance, mixed, capacity cloud disk or persistent volume of virtual machine/container, and remote disks of physical machines
+- high-performance separation of storage and computation architecture: high-performance and low latency architecture based on RDMA+SPDK, supporting the separation deployment structure of various databases such as MySQL and Kafka
+
+
+The core application scenarios of CurveFS mainly include:
+- the cost-effective storage in AI training scene
+- the hot and cold data automation layered storage in big data scenarios
+- the cost-effective shared file storage on the public cloud: It can be used for business scenarios such as AI, big data, file sharing
+- Hybrid storage: Hot data is stored in the local IDC, cold data is stored in public cloud
 
 <details>
   <summary><b><font=5>High Performance | More stable | Easy Operation | Cloud Native</b></font></summary>
@@ -111,7 +126,7 @@
 
 </details>
 <details>
-  <summary><b><font=5>Docking Kubernates</b></font></summary>
+  <summary><b><font=5>Docking Kubernetes</b></font></summary>
 
 - Use [Curve CSI Driver](https://github.com/opencurve/curve-csi), The plugin implements the Container Storage Interface(CSI) between Container Orchestrator(CO) and Curve cluster. It allows dynamically provisioning curve volumes and attaching them to workloads.
 - For details of the documentation, see [CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md).
@@ -143,7 +158,7 @@ Curve supports deployment in private and public cloud environments, and can also
 <div align=center> <image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
 <div align=left>
 
-One of them, CurveFS shared file storage system, can be elasticly scaled to public cloud storage, which can provide users with greater capacity elasticity, lower cost, and better performance experience.
+One of them, CurveFS shared file storage system, can be elastically scaled to public cloud storage, which can provide users with greater capacity elasticity, lower cost, and better performance experience.
 
 </details>
 
@@ -199,6 +214,8 @@ Fio Curve engine is added, you can clone https://github.com/opencurve/fio and co
 $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
 ```
 
+If you have any questions during performance testing, please check the [Curve block storage performance tuning guide](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md).
+
 ##  CurveFS quick start
 Please use [CurveAdm](https://github.com/opencurve/curveadm/wiki) tool to deploy CurveFS，see [CurveFS Deployment Process](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), and the [CurveFS Command Instructions](curvefs/src/tools#readme).
 
@@ -208,12 +225,13 @@ Please refer to the [Test environment configuration](docs/cn/测试环境配置
 
 ## Practical
 - [CurveBS+NFS Build NFS Server](docs/practical/curvebs_nfs.md)
+- [CurveFS+MinIO S3 Gateway](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
 
 ## Governance
 See [Governance](https://github.com/opencurve/community/blob/master/GOVERNANCE.md).
 
 ##  Contribute us
-Participation in the Curve project is described in the [Curve Open Source Community Guidelines](Community_Guidelines.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
+Participation in the Curve project is described in the [Curve Developers Guidelines](developers_guide.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
 We welcome your contribution!
 
 ## Code of Conduct
@@ -230,6 +248,19 @@ Curve is distributed under the [**Apache 2.0 LICENSE**](LICENSE).
 ## Branch
 All the developments will be done under master branch. If there's any new version to establish, a new branch release-x.y will be pulled from the master, and the new version will be released from this branch.
 
+## Contributors
+This project exists thanks to all the people who contribute.
+
+<a href="https://github.com/opencurve/curve/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=opencurve/curve" />
+</a>
+
+Made with [contrib.rocks](https://contrib.rocks).
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=opencurve/curve&type=Date)](https://star-history.com/#opencurve/curve&Date)
+
 ## Feedback & Contact
 - [Github Issues](https://github.com/openCURVE/CURVE/issues)：You are sincerely welcomed to issue any bugs you came across or any suggestions through Github issues. If you have any question you can refer to our FAQ or join our user group for more details.
 - [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：Frequently asked question in our user group, and we'll keep working on it.

README_cn.md

@@ -4,7 +4,9 @@
 
 <div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
 
-**A could-native distributed storage system**
+**云原生高性能分布式存储系统**
+
+**CNCF基金会的沙箱托管项目**
 
 #### [English](README.md) | 简体中文
 ### 📄 [文档](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [官网](https://www.opencurve.io/Curve/HOME) || 🏠 [论坛](https://ask.opencurve.io/t/topic/7)
@@ -48,6 +50,18 @@
 ##  关于 Curve
 **Curve** 是网易主导自研的现代化存储系统, 目前支持文件存储(CurveFS)和块存储(CurveBS)。现作为沙箱项目托管于[CNCF](https://www.cncf.io/)。
 
+CurveBS的核心应用场景主要包括：
+- 虚拟机/容器的性能型、混合型、容量型云盘或持久化卷，以及物理机的远程存储盘
+- 高性能存算分离架构：基于RDMA+SPDK的高性能低时延架构，支撑MySQL、kafka等各类数据库、中间件的存算分离部署架构，提升实例交付效率和资源利用率
+
+
+CurveFS的核心应用场景主要包括：
+- AI训练（含机器学习等）场景下的高性价比存储
+- 大数据场景下的冷热数据自动化分层存储
+- 公有云上高性价比的共享文件存储：可用于AI、大数据、文件共享等业务场景
+- 混合云存储：热数据存储在本地IDC，冷数据存储在公有云
+
+
 <details>
   <summary><b><font=5>高性能 | 更稳定 | 易运维 | 云原生</b></font></summary>
 
@@ -112,7 +126,7 @@
 
 </details>
 <details>
-  <summary><b><font=5>对接 Kubernates</b></font></summary>
+  <summary><b><font=5>对接 Kubernetes</b></font></summary>
 
 - 使用 [Curve CSI Driver](https://github.com/opencurve/curve-csi) 插件在 Container Orchestrator (CO) 与 Curve 集群中实现了 Container Storage Interface(CSI)。
 - 文档详见[CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md)。
@@ -196,6 +210,8 @@ fio的Curve块存储引擎代码已经上传到 https://github.com/opencurve/fio
 $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
 ```
 
+在性能测试过程中有任何问题，请查看[Curve块存储性能调优指南](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md)
+
 ##  CurveFS 快速体验
 请使用 [CurveAdm](https://github.com/opencurve/curveadm/wiki) 工具进行 CurveFS 的部署，具体流程见：[CurveFS部署流程](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), 以及[CurveFS命令行工具说明](curvefs/src/tools#readme)。
 
@@ -207,10 +223,11 @@ $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest
 
 ##  贡献我们
 
-参与 Curve 项目开发详见[Curve 开源社区指南](Community_Guidelines_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
+参与 Curve 项目开发详见[Curve 开发者指南](developers_guide_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
 
 ## 最佳实践
 - [CurveBS+NFS搭建NFS存储](docs/practical/curvebs_nfs.md)
+- [CurveFS+S3网关部署实践](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
 
 ##  行为守则
 Curve 的行为守则遵循[CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md)。

WORKSPACE

@@ -75,11 +75,14 @@ bind(
 )
 
 #import the  gtest files.
-new_git_repository(
+http_archive(
     name = "com_google_googletest",
-    build_file = "//:thirdparties/gmock.BUILD",
-    remote = "https://github.com/google/googletest",
-    tag = "release-1.8.0",
+    urls = [
+        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
+        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
+    ],
+    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
+    strip_prefix = "googletest-release-1.12.1",
 )
 
 bind(
@@ -228,6 +231,24 @@ http_archive(
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
 
+# fmt
+http_archive(
+  name = "fmt",
+  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
+  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
+  strip_prefix = "fmt-9.1.0",
+  build_file = "//:thirdparties/fmt.BUILD",
+)
+
+# spdlog
+http_archive(
+  name = "spdlog",
+  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
+  strip_prefix = "spdlog-1.11.0",
+  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
+  build_file = "//:thirdparties/spdlog.BUILD",
+)
+
 # Bazel platform rules.
 http_archive(
     name = "platforms",
@@ -248,14 +269,14 @@ new_local_repository(
 http_archive(
     name = "hedron_compile_commands",
 
-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. 
+    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
     # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
     urls = [
         "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
         "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
     ],
     strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." 
+    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()

buildfs.sh

@@ -7,12 +7,22 @@ then
     exit
 fi
 
+if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
+then
+    bazelflags=''
+else
+    bazelflags='--copt -faligned-new'
+fi
+
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi
 
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
+--define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
+-DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
+
 if [ $? -ne 0 ]
 then
     echo "build curvefs failed"
@@ -34,4 +44,5 @@ then
     echo "mds_test failed"
     exit
 fi
-fi
+fi
+echo "end compile"

conf/chunkserver.conf

@@ -9,13 +9,20 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.block_size=4096
+
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +115,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/chunkserver.conf.example

@@ -13,9 +13,12 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +111,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/client.conf

@@ -168,13 +168,6 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
 
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
-
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@@ -188,4 +181,3 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
-

conf/cs_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/mds.conf

@@ -156,7 +156,7 @@ mds.topology.CreateCopysetRpcRetryTimes=20
 # 请求chunkserver上创建copyset重试间隔
 mds.topology.CreateCopysetRpcRetrySleepTimeMs=1000
 # Topology模块刷新metric时间间隔
-mds.topology.UpdateMetricIntervalSec=60
+mds.topology.UpdateMetricIntervalSec=10
 #和mds.chunkserver.failure.tolerance设置有关,一个zone 标准配置20台节点，如果允许3台节点failover,
 #那么剩余17台机器需要承载原先20台机器的空间,17/20=0.85,即使用量超过这个值即不再往这个池分配,
 #具体分为来两种情况, 当不使用chunkfilepool,物理池限制使用百分比,当使用 chunkfilepool 进行chunkfilepool分配时需预留failover空间,
@@ -197,6 +197,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -237,3 +239,16 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
+
+#
+## poolset rules
+#
+# for backward compatibility, rules are applied for select poolset when creating file
+#
+# for example
+#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
+#
+# when creating file reqeust doesn't have poolset, above rules are used to select poolset
+# - if filename is /dir1/file, then poolset1 is select
+# - if filename is /dir1/sub/file, then sub is select
+mds.poolset.rules=

conf/py_client.conf

@@ -153,10 +153,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/snap_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

curve-ansible/roles/generate_config/defaults/main.yml

@@ -164,7 +164,6 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
-chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -244,8 +243,6 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
-client_alignment_common: 512
-client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -14,8 +14,6 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
-# minimum alignment for io request
-global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -175,10 +175,3 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume={{ client_alignment_common }}
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume={{ client_alignment_clone }}

curvefs/conf/client.conf

@@ -77,16 +77,8 @@ rpc.healthCheckIntervalSec=0
 
 #### fuseClient
 # TODO(xuchaojie): add unit
-fuseClient.attrTimeOut=1.0
-fuseClient.entryTimeOut=1.0
 fuseClient.listDentryLimit=65536
-fuseClient.flushPeriodSec=5
-fuseClient.maxNameLength=255
-fuseClient.iCacheLruSize=65536
-fuseClient.dCacheLruSize=1000000
-fuseClient.enableICacheMetrics=true
-fuseClient.enableDCacheMetrics=true
-fuseClient.cto=true
+fuseClient.downloadMaxRetryTimes=3
 
 ### kvcache opt
 fuseClient.supportKVcache=false
@@ -110,6 +102,62 @@ fuseClient.disableXattr=false
 fuseClient.maxDataSize=1024
 # default refresh data interval 30s
 fuseClient.refreshDataIntervalSec=30
+fuseClient.warmupThreadsNum=10
+
+# the write throttle bps of fuseClient, default no limit
+fuseClient.throttle.avgWriteBytes=0
+# the write burst bps of fuseClient, default no limit
+fuseClient.throttle.burstWriteBytes=0
+# the times that write burst bps can continue, default 180s
+fuseClient.throttle.burstWriteBytesSecs=180
+
+# the write throttle iops of fuseClient, default no limit
+fuseClient.throttle.avgWriteIops=0
+# the write burst iops of fuseClient, default no limit
+fuseClient.throttle.burstWriteIops=0
+# the times that write burst Iops can continue, default 180s
+fuseClient.throttle.burstWriteIopsSecs=180
+
+# the read throttle bps of fuseClient, default no limit
+fuseClient.throttle.avgReadBytes=0
+# the read burst bps of fuseClient, default no limit
+fuseClient.throttle.burstReadBytes=0
+# the times that read burst bps can continue, default 180s
+fuseClient.throttle.burstReadBytesSecs=180
+
+# the read throttle iops of fuseClient, default no limit
+fuseClient.throttle.avgReadIops=0
+# the read burst Iops of fuseClient, default no limit
+fuseClient.throttle.burstReadIops=0
+# the times that read burst Iops can continue, default 180s
+fuseClient.throttle.burstReadIopsSecs=180
+
+#### filesystem metadata
+# {
+# fs.disableXattr:
+#   if you want to get better metadata performance,
+#   you can mount fs with |fs.disableXattr| is true
+#
+# fs.lookupCache.negativeTimeoutSec:
+#   entry which not found will be cached if |timeout| > 0
+fs.cto=true
+fs.maxNameLength=255
+fs.disableXattr=false
+fs.accessLogging=true
+fs.kernelCache.attrTimeoutSec=3600
+fs.kernelCache.dirAttrTimeoutSec=3600
+fs.kernelCache.entryTimeoutSec=3600
+fs.kernelCache.dirEntryTimeoutSec=3600
+fs.lookupCache.negativeTimeoutSec=0
+fs.lookupCache.minUses=1
+fs.lookupCache.lruSize=100000
+fs.dirCache.lruSize=5000000
+fs.openFile.lruSize=65536
+fs.attrWatcher.lruSize=5000000
+fs.rpc.listDentryLimit=65536
+fs.deferSync.delay=3
+fs.deferSync.deferDirMtime=false
+# }
 
 #### volume
 volume.bigFileSize=1048576
@@ -132,8 +180,6 @@ volume.blockGroup.allocateOnce=4
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false
-# the max size that fuse send
-s3.fuseMaxSize=131072
 s3.pageSize=65536
 # prefetch blocks that disk cache use
 s3.prefetchBlocks=1
@@ -148,8 +194,11 @@ s3.baseSleepUs=500
 s3.threadScheduleInterval=3
 # data cache flush wait time
 s3.cacheFlushIntervalSec=5
+# write cache < 8,388,608 (8MB) is not allowed
 s3.writeCacheMaxByte=838860800
 s3.readCacheMaxByte=209715200
+# file cache read thread num
+s3.readCacheThreads=5
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=False

curvefs/conf/mds.conf

@@ -6,6 +6,9 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
 mds.loglevel=0
+# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+mds.server.idleTimeoutSec=-1
 
 #
 # space options

curvefs/conf/metaserver.conf

@@ -128,16 +128,25 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000
 
-### apply queue options for each copyset
-### apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding worker queue by certain rules
-# number of apply queue workers for each, each worker will start a indepent thread
-applyqueue.worker_count=3
+#
+# Concurrent apply queue
+### concurrent apply queue options for each copyset
+### concurrent apply queue is used to isolate raft threads, each worker has its own queue
+### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
 
-# apply queue depth for each copyset
+# worker_count: number of apply queue workers for each, each worker will start a indepent thread
+# queue_depth: apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-applyqueue.queue_depth=1
+# write apply queue workers count
+applyqueue.write_worker_count=3
+# write apply queue depth
+applyqueue.write_queue_depth=1
+# read apply queue workers count
+applyqueue.read_worker_count=2
+# read apply queue depth
+applyqueue.read_queue_depth=1
+
 
 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@@ -145,6 +154,10 @@ applyqueue.queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto
 
+# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+server.idleTimeoutSec=-1
+
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need
@@ -267,3 +280,43 @@ storage.rocksdb.perf_sampling_ratio=0
 # we will sending its with rpc streaming instead of
 # padding its into inode (default: 25000, about 25000 * 41 (byte) = 1MB)
 storage.s3_meta_inside_inode.limit_size=25000
+
+# recycle options
+# metaserver scan recycle period, default 1h
+recycle.manager.scanPeriodSec=3600
+# metaserver recycle cleaner scan list dentry limit, default 1000
+recycle.cleaner.scanLimit=1000
+
+#### excutorOpt
+# excutorOpt rpc with metaserver
+# rpc retry times with metaserver
+excutorOpt.maxRetry=4294967295
+# internal rpc retry times with metaserver
+excutorOpt.maxInternalRetry = 3
+# Retry sleep time between failed RPCs
+excutorOpt.retryIntervalUS=100000
+# RPC timeout for communicating with metaserver
+excutorOpt.rpcTimeoutMS=1000
+# RPC stream idle timeout
+excutorOpt.rpcStreamIdleTimeoutMS=500
+# The maximum timeout RPC time of the retry request.
+# The timeout time will follow the exponential backoff policy.
+# Because the timeout occurs when the network is congested, the RPC timeout needs to be increased
+excutorOpt.maxRPCTimeoutMS=8000
+# Maximum sleep time between retry requests.
+# when the network is congested or the metaserver is overloaded,
+# it is necessary to increase the sleep time
+excutorOpt.maxRetrySleepIntervalUS=8000000
+excutorOpt.minRetryTimesForceTimeoutBackoff=5
+excutorOpt.maxRetryTimesBeforeConsiderSuspend=20
+# batch limit of get inode attr and xattr
+excutorOpt.batchInodeAttrLimit=10000
+excutorOpt.enableMultiMountPointRename=true
+
+#### metaCacheOpt
+# Gets the number of retries for the leader
+metaCacheOpt.metacacheGetLeaderRetry=3
+# Need to sleep for a period of time before each get leader retry
+metaCacheOpt.metacacheRPCRetryIntervalUS=100000
+# RPC timeout of get leader
+metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000

curvefs/conf/tools.conf

@@ -24,6 +24,7 @@ volumeName=volume
 volumeUser=user
 volumePassword=password
 volumeBlockGroupSize=134217728
+volumeCluster=127.0.0.1:6666,127.0.0.1:6667,127.0.0.1:6668
 # support |AtStart| and |AtEnd|
 volumeBitmapLocation=AtStart
 # s3
@@ -34,5 +35,12 @@ s3.bucket_name=bucket
 s3.blocksize=4194304
 s3.chunksize=67108864
 s3.useVirtualAddressing=false
+# s3 objectPrefix, if set 0, means no prefix, if set 1, means inode prefix
+# if set 2 and other values mean hash prefix
+s3.objectPrefix=0
 # statistic info in xattr, hardlink will not be supported when enable
 enableSumInDir=true
+
+# fs recycle, if set 0, disable fs recycle, delete files directly,
+# if set not 0, enable fs recycle, delete files after a period of time
+recycleTimeHour=0

curvefs/docker/debian10/Dockerfile

@@ -1,11 +0,0 @@
-FROM opencurvedocker/curve-base:debian10
-ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
-COPY curvefs /curvefs
-COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libmemcached.so /usr/lib/
-COPY libmemcached.so.11 /usr/lib/
-COPY libhashkit.so.2 /usr/lib
-RUN chmod a+x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian10/base/Dockerfile

@@ -1,39 +0,0 @@
-FROM opencurvedocker/curve-base:debian10
-
-RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        gcc \
-        gdb \
-        make \
-        openssl \
-        net-tools \
-        libcurl3-gnutls \
-        perl \
-        linux-perf \
-        vim \
-        curl \
-        cron \
-        procps \
-        lsof \
-        nginx \
-        less \
-        fuse3 \
-        libnl-3-200 \
-        libnl-genl-3-200 \
-        libjemalloc2 \
-        libsnappy1v5 \
-        liblz4-1 \
-        zlib1g \
-    && ln -sf /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
-
-COPY libetcdclient.so /usr/lib/

curvefs/docker/debian10/base/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:debian10 .

curvefs/docker/debian10/compile/Dockerfile

@@ -1,52 +0,0 @@
-FROM opencurvedocker/curve-base:build-debian10
-
-RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        wget \
-        git \
-        gcc \
-        g++ \
-        clang \
-        libssl-dev \
-        libnl-genl-3-dev \
-        libcurl4-gnutls-dev \
-        uuid-dev \
-        libfiu-dev \
-        libfuse3-dev \
-        zlib1g-dev \
-        make \
-        openjdk-11-jdk \
-        sudo \
-        zlib1g-dev \
-        libsnappy-dev \
-        liblz4-dev \
-        cmake \
-        ca-certificates \
-        curl \
-        gnupg \
-        lsb-release \
-        curl \
-        git \
-    && mkdir -p /etc/apt/keyrings \
-    && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
-    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
-  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
-    && apt-get update \
-    && apt-get install -y docker-ce docker-ce-cli containerd.io \
-    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
-    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
-    && ln -sf /usr/bin/bazelisk /usr/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
-    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
-    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk

curvefs/docker/debian10/compile/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:build-debian10 .

curvefs/docker/debian10/entrypoint.sh

@@ -1,126 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-############################  GLOBAL VARIABLES
-g_role=""
-g_args=""
-g_prefix=""
-g_binary=""
-g_start_args=""
-
-############################  BASIC FUNCTIONS
-function msg() {
-    printf '%b' "$1" >&2
-}
-
-function success() {
-    msg "\33[32m[✔]\33[0m ${1}${2}"
-}
-
-function die() {
-    msg "\33[31m[✘]\33[0m ${1}${2}"
-    exit 1
-}
-
-############################ FUNCTIONS
-function usage () {
-    cat << _EOC_
-Usage:
-    entrypoint.sh --role=ROLE
-    entrypoint.sh --role=ROLE --args=ARGS
-
-Examples:
-    entrypoint.sh --role=etcd
-    entrypoint.sh --role=client --args="-o default_permissions"
-_EOC_
-}
-
-function get_options() {
-    local long_opts="role:,args:,help"
-    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
-    eval set -- "${args}"
-    while true
-    do
-        case "$1" in
-            -r|--role)
-                g_role=$2
-                shift 2
-                ;;
-            -a|--args)
-                g_args=$2
-                shift 2
-                ;;
-            -h)
-                usage
-                exit 1
-                ;;
-            --)
-                shift
-                break
-                ;;
-            *)
-                exit 1
-                ;;
-        esac
-    done
-}
-
-function prepare() {
-    g_prefix="/curvefs/$g_role"
-    conf_path="$g_prefix/conf/$g_role.conf"
-
-    case $g_role in
-        etcd)
-            g_binary="$g_prefix/sbin/etcd"
-            g_start_args="--config-file $conf_path"
-            ;;
-        mds)
-            g_binary="$g_prefix/sbin/curvefs-mds"
-            g_start_args="--confPath $conf_path"
-            ;;
-        metaserver)
-            g_binary="$g_prefix/sbin/curvefs-metaserver"
-            g_start_args="--confPath $conf_path"
-            ;;
-        client)
-            g_binary="$g_prefix/sbin/curve-fuse"
-            g_start_args="--confPath $conf_path"
-            ;;
-        *)
-            usage
-            exit 1
-            ;;
-    esac
-
-    if [ "$g_args" != "" ]; then
-        g_start_args=$g_args
-    fi
-}
-
-function create_directory() {
-    chmod 700 "$g_prefix/data"
-    if [ "$g_role" == "etcd" ]; then
-        mkdir -p "$g_prefix/data/wal"
-    elif [ "$g_role" == "metaserver" ]; then
-        mkdir -p "$g_prefix/data/storage"
-    elif [ "$g_role" == "client" ]; then
-        mkdir -p "$g_prefix/mnt"
-    fi
-}
-
-function main() {
-    get_options "$@"
-
-    prepare
-    create_directory
-    [[ $(command -v crontab) ]] && cron
-    if [ $g_role == "etcd" ]; then
-        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
-    else
-        exec $g_binary $g_start_args
-    fi
-}
-
-############################  MAIN()
-main "$@"

curvefs/docker/debian11/Dockerfile

@@ -1,11 +0,0 @@
-FROM opencurvedocker/curve-base:debian11
-ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
-COPY curvefs /curvefs
-COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libmemcached.so /usr/lib/
-COPY libmemcached.so.11 /usr/lib/
-COPY libhashkit.so.2 /usr/lib
-RUN chmod a+x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian11/base/Dockerfile

@@ -1,39 +0,0 @@
-FROM opencurvedocker/curve-base:debian11
-
-RUN echo "deb http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        gcc \
-        gdb \
-        make \
-        openssl \
-        net-tools \
-        libcurl3-gnutls \
-        perl \
-        linux-perf \
-        vim \
-        curl \
-        cron \
-        procps \
-        lsof \
-        nginx \
-        less \
-        fuse3 \
-        libnl-3-200 \
-        libnl-genl-3-200 \
-        libjemalloc2 \
-        libsnappy1v5 \
-        liblz4-1 \
-        zlib1g
-    && ln -sf /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
-
-COPY libetcdclient.so /usr/lib/

curvefs/docker/debian11/base/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:debian11 .

curvefs/docker/debian11/compile/Dockerfile

@@ -1,51 +0,0 @@
-FROM opencurvedocker/curve-base:build-debian11 
-
-RUN echo "deb http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        wget \
-        git \
-        gcc \
-        g++ \
-        clang \
-        libssl-dev \
-        libnl-genl-3-dev \
-        libcurl4-gnutls-dev \
-        uuid-dev \
-        libfiu-dev \
-        libfuse3-dev \
-        zlib1g-dev \
-        make \
-        openjdk-11-jdk \
-        sudo \
-        libsnappy-dev \
-        liblz4-dev \
-        cmake \
-        ca-certificates \
-        curl \
-        gnupg \
-        lsb-release \
-        curl \
-        git \
-    && mkdir -p /etc/apt/keyrings \
-    # && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
-    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
-  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
-    && apt-get update \
-    && apt-get install -y docker-ce docker-ce-cli containerd.io \
-    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
-    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
-    && ln -sf /usr/bin/bazelisk /usr/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
-    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
-    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk

curvefs/docker/debian11/compile/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:build-debian11 .

curvefs/docker/debian11/entrypoint.sh

@@ -1,126 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-############################  GLOBAL VARIABLES
-g_role=""
-g_args=""
-g_prefix=""
-g_binary=""
-g_start_args=""
-
-############################  BASIC FUNCTIONS
-function msg() {
-    printf '%b' "$1" >&2
-}
-
-function success() {
-    msg "\33[32m[✔]\33[0m ${1}${2}"
-}
-
-function die() {
-    msg "\33[31m[✘]\33[0m ${1}${2}"
-    exit 1
-}
-
-############################ FUNCTIONS
-function usage () {
-    cat << _EOC_
-Usage:
-    entrypoint.sh --role=ROLE
-    entrypoint.sh --role=ROLE --args=ARGS
-
-Examples:
-    entrypoint.sh --role=etcd
-    entrypoint.sh --role=client --args="-o default_permissions"
-_EOC_
-}
-
-function get_options() {
-    local long_opts="role:,args:,help"
-    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
-    eval set -- "${args}"
-    while true
-    do
-        case "$1" in
-            -r|--role)
-                g_role=$2
-                shift 2
-                ;;
-            -a|--args)
-                g_args=$2
-                shift 2
-                ;;
-            -h)
-                usage
-                exit 1
-                ;;
-            --)
-                shift
-                break
-                ;;
-            *)
-                exit 1
-                ;;
-        esac
-    done
-}
-
-function prepare() {
-    g_prefix="/curvefs/$g_role"
-    conf_path="$g_prefix/conf/$g_role.conf"
-
-    case $g_role in
-        etcd)
-            g_binary="$g_prefix/sbin/etcd"
-            g_start_args="--config-file $conf_path"
-            ;;
-        mds)
-            g_binary="$g_prefix/sbin/curvefs-mds"
-            g_start_args="--confPath $conf_path"
-            ;;
-        metaserver)
-            g_binary="$g_prefix/sbin/curvefs-metaserver"
-            g_start_args="--confPath $conf_path"
-            ;;
-        client)
-            g_binary="$g_prefix/sbin/curve-fuse"
-            g_start_args="--confPath $conf_path"
-            ;;
-        *)
-            usage
-            exit 1
-            ;;
-    esac
-
-    if [ "$g_args" != "" ]; then
-        g_start_args=$g_args
-    fi
-}
-
-function create_directory() {
-    chmod 700 "$g_prefix/data"
-    if [ "$g_role" == "etcd" ]; then
-        mkdir -p "$g_prefix/data/wal"
-    elif [ "$g_role" == "metaserver" ]; then
-        mkdir -p "$g_prefix/data/storage"
-    elif [ "$g_role" == "client" ]; then
-        mkdir -p "$g_prefix/mnt"
-    fi
-}
-
-function main() {
-    get_options "$@"
-
-    prepare
-    create_directory
-    [[ $(command -v crontab) ]] && cron
-    if [ $g_role == "etcd" ]; then
-        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
-    else
-        exec $g_binary $g_start_args
-    fi
-}
-
-############################  MAIN()
-main "$@"

curvefs/docker/debian9/Dockerfile

@@ -1,11 +1,8 @@
 FROM opencurvedocker/curve-base:debian9
-ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
-COPY curvefs /curvefs
 COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libmemcached.so /usr/lib/
-COPY libmemcached.so.11 /usr/lib/
-COPY libhashkit.so.2 /usr/lib
-RUN chmod a+x /entrypoint.sh
+COPY curvefs /curvefs
+COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 /usr/lib/
+RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \
+    && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \
+    && cp /curvefs/tools-v2/sbin/curve /usr/bin/
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian9/base/Dockerfile

@@ -1,38 +0,0 @@
-FROM opencurvedocker/curve-base:debian9
-
-RUN echo "deb http://mirrors.163.com/debian/ stretch main\n" \
-         "deb http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ stretch/updates main non-free contrib\n" \
-         "deb http://httpredir.debian.org/debian stretch-backports main contrib non-free\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        gcc \
-        gdb \
-        make \
-        openssl \
-        net-tools \
-        libcurl3-gnutls \
-        podlators-perl \
-        linux-perf \
-        vim \
-        curl \
-        cron \
-        procps \
-        lsof \
-        nginx \
-        less \
-        zlib1g \
-        libsnappy1v5 \
-        liblz4-1 \
-        python \
-        wget \
-        python3 \
-    && wget https://curve-build.nos-eastchina1.126.net/curve-base.tar.gz \
-    && tar -xzvf curve-base.tar.gz
-COPY fusermount3 /usr/local/bin
-COPY libetcdclient.so /usr/lib/
-COPY libfuse3.so libfuse3.so.3 libfuse3.so.3.10.5 libnl-3.so.200 libnl-genl-3.so.200 /usr/lib/x86_64-linux-gnu/
-COPY libjemalloc.so libjemalloc.so.1 libjemalloc.so.2 /usr/local/lib/

curvefs/docker/debian9/base/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:debian9 .

curvefs/docker/debian9/compile/Dockerfile

@@ -1,50 +0,0 @@
-FROM opencurvedocker/curve-base:build-debian9 
-
-RUN apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        g++ \
-        wget \
-        clang \
-        git \
-        libnl-genl-3-dev \
-        libssl-dev \
-        uuid-dev \
-        libcurl4-gnutls-dev \
-        libfiu-dev \
-        openjdk-8-jdk \
-        sudo \
-        python-dev \
-        python-setuptools \
-        python-wheel \
-        python3-dev \
-        python3-setuptools \
-        python3-wheel \
-        zlib1g-dev \
-        libsnappy-dev \
-        liblz4-dev \
-        cmake \
-        ca-certificates \
-        curl \
-        gnupg \
-        lsb-release \
-        curl \
-        apt-transport-https \
-        git \
-    && mkdir -p /etc/apt/keyrings \
-    && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
-    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
-  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
-    && apt-get update \
-    && apt-get install -y docker-ce docker-ce-cli containerd.io \
-    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
-    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
-    && ln -sf /usr/bin/bazelisk /usr/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
-    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/0.17.2/bazel-0.17.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin \
-    && mv bazel-0.17.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel \
-    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk
-COPY fuse3/* /usr/include/fuse3/

curvefs/docker/debian9/compile/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:build-debian9 .

curvefs/docker/debian9/entrypoint.sh

@@ -8,6 +8,7 @@ g_args=""
 g_prefix=""
 g_binary=""
 g_start_args=""
+g_preexec="/curvefs/tools-v2/sbin/daemon"
 
 ############################  BASIC FUNCTIONS
 function msg() {
@@ -119,6 +120,7 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
+    [[ ! -z $g_preexec ]] && $g_preexec &
     if [ $g_role == "etcd" ]; then
         exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
     elif [ $g_role == "monitor" ]; then

curvefs/proto/common.proto

@@ -58,6 +58,7 @@ message S3Info {
     required string bucketname = 4;
     required uint64 blockSize = 5;
     required uint64 chunkSize = 6;
+    optional uint32 objectPrefix = 7;
 }
 
 enum PartitionStatus {
@@ -83,6 +84,7 @@ message PartitionInfo {
     optional uint64 inodeNum = 10;  // hearbeat upload this value to mds topo, topo/metaserver needn't persist this value
     optional uint64 dentryNum = 11;  // hearbeat upload this value to mds topo, topo/metaserver needn't persist this value
     map<int32, uint64> fileType2inodeNum = 12;
+    optional bool manageFlag = 13; // if a partition has recyclebin inode, set this flag true
 }
 
 message Peer {

curvefs/proto/mds.proto

@@ -56,6 +56,10 @@ enum FSStatusCode {
     VOLUME_INFO_ERROR = 31;
     MOUNT_POINT_CONFLICT = 32;
     FSNAME_INVALID = 33;
+    INSERT_DENTRY_FAIL = 34;
+    INSERT_MANAGE_INODE_FAIL = 35;
+    DELETE_DENTRY_FAIL = 36;
+    UPDATE_FS_FAIL = 37;
 }
 
 // fs interface
@@ -97,6 +101,7 @@ message FsInfo {
     required string owner = 12;
     optional uint64 txSequence = 13;
     optional string txOwner = 14;
+    optional uint32 recycleTimeHour =15;
 }
 
 message GetFsInfoResponse {
@@ -112,6 +117,7 @@ message CreateFsRequest {
     required bool enableSumInDir = 5;
     required string owner = 6;
     required uint64 capacity = 7;
+    optional uint32 recycleTimeHour = 8;
 }
 
 message CreateFsResponse {
@@ -185,6 +191,7 @@ message RefreshSessionRequest {
 message RefreshSessionResponse {
     required FSStatusCode statusCode = 1;
     repeated topology.PartitionTxId latestTxIdList = 2;
+    optional bool enableSumInDir = 3;
 }
 
 message DLockValue {

curvefs/proto/metaserver.proto

@@ -218,7 +218,7 @@ message Inode {
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index
     optional uint32 dtime = 19;
     optional uint32 openmpcount = 20; // openmpcount mount points had the file open
-    map<string, string> xattr = 21;
+    map<string, bytes> xattr = 21;
     repeated uint64 parent = 22;
 }
 
@@ -272,6 +272,28 @@ message CreateRootInodeResponse {
     optional uint64 appliedIndex = 2;
 }
 
+enum ManageInodeType {
+    TYPE_NOT_MANAGE = 0;
+    TYPE_RECYCLE = 1;
+};
+
+message CreateManageInodeRequest {
+    required uint32 poolId = 1;
+    required uint32 copysetId = 2;
+    required uint32 partitionId = 3;
+    required uint32 fsId = 4;
+    required uint32 uid = 5;
+    required uint32 gid = 6;
+    required uint32 mode = 7;
+    required ManageInodeType manageType = 8;
+}
+
+message CreateManageInodeResponse {
+    required MetaStatusCode statusCode = 1;
+    optional Inode inode = 2;
+    optional uint64 appliedIndex = 3;
+}
+
 message UpdateInodeRequest {
     required uint32 poolId = 1;
     required uint32 copysetId = 2;
@@ -292,7 +314,7 @@ message UpdateInodeRequest {
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 17;
     optional uint32 nlink = 18;
     // field 19 is left for compatibility
-    map<string, string> xattr = 20;
+    map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
     optional VolumeExtentList volumeExtents = 23;
@@ -385,7 +407,7 @@ message InodeAttr {
     optional uint64 rdev = 16;
     optional uint32 dtime = 17;
     optional uint32 openmpcount = 18;
-    map<string, string> xattr = 19;
+    map<string, bytes> xattr = 19;
     repeated uint64 parent = 20;
 }
 
@@ -416,7 +438,7 @@ message BatchGetXAttrRequest {
 message XAttr {
     required uint64 inodeId = 1;
     required uint32 fsId = 2;
-    map<string, string> xAttrInfos = 3;
+    map<string, bytes> xAttrInfos = 3;
 }
 
 message BatchGetXAttrResponse {
@@ -474,6 +496,7 @@ service MetaServerService {
     rpc DeleteInode(DeleteInodeRequest) returns (DeleteInodeResponse);
     rpc CreateRootInode(CreateRootInodeRequest) returns
                                             (CreateRootInodeResponse);
+    rpc CreateManageInode(CreateManageInodeRequest) returns (CreateManageInodeResponse);
     rpc GetOrModifyS3ChunkInfo(GetOrModifyS3ChunkInfoRequest) returns (GetOrModifyS3ChunkInfoResponse);
     rpc BatchGetInodeAttr(BatchGetInodeAttrRequest) returns (BatchGetInodeAttrResponse);
     rpc BatchGetXAttr(BatchGetXAttrRequest) returns (BatchGetXAttrResponse);

curvefs/src/client/BUILD

@@ -43,6 +43,10 @@ cc_library(
             "s3/*.h",
             "volume/*.cpp",
             "volume/*.h",
+            "filesystem/*.cpp",
+            "filesystem/*.h",
+            "warmup/*.h",
+            "warmup/*.cpp",
         ],
         exclude = ["main.cpp"],
     ),
@@ -69,10 +73,14 @@ cc_library(
         "//curvefs/src/volume",
         "//curvefs/src/common:metric_utils",
         "//curvefs/src/common:dynamic_vlog",
+        "//curvefs/src/common:threading",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/synchronization",
+        "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",
         "@com_google_googletest//:gtest_prod",
+        "@spdlog//:spdlog",
     ],
 )

curvefs/src/client/async_request_closure.cpp

@@ -28,7 +28,7 @@
 #include <mutex>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 namespace curvefs {
@@ -63,7 +63,7 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
         cond_.wait(lk);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(GetStatusCode());
+    return ToFSError(GetStatusCode());
 }
 
 void UpdateVolumeExtentClosure::Run() {
@@ -101,7 +101,7 @@ void UpdateInodeAttrAndExtentClosure::Run() {
                    << ", inode: " << inode_->GetInodeId();
         inode_->MarkInodeError();
     }
-
+    inode_->ClearDirty();
     inode_->syncingVolumeExtentsMtx_.unlock();
     inode_->ReleaseSyncingInode();
 

curvefs/src/client/async_request_closure.h

@@ -28,12 +28,15 @@
 
 #include <memory>
 
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
+using ::curvefs::client::filesystem::ToFSError;
+
 class InodeWrapper;
 
 namespace internal {

curvefs/src/client/client_operator.cpp

@@ -24,13 +24,14 @@
 
 #include "src/common/uuid.h"
 #include "curvefs/src/client/client_operator.h"
-
+#include "curvefs/src/client/filesystem/error.h"
 namespace curvefs {
 namespace client {
 
 using ::curve::common::UUIDGenerator;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::mds::topology::PartitionTxId;
+using ::curvefs::client::filesystem::ToFSError;
 
 #define LOG_ERROR(action, rc) \
     LOG(ERROR) << action << " failed, retCode = " << rc \
@@ -95,7 +96,7 @@ CURVEFS_ERROR RenameOperator::GetTxId(uint32_t fsId,
     if (rc != MetaStatusCode::OK) {
         LOG_ERROR("GetTxId", rc);
     }
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::GetLatestTxIdWithLock() {
@@ -204,7 +205,7 @@ CURVEFS_ERROR RenameOperator::PrepareRenameTx(
         LOG_ERROR("PrepareRenameTx", rc);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::PrepareTx() {
@@ -396,9 +397,28 @@ CURVEFS_ERROR RenameOperator::UpdateInodeParent() {
     return rc;
 }
 
+CURVEFS_ERROR RenameOperator::UpdateInodeCtime() {
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    auto rc = inodeManager_->GetInode(srcDentry_.inodeid(), inodeWrapper);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG_ERROR("GetInode", rc);
+        return rc;
+    }
+
+    curve::common::UniqueLock lk = inodeWrapper->GetUniqueLock();
+    inodeWrapper->UpdateTimestampLocked(kChangeTime);
+
+    rc = inodeWrapper->SyncAttr();
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG_ERROR("UpdateInodeCtime", rc);
+        return rc;
+    }
+
+    LOG(INFO) << "UpdateInodeCtime inodeid = " << srcDentry_.inodeid();
+    return rc;
+}
+
 void RenameOperator::UpdateCache() {
-    dentryManager_->DeleteCache(parentId_, name_);
-    dentryManager_->InsertOrReplaceCache(newDentry_);
     SetTxId(srcPartitionId_, srcTxId_ + 1);
     SetTxId(dstPartitionId_, dstTxId_ + 1);
 }

curvefs/src/client/client_operator.h

@@ -59,6 +59,7 @@ class RenameOperator {
     CURVEFS_ERROR UnlinkSrcParentInode();
     void UnlinkOldInode();
     CURVEFS_ERROR UpdateInodeParent();
+    CURVEFS_ERROR UpdateInodeCtime();
     void UpdateCache();
 
     void GetOldInode(uint64_t *oldInodeId, int64_t *oldInodeSize,

curvefs/src/client/common/common.cpp

@@ -22,6 +22,8 @@
  */
 
 #include "curvefs/src/client/common/common.h"
+#include "src/common/string_util.h"
+
 namespace curvefs {
 namespace client {
 namespace common {
@@ -77,6 +79,7 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) {
 }
 
 const char kCurveFsWarmupOpAdd[] = "add";
+const char kCurveFsWarmupOpQuery[] = "query";
 const char kCurveFsWarmupTypeList[] = "list";
 const char kCurveFsWarmupTypeSingle[] = "single";
 
@@ -85,6 +88,9 @@ WarmupOpType GetWarmupOpType(const std::string& op) {
     if (op == kCurveFsWarmupOpAdd) {
         ret = WarmupOpType::kWarmupOpAdd;
     }
+    if (op == kCurveFsWarmupOpQuery) {
+        ret = WarmupOpType::kWarmupOpQuery;
+    }
     return ret;
 }
 
@@ -99,6 +105,59 @@ WarmupType GetWarmupType(const std::string& type) {
     return ret;
 }
 
+const char kCurveFsWarmupStorageDisk[] = "disk";
+const char kCurveFsWarmupStorageKvclient[] = "kvclient";
+
+WarmupStorageType GetWarmupStorageType(const std::string &type) {
+    auto ret = WarmupStorageType::kWarmupStorageTypeUnknown;
+    if (type == kCurveFsWarmupStorageDisk) {
+        ret = WarmupStorageType::kWarmupStorageTypeDisk;
+    } else if (type == kCurveFsWarmupStorageKvclient) {
+        ret = WarmupStorageType::kWarmupStorageTypeKvClient;
+    }
+    return ret;
+}
+
+using ::curve::common::StringToUll;
+
+// if direction is true means '+', false means '-'
+// is direction is true, add second to first
+// if direction is false, sub second from first
+bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction) {
+    uint64_t firstNum = 0;
+    uint64_t secondNum = second;
+    if (StringToUll(*first, &firstNum)) {
+        if (direction) {
+            *first = std::to_string(firstNum + secondNum);
+        } else {
+            if (firstNum < secondNum) {
+                *first = std::to_string(0);
+                LOG(WARNING) << "AddUllStringToFirst failed when minus,"
+                             << " first = " << firstNum
+                             << ", second = " << secondNum;
+                return false;
+            }
+            *first = std::to_string(firstNum - secondNum);
+        }
+    } else {
+        LOG(ERROR) << "StringToUll failed, first = " << *first
+                   << ", second = " << second;
+        return false;
+    }
+    return true;
+}
+
+bool AddUllStringToFirst(uint64_t *first, const std::string &second) {
+    uint64_t secondNum = 0;
+    if (StringToUll(second, &secondNum)) {
+        *first += secondNum;
+        return true;
+    }
+
+    LOG(ERROR) << "StringToUll failed, second = " << second;
+    return false;
+}
+
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/common/common.h

@@ -59,17 +59,23 @@ enum class MetaServerOpType {
     GetOrModifyS3ChunkInfo,
     GetVolumeExtent,
     UpdateVolumeExtent,
+    CreateManageInode,
 };
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);
 
-const uint32_t MAXXATTRLENGTH = 256;
+const uint32_t MAX_XATTR_NAME_LENGTH = 255;
+const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024;
 
 const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op";
 
+
+constexpr int kWarmupOpNum = 4;
+
 enum class WarmupOpType {
     kWarmupOpUnknown = 0,
     kWarmupOpAdd = 1,
+    kWarmupOpQuery = 2,
 };
 
 WarmupOpType GetWarmupOpType(const std::string& op);
@@ -80,7 +86,15 @@ enum class WarmupType {
     kWarmupTypeSingle = 2,
 };
 
-WarmupType GetWarmupType(const std::string& type);
+WarmupType GetWarmupType(const std::string &type);
+
+enum class WarmupStorageType {
+    kWarmupStorageTypeUnknown = 0,
+    kWarmupStorageTypeDisk = 1,
+    kWarmupStorageTypeKvClient = 2,
+};
+
+WarmupStorageType GetWarmupStorageType(const std::string &type);
 
 enum class FileHandle : uint64_t {
     kDefaultValue = 0,
@@ -92,6 +106,11 @@ enum class NlinkChange : int32_t {
     kSubOne = -1,
 };
 
+// if direction is true means '+', false means '-'
+// is direction is true, add second to first
+// if direction is false, sub second from first
+bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction);
+bool AddUllStringToFirst(uint64_t *first, const std::string &second);
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/common/config.cpp

@@ -50,6 +50,56 @@ DEFINE_bool(enableCto, true, "acheieve cto consistency");
 DEFINE_bool(useFakeS3, false,
             "Use fake s3 to inject more metadata for testing metaserver");
 DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing");
+DEFINE_bool(access_logging, true, "enable access log");
+
+/**
+ * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true
+ * for dynamic parameter configuration
+ */
+static bool pass_uint64(const char *, uint64_t) { return true; }
+
+DEFINE_uint64(fuseClientAvgWriteBytes, 0,
+              "the write throttle bps of fuse client");
+DEFINE_validator(fuseClientAvgWriteBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteBytes, 0,
+              "the write burst bps of fuse client");
+DEFINE_validator(fuseClientBurstWriteBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteBytesSecs, 180,
+              "the times that write burst bps can continue");
+DEFINE_validator(fuseClientBurstWriteBytesSecs, &pass_uint64);
+
+
+DEFINE_uint64(fuseClientAvgWriteIops, 0,
+              "the write throttle iops of fuse client");
+DEFINE_validator(fuseClientAvgWriteIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteIops, 0,
+              "the write burst iops of fuse client");
+DEFINE_validator(fuseClientBurstWriteIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteIopsSecs, 180,
+              "the times that write burst iops can continue");
+DEFINE_validator(fuseClientBurstWriteIopsSecs, &pass_uint64);
+
+
+DEFINE_uint64(fuseClientAvgReadBytes, 0,
+              "the Read throttle bps of fuse client");
+DEFINE_validator(fuseClientAvgReadBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadBytes, 0,
+              "the Read burst bps of fuse client");
+DEFINE_validator(fuseClientBurstReadBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadBytesSecs, 180,
+              "the times that Read burst bps can continue");
+DEFINE_validator(fuseClientBurstReadBytesSecs, &pass_uint64);
+
+
+DEFINE_uint64(fuseClientAvgReadIops, 0,
+              "the Read throttle iops of fuse client");
+DEFINE_validator(fuseClientAvgReadIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadIops, 0,
+              "the Read burst iops of fuse client");
+DEFINE_validator(fuseClientBurstReadIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadIopsSecs, 180,
+              "the times that Read burst iops can continue");
+DEFINE_validator(fuseClientBurstReadIopsSecs, &pass_uint64);
 
 void InitMdsOption(Configuration *conf, MdsOption *mdsOpt) {
     conf->GetValueFatalIfFail("mdsOpt.mdsMaxRetryMS", &mdsOpt->mdsMaxRetryMS);
@@ -157,8 +207,6 @@ void InitDiskCacheOption(Configuration *conf,
 
 void InitS3Option(Configuration *conf, S3Option *s3Opt) {
     conf->GetValueFatalIfFail("s3.fakeS3", &FLAGS_useFakeS3);
-    conf->GetValueFatalIfFail("s3.fuseMaxSize",
-                              &s3Opt->s3ClientAdaptorOpt.fuseMaxSize);
     conf->GetValueFatalIfFail("s3.pageSize",
                               &s3Opt->s3ClientAdaptorOpt.pageSize);
     conf->GetValueFatalIfFail("s3.prefetchBlocks",
@@ -175,6 +223,8 @@ void InitS3Option(Configuration *conf, S3Option *s3Opt) {
                               &s3Opt->s3ClientAdaptorOpt.writeCacheMaxByte);
     conf->GetValueFatalIfFail("s3.readCacheMaxByte",
                               &s3Opt->s3ClientAdaptorOpt.readCacheMaxByte);
+    conf->GetValueFatalIfFail("s3.readCacheThreads",
+                              &s3Opt->s3ClientAdaptorOpt.readCacheThreads);
     conf->GetValueFatalIfFail("s3.nearfullRatio",
                               &s3Opt->s3ClientAdaptorOpt.nearfullRatio);
     conf->GetValueFatalIfFail("s3.baseSleepUs",
@@ -243,6 +293,55 @@ void InitKVClientManagerOpt(Configuration *conf,
                               &config->getThreadPooln);
 }
 
+void InitFileSystemOption(Configuration* c, FileSystemOption* option) {
+    c->GetValueFatalIfFail("fs.cto", &option->cto);
+    c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto);
+    c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr);
+    c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength);
+    c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging);
+    {  // kernel cache option
+        auto o = &option->kernelCacheOption;
+        c->GetValueFatalIfFail("fs.kernelCache.attrTimeoutSec",
+                               &o->attrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeoutSec",
+                               &o->dirAttrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.entryTimeoutSec",
+                               &o->entryTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeoutSec",
+                               &o->dirEntryTimeoutSec);
+    }
+    {  // lookup cache option
+        auto o = &option->lookupCacheOption;
+        c->GetValueFatalIfFail("fs.lookupCache.lruSize",
+                               &o->lruSize);
+        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeoutSec",
+                               &o->negativeTimeoutSec);
+        c->GetValueFatalIfFail("fs.lookupCache.minUses",
+                               &o->minUses);
+    }
+    {  // dir cache option
+        auto o = &option->dirCacheOption;
+        c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize);
+    }
+    {  // open file option
+        auto o = &option->openFilesOption;
+        c->GetValueFatalIfFail("fs.openFile.lruSize", &o->lruSize);
+    }
+    {  // attr watcher option
+        auto o = &option->attrWatcherOption;
+        c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize);
+    }
+    {  // rpc option
+        auto o = &option->rpcOption;
+        c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit);
+    }
+    {  // defer sync option
+        auto o = &option->deferSyncOption;
+        c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay);
+        c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime);
+    }
+}
+
 void SetBrpcOpt(Configuration *conf) {
     curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy;
     dummy.Load(conf, "defer_close_second", "rpc.defer.close.second",
@@ -263,47 +362,52 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
     InitLeaseOpt(conf, &clientOption->leaseOpt);
     InitRefreshDataOpt(conf, &clientOption->refreshDataOption);
     InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt);
+    InitFileSystemOption(conf, &clientOption->fileSystemOption);
 
-    conf->GetValueFatalIfFail("fuseClient.attrTimeOut",
-                              &clientOption->attrTimeOut);
-    conf->GetValueFatalIfFail("fuseClient.entryTimeOut",
-                              &clientOption->entryTimeOut);
     conf->GetValueFatalIfFail("fuseClient.listDentryLimit",
                               &clientOption->listDentryLimit);
     conf->GetValueFatalIfFail("fuseClient.listDentryThreads",
                               &clientOption->listDentryThreads);
-    conf->GetValueFatalIfFail("fuseClient.flushPeriodSec",
-                              &clientOption->flushPeriodSec);
-    conf->GetValueFatalIfFail("fuseClient.maxNameLength",
-                              &clientOption->maxNameLength);
-    conf->GetValueFatalIfFail("fuseClient.iCacheLruSize",
-                              &clientOption->iCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.dCacheLruSize",
-                              &clientOption->dCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.enableICacheMetrics",
-                              &clientOption->enableICacheMetrics);
-    conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
-                              &clientOption->enableDCacheMetrics);
     conf->GetValueFatalIfFail("client.dummyServer.startPort",
                               &clientOption->dummyServerStartPort);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                               &clientOption->enableMultiMountPointRename);
-    conf->GetValueFatalIfFail("fuseClient.disableXattr",
-                              &clientOption->disableXattr);
-    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
-
-
+    conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
+                              &clientOption->downloadMaxRetryTimes);
+    conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
+                              &clientOption->warmupThreadsNum);
     LOG_IF(WARNING, conf->GetBoolValue("fuseClient.enableSplice",
                                        &clientOption->enableFuseSplice))
         << "Not found `fuseClient.enableSplice` in conf, use default value `"
         << std::boolalpha << clientOption->enableFuseSplice << '`';
 
-    // if enableCto, attr and entry cache must invalid
-    if (FLAGS_enableCto) {
-        clientOption->attrTimeOut = 0;
-        clientOption->entryTimeOut = 0;
-    }
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteBytes",
+                              &FLAGS_fuseClientAvgWriteBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytes",
+                              &FLAGS_fuseClientBurstWriteBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytesSecs",
+                              &FLAGS_fuseClientBurstWriteBytesSecs);
 
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteIops",
+                              &FLAGS_fuseClientAvgWriteIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIops",
+                              &FLAGS_fuseClientBurstWriteIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIopsSecs",
+                              &FLAGS_fuseClientBurstWriteIopsSecs);
+
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadBytes",
+                              &FLAGS_fuseClientAvgReadBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytes",
+                              &FLAGS_fuseClientBurstReadBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytesSecs",
+                              &FLAGS_fuseClientBurstReadBytesSecs);
+
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadIops",
+                              &FLAGS_fuseClientAvgReadIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIops",
+                              &FLAGS_fuseClientBurstReadIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIopsSecs",
+                              &FLAGS_fuseClientBurstReadIopsSecs);
     SetBrpcOpt(conf);
 }
 
@@ -311,6 +415,7 @@ void SetFuseClientS3Option(FuseClientOption *clientOption,
     const S3InfoOption &fsS3Opt) {
     clientOption->s3Opt.s3ClientAdaptorOpt.blockSize = fsS3Opt.blockSize;
     clientOption->s3Opt.s3ClientAdaptorOpt.chunkSize = fsS3Opt.chunkSize;
+    clientOption->s3Opt.s3ClientAdaptorOpt.objectPrefix = fsS3Opt.objectPrefix;
     clientOption->s3Opt.s3AdaptrOpt.s3Address = fsS3Opt.s3Address;
     clientOption->s3Opt.s3AdaptrOpt.ak = fsS3Opt.ak;
     clientOption->s3Opt.s3AdaptrOpt.sk = fsS3Opt.sk;
@@ -325,6 +430,7 @@ void S3Info2FsS3Option(const curvefs::common::S3Info& s3,
     fsS3Opt->bucketName = s3.bucketname();
     fsS3Opt->blockSize = s3.blocksize();
     fsS3Opt->chunkSize = s3.chunksize();
+    fsS3Opt->objectPrefix = s3.has_objectprefix() ? s3.objectprefix() : 0;
 }
 
 }  // namespace common

curvefs/src/client/common/config.h

@@ -32,15 +32,13 @@
 #include "src/common/configuration.h"
 #include "src/common/s3_adapter.h"
 
+namespace curvefs {
+namespace client {
+namespace common {
 using ::curve::common::Configuration;
 using ::curve::common::S3AdapterOption;
 using ::curvefs::client::common::DiskCacheType;
 using ::curve::common::S3InfoOption;
-
-namespace curvefs {
-namespace client {
-namespace common {
-
 using MdsOption = ::curve::client::MetaServerOption;
 
 struct BlockDeviceClientOptions {
@@ -106,7 +104,7 @@ struct DiskCacheOption {
     // the max time system command can run
     uint32_t cmdTimeoutSec;
     // threads for disk cache
-    uint32_t threads;
+    uint32_t threads = 10;
     // the write throttle bps of disk cache
     uint64_t avgFlushBytes;
     // the write burst bps of disk cache
@@ -124,7 +122,6 @@ struct DiskCacheOption {
 struct S3ClientAdaptorOption {
     uint64_t blockSize;
     uint64_t chunkSize;
-    uint32_t fuseMaxSize;
     uint64_t pageSize;
     uint32_t prefetchBlocks;
     uint32_t prefetchExecQueueNum;
@@ -133,10 +130,12 @@ struct S3ClientAdaptorOption {
     uint32_t flushIntervalSec;
     uint64_t writeCacheMaxByte;
     uint64_t readCacheMaxByte;
+    uint32_t readCacheThreads;
     uint32_t nearfullRatio;
     uint32_t baseSleepUs;
     uint32_t maxReadRetryIntervalMs;
     uint32_t readRetryIntervalMs;
+    uint32_t objectPrefix;
     DiskCacheOption diskCacheOpt;
 };
 
@@ -175,6 +174,59 @@ struct RefreshDataOption {
     uint64_t maxDataSize = 1024;
     uint32_t refreshDataIntervalSec = 30;
 };
+
+// { filesystem option
+struct KernelCacheOption {
+    uint32_t entryTimeoutSec;
+    uint32_t dirEntryTimeoutSec;
+    uint32_t attrTimeoutSec;
+    uint32_t dirAttrTimeoutSec;
+};
+
+struct LookupCacheOption {
+    uint64_t lruSize;
+    uint32_t negativeTimeoutSec;
+    uint32_t minUses;
+};
+
+struct DirCacheOption {
+    uint64_t lruSize;
+    uint32_t timeoutSec;
+};
+
+struct AttrWatcherOption {
+    uint64_t lruSize;
+};
+
+struct OpenFilesOption {
+    uint64_t lruSize;
+    uint32_t deferSyncSecond;
+};
+
+struct RPCOption {
+    uint32_t listDentryLimit;
+};
+
+struct DeferSyncOption {
+    uint32_t delay;
+    bool deferDirMtime;
+};
+
+struct FileSystemOption {
+    bool cto;
+    bool disableXattr;
+    uint32_t maxNameLength;
+    uint32_t blockSize = 0x10000u;
+    KernelCacheOption kernelCacheOption;
+    LookupCacheOption lookupCacheOption;
+    DirCacheOption dirCacheOption;
+    OpenFilesOption openFilesOption;
+    AttrWatcherOption attrWatcherOption;
+    RPCOption rpcOption;
+    DeferSyncOption deferSyncOption;
+};
+// }
+
 struct FuseClientOption {
     MdsOption mdsOpt;
     MetaCacheOpt metaCacheOpt;
@@ -188,21 +240,15 @@ struct FuseClientOption {
     LeaseOpt leaseOpt;
     RefreshDataOption refreshDataOption;
     KVClientManagerOpt kvClientManagerOpt;
+    FileSystemOption fileSystemOption;
 
-    double attrTimeOut;
-    double entryTimeOut;
     uint32_t listDentryLimit;
     uint32_t listDentryThreads;
-    uint32_t flushPeriodSec;
-    uint32_t maxNameLength;
-    uint64_t iCacheLruSize;
-    uint64_t dCacheLruSize;
-    bool enableICacheMetrics;
-    bool enableDCacheMetrics;
     uint32_t dummyServerStartPort;
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;
-    bool disableXattr = false;
+    uint32_t downloadMaxRetryTimes;
+    uint32_t warmupThreadsNum = 10;
 };
 
 void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption);

curvefs/src/client/curve_fuse_op.h

@@ -37,10 +37,9 @@
 extern "C" {
 #endif
 
-int InitGlog(const char *confPath, const char *argv0);
+int InitLog(const char *confPath, const char *argv0);
 
-int InitFuseClient(const char *confPath, const char *fsName,
-        const char* fsType, const char* mdsAddr);
+int InitFuseClient(const struct MountOption *mountOption);
 
 void UnInitFuseClient();
 

curvefs/src/client/dentry_cache_manager.cpp

@@ -44,31 +44,13 @@ namespace client {
 
 using curve::common::WriteLockGuard;
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
-
-void DentryCacheManagerImpl::InsertOrReplaceCache(const Dentry &dentry) {
-    std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name());
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        NameLockGuard lock(nameLock_, key);
-        dCache_->Put(key, dentry);
-    }
-}
-
-void DentryCacheManagerImpl::DeleteCache(uint64_t parentId,
-                                         const std::string &name) {
-    std::string key = GetDentryCacheKey(parentId, name);
-    NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
-}
+using ::curvefs::client::filesystem::ToFSError;
 
 CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
                                                 const std::string &name,
                                                 Dentry *out) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    bool ok = dCache_->Get(key, out);
-    if (ok) {
-        return CURVEFS_ERROR::OK;
-    }
 
     MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out);
     if (ret != MetaStatusCode::OK) {
@@ -76,11 +58,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
             << "metaClient_ GetDentry failed, MetaStatusCode = " << ret
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
             << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
-    }
-
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, *out);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -95,12 +73,9 @@ CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) {
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << dentry.parentinodeid()
                    << ", name = " << dentry.name();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, dentry);
-    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -109,14 +84,13 @@ CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent,
                                                    FsFileType type) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
 
     MetaStatusCode ret = metaClient_->DeleteDentry(fsId_, parent, name, type);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -150,7 +124,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", parent = " << parent << ", last = " << last
                        << ", count = " << limit << ", onlyDir = " << onlyDir;
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
 
         if (!onlyDir) {

curvefs/src/client/dentry_cache_manager.h

@@ -33,20 +33,18 @@
 #include <utility>
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
-#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
-#include "src/common/lru_cache.h"
 #include "src/common/concurrent/name_lock.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Dentry;
-using ::curve::common::LRUCache;
-using ::curve::common::CacheMetrics;
 
 namespace curvefs {
 namespace client {
 
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 static const char* kDentryKeyDelimiter = ":";
 
@@ -59,12 +57,6 @@ class DentryCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) = 0;
-
-    virtual void InsertOrReplaceCache(const Dentry& dentry) = 0;
-
-    virtual void DeleteCache(uint64_t parentId, const std::string& name) = 0;
-
     virtual CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) = 0;
 
@@ -85,29 +77,11 @@ class DentryCacheManager {
 class DentryCacheManagerImpl : public DentryCacheManager {
  public:
     DentryCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()),
-        dCache_(nullptr) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
 
     explicit DentryCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient),
-        dCache_(nullptr) {}
-
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) override {
-        if (enableCacheMetrics) {
-            dCache_ = std::make_shared<
-                LRUCache<std::string, Dentry>>(cacheSize,
-                    std::make_shared<CacheMetrics>("dcache"));
-        } else {
-            dCache_ = std::make_shared<
-                LRUCache<std::string, Dentry>>(cacheSize);
-        }
-        return CURVEFS_ERROR::OK;
-    }
-
-    void InsertOrReplaceCache(const Dentry& dentry) override;
-
-    void DeleteCache(uint64_t parentId, const std::string& name) override;
+      : metaClient_(metaClient) {}
 
     CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) override;
@@ -128,8 +102,6 @@ class DentryCacheManagerImpl : public DentryCacheManager {
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    // key is parentId + name
-    std::shared_ptr<LRUCache<std::string, Dentry>> dCache_;
     curve::common::GenericNameLock<Mutex> nameLock_;
 };
 

curvefs/src/client/error_code.cpp

@@ -1,98 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-/*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
- */
-
-#include "curvefs/src/client/error_code.h"
-
-#include <map>
-#include <string>
-
-namespace curvefs {
-namespace client {
-
-using ::curvefs::metaserver::MetaStatusCode;
-
-static const std::map<CURVEFS_ERROR, std::string> err2Msg = {
-    {CURVEFS_ERROR::OK, "OK"},
-    {CURVEFS_ERROR::INTERNAL, "internal error"},
-    {CURVEFS_ERROR::UNKNOWN, "unknown"},
-    {CURVEFS_ERROR::EXISTS, "inode or dentry already exist"},
-    {CURVEFS_ERROR::NOTEXIST, "inode or dentry not exist"},
-    {CURVEFS_ERROR::NO_SPACE, "no space to alloc"},
-    {CURVEFS_ERROR::BAD_FD, "bad fd"},
-    {CURVEFS_ERROR::INVALIDPARAM, "invalid param"},
-    {CURVEFS_ERROR::NOPERMISSION, "no permission"},
-    {CURVEFS_ERROR::NOTEMPTY, "dir not empty"},
-    {CURVEFS_ERROR::NOFLUSH, "no flush"},
-    {CURVEFS_ERROR::NOTSUPPORT, "not support"},
-    {CURVEFS_ERROR::NAMETOOLONG, "name too long"},
-    {CURVEFS_ERROR::MOUNT_POINT_EXIST, "mount point already exist"},
-    {CURVEFS_ERROR::MOUNT_FAILED, "mount failed"},
-};
-
-std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
-    os << static_cast<int>(code) << "[" << [code]() {
-        auto it = err2Msg.find(code);
-        if (it != err2Msg.end()) {
-            return it->second;
-        }
-
-        return std::string{"Unknown"};
-    }() << "]";
-
-    return os;
-}
-
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    MetaStatusCode code) {
-    CURVEFS_ERROR ret = CURVEFS_ERROR::UNKNOWN;
-    switch (code) {
-        case MetaStatusCode::OK:
-            ret = CURVEFS_ERROR::OK;
-            break;
-
-        case MetaStatusCode::NOT_FOUND:
-            ret = CURVEFS_ERROR::NOTEXIST;
-            break;
-
-        case MetaStatusCode::PARAM_ERROR:
-            ret = CURVEFS_ERROR::INVALIDPARAM;
-            break;
-
-        case MetaStatusCode::INODE_EXIST:
-        case MetaStatusCode::DENTRY_EXIST:
-            ret = CURVEFS_ERROR::EXISTS;
-            break;
-
-        case MetaStatusCode::SYM_LINK_EMPTY:
-        case MetaStatusCode::RPC_ERROR:
-            ret = CURVEFS_ERROR::INTERNAL;
-            break;
-
-        default:
-            ret = CURVEFS_ERROR::UNKNOWN;
-    }
-    return ret;
-}
-
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/README.md

@@ -0,0 +1,78 @@
+CurveFS Client Metadata Cache Design
+===
+
+```
++----------------+
++  Kernel Cache  |
++----------------+
+    |       ^
+(1) |       | (4)
+    v       |
++----------------+
++  CurveFS Fuse  |
++----------------+
+    |       ^
+(2) |       | (3)
+    v       |
++----------------+
++   MetaServer   |
++----------------+
+```
+
+(1) kernel cache timeout
+---
+
+* requests are always sent to fuse layer, except `lookup` and `getattr` request which affected by caching.
+* if entry or attribute cache timeout, vfs layer will launch request to fuse layer.
+
+(2) revalidate cache
+---
+
+* for `open` and `opendir` request, fuse layer should revalidate cache by comparing mtime, if modified, fuse layer should:
+    * `open`: return **ESTALE** to trigger vfs layer to invoke the `open` again with ignoring cache.
+    * `opendir`: drop all directory cache.
+* others, proxy request to metaserver directly.
+
+
+(3) retrive fresh metadata
+---
+
+* for `readdir` request, fuse layer should cache direcoty entries and their attributes.
+* others, no caching.
+
+(4) reply with timeout
+---
+
+* reply entry or attribute to vfs layer with corresponding cache timeout.
+* four type timeouts provided: `entryTimeout`, `dirEntryTimeout`, `attrTimeout`, `dirAtttTimeout`.
+* fuse layer should remeber the `mtime` of attribute while reply it to vfs layer.
+
+Cache Layer Level
+===
+
+```
++-------------------------------------------------------+
+|                       Kernel Cache                    |  (L1 Cache)
++-------------------------------------------------------+
+                            ^  (update length)
+                            |
++--------------------------------------------------------+
+|                       Open File Cache                  |  (L2 Cache)
+|                   (only length and chunk)              |
++--------------------------------------------------------+
+         ^                            ^
+         | (reply entry/attr)         |  (reply entry/attr)
++-----------------+                   |
+| Directory Cache |  (L3 Cache)       |
++-----------------+                   |
+         ^                            |
+         |                            |
++-----------------+ +------------------------------------+
+|     readdir     | | lookup、getattr、setattr、create... |
++-----------------+ +------------------------------------+
+         ^                            ^
+         |                            |
++--------------------------------------------------------+
+|                        MetaServer                      |
++--------------------------------------------------------+
+```

curvefs/src/client/filesystem/access_log.h

@@ -0,0 +1,89 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-17
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <unistd.h>
+#include <butil/time.h>
+#include <spdlog/spdlog.h>
+#include <spdlog/sinks/basic_file_sink.h>
+#include <spdlog/sinks/daily_file_sink.h>
+
+#include <string>
+#include <memory>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/common/config.h"
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+
+namespace curvefs {
+namespace client {
+namespace common {
+
+DECLARE_bool(access_logging);
+
+}
+namespace filesystem {
+
+using ::absl::StrFormat;
+using ::curvefs::client::common::FLAGS_access_logging;
+using MessageHandler = std::function<std::string()>;
+
+static std::shared_ptr<spdlog::logger> Logger;
+
+bool InitAccessLog(const std::string& prefix) {
+    std::string filename = StrFormat("%s/access.%d.log", prefix, getpid());
+    Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0);
+    spdlog::flush_every(std::chrono::seconds(1));
+    return true;
+}
+
+struct AccessLogGuard {
+    explicit AccessLogGuard(MessageHandler handler)
+        : enable(FLAGS_access_logging),
+          handler(handler) {
+        if (!enable) {
+            return;
+        }
+
+        timer.start();
+    }
+
+    ~AccessLogGuard() {
+        if (!enable) {
+            return;
+        }
+
+        timer.stop();
+        Logger->info("{0} <{1:.6f}>", handler(), timer.u_elapsed() / 1e6);
+    }
+
+    bool enable;
+    MessageHandler handler;
+    butil::Timer timer;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_

curvefs/src/client/filesystem/attr_watcher.cpp

@@ -0,0 +1,117 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+AttrWatcher::AttrWatcher(AttrWatcherOption option,
+                         std::shared_ptr<OpenFiles> openFiles,
+                         std::shared_ptr<DirCache> dirCache)
+    : modifiedAt_(std::make_shared<LRUType>(option.lruSize)),
+      openFiles_(openFiles),
+      dirCache_(dirCache) {}
+
+void AttrWatcher::RemeberMtime(const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    modifiedAt_->Put(attr.inodeid(), AttrMtime(attr));
+}
+
+bool AttrWatcher::GetMtime(Ino ino, TimeSpec* time) {
+    ReadLockGuard lk(rwlock_);
+    return modifiedAt_->Get(ino, time);
+}
+
+void AttrWatcher::UpdateDirEntryAttr(Ino ino, const InodeAttr& attr) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : attr.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateAttr(ino, attr);
+
+        VLOG(1) << "Write back attribute to dir entry cache: ino = " << ino
+                << ", attr = " << attr.ShortDebugString();
+    }
+}
+
+void AttrWatcher::UpdateDirEntryLength(Ino ino, const InodeAttr& open) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : open.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateLength(ino, open);
+
+        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
+                << ", attr = " << open.ShortDebugString();
+    }
+}
+
+AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                                   InodeAttr* attr,
+                                   ReplyType type,
+                                   bool writeBack)
+    : watcher(watcher), attr(attr), type(type), writeBack(writeBack) {
+    InodeAttr open;
+    Ino ino = attr->inodeid();
+    bool yes = watcher->openFiles_->GetFileAttr(ino, &open);
+    if (!yes) {
+        return;
+    }
+
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+}
+
+AttrWatcherGuard::~AttrWatcherGuard() {
+    switch (type) {
+        case ReplyType::ATTR:
+            watcher->RemeberMtime(*attr);
+            if (writeBack) {
+                watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
+            }
+            break;
+
+        case ReplyType::ONLY_LENGTH:
+            if (writeBack) {
+                watcher->UpdateDirEntryLength(attr->inodeid(), *attr);
+            }
+            break;
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/attr_watcher.h

@@ -0,0 +1,105 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::AttrWatcherOption;
+
+class AttrWatcher {
+ public:
+    using LRUType = LRUCache<Ino, struct TimeSpec>;
+
+ public:
+    AttrWatcher(AttrWatcherOption option,
+                std::shared_ptr<OpenFiles> openFiles,
+                std::shared_ptr<DirCache> dirCache);
+
+    void RemeberMtime(const InodeAttr& attr);
+
+    bool GetMtime(Ino ino, TimeSpec* time);
+
+    void UpdateDirEntryAttr(Ino ino, const InodeAttr& attr);
+
+    void UpdateDirEntryLength(Ino ino, const InodeAttr& open);
+
+ private:
+    friend class AttrWatcherGuard;
+
+ private:
+    RWLock rwlock_;
+    std::shared_ptr<LRUType> modifiedAt_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<DirCache> dirCache_;
+};
+
+
+enum class ReplyType {
+    ATTR,
+    ONLY_LENGTH
+};
+
+/*
+ * each attribute reply to kernel, the watcher will:
+ *  before reply:
+ *    1) set attibute length if the corresponding file is opened
+ *  after reply:
+ *    1) remeber attribute modified time.
+ *    2) write back attribute to dir entry cache if |writeBack| is true,
+ *       because the dir-entry attribute maybe stale.
+ */
+struct AttrWatcherGuard {
+ public:
+    AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                     InodeAttr* attr,
+                     ReplyType type,
+                     bool writeBack);
+
+    ~AttrWatcherGuard();
+
+ private:
+    std::shared_ptr<AttrWatcher> watcher;
+    InodeAttr* attr;
+    ReplyType type;
+    bool writeBack;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_

curvefs/src/client/filesystem/defer_sync.cpp

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <vector>
+#include <memory>
+
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DeferSync::DeferSync(DeferSyncOption option)
+    : option_(option),
+      mutex_(),
+      running_(false),
+      thread_(),
+      sleeper_(),
+      inodes_() {
+}
+
+void DeferSync::Start() {
+    if (!running_.exchange(true)) {
+        thread_ = std::thread(&DeferSync::SyncTask, this);
+        LOG(INFO) << "Defer sync thread start success";
+    }
+}
+
+void DeferSync::Stop() {
+    if (running_.exchange(false)) {
+        LOG(INFO) << "Stop defer sync thread...";
+        sleeper_.interrupt();
+        thread_.join();
+        LOG(INFO) << "Defer sync thread stopped";
+    }
+}
+
+void DeferSync::SyncTask() {
+    std::vector<std::shared_ptr<InodeWrapper>> inodes;
+    for ( ;; ) {
+        bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay));
+
+        {
+            LockGuard lk(mutex_);
+            inodes.swap(inodes_);
+        }
+        for (const auto& inode : inodes) {
+            UniqueLock lk(inode->GetUniqueLock());
+            inode->Async(nullptr, true);
+        }
+        inodes.clear();
+
+        if (!running) {
+            break;
+        }
+    }
+}
+
+void DeferSync::Push(const std::shared_ptr<InodeWrapper>& inode) {
+    LockGuard lk(mutex_);
+    inodes_.emplace_back(inode);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/defer_sync.h

@@ -0,0 +1,70 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+
+#include <atomic>
+#include <vector>
+#include <memory>
+
+#include "src/common/interruptible_sleeper.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::DeferSyncOption;
+
+using ::curve::common::Mutex;
+using ::curve::common::LockGuard;
+using ::curve::common::InterruptibleSleeper;
+
+class DeferSync {
+ public:
+    explicit DeferSync(DeferSyncOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Push(const std::shared_ptr<InodeWrapper>& inode);
+
+ private:
+    void SyncTask();
+
+ private:
+    DeferSyncOption option_;
+    Mutex mutex_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    InterruptibleSleeper sleeper_;
+    std::vector<std::shared_ptr<InodeWrapper>> inodes_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_

curvefs/src/client/filesystem/dir_cache.cpp

@@ -0,0 +1,189 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <utility>
+
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DirEntryList::DirEntryList()
+    : rwlock_(),
+      mtime_(),
+      entries_(),
+      attrs_() {}
+
+size_t DirEntryList::Size() {
+    ReadLockGuard lk(rwlock_);
+    return entries_.size();
+}
+
+void DirEntryList::Add(const DirEntry& dirEntry) {
+    WriteLockGuard lk(rwlock_);
+    entries_.push_back(std::move(dirEntry));
+    attrs_[dirEntry.ino] = &entries_.back();
+}
+
+bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) {
+    ReadLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+    *dirEntry = *iter->second;
+    return true;
+}
+
+bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    DirEntry* dirEntry = iter->second;
+    dirEntry->attr = std::move(attr);
+    return true;
+}
+
+bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    InodeAttr* attr = &iter->second->attr;
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+    return true;
+}
+
+void DirEntryList::Iterate(IterateHandler handler) {
+    ReadLockGuard lk(rwlock_);
+    for (auto iter = entries_.begin(); iter != entries_.end(); iter++) {
+        handler(&(*iter));
+    }
+}
+
+void DirEntryList::Clear() {
+    WriteLockGuard lk(rwlock_);
+    entries_.clear();
+    attrs_.clear();
+}
+
+void DirEntryList::SetMtime(TimeSpec mtime) {
+    WriteLockGuard lk(rwlock_);
+    mtime_ = mtime;
+}
+
+TimeSpec DirEntryList::GetMtime() {
+    ReadLockGuard lk(rwlock_);
+    return mtime_;
+}
+
+DirCache::DirCache(DirCacheOption option)
+    : rwlock_(),
+      nentries_(0),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(0);  // control size by ourself
+    mq_ = std::make_shared<MessageQueueType>("dircache", 10000);
+    mq_->Subscribe([&](const std::shared_ptr<DirEntryList>& entries){
+        entries->Clear();
+    });
+
+    LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize;
+}
+
+void DirCache::Start() {
+    mq_->Start();
+}
+
+void DirCache::Stop() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+    mq_->Stop();
+}
+
+void DirCache::Delete(Ino parent,
+                      std::shared_ptr<DirEntryList> entries,
+                      bool evit) {
+    nentries_ -= entries->Size();
+    mq_->Publish(entries);  // clear entries in background
+    lru_->Remove(parent);
+
+    VLOG(1) << "Delete directory cache (evit=" << evit << "): "
+            << "parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
+}
+
+void DirCache::Evit(size_t size) {
+    Ino parent;
+    std::shared_ptr<DirEntryList> entries;
+    while (nentries_ + size >= option_.lruSize) {
+        bool yes = lru_->GetLast(&parent, &entries);
+        if (!yes) {
+            break;
+        }
+        Delete(parent, entries, true);
+    }
+}
+
+void DirCache::Put(Ino parent, std::shared_ptr<DirEntryList> entries) {
+    WriteLockGuard lk(rwlock_);
+    Evit(entries->Size());  // it guarantee put entries success
+    lru_->Put(parent, entries);
+    nentries_ += entries->Size();
+
+    VLOG(1) << "Insert directory cache: parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
+}
+
+bool DirCache::Get(Ino parent, std::shared_ptr<DirEntryList>* entries) {
+    ReadLockGuard lk(rwlock_);
+    return lru_->Get(parent, entries);
+}
+
+void DirCache::Drop(Ino parent) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = lru_->Get(parent, &entries);
+    if (yes) {
+        Delete(parent, entries, false);
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/dir_cache.h

@@ -0,0 +1,114 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+
+#include <map>
+#include <list>
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/message_queue.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::DirCacheOption;
+
+class DirEntryList {
+ public:
+    using IterateHandler = std::function<void(DirEntry* dirEntry)>;
+
+ public:
+    DirEntryList();
+
+    size_t Size();
+
+    void Add(const DirEntry& dirEntry);
+
+    void Iterate(IterateHandler handler);
+
+    bool Get(Ino ino, DirEntry* dirEntry);
+
+    bool UpdateAttr(Ino ino, const InodeAttr& attr);
+
+    bool UpdateLength(Ino ino, const InodeAttr& open);
+
+    void Clear();
+
+    void SetMtime(TimeSpec mtime);
+
+    TimeSpec GetMtime();
+
+ private:
+    RWLock rwlock_;
+    TimeSpec mtime_;
+    std::list<DirEntry> entries_;
+    std::map<Ino, DirEntry*> attrs_;
+};
+
+class DirCache {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<DirEntryList>>;
+    using MessageType = std::shared_ptr<DirEntryList>;
+    using MessageQueueType = MessageQueue<MessageType>;
+
+ public:
+    explicit DirCache(DirCacheOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Put(Ino parent, std::shared_ptr<DirEntryList> entries);
+
+    bool Get(Ino parent, std::shared_ptr<DirEntryList>* entries);
+
+    void Drop(Ino parent);
+
+ private:
+    void Delete(Ino parent, std::shared_ptr<DirEntryList> entries, bool evit);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    size_t nentries_;
+    DirCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+    std::shared_ptr<MessageQueueType> mq_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_

curvefs/src/client/filesystem/error.cpp

@@ -0,0 +1,106 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <utility>
+#include <ostream>
+
+#include "curvefs/src/client/filesystem/error.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+static const std::map<CURVEFS_ERROR, std::pair<int, std::string>> errors = {
+    { CURVEFS_ERROR::OK, { 0, "OK"} },
+    { CURVEFS_ERROR::INTERNAL,  { EIO, "internal error" } },
+    { CURVEFS_ERROR::UNKNOWN,  { -1, "unknown" } },
+    { CURVEFS_ERROR::EXISTS,  { EEXIST, "inode or dentry already exist" } },
+    { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } },
+    { CURVEFS_ERROR::NO_SPACE,  { ENOSPC, "no space to alloc" } },
+    { CURVEFS_ERROR::BAD_FD,  { EBADF, "bad file number" } },
+    { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } },
+    { CURVEFS_ERROR::NOPERMISSION,  { EACCES, "permission denied" } },
+    { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } },
+    { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } },
+    { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } },
+    { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } },
+    { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } },
+    { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } },
+    { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } },
+    { CURVEFS_ERROR::NODATA, { ENODATA, "no data available" } },
+    { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } },
+    { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } },
+    { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } },
+};
+
+std::string StrErr(CURVEFS_ERROR code) {
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        return it->second.second;
+    }
+    return "unknown";
+}
+
+int SysErr(CURVEFS_ERROR code) {
+    int syscode = -1;
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        syscode = it->second.first;
+    }
+    return (syscode == -1) ? EIO : syscode;
+}
+
+std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
+    os << static_cast<int>(code) << "[" << [code]() {
+        auto it = errors.find(code);
+        if (it != errors.end()) {
+            return it->second.second;
+        }
+
+        return std::string{"Unknown"};
+    }() << "]";
+
+    return os;
+}
+
+CURVEFS_ERROR ToFSError(MetaStatusCode code) {
+    static std::map<MetaStatusCode, CURVEFS_ERROR> errs = {
+        { MetaStatusCode::OK, CURVEFS_ERROR::OK },
+        { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST },
+        { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM },
+        { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL },
+        { MetaStatusCode::RPC_ERROR, CURVEFS_ERROR::INTERNAL },
+    };
+
+    auto it = errs.find(code);
+    if (it != errs.end()) {
+        return it->second;
+    }
+    return CURVEFS_ERROR::UNKNOWN;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/error.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2021 NetEase Inc.
+ *  Copyright (c) 2023 NetEase Inc.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -14,22 +14,27 @@
  *  limitations under the License.
  */
 
-
 /*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
  */
 
-#ifndef CURVEFS_SRC_CLIENT_ERROR_CODE_H_
-#define CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#include <map>
+#include <string>
+#include <iostream>
 
 #include "curvefs/proto/metaserver.pb.h"
 
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+
 namespace curvefs {
 namespace client {
+namespace filesystem {
+
+using ::curvefs::metaserver::MetaStatusCode;
 
-// notice : the error code should be negative.
 enum class CURVEFS_ERROR {
     OK = 0,
     INTERNAL = -1,
@@ -49,15 +54,21 @@ enum class CURVEFS_ERROR {
     OUT_OF_RANGE = -15,
     NODATA = -16,
     IO_ERROR = -17,
+    CACHETOOSMALL = -18,
+    STALE = -19,
+    NOSYS = -20,
 };
 
+std::string StrErr(CURVEFS_ERROR code);
+
+int SysErr(CURVEFS_ERROR code);
 
 std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code);
 
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    ::curvefs::metaserver::MetaStatusCode code);
+CURVEFS_ERROR ToFSError(MetaStatusCode code);
 
+}  // namespace filesystem
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_

curvefs/src/client/filesystem/filesystem.cpp

@@ -0,0 +1,353 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+
+#include "curvefs/src/client/filesystem/filesystem.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
+    : option_(option), member(member) {
+    deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
+    negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
+    dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
+    openFiles_ = std::make_shared<OpenFiles>(option_.openFilesOption,
+                                             deferSync_);
+    attrWatcher_ = std::make_shared<AttrWatcher>(option_.attrWatcherOption,
+                                                 openFiles_, dirCache_);
+    handlerManager_ = std::make_shared<HandlerManager>();
+    rpc_ = std::make_shared<RPCClient>(option.rpcOption, member);
+}
+
+void FileSystem::Run() {
+    deferSync_->Start();
+    dirCache_->Start();
+}
+
+void FileSystem::Destory() {
+    openFiles_->CloseAll();
+    deferSync_->Stop();
+    dirCache_->Stop();
+}
+
+void FileSystem::Attr2Stat(InodeAttr* attr, struct stat* stat) {
+    std::memset(stat, 0, sizeof(struct stat));
+    stat->st_ino = attr->inodeid();  //  inode number
+    stat->st_mode = attr->mode();  // permission mode
+    stat->st_nlink = attr->nlink();  // number of links
+    stat->st_uid = attr->uid();  // user ID of owner
+    stat->st_gid = attr->gid();  // group ID of owner
+    stat->st_size = attr->length();  // total size, in bytes
+    stat->st_rdev = attr->rdev();  // device ID (if special file)
+    stat->st_atim.tv_sec = attr->atime();  // time of last access
+    stat->st_atim.tv_nsec = attr->atime_ns();
+    stat->st_mtim.tv_sec = attr->mtime();  // time of last modification
+    stat->st_mtim.tv_nsec = attr->mtime_ns();
+    stat->st_ctim.tv_sec = attr->ctime();  // time of last status change
+    stat->st_ctim.tv_nsec = attr->ctime_ns();
+    stat->st_blksize = option_.blockSize;  // blocksize for file system I/O
+    stat->st_blocks = 0;  // number of 512B blocks allocated
+    if (IsS3File(*attr)) {
+        stat->st_blocks = (attr->length() + 511) / 512;
+    }
+}
+
+void FileSystem::Entry2Param(EntryOut* entryOut,
+                             fuse_entry_param* e) {
+    std::memset(e, 0, sizeof(fuse_entry_param));
+    e->ino = entryOut->attr.inodeid();
+    e->generation = 0;
+    Attr2Stat(&entryOut->attr, &e->attr);
+    e->entry_timeout = entryOut->entryTimeout;
+    e->attr_timeout = entryOut->attrTimeout;
+}
+
+void FileSystem::SetEntryTimeout(EntryOut* entryOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(entryOut->attr)) {
+        entryOut->entryTimeout = option.dirEntryTimeoutSec;
+        entryOut->attrTimeout = option.dirAttrTimeoutSec;
+    } else {
+        entryOut->entryTimeout = option.entryTimeoutSec;
+        entryOut->attrTimeout = option.attrTimeoutSec;
+    }
+}
+
+void FileSystem::SetAttrTimeout(AttrOut* attrOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(attrOut->attr)) {
+        attrOut->attrTimeout = option.dirAttrTimeoutSec;
+    } else {
+        attrOut->attrTimeout = option.attrTimeoutSec;
+    }
+}
+
+// fuse reply*
+void FileSystem::ReplyError(Request req, CURVEFS_ERROR code) {
+    fuse_reply_err(req, SysErr(code));
+}
+
+void FileSystem::ReplyEntry(Request req,
+                            EntryOut* entryOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_entry(req, &e);
+}
+
+void FileSystem::ReplyAttr(Request req,
+                           AttrOut* attrOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr,
+                             ReplyType::ATTR, true);
+    struct stat stat;
+    SetAttrTimeout(attrOut);
+    Attr2Stat(&attrOut->attr, &stat);
+    fuse_reply_attr(req, &stat, attrOut->attrTimeout);
+}
+
+void FileSystem::ReplyReadlink(Request req, const std::string& link) {
+    fuse_reply_readlink(req, link.c_str());
+}
+
+void FileSystem::ReplyOpen(Request req, FileInfo* fi) {
+    fuse_reply_open(req, fi);
+}
+
+void FileSystem::ReplyOpen(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_open(req, fileOut->fi);
+}
+
+void FileSystem::ReplyData(Request req,
+                           struct fuse_bufvec *bufv,
+                           enum fuse_buf_copy_flags flags) {
+    fuse_reply_data(req, bufv, flags);
+}
+
+void FileSystem::ReplyWrite(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_write(req, fileOut->nwritten);
+}
+
+void FileSystem::ReplyBuffer(Request req, const char *buf, size_t size) {
+    fuse_reply_buf(req, buf, size);
+}
+
+void FileSystem::ReplyStatfs(Request req, const struct statvfs *stbuf) {
+    fuse_reply_statfs(req, stbuf);
+}
+
+void FileSystem::ReplyXattr(Request req, size_t size) {
+    fuse_reply_xattr(req, size);
+}
+
+void FileSystem::ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_create(req, &e, fi);
+}
+
+void FileSystem::AddDirEntry(Request req,
+                             DirBufferHead* buffer,
+                             DirEntry* dirEntry) {
+    struct stat stat;
+    std::memset(&stat, 0, sizeof(stat));
+    stat.st_ino = dirEntry->ino;
+
+    // add a directory entry to the buffer
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry(req,
+                      buffer->p + oldsize,  // char* buf
+                      buffer->size - oldsize,  // size_t bufisze
+                      name, &stat, buffer->size);
+}
+
+void FileSystem::AddDirEntryPlus(Request req,
+                                 DirBufferHead* buffer,
+                                 DirEntry* dirEntry) {
+    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr,
+                             ReplyType::ATTR, false);
+    struct fuse_entry_param e;
+    EntryOut entryOut(dirEntry->attr);
+    SetEntryTimeout(&entryOut);
+    Entry2Param(&entryOut, &e);
+
+    // add a directory entry to the buffer with the attributes
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry_plus(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry_plus(req,
+                           buffer->p + oldsize,  // char* buf
+                           buffer->size - oldsize,  // size_t bufisze
+                           name, &e, buffer->size);
+}
+
+// handler*
+std::shared_ptr<FileHandler> FileSystem::NewHandler() {
+    return handlerManager_->NewHandler();
+}
+
+std::shared_ptr<FileHandler> FileSystem::FindHandler(uint64_t fh) {
+    return handlerManager_->FindHandler(fh);
+}
+
+void FileSystem::ReleaseHandler(uint64_t fh) {
+    return handlerManager_->ReleaseHandler(fh);
+}
+
+FileSystemMember FileSystem::BorrowMember() {
+    return FileSystemMember(deferSync_, openFiles_, attrWatcher_);
+}
+
+// fuse request*
+CURVEFS_ERROR FileSystem::Lookup(Request req,
+                                 Ino parent,
+                                 const std::string& name,
+                                 EntryOut* entryOut) {
+    if (name.size() > option_.maxNameLength) {
+        return CURVEFS_ERROR::NAMETOOLONG;
+    }
+
+    bool yes = negative_->Get(parent, name);
+    if (yes) {
+        return CURVEFS_ERROR::NOTEXIST;
+    }
+
+    auto rc = rpc_->Lookup(parent, name, entryOut);
+    if (rc == CURVEFS_ERROR::OK) {
+        negative_->Delete(parent, name);
+    } else if (rc == CURVEFS_ERROR::NOTEXIST) {
+        negative_->Put(parent, name);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) {
+    InodeAttr attr;
+    auto rc = rpc_->GetAttr(ino, &attr);
+    if (rc == CURVEFS_ERROR::OK) {
+        *attrOut = AttrOut(attr);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) {
+    InodeAttr attr;
+    CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    // revalidate directory cache
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = dirCache_->Get(ino, &entries);
+    if (yes) {
+        if (entries->GetMtime() != AttrMtime(attr)) {
+            dirCache_->Drop(ino);
+        }
+    }
+
+    auto handler = NewHandler();
+    handler->mtime = AttrMtime(attr);
+    fi->fh = handler->fh;
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReadDir(Request req,
+                                  Ino ino,
+                                  FileInfo* fi,
+                                  std::shared_ptr<DirEntryList>* entries) {
+    bool yes = dirCache_->Get(ino, entries);
+    if (yes) {
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->ReadDir(ino, entries);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    (*entries)->SetMtime(FindHandler(fi->fh)->mtime);
+    dirCache_->Put(ino, *entries);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) {
+    ReleaseHandler(fi->fh);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) {
+    std::shared_ptr<InodeWrapper> inode;
+    bool yes = openFiles_->IsOpened(ino, &inode);
+    if (yes) {
+        openFiles_->Open(ino, inode);
+        // fi->keep_cache = 1;
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->Open(ino, &inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    TimeSpec mtime;
+    yes = attrWatcher_->GetMtime(ino, &mtime);
+    if (!yes) {
+        // It is rare which only arise when attribute evited for attr-watcher.
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ": attribute not found in wacther";
+        return CURVEFS_ERROR::STALE;
+    } else if (mtime != InodeMtime(inode)) {
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ", cache(" << mtime << ") vs remote("
+                     << InodeMtime(inode) << ")";
+        return CURVEFS_ERROR::STALE;
+    }
+
+    openFiles_->Open(ino, inode);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) {
+    openFiles_->Close(ino);
+    return CURVEFS_ERROR::OK;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/filesystem.h

@@ -0,0 +1,167 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+
+#include <gtest/gtest_prod.h>
+
+#include <memory>
+#include <string>
+
+#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+#include "curvefs/src/client/filesystem/rpc_client.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::FileSystemOption;
+
+struct FileSystemMember {
+    FileSystemMember(std::shared_ptr<DeferSync> deferSync,
+                     std::shared_ptr<OpenFiles> openFiles,
+                     std::shared_ptr<AttrWatcher> attrWatcher)
+        : deferSync(deferSync),
+          openFiles(openFiles),
+          attrWatcher(attrWatcher) {}
+
+    std::shared_ptr<DeferSync> deferSync;
+    std::shared_ptr<OpenFiles> openFiles;
+    std::shared_ptr<AttrWatcher> attrWatcher;
+};
+
+class FileSystem {
+ public:
+    FileSystem(struct FileSystemOption option, ExternalMember member);
+
+    void Run();
+
+    void Destory();
+
+    // fuse request
+    CURVEFS_ERROR Lookup(Request req,
+                         Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut);
+
+    CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR ReadDir(Request req,
+                          Ino ino,
+                          FileInfo* fi,
+                          std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Release(Request req, Ino ino);
+
+    // fuse reply: we control all replies to vfs layer in same entrance.
+    void ReplyError(Request req, CURVEFS_ERROR code);
+
+    void ReplyEntry(Request req, EntryOut* entryOut);
+
+    void ReplyAttr(Request req, AttrOut* attrOut);
+
+    void ReplyReadlink(Request req, const std::string& link);
+
+    void ReplyOpen(Request req, FileInfo *fi);
+
+    void ReplyOpen(Request req, FileOut* fileOut);
+
+    void ReplyData(Request req,
+                   struct fuse_bufvec *bufv,
+                   enum fuse_buf_copy_flags flags);
+
+    void ReplyWrite(Request req, FileOut* fileOut);
+
+    void ReplyBuffer(Request req, const char *buf, size_t size);
+
+    void ReplyStatfs(Request req, const struct statvfs *stbuf);
+
+    void ReplyXattr(Request req, size_t size);
+
+    void ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi);
+
+    void AddDirEntry(Request req,
+                     DirBufferHead* buffer,
+                     DirEntry* dirEntry);
+
+    void AddDirEntryPlus(Request req,
+                         DirBufferHead* buffer,
+                         DirEntry* dirEntry);
+
+    // utility: file handler
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t fh);
+
+    void ReleaseHandler(uint64_t fh);
+
+    // utility: others
+    FileSystemMember BorrowMember();
+
+ private:
+    FRIEND_TEST(FileSystemTest, Attr2Stat);
+    FRIEND_TEST(FileSystemTest, Entry2Param);
+    FRIEND_TEST(FileSystemTest, SetEntryTimeout);
+    FRIEND_TEST(FileSystemTest, SetAttrTimeout);
+
+    // utility: convert to system type.
+    void Attr2Stat(InodeAttr* attr, struct stat* stat);
+
+    void Entry2Param(EntryOut* entryOut, fuse_entry_param* e);
+
+    // utility: set entry/attribute timeout
+    void SetEntryTimeout(EntryOut* entryOut);
+
+    void SetAttrTimeout(AttrOut* attrOut);
+
+ private:
+    FileSystemOption option_;
+    ExternalMember member;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<LookupCache> negative_;
+    std::shared_ptr<DirCache> dirCache_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<AttrWatcher> attrWatcher_;
+    std::shared_ptr<HandlerManager> handlerManager_;
+    std::shared_ptr<RPCClient> rpc_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_

curvefs/src/client/filesystem/lookup_cache.cpp

@@ -0,0 +1,104 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include <ctime>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+#define RETURN_FALSE_IF_DISABLED() \
+    do {                           \
+        if (!enable_) {            \
+            return false;          \
+        }                          \
+    } while (0)
+
+
+LookupCache::LookupCache(LookupCacheOption option)
+    : enable_(option.negativeTimeoutSec > 0),
+      rwlock_(),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(option.lruSize);
+    if (enable_) {
+        LOG(INFO) << "Using lookup negative lru cache"
+                  << ", timeout = " << option.negativeTimeoutSec
+                  << ", capacity = " << option.lruSize;
+    }
+}
+
+std::string LookupCache::CacheKey(Ino parent, const std::string& name) {
+    return absl::StrFormat("%d:%s", parent, name);
+}
+
+bool LookupCache::Get(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    ReadLockGuard lk(rwlock_);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
+    if (!yes) {
+        VLOG(1) << absl::StrFormat("Lookup cache not found: key(%d,%s)",
+                                   parent, name);
+        return false;
+    } else if (entry.uses < option_.minUses) {
+        return false;
+    } else if (entry.expireTime < Now()) {
+        return false;
+    }
+    return true;
+}
+
+bool LookupCache::Put(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    WriteLockGuard lk(rwlock_);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
+    if (yes) {
+        entry.uses++;
+    } else {
+        entry.uses = 0;
+    }
+
+    entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0);
+    lru_->Put(key, entry);
+    return true;
+}
+
+bool LookupCache::Delete(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    WriteLockGuard lk(rwlock_);
+    auto key = CacheKey(parent, name);
+    lru_->Remove(key);
+    return true;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/lookup_cache.h

@@ -0,0 +1,77 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+
+#include <memory>
+#include <string>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::LookupCacheOption;
+
+// memory cache for lookup result, now we only support cache negative result,
+// and other positive entry will be cached in kernel.
+class LookupCache {
+ public:
+    struct CacheEntry {
+        uint32_t uses;
+        TimeSpec expireTime;
+    };
+
+    using LRUType = LRUCache<std::string, CacheEntry>;
+
+ public:
+    explicit LookupCache(LookupCacheOption option);
+
+    bool Get(Ino parent, const std::string& name);
+
+    bool Put(Ino parent, const std::string& name);
+
+    bool Delete(Ino parent, const std::string& name);
+
+ private:
+    std::string CacheKey(Ino parent, const std::string& name);
+
+ private:
+    bool enable_;
+    RWLock rwlock_;
+    LookupCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_

curvefs/src/client/filesystem/message_queue.h

@@ -0,0 +1,118 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+
+#include <glog/logging.h>
+
+#include <string>
+#include <atomic>
+#include <thread>
+
+#include "src/common/concurrent/task_queue.h"
+#include "curvefs/src/common/threading.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::TaskQueue;
+using ::curvefs::common::SetThreadName;
+
+template <typename MessageT>
+class MessageQueue {
+ public:
+    using MessageHandler = std::function<void(const MessageT& message)>;
+
+ public:
+    MessageQueue(const std::string& name, size_t bufferSize)
+        : name_(name),
+          running_(false),
+          thread_(),
+          handler_(),
+          queue_(bufferSize) {}
+
+    void Start() {
+        if (running_.exchange(true)) {
+            return;
+        }
+
+        thread_ = std::thread(&MessageQueue::Consumer, this);
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread start success";
+    }
+
+    void Stop() {
+        if (!running_.exchange(false)) {
+            return;
+        }
+
+        auto wakeup = []() {};
+        queue_.Push(wakeup);
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stoping...";
+
+        thread_.join();
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stopped";
+    }
+
+    void Publish(MessageT message) {
+        if (handler_ != nullptr) {
+            queue_.Push([this, message](){
+                this->handler_(message);
+            });
+        }
+    }
+
+    void Subscribe(MessageHandler handler) {
+        handler_ = handler;
+    }
+
+ private:
+    void Consumer() {
+        SetThreadName(name_.c_str());
+        while (running_.load(std::memory_order_relaxed)) {
+            queue_.Pop()();
+        }
+
+        while (queue_.Size() > 0) {
+            queue_.Pop()();
+        }
+    }
+
+ private:
+    std::string name_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    MessageHandler handler_;
+    TaskQueue queue_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_

curvefs/src/client/filesystem/meta.cpp

@@ -0,0 +1,129 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <sys/stat.h>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+HandlerManager::HandlerManager()
+    : mutex_(),
+      dirBuffer_(std::make_shared<DirBuffer>()),
+      handlers_() {}
+
+HandlerManager::~HandlerManager() {
+    dirBuffer_->DirBufferFreeAll();
+}
+
+std::shared_ptr<FileHandler> HandlerManager::NewHandler() {
+    UniqueLock lk(mutex_);
+    auto handler = std::make_shared<FileHandler>();
+    handler->fh = dirBuffer_->DirBufferNew();
+    handler->buffer = dirBuffer_->DirBufferGet(handler->fh);
+    handler->padding = false;
+    handlers_.emplace(handler->fh, handler);
+    return handler;
+}
+
+std::shared_ptr<FileHandler> HandlerManager::FindHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    auto iter = handlers_.find(fh);
+    if (iter == handlers_.end()) {
+        return nullptr;
+    }
+    return iter->second;
+}
+
+void HandlerManager::ReleaseHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    dirBuffer_->DirBufferRelease(fh);
+    handlers_.erase(fh);
+}
+
+std::string StrMode(uint16_t mode) {
+    static std::map<uint16_t, char> type2char = {
+        { S_IFSOCK, 's' },
+        { S_IFLNK, 'l' },
+        { S_IFREG, '-' },
+        { S_IFBLK, 'b' },
+        { S_IFDIR, 'd' },
+        { S_IFCHR, 'c' },
+        { S_IFIFO, 'f' },
+        { 0, '?' },
+    };
+
+    std::string s("?rwxrwxrwx");
+    s[0] = type2char[mode & (S_IFMT & 0xffff)];
+    if (mode & S_ISUID) {
+        s[3] = 's';
+    }
+    if (mode & S_ISGID) {
+        s[6] = 's';
+    }
+    if (mode & S_ISVTX) {
+        s[9] = 't';
+    }
+
+    for (auto i = 0; i < 9; i++) {
+        if ((mode & (1 << i)) == 0) {
+            if ((s[9-i] == 's') || (s[9-i] == 't')) {
+                s[9-i] &= 0xDF;
+            } else {
+                s[9-i] = '-';
+            }
+        }
+    }
+    return s;
+}
+
+namespace {
+
+std::string Attr2Str(const InodeAttr& attr) {
+    if (!attr.IsInitialized()) {
+        return "";
+    }
+
+    std::string smode;
+    return absl::StrFormat(" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])",
+        attr.inodeid(), StrMode(attr.mode()).c_str(), attr.mode(), attr.nlink(),
+        attr.uid(), attr.gid(),
+        attr.atime(), attr.mtime(), attr.ctime(),
+        attr.length());
+}
+
+}  // namespace
+
+std::string StrEntry(EntryOut entryOut) {
+    return Attr2Str(entryOut.attr);
+}
+
+std::string StrAttr(AttrOut attrOut) {
+    return Attr2Str(attrOut.attr);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/meta.h

@@ -0,0 +1,178 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <iostream>
+
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/client/fuse_common.h"
+#include "curvefs/src/client/dir_buffer.h"
+#include "curvefs/src/client/inode_wrapper.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::Mutex;
+using ::curve::common::RWLock;
+using ::curve::common::UniqueLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::metaserver::XAttr;
+using ::curvefs::metaserver::Dentry;
+using ::curvefs::metaserver::InodeAttr;
+using ::curvefs::metaserver::FsFileType;
+using ::curvefs::client::DirBufferHead;
+using ::curvefs::client::InodeWrapper;
+
+using Ino = fuse_ino_t;
+using Request = fuse_req_t;
+using FileInfo = struct fuse_file_info;
+
+struct EntryOut {
+    EntryOut() = default;
+
+    explicit EntryOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double entryTimeout;
+    double attrTimeout;
+};
+
+struct AttrOut {
+    AttrOut() = default;
+
+    explicit AttrOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double attrTimeout;
+};
+
+struct DirEntry {
+    DirEntry() = default;
+
+    DirEntry(Ino ino, const std::string& name, InodeAttr attr)
+        : ino(ino), name(name), attr(attr) {}
+
+    Ino ino;
+    std::string name;
+    InodeAttr attr;
+};
+
+struct FileOut {
+    FileOut() = default;
+
+    FileOut(FileInfo* fi, InodeAttr attr)
+        : fi(fi), attr(attr), nwritten(0) {}
+
+    FileOut(InodeAttr attr, size_t nwritten)
+        : fi(nullptr), attr(attr), nwritten(nwritten) {}
+
+    FileInfo* fi;
+    InodeAttr attr;
+    size_t nwritten;
+};
+
+struct TimeSpec {
+    TimeSpec() : seconds(0), nanoSeconds(0) {}
+
+    TimeSpec(uint64_t seconds, uint32_t nanoSeconds)
+        : seconds(seconds), nanoSeconds(nanoSeconds) {}
+
+    TimeSpec(const TimeSpec& time)
+        : seconds(time.seconds), nanoSeconds(time.nanoSeconds) {}
+
+    TimeSpec& operator=(const TimeSpec& time) = default;
+
+    TimeSpec operator+(const TimeSpec& time) const {
+        return TimeSpec(seconds + time.seconds, nanoSeconds + time.nanoSeconds);
+    }
+
+    uint64_t seconds;
+    uint32_t nanoSeconds;
+};
+
+struct FileHandler {
+    uint64_t fh;
+    DirBufferHead* buffer;
+    TimeSpec mtime;
+    bool padding;  // padding buffer
+};
+
+class HandlerManager {
+ public:
+    HandlerManager();
+
+    ~HandlerManager();
+
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t id);
+
+    void ReleaseHandler(uint64_t id);
+
+ private:
+    Mutex mutex_;
+    std::shared_ptr<DirBuffer> dirBuffer_;
+    std::map<uint64_t, std::shared_ptr<FileHandler>> handlers_;
+};
+
+inline bool operator==(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds == rhs.seconds) &&
+        (lhs.nanoSeconds == rhs.nanoSeconds);
+}
+
+inline bool operator!=(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return !(lhs == rhs);
+}
+
+inline bool operator<(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds < rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds < rhs.nanoSeconds);
+}
+
+inline bool operator>(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds > rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds > rhs.nanoSeconds);
+}
+
+inline std::ostream &operator<<(std::ostream &os, const TimeSpec& time) {
+    return os << time.seconds << "." << time.nanoSeconds;
+}
+
+std::string StrMode(uint16_t mode);
+
+std::string StrEntry(EntryOut entryOut);
+
+std::string StrAttr(AttrOut attrOut);
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_

curvefs/src/client/filesystem/openfile.cpp

@@ -0,0 +1,141 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+OpenFiles::OpenFiles(OpenFilesOption option,
+                     std::shared_ptr<DeferSync> deferSync)
+    : rwlock_(),
+      option_(option),
+      deferSync_(deferSync),
+      files_(std::make_shared<LRUType>(option.lruSize)) {
+    LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize;
+}
+
+/*
+ * Delete(...) does:
+ *   1) publish to message queue which will flush file to server with async
+ *   2) delete file from lru cache
+ */
+void OpenFiles::Delete(Ino ino,
+                       const std::shared_ptr<OpenFile>& file,
+                       bool flush) {
+    if (flush) {
+        deferSync_->Push(file->inode);
+    }
+    files_->Remove(ino);
+
+    VLOG(1) << "Delete open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+void OpenFiles::Evit(size_t size) {
+    Ino ino;
+    std::shared_ptr<OpenFile> file;
+    while (files_->Size() + size >= option_.lruSize) {
+        bool yes = files_->GetLast(&ino, &file);
+        if (!yes) {
+            break;
+        }
+        Delete(ino, file, true);
+    }
+}
+
+void OpenFiles::Open(Ino ino, std::shared_ptr<InodeWrapper> inode) {
+    WriteLockGuard lk(rwlock_);
+
+    Evit(1);
+
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (yes) {
+        file->refs++;
+        return;
+    }
+
+    file = std::make_shared<OpenFile>(inode);
+    file->refs++;
+    files_->Put(ino, file);
+
+    VLOG(1) << "Insert open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+bool OpenFiles::IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+    *inode = file->inode;
+    return true;
+}
+
+void OpenFiles::Close(Ino ino) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return;
+    }
+
+    if (file->refs > 0) {
+        file->refs--;
+    }
+
+    if (file->refs == 0) {
+        Delete(ino, file, false);  // file already flushed before close
+    }
+}
+
+/*
+ * CloseAll() does:
+ *   flush all file to server and delete from LRU cache
+ */
+void OpenFiles::CloseAll() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+}
+
+bool OpenFiles::GetFileAttr(Ino ino, InodeAttr* attr) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+
+    file->inode->GetInodeAttr(attr);
+    return true;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/openfile.h

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::InodeWrapper;
+using ::curvefs::client::common::OpenFilesOption;
+
+struct OpenFile {
+    explicit OpenFile(std::shared_ptr<InodeWrapper> inode)
+        : inode(inode), refs(0) {}
+
+    std::shared_ptr<InodeWrapper> inode;
+    uint64_t refs;
+};
+
+class OpenFiles {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<OpenFile>>;
+
+ public:
+    explicit OpenFiles(OpenFilesOption option,
+                       std::shared_ptr<DeferSync> deferSync);
+
+    void Open(Ino ino, std::shared_ptr<InodeWrapper> inode);
+
+    bool IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+    void Close(Ino ino);
+
+    void CloseAll();
+
+    bool GetFileAttr(Ino ino, InodeAttr* inode);
+
+ private:
+    void Delete(Ino ino, const std::shared_ptr<OpenFile>& file, bool flush);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    OpenFilesOption option_;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<LRUType> files_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_

curvefs/src/client/filesystem/package.h

@@ -0,0 +1,51 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+
+#include <memory>
+
+#include "curvefs/src/client/dentry_cache_manager.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/xattr_manager.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+struct ExternalMember {  // external member depended by FileSystem
+    ExternalMember() = delete;
+    ExternalMember(std::shared_ptr<DentryCacheManager> dentryManager,
+                   std::shared_ptr<InodeCacheManager> inodeManager)
+        : dentryManager(dentryManager),
+          inodeManager(inodeManager) {}
+
+    std::shared_ptr<DentryCacheManager> dentryManager;
+    std::shared_ptr<InodeCacheManager> inodeManager;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_

curvefs/src/client/filesystem/rpc_client.cpp

@@ -0,0 +1,128 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <set>
+#include <list>
+#include <string>
+#include <memory>
+#include <utility>
+
+#include "curvefs/src/client/filesystem/rpc_client.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+RPCClient::RPCClient(RPCOption option,
+                     ExternalMember member)
+    : option_(option),
+      inodeManager_(member.inodeManager),
+      dentryManager_(member.dentryManager) {}
+
+CURVEFS_ERROR RPCClient::GetAttr(Ino ino, InodeAttr* attr) {
+    CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(ino, attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(getattr::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::Lookup(Ino parent,
+                                const std::string& name,
+                                EntryOut* entryOut) {
+    Dentry dentry;
+    CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry);
+    if (rc != CURVEFS_ERROR::OK) {
+        if (rc != CURVEFS_ERROR::NOTEXIST) {
+            LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc
+                       << ", parent = " << parent << ", name = " << name;
+        }
+        return rc;
+    }
+
+    Ino ino = dentry.inodeid();
+    rc = inodeManager_->GetInodeAttr(ino, &entryOut->attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(lookup::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
+                                 std::shared_ptr<DirEntryList>* entries) {
+    uint32_t limit = option_.listDentryLimit;
+
+    std::list<Dentry> dentries;
+    CURVEFS_ERROR rc = dentryManager_->ListDentry(ino, &dentries, limit);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc
+                   << ", ino = " << ino;
+        return rc;
+    }
+
+    std::set<uint64_t> inos;
+    std::map<uint64_t, InodeAttr> attrs;
+    std::for_each(dentries.begin(), dentries.end(), [&](Dentry& dentry){
+        inos.emplace(dentry.inodeid());
+    });
+    rc = inodeManager_->BatchGetInodeAttrAsync(ino, &inos, &attrs);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::BatchGetInodeAttrAsync) failed"
+                   << ", retCode = " << rc << ", ino = " << ino;
+        return rc;
+    }
+
+    DirEntry dirEntry;
+    for (const auto& dentry : dentries) {
+        Ino ino = dentry.inodeid();
+        auto iter = attrs.find(ino);
+        if (iter == attrs.end()) {
+            LOG(WARNING) << "rpc(readdir::BatchGetInodeAttrAsync) "
+                         << "missing attribute, ino = " << ino;
+            continue;
+        }
+
+        // NOTE: we can't use std::move() for attribute for hard link
+        // which will sharing inode attribute.
+        dirEntry.ino = ino;
+        dirEntry.name = std::move(dentry.name());
+        dirEntry.attr = iter->second;
+        (*entries)->Add(dirEntry);
+    }
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR RPCClient::Open(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    CURVEFS_ERROR rc = inodeManager_->GetInode(ino, *inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(open/GetInode) failed" << ", retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/rpc_client.h

@@ -0,0 +1,65 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+
+#include <memory>
+#include <string>
+
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::RPCOption;
+
+class RPCClient {
+ public:
+    RPCClient(RPCOption option,
+              ExternalMember member);
+
+    CURVEFS_ERROR GetAttr(Ino ino, InodeAttr* attr);
+
+    CURVEFS_ERROR Lookup(Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR Open(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+ private:
+    RPCOption option_;
+    std::shared_ptr<InodeCacheManager> inodeManager_;
+    std::shared_ptr<DentryCacheManager> dentryManager_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_

curvefs/src/client/filesystem/utils.cpp

@@ -0,0 +1,69 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+bool IsDir(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_DIRECTORY;
+}
+
+bool IsS3File(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_S3;
+}
+
+bool IsVolmeFile(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_FILE;
+}
+
+bool IsSymLink(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_SYM_LINK;
+}
+
+TimeSpec AttrMtime(const InodeAttr& attr) {
+    return TimeSpec(attr.mtime(), attr.mtime_ns());
+}
+
+TimeSpec AttrCtime(const InodeAttr& attr) {
+    return TimeSpec(attr.ctime(), attr.ctime_ns());
+}
+
+TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode) {
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    return AttrMtime(attr);
+}
+
+TimeSpec Now() {
+    struct timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+    return TimeSpec(now.tv_sec, now.tv_nsec);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/utils.h

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+
+#include <memory>
+
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+// directory
+bool IsDir(const InodeAttr& attr);
+
+// file which data is stored in s3
+bool IsS3File(const InodeAttr& attr);
+
+// file which data is stored in volume
+bool IsVolmeFile(const InodeAttr& attr);
+
+// symbol link
+bool IsSymLink(const InodeAttr& attr);
+
+struct TimeSpec AttrMtime(const InodeAttr& attr);
+
+struct TimeSpec AttrCtime(const InodeAttr& attr);
+
+struct TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode);
+
+struct TimeSpec Now();
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_

curvefs/src/client/fuse_client.h

@@ -25,6 +25,7 @@
 
 #include <unistd.h>
 #include <sys/stat.h>
+#include <bthread/unstable.h>
 
 #include <map>
 #include <memory>
@@ -32,6 +33,7 @@
 #include <list>
 #include <utility>
 #include <vector>
+#include <atomic>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/mds.pb.h"
@@ -52,38 +54,46 @@
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/lease/lease_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
+#include "curvefs/src/client/warmup/warmup_manager.h"
+#include "src/common/throttle.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/filesystem.h"
 
 #define DirectIOAlignment 512
-#define WARMUP_CHECKINTERVAL_US 1000*1000
-#define WARMUP_THREADS 10
 
 using ::curve::common::Atomic;
 using ::curve::common::InterruptibleSleeper;
 using ::curve::common::Thread;
 using ::curvefs::common::FSType;
 using ::curvefs::metaserver::DentryFlag;
+using ::curvefs::metaserver::ManageInodeType;
 using ::curvefs::client::metric::FSMetric;
 
 namespace curvefs {
 namespace client {
 
+
+namespace warmup {
+class WarmupManager;
+}
+
 using common::FuseClientOption;
 using rpcclient::MDSBaseClient;
 using rpcclient::MdsClient;
 using rpcclient::MdsClientImpl;
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::FileSystem;
+using ::curvefs::client::filesystem::EntryOut;
+using ::curvefs::client::filesystem::AttrOut;
+using ::curvefs::client::filesystem::FileOut;
 
 using curvefs::common::is_aligned;
 
 const uint32_t kMaxHostNameLength = 255u;
 
 using mds::Mountpoint;
-typedef struct WarmUpFileContext {
-    fuse_ino_t inode;
-    uint64_t fileLen;
-    bool exist;
-} WarmUpFileContext_t;
+
 class FuseClient {
  public:
     FuseClient()
@@ -91,31 +101,30 @@ class FuseClient {
         metaClient_(std::make_shared<MetaServerClientImpl>()),
         inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
         dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
-        dirBuf_(std::make_shared<DirBuffer>()),
         fsInfo_(nullptr),
-        mdsBase_(nullptr),
-        isStop_(true),
         init_(false),
-        mounted_(false),
-        enableSumInDir_(false) {}
+        enableSumInDir_(false),
+        warmupManager_(nullptr),
+        mdsBase_(nullptr),
+        isStop_(true) {}
 
     virtual ~FuseClient() {}
 
     FuseClient(const std::shared_ptr<MdsClient> &mdsClient,
         const std::shared_ptr<MetaServerClient> &metaClient,
         const std::shared_ptr<InodeCacheManager> &inodeManager,
-        const std::shared_ptr<DentryCacheManager> &dentryManager)
+        const std::shared_ptr<DentryCacheManager> &dentryManager,
+        const std::shared_ptr<warmup::WarmupManager> &warmupManager)
           : mdsClient_(mdsClient),
             metaClient_(metaClient),
             inodeManager_(inodeManager),
             dentryManager_(dentryManager),
-            dirBuf_(std::make_shared<DirBuffer>()),
             fsInfo_(nullptr),
-            mdsBase_(nullptr),
-            isStop_(true),
             init_(false),
-            mounted_(false),
-            enableSumInDir_(false) {}
+            enableSumInDir_(false),
+            warmupManager_(warmupManager),
+            mdsBase_(nullptr),
+            isStop_(true) {}
 
     virtual CURVEFS_ERROR Init(const FuseClientOption &option);
 
@@ -133,31 +142,42 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                       const char* buf, size_t size, off_t off,
                                       struct fuse_file_info* fi,
-                                      size_t* wSize) = 0;
+                                      FileOut* fileOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                      size_t size, off_t off,
                                      struct fuse_file_info* fi, char* buffer,
                                      size_t* rSize) = 0;
 
-    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
-                                     struct fuse_file_info* fi);
+    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     struct fuse_file_info* fi,
+                                     FileOut* fileOut);
 
-    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, mode_t mode,
+    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       mode_t mode,
                                        struct fuse_file_info* fi,
-                                       fuse_entry_param* e) = 0;
+                                       EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode, dev_t rdev,
-                                      fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      dev_t rdev,
+                                      EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode,
-                                      fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      EntryOut* entryOut);
 
     virtual CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                        const char* name) = 0;
@@ -165,33 +185,41 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
                                       const char* name);
 
-    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req,
+                                           fuse_ino_t ino,
                                            struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
-                                            size_t size, off_t off,
-                                            struct fuse_file_info* fi,
-                                            char** buffer, size_t* rSize,
-                                            bool cacheDir);
+    virtual CURVEFS_ERROR FuseOpReadDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        size_t size,
+                                        off_t off,
+                                        struct fuse_file_info* fi,
+                                        char** bufferOut,
+                                        size_t* rSize,
+                                        bool plus);
 
-    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_ino_t newparent,
-                                       const char* newname);
+    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       unsigned int flags);
 
     virtual CURVEFS_ERROR FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info* fi,
-                                        struct stat* attr);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct stat* attr, int to_set,
                                         struct fuse_file_info* fi,
-                                        struct stat* attrOut);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                          size_t size);
 
     virtual CURVEFS_ERROR FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
@@ -201,13 +229,17 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 
-    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req, const char* link,
-                                        fuse_ino_t parent, const char* name,
-                                        fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req,
+                                        const char* link,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char* newname,
-                                     fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     fuse_ino_t newparent,
+                                     const char* newname,
+                                     EntryOut* entryOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string* linkStr);
@@ -220,11 +252,16 @@ class FuseClient {
                                       struct fuse_file_info* fi) = 0;
     virtual CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                       struct fuse_file_info *fi) {
+        (void)req;
+        (void)ino;
+        (void)fi;
         return CURVEFS_ERROR::OK;
     }
 
     virtual CURVEFS_ERROR FuseOpStatFs(fuse_req_t req, fuse_ino_t ino,
                                        struct statvfs* stbuf) {
+        (void)req;
+        (void)ino;
         // TODO(chengyi01,wuhanqing): implement in s3 and volume client
         stbuf->f_frsize = stbuf->f_bsize = fsInfo_->blocksize();
         stbuf->f_blocks = 10UL << 30;
@@ -236,31 +273,30 @@ class FuseClient {
         stbuf->f_fsid = fsInfo_->fsid();
 
         stbuf->f_flag = 0;
-        stbuf->f_namemax = option_.maxNameLength;
+        stbuf->f_namemax = option_.fileSystemOption.maxNameLength;
         return CURVEFS_ERROR::OK;
     }
 
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
+
     void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
         fsInfo_ = fsInfo;
         init_ = true;
     }
 
     void SetMounted(bool mounted) {
-        mounted_ = mounted;
-    }
-
-    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable> &
-      GetTaskFetchPool() {
-        return taskFetchMetaPool_;
+        if (warmupManager_ != nullptr) {
+            warmupManager_->SetMounted(mounted);
+        }
     }
 
     std::shared_ptr<FsInfo> GetFsInfo() {
         return fsInfo_;
     }
 
-    virtual void FlushInode();
-
-    virtual void FlushInodeAll();
+    std::shared_ptr<FileSystem> GetFileSystem() {
+        return fs_;
+    }
 
     virtual void FlushAll();
 
@@ -268,47 +304,77 @@ class FuseClient {
     void SetEnableSumInDir(bool enable) {
         enableSumInDir_ = enable;
     }
-    std::list<fuse_ino_t>& GetReadAheadFiles() {
-        std::unique_lock<std::mutex> lck(fetchMtx_);
-        return readAheadFiles_;
+
+    bool PutWarmFilelistTask(fuse_ino_t key, common::WarmupStorageType type) {
+        if (fsInfo_->fstype() == FSType::TYPE_S3) {
+            return warmupManager_->AddWarmupFilelist(key, type);
+        }  // only support s3
+        return true;
     }
 
-    void GetWarmUpFile(WarmUpFileContext_t* warmUpFile) {
-        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
-        *warmUpFile = std::move(warmUpFile_);
-        warmUpFile_.exist = false;
-        return;
-    }
-    void SetWarmUpFile(WarmUpFileContext_t warmUpFile) {
-        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
-        warmUpFile_ = warmUpFile;
-        warmUpFile_.exist = true;
-    }
-    bool hasWarmUpTask() {
-        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
-        return warmUpFile_.exist;
+    bool PutWarmFileTask(fuse_ino_t key, const std::string &path,
+                         common::WarmupStorageType type) {
+        if (fsInfo_->fstype() == FSType::TYPE_S3) {
+            return warmupManager_->AddWarmupFile(key, path, type);
+        }  // only support s3
+        return true;
     }
 
-    void FetchDentryEnqueue(std::string file);
-
-    void PutWarmTask(const std::string& warmUpTask) {
-        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
-        warmUpTasks_.push_back(warmUpTask);
-        WarmUpRun();
+    bool GetWarmupProgress(fuse_ino_t key, warmup::WarmupProgress *progress) {
+        if (fsInfo_->fstype() == FSType::TYPE_S3) {
+            return warmupManager_->QueryWarmupProgress(key, progress);
+        }
+        return false;
     }
 
+    CURVEFS_ERROR SetMountStatus(const struct MountOption *mountOption);
+
+    void Add(bool isRead, size_t size) { throttle_.Add(isRead, size); }
+
+    void InitQosParam();
+
  protected:
-    CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
-                           mode_t mode, FsFileType type, dev_t rdev,
-                           fuse_entry_param* e);
+    CURVEFS_ERROR MakeNode(fuse_req_t req,
+                           fuse_ino_t parent,
+                           const char* name,
+                           mode_t mode,
+                           FsFileType type,
+                           dev_t rdev,
+                           bool internal,
+                           std::shared_ptr<InodeWrapper>& InodeWrapper);  // NOLINT
 
     CURVEFS_ERROR RemoveNode(fuse_req_t req, fuse_ino_t parent,
                              const char* name, FsFileType type);
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                             fuse_ino_t newparent, const char* newname,
+    CURVEFS_ERROR CreateManageNode(fuse_req_t req,
+                                   uint64_t parent,
+                                   const char* name,
+                                   mode_t mode,
+                                   ManageInodeType manageType,
+                                   EntryOut* entryOut);
+
+    CURVEFS_ERROR GetOrCreateRecycleDir(fuse_req_t req, Dentry *out);
+
+    CURVEFS_ERROR DeleteNode(fuse_ino_t ino, fuse_ino_t parent,
+                             const char* name, FsFileType type);
+
+    CURVEFS_ERROR MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
+                                fuse_ino_t parent, const char* name,
+                                FsFileType type);
+
+    bool ShouldMoveToRecycle(fuse_ino_t parent);
+
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char* newname,
                              FsFileType type,
-                             fuse_entry_param* e);
+                             EntryOut* entryOut);
+
+    CURVEFS_ERROR HandleOpenFlags(fuse_req_t req,
+                                  fuse_ino_t ino,
+                                  struct fuse_file_info* fi,
+                                  FileOut* fileOut);
 
     int SetHostPortInMountPoint(Mountpoint* out) {
         char hostname[kMaxHostNameLength];
@@ -323,54 +389,24 @@ class FuseClient {
         return 0;
     }
 
-    void splitStr(const std::string& srcStr, const std::string& delimiter,
-      std::vector<std::string>* splitPath) {
-        char* pToken = nullptr;
-        char* pSave = nullptr;
-        pToken = strtok_r(const_cast<char*>(srcStr.c_str()),
-          const_cast<char*>(delimiter.c_str()), &pSave);
-        if (nullptr == pToken) {
-            VLOG(6) << "del lookpath end";
-            return;
-        }
-        splitPath->push_back(pToken);
-        while (true) {
-            pToken = strtok_r(NULL, const_cast<char*>(
-              delimiter.c_str()), &pSave);
-            if (nullptr == pToken) {
-                VLOG(6) << "del lookpath end";
-                break;
-            }
-            VLOG(9) << "del pToken is:" << pToken
-                    << "pSave:" << pSave;
-            splitPath->push_back(pToken);
-        }
-    }
 
  private:
-    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
-
     virtual void FlushData() = 0;
 
     CURVEFS_ERROR UpdateParentMCTimeAndNlink(
         fuse_ino_t parent, FsFileType type,  NlinkChange nlink);
 
-    void WarmUpTask();
+    std::string GenerateNewRecycleName(fuse_ino_t ino,
+            fuse_ino_t parent, const char* name) {
+        std::string newName(name);
+        newName = std::to_string(parent) + "_" + std::to_string(ino)
+                    + "_" + newName;
+        if (newName.length() > option_.fileSystemOption.maxNameLength) {
+            newName = newName.substr(0, option_.fileSystemOption.maxNameLength);
+        }
 
-    void WarmUpRun() {
-        std::lock_guard<std::mutex> lk(mtx_);
-        runned_ = true;
-        cond_.notify_one();
+        return newName;
     }
-    void WaitWarmUp() {
-        std::unique_lock<std::mutex> lk(mtx_);
-        cond_.wait(lk, [this]() { return runned_; });
-        runned_ = false;
-    }
-    std::mutex mtx_;
-    std::condition_variable cond_;
-    bool runned_ = false;
-
 
  protected:
     // mds client
@@ -390,9 +426,6 @@ class FuseClient {
 
     std::shared_ptr<LeaseExecutor> leaseExecutor_;
 
-    // dir buffer
-    std::shared_ptr<DirBuffer> dirBuf_;
-
     // filesystem info
     std::shared_ptr<FsInfo> fsInfo_;
 
@@ -401,15 +434,18 @@ class FuseClient {
     // init flags
     bool init_;
 
-    std::atomic<bool>  mounted_;
-
     // enable record summary info in dir inode xattr
-    bool enableSumInDir_;
+    std::atomic<bool> enableSumInDir_;
 
     std::shared_ptr<FSMetric> fsMetric_;
 
     Mountpoint mountpoint_;
 
+    // warmup manager
+    std::shared_ptr<warmup::WarmupManager> warmupManager_;
+
+    std::shared_ptr<FileSystem> fs_;
+
  private:
     MDSBaseClient* mdsBase_;
 
@@ -417,39 +453,9 @@ class FuseClient {
 
     curve::common::Mutex renameMutex_;
 
-    Thread bgCmdTaskThread_;
-    std::atomic<bool> bgCmdStop_;
-    std::mutex cmdMtx_;
+    Throttle throttle_;
 
-    void FetchChildDentryEnqueue(fuse_ino_t  ino);
-    void FetchChildDentry(fuse_ino_t ino);
-    void FetchDentry(fuse_ino_t ino, std::string file);
-    void LookPath(std::string file);
-    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
-        taskFetchMetaPool_;
-
-    // need warmup files
-    std::list<fuse_ino_t> readAheadFiles_;
-    std::mutex fetchMtx_;
-
-    //  one warmup file provided by the user
-    WarmUpFileContext_t warmUpFile_;
-    std::mutex warmUpFileMtx_;
-
-    std::list<std::string> warmUpTasks_;  // todo: need size control ?
-    std::mutex warmUpTaskMtx_;
-
-    void GetwarmTask(std::string *warmUpTask) {
-        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
-        if (warmUpTasks_.empty())
-            return;
-        *warmUpTask = std::move(warmUpTasks_.front());
-        warmUpTasks_.pop_front();
-    }
-    bool hasWarmTask() {
-        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
-        return !warmUpTasks_.empty();
-    }
+    bthread_timer_t throttleTimer_;
 };
 
 }  // namespace client

curvefs/src/client/fuse_common.h

@@ -34,9 +34,9 @@ extern "C" {
 #endif
 
 struct MountOption {
-    char* mountPoint;
-    char* fsName;
-    char* fsType;
+    const char* mountPoint;
+    const char* fsName;
+    const char* fsType;
     char* conf;
     char* mdsAddr;
 };

curvefs/src/client/fuse_s3_client.cpp

@@ -48,7 +48,6 @@ using curvefs::mds::topology::MemcacheServerInfo;
 
 CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
     FuseClientOption opt(option);
-    initbgFetchThread_ = false;
 
     CURVEFS_ERROR ret = FuseClient::Init(opt);
     if (ret != CURVEFS_ERROR::OK) {
@@ -68,10 +67,20 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
 
     auto s3Client = std::make_shared<S3ClientImpl>();
     s3Client->Init(opt.s3Opt.s3AdaptrOpt);
+
+    const uint64_t writeCacheMaxByte =
+        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
+    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
+        LOG(ERROR) << "writeCacheMaxByte is too small"
+                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
+                      ", writeCacheMaxByte = " << writeCacheMaxByte;
+        return CURVEFS_ERROR::CACHETOOSMALL;
+    }
+
     auto fsCacheManager = std::make_shared<FsCacheManager>(
         dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
-        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte);
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte, writeCacheMaxByte,
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheThreads, kvClientManager_);
     if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
         DiskCacheType::Disable) {
         auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
@@ -85,17 +94,12 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
             diskCacheManager, s3DiskCacheClient);
         ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
                                inodeManager_, mdsClient_, fsCacheManager,
-                               diskCacheManagerImpl, true);
+                               diskCacheManagerImpl, kvClientManager_, true);
     } else {
         ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
                                inodeManager_, mdsClient_, fsCacheManager,
-                               nullptr, true);
+                               nullptr, kvClientManager_, true);
     }
-
-    bgFetchStop_.store(false, std::memory_order_release);
-    bgFetchThread_ = Thread(&FuseS3Client::BackGroundFetch, this);
-    initbgFetchThread_ = true;
-    GetTaskFetchPool();
     return ret;
 }
 
@@ -118,264 +122,21 @@ bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
         return false;
     }
 
-    g_kvClientManager = new KVClientManager();
-    if (!g_kvClientManager->Init(opt, memcacheClient)) {
+    kvClientManager_ = std::make_shared<KVClientManager>();
+    if (!kvClientManager_->Init(opt, memcacheClient)) {
         LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache
                    << ", but init kvClientManager fail";
         return false;
     }
 
+    if (warmupManager_ != nullptr) {
+        warmupManager_->SetKVClientManager(kvClientManager_);
+    }
+
     return true;
 }
 
-void FuseS3Client::GetWarmUpFileList(const WarmUpFileContext_t&warmUpFile,
-  std::vector<std::string>& warmUpFilelist) {
-        struct fuse_file_info fi{};
-        fi.flags &= ~O_DIRECT;
-        size_t rSize = 0;
-        std::unique_ptr<char[]> data(new char[warmUpFile.fileLen+1]);
-        std::memset(data.get(), 0, warmUpFile.fileLen);
-        data[warmUpFile.fileLen] = '\n';
-        FuseOpRead(nullptr, warmUpFile.inode,
-            warmUpFile.fileLen, 0, &fi, data.get(), &rSize);
-        std::string file = data.get();
-        VLOG(9) << "file is: " << file;
-        // remove enter, newline, blank
-        std::string blanks("\r\n ");
-        file.erase(0, file.find_first_not_of(blanks));
-        file.erase(file.find_last_not_of(blanks) + 1);
-        VLOG(9) << "after del file is: " << file;
-        splitStr(file, "\n", &warmUpFilelist);
-}
-
-void FuseS3Client::BackGroundFetch() {
-    while (!bgFetchStop_.load(std::memory_order_acquire)) {
-        LOG_EVERY_N(WARNING, 100)
-            << "fetch thread start.";
-        if (hasWarmUpTask()) {  // new warmup task
-            WarmUpFileContext_t warmUpFile;
-            GetWarmUpFile(&warmUpFile);
-            VLOG(9) << " len is: " << warmUpFile.fileLen
-                << "ino is: " << warmUpFile.inode;
-
-            std::vector<std::string> warmUpFilelist;
-            GetWarmUpFileList(warmUpFile, warmUpFilelist);
-            for (auto filePath : warmUpFilelist) {
-                FetchDentryEnqueue(filePath);
-            }
-        }
-        {   // file need warmup
-            std::list<fuse_ino_t> readAheadFiles;
-            readAheadFiles.swap(GetReadAheadFiles());
-            for (auto iter : readAheadFiles) {
-                VLOG(9) << "BackGroundFetch: " << iter;
-                fetchDataEnqueue(iter);
-            }
-        }
-        LOG_EVERY_N(WARNING, 100)
-            << "fetch thread end.";
-        usleep(WARMUP_CHECKINTERVAL_US);
-    }
-    return;
-}
-
-void FuseS3Client::fetchDataEnqueue(fuse_ino_t ino) {
-    VLOG(9) << "fetchDataEnqueue start: " << ino;
-    auto task = [this, ino]() {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                    << ", inodeid = " << ino;
-            return;
-        }
-        google::protobuf::Map<uint64_t, S3ChunkInfoList> *s3ChunkInfoMap
-          = nullptr;
-        {
-            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-            s3ChunkInfoMap = inodeWrapper->GetChunkInfoMap();
-        }
-        if (nullptr == s3ChunkInfoMap ||
-          s3ChunkInfoMap->empty()) {
-            return;
-        }
-        travelChunks(ino, s3ChunkInfoMap);
-    };
-    GetTaskFetchPool().Enqueue(task);
-}
-
-// travel and download all objs belong to the chunk
-void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
-  std::list<std::pair<std::string, uint64_t>>* prefetchObjs) {
-    uint64_t blockSize = s3Adaptor_->GetBlockSize();
-    uint64_t chunkSize = s3Adaptor_->GetChunkSize();
-    uint64_t offset, len, chunkid, compaction;
-    for (size_t i = 0; i < chunkInfo.s3chunks_size(); i++) {
-        auto chunkinfo = chunkInfo.mutable_s3chunks(i);
-        auto fsId = fsInfo_->fsid();
-        chunkid = chunkinfo->chunkid();
-        compaction = chunkinfo->compaction();
-        offset = chunkinfo->offset();
-        len = chunkinfo->len();
-        // the offset in the chunk
-        uint64_t chunkPos = offset % chunkSize;
-        // the offset in the block
-        uint64_t blockPos = chunkPos % blockSize;
-        // the first blockIndex
-        uint64_t blockIndexBegin = chunkPos / blockSize;
-
-        if (len < blockSize) {  // just one block
-            auto objectName = curvefs::common::s3util::GenObjName(
-                chunkid, blockIndexBegin, compaction, fsId, ino);
-            prefetchObjs->push_back(std::make_pair(objectName, len));
-        } else {
-            // the offset in the block
-            uint64_t blockPos = chunkPos % blockSize;
-
-            // firstly, let's get the size in the first block
-            // then, subtract the length in the first block
-            // to obtain the remaining length
-            // lastly, We need to judge the last block is full or not
-            uint64_t firstBlockSize = (blockPos != 0) ?
-              blockSize - blockPos : blockSize;
-            uint64_t leftSize = len - firstBlockSize;
-            uint32_t blockCounts = (leftSize % blockSize == 0) ?
-              (leftSize / blockSize + 1) : (leftSize / blockSize + 1 + 1);
-            // so we can get the last blockIndex
-            // because the bolck Index is cumulative
-            uint64_t blockIndexEnd = blockIndexBegin + blockCounts - 1;
-
-            // the size of the last block
-            uint64_t lastBlockSize = leftSize % blockSize;
-            // whether the first block or the last block is full or not
-            bool firstBlockFull = (blockPos == 0) ? true : false;
-            bool lastBlockFull = (lastBlockSize == 0) ? true : false;
-            // the start and end block Index that need travel
-            uint64_t travelStartIndex, travelEndIndex;
-            // if the block is full, the size is needed download
-            // of the obj is blockSize. Otherwise, the value is special.
-            if (!firstBlockFull) {
-                travelStartIndex = blockIndexBegin + 1;
-                auto objectName = curvefs::common::s3util::GenObjName(
-                  chunkid, blockIndexBegin, compaction, fsId, ino);
-                prefetchObjs->push_back(std::make_pair(
-                  objectName, firstBlockSize));
-            } else {
-                travelStartIndex = blockIndexBegin;
-            }
-            if (!lastBlockFull) {
-                // block index is greater than or equal to 0
-                travelEndIndex = (blockIndexEnd == blockIndexBegin) ?
-                  blockIndexEnd : blockIndexEnd - 1;
-                auto objectName = curvefs::common::s3util::GenObjName(
-                  chunkid, blockIndexEnd, compaction, fsId, ino);
-                // there is no need to care about the order
-                // in which objects are downloaded
-                prefetchObjs->push_back(
-                  std::make_pair(objectName, lastBlockSize));
-            } else {
-                travelEndIndex = blockIndexEnd;
-            }
-            VLOG(9) << "travel obj, ino: " << ino
-                << ", chunkid: " << chunkid
-                << ", blockCounts: " << blockCounts
-                << ", compaction: " << compaction
-                << ", blockSize: " << blockSize
-                << ", chunkSize: " << chunkSize
-                << ", offset: "  << offset
-                << ", blockIndexBegin: " << blockIndexBegin
-                << ", blockIndexEnd: " << blockIndexEnd
-                << ", len: " << len
-                << ", firstBlockSize: " << firstBlockSize
-                << ", lastBlockSize: " << lastBlockSize
-                << ", blockPos: " << blockPos
-                << ", chunkPos: " << chunkPos;
-            for (auto blockIndex = travelStartIndex;
-              blockIndex <= travelEndIndex ; blockIndex++) {
-                auto objectName = curvefs::common::s3util::GenObjName(
-                    chunkid, blockIndex, compaction, fsId, ino);
-                prefetchObjs->push_back(std::make_pair(objectName, blockSize));
-            }
-        }
-    }
-}
-
-// TODO(hzwuhongsong): These logics are very similar to other place,
-// try to merge it
-void FuseS3Client::WarmUpAllObjs(
-  const std::list<std::pair<std::string, uint64_t>> &prefetchObjs) {
-    std::atomic<uint64_t> pendingReq(0);
-    curve::common::CountDownEvent cond(1);
-    // callback function
-    GetObjectAsyncCallBack cb =
-        [&](const S3Adapter *adapter,
-            const std::shared_ptr<GetObjectAsyncContext> &context) {
-            if (context->retCode == 0) {
-                VLOG(9) << "Get Object success: " << context->key;
-                int ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(
-                  context->key, context->buf, context->len);
-                if (ret < 0) {
-                    LOG_EVERY_SECOND(INFO) <<
-                    "write read directly failed, key: " << context->key;
-                }
-                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
-                    VLOG(6) << "pendingReq is over";
-                    cond.Signal();
-                }
-                delete []context->buf;
-                return;
-            }
-            // todo: retry
-            LOG(WARNING) << "Get Object failed, key: " << context->key
-                         << ", offset: " << context->offset;
-            s3Adaptor_->GetS3Client()->DownloadAsync(context);
-    };
-
-    pendingReq.fetch_add(prefetchObjs.size(), std::memory_order_seq_cst);
-    if (pendingReq.load(std::memory_order_seq_cst)) {
-        VLOG(9) << "wait for pendingReq";
-        for (auto iter : prefetchObjs) {
-            VLOG(9) << "download start: " << iter.first;
-            std::string name = iter.first;
-            uint64_t readLen = iter.second;
-            if (s3Adaptor_->GetDiskCacheManager()->IsCached(name)) {
-                pendingReq.fetch_sub(1);
-                continue;
-            }
-            char *cacheS3 = new char[readLen];
-            memset(cacheS3, 0, readLen);
-            auto context = std::make_shared<GetObjectAsyncContext>();
-            context->key = name;
-            context->buf = cacheS3;
-            context->offset = 0;
-            context->len = readLen;
-            context->cb = cb;
-            s3Adaptor_->GetS3Client()->DownloadAsync(context);
-        }
-        if (pendingReq.load())
-            cond.Wait();
-    }
-}
-
-void FuseS3Client::travelChunks(fuse_ino_t ino, google::protobuf::Map<uint64_t,
-    S3ChunkInfoList> *s3ChunkInfoMap) {
-    VLOG(9) << "travel chunk start: " << ino
-            << ", size: " << s3ChunkInfoMap->size();
-    std::list<std::pair<std::string, uint64_t>> prefetchObjs;
-    for (auto &iter : *s3ChunkInfoMap) {
-        VLOG(9) << "travel chunk: " << iter.first;
-        travelChunk(ino, iter.second, &prefetchObjs);
-    }
-    WarmUpAllObjs(prefetchObjs);
-    VLOG(9) << "travel chunks end";
-    return;
-}
-
 void FuseS3Client::UnInit() {
-    bgFetchStop_.store(true, std::memory_order_release);
-    if (initbgFetchThread_) {
-        bgFetchThread_.join();
-    }
     FuseClient::UnInit();
     s3Adaptor_->Stop();
     curve::common::S3Adapter::Shutdown();
@@ -394,7 +155,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
 CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                         const char *buf, size_t size, off_t off,
                                         struct fuse_file_info *fi,
-                                        size_t *wSize) {
+                                        FileOut* fileOut) {
+    size_t *wSize = &fileOut->nwritten;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -456,6 +218,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
             }
         }
     }
+
+    inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
     return ret;
 }
 
@@ -463,6 +227,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                        size_t size, off_t off,
                                        struct fuse_file_info *fi, char *buffer,
                                        size_t *rSize) {
+    (void)req;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -481,7 +246,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
     uint64_t fileSize = inodeWrapper->GetLength();
 
     size_t len = 0;
-    if (fileSize <= off) {
+    if (static_cast<int64_t>(fileSize) <= off) {
         *rSize = 0;
         return CURVEFS_ERROR::OK;
     } else if (fileSize < off + size) {
@@ -517,32 +282,56 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                          const char *name, mode_t mode,
                                          struct fuse_file_info *fi,
-                                         fuse_entry_param *e) {
+                                         EntryOut* entryOut) {
     VLOG(1) << "FuseOpCreate, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
+
+    std::shared_ptr<InodeWrapper> inode;
     CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, e);
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    return FuseOpOpen(req, e->ino, fi);
+
+    auto openFiles = fs_->BorrowMember().openFiles;
+    openFiles->Open(inode->GetInodeId(), inode);
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                        const char *name, mode_t mode,
-                                        dev_t rdev, fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        mode_t mode,
+                                        dev_t rdev,
+                                        EntryOut* entryOut) {
     VLOG(1) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, rdev, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
+                                FsFileType::TYPE_S3, rdev, false,
+                                inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    *entryOut = EntryOut(attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
-                                     fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req,
+                                       fuse_ino_t ino,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       EntryOut* entryOut) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_S3, e);
+        req, ino, newparent, newname, FsFileType::TYPE_S3, entryOut);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
@@ -554,6 +343,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
 CURVEFS_ERROR FuseS3Client::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                         int datasync,
                                         struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(1) << "FuseOpFsync, ino: " << ino << ", datasync: " << datasync;
 
     CURVEFS_ERROR ret = s3Adaptor_->Flush(ino);
@@ -582,6 +373,8 @@ CURVEFS_ERROR FuseS3Client::Truncate(InodeWrapper *inode, uint64_t length) {
 
 CURVEFS_ERROR FuseS3Client::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(1) << "FuseOpFlush, ino: " << ino;
     CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
 

curvefs/src/client/fuse_s3_client.h

@@ -32,6 +32,8 @@
 
 #include "curvefs/src/client/fuse_client.h"
 #include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "curvefs/src/client/warmup/warmup_manager.h"
+#include "curvefs/src/volume/common.h"
 #include "src/common/s3_adapter.h"
 
 namespace curvefs {
@@ -39,18 +41,35 @@ namespace client {
 
 using curve::common::GetObjectAsyncContext;
 using curve::common::GetObjectAsyncCallBack;
+using curvefs::volume::kMiB;
+namespace warmup {
+class WarmupManager;
+class WarmupManagerS3Impl;
+}  // namespace warmup
+
 
 class FuseS3Client : public FuseClient {
  public:
     FuseS3Client()
-        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {}
+        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {
+        auto readFunc = [this](fuse_req_t req, fuse_ino_t ino, size_t size,
+                               off_t off, struct fuse_file_info *fi,
+                               char *buffer, size_t *rSize) {
+            return FuseOpRead(req, ino, size, off, fi, buffer, rSize);
+        };
+        warmupManager_ = std::make_shared<warmup::WarmupManagerS3Impl>(
+            metaClient_, inodeManager_, dentryManager_, fsInfo_, readFunc,
+            s3Adaptor_, nullptr);
+    }
 
     FuseS3Client(const std::shared_ptr<MdsClient> &mdsClient,
                  const std::shared_ptr<MetaServerClient> &metaClient,
                  const std::shared_ptr<InodeCacheManager> &inodeManager,
                  const std::shared_ptr<DentryCacheManager> &dentryManager,
-                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor)
-        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager),
+                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor,
+                 const std::shared_ptr<warmup::WarmupManager> &warmupManager)
+        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
+                     warmupManager),
           s3Adaptor_(s3Adaptor) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
@@ -62,7 +81,7 @@ class FuseS3Client : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, size_t *wSize) override;
+        struct fuse_file_info *fi, FileOut* fileOut) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
         fuse_ino_t ino, size_t size, off_t off,
@@ -70,17 +89,25 @@ class FuseS3Client : public FuseClient {
         char *buffer,
         size_t *rSize) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, struct fuse_file_info *fi,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                               fuse_ino_t parent,
+                               const char* name,
+                               mode_t mode,
+                               struct fuse_file_info *fi,
+                               EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, dev_t rdev,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                              fuse_ino_t parent,
+                              const char* name,
+                              mode_t mode,
+                              dev_t rdev,
+                              EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-        fuse_ino_t newparent, const char *newname,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char *newname,
+                             EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
         const char *name) override;
@@ -91,37 +118,19 @@ class FuseS3Client : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
         struct fuse_file_info *fi) override;
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
  private:
     bool InitKVCache(const KVClientManagerOpt &opt);
 
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
     void FlushData() override;
-    // get the warmUp filelist
-    void GetWarmUpFileList(const WarmUpFileContext_t&,
-      std::vector<std::string>&);
-    void BackGroundFetch();
-    // put the file needed warmup to queue,
-    // then can downlaod the objs belong to it
-    void fetchDataEnqueue(fuse_ino_t ino);
-    // travel all chunks
-    void travelChunks(fuse_ino_t ino, google::protobuf::Map<uint64_t,
-      S3ChunkInfoList> *s3ChunkInfoMap);
-    // travel and download all objs belong to the chunk
-    void travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo,
-      std::list<std::pair<std::string, uint64_t>>* prefetchObjs);
-    // warmup all the prefetchObjs
-    void WarmUpAllObjs(const std::list<
-      std::pair<std::string, uint64_t>> &prefetchObjs);
 
  private:
     // s3 adaptor
     std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
+    std::shared_ptr<KVClientManager> kvClientManager_;
 
-    bool initbgFetchThread_;
-    Thread bgFetchThread_;
-    std::atomic<bool> bgFetchStop_;
-    std::mutex fetchMtx_;
+    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };
 
 

curvefs/src/client/fuse_volume_client.cpp

@@ -32,7 +32,7 @@
 #include "absl/cleanup/cleanup.h"
 #include "absl/memory/memory.h"
 #include "curvefs/proto/mds.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/volume/default_volume_storage.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/volume/common.h"
@@ -133,10 +133,11 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
                                             size_t size,
                                             off_t off,
                                             struct fuse_file_info *fi,
-                                            size_t *wSize) {
+                                            FileOut* fileOut) {
     VLOG(9) << "write start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
+    size_t* wSize = &fileOut->nwritten;
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
               is_aligned(size, DirectIOAlignment))) {
@@ -148,7 +149,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
     butil::Timer timer;
     timer.start();
 
-    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf);
+    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf, fileOut);
     if (ret != CURVEFS_ERROR::OK) {
         if (fsMetric_) {
             fsMetric_->userWrite.eps.count << 1;
@@ -189,6 +190,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
                                            struct fuse_file_info *fi,
                                            char *buffer,
                                            size_t *rSize) {
+    (void)req;
     VLOG(3) << "read start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
@@ -233,33 +235,57 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
 CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                              const char *name, mode_t mode,
                                              struct fuse_file_info *fi,
-                                             fuse_entry_param *e) {
+                                             EntryOut* entryOut) {
     VLOG(3) << "FuseOpCreate, parent: " << parent
               << ", name: " << name
               << ", mode: " << mode;
-    CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR ret = MakeNode(
+        req, parent, name, mode, FsFileType::TYPE_FILE, 0, false, inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    return FuseOpOpen(req, e->ino, fi);
+
+    auto openFiles = fs_->BorrowMember().openFiles;
+    openFiles->Open(inode->GetInodeId(), inode);
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                            const char *name, mode_t mode,
-                                            dev_t rdev, fuse_entry_param *e) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req,
+                                            fuse_ino_t parent,
+                                            const char* name,
+                                            mode_t mode,
+                                            dev_t rdev,
+                                            EntryOut* entryOut) {
     VLOG(3) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, rdev, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
+                                FsFileType::TYPE_FILE, rdev,
+                                false, inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    *entryOut = EntryOut(attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
-                                     fuse_entry_param *e) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req,
+                                           fuse_ino_t ino,
+                                           fuse_ino_t newparent,
+                                           const char* newname,
+                                           EntryOut* entryOut) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_FILE, e);
+        req, ino, newparent, newname, FsFileType::TYPE_FILE, entryOut);
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
@@ -271,6 +297,8 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
 CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                             int datasync,
                                             struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(3) << "FuseOpFsync start, ino: " << ino << ", datasync: " << datasync;
 
     CURVEFS_ERROR ret = storage_->Flush(ino);
@@ -297,12 +325,16 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
 }
 
 CURVEFS_ERROR FuseVolumeClient::Truncate(InodeWrapper *inode, uint64_t length) {
+    (void)inode;
+    (void)length;
     // Todo: call volume truncate
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                             struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(9) << "FuseOpFlush, ino: " << ino;
 
     CURVEFS_ERROR ret = storage_->Flush(ino);

curvefs/src/client/fuse_volume_client.h

@@ -52,7 +52,8 @@ class FuseVolumeClient : public FuseClient {
         const std::shared_ptr<InodeCacheManager> &inodeManager,
         const std::shared_ptr<DentryCacheManager> &dentryManager,
         const std::shared_ptr<BlockDeviceClient> &blockDeviceClient)
-        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager),
+        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
+                     nullptr),
           blockDeviceClient_(blockDeviceClient) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
@@ -64,24 +65,32 @@ class FuseVolumeClient : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, size_t *wSize) override;
+        struct fuse_file_info *fi, FileOut* fileOut) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
             fuse_ino_t ino, size_t size, off_t off,
             struct fuse_file_info *fi,
             char *buffer,
             size_t *rSize) override;
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, struct fuse_file_info *fi,
-        fuse_entry_param *e) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, dev_t rdev,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                               fuse_ino_t parent,
+                               const char* name,
+                               mode_t mode,
+                               struct fuse_file_info* fi,
+                               EntryOut* entryOut) override;
+
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                              fuse_ino_t parent,
+                              const char* name,
+                              mode_t mode,
+                              dev_t rdev,
+                              EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-        fuse_ino_t newparent, const char *newname,
-        fuse_entry_param *e) override;
+                             fuse_ino_t newparent,
+                             const char *newname,
+                             EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                const char *name) override;
@@ -92,13 +101,13 @@ class FuseVolumeClient : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                               struct fuse_file_info *fi) override;
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
     void SetSpaceManagerForTesting(SpaceManager *manager);
 
     void SetVolumeStorageForTesting(VolumeStorage *storage);
 
  private:
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
     void FlushData() override;
 
  private:

curvefs/src/client/inode_cache_manager.cpp

@@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 using ::curvefs::metaserver::Inode;
@@ -46,36 +46,11 @@ DECLARE_bool(enableCto);
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::ToFSError;
+
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
 using curvefs::client::common::FLAGS_enableCto;
 
-class TrimICacheAsyncDone : public MetaServerClientDone {
- public:
-    explicit TrimICacheAsyncDone(
-        const std::shared_ptr<InodeWrapper> &inodeWrapper,
-        const std::shared_ptr<InodeCacheManagerImpl> &inodeCacheManager)
-        : inodeWrapper_(inodeWrapper), inodeCacheManager_(inodeCacheManager) {}
-
-    void Run() override {
-        std::unique_ptr<TrimICacheAsyncDone> self_guard(this);
-        MetaStatusCode ret = GetStatusCode();
-        if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
-            LOG(ERROR) << "metaClient_ UpdateInode failed, "
-                       << "MetaStatusCode: " << ret
-                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
-                       << ", inodeid: " << inodeWrapper_->GetInodeId();
-            inodeWrapper_->MarkInodeError();
-        }
-        VLOG(9) << "Trim inode " << inodeWrapper_->GetInodeId()
-                << " async success.";
-        inodeCacheManager_->RemoveICache(inodeWrapper_);
-    };
-
- private:
-    std::shared_ptr<InodeWrapper> inodeWrapper_;
-    std::shared_ptr<InodeCacheManagerImpl> inodeCacheManager_;
-};
-
 #define GET_INODE_REMOTE(FSID, INODEID, OUT, STREAMING)                        \
     MetaStatusCode ret = metaClient_->GetInode(FSID, INODEID, OUT, STREAMING); \
     if (ret != MetaStatusCode::OK) {                                           \
@@ -83,17 +58,7 @@ class TrimICacheAsyncDone : public MetaServerClientDone {
             << "metaClient_ GetInode failed, MetaStatusCode = " << ret         \
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)          \
             << ", inodeid = " << INODEID;                                      \
-        return MetaStatusCodeToCurvefsErrCode(ret);                            \
-    }
-
-#define PUT_INODE_CACHE(INODEID, INODEWRAPPER)                                 \
-    std::shared_ptr<InodeWrapper> eliminatedOne;                               \
-    bool eliminated = iCache_->Put(INODEID, INODEWRAPPER, &eliminatedOne);     \
-    if (eliminated) {                                                          \
-        VLOG(3) << "GetInode eliminate one inode, ino: "                       \
-                << eliminatedOne->GetInodeId()                                 \
-                << ", iCache does not evict inodes via put interface ";        \
-        assert(0);                                                             \
+        return ToFSError(ret);                            \
     }
 
 #define REFRESH_DATA_REMOTE(OUT, STREAMING)                                    \
@@ -106,15 +71,8 @@ CURVEFS_ERROR
 InodeCacheManagerImpl::GetInode(uint64_t inodeId,
                                 std::shared_ptr<InodeWrapper> &out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    // get inode from cache
-    bool ok = iCache_->Get(inodeId, &out);
-    if (ok && NeedUseCahce(inodeId, out->IsDirty())) {
-        curve::common::UniqueLock lgGuard = out->GetUniqueLock();
-        if (out->GetType() == FsFileType::TYPE_FILE) {
-            return CURVEFS_ERROR::OK;
-        }
-
-        REFRESH_DATA_REMOTE(out, out->NeedRefreshData());
+    bool yes = openFiles_->IsOpened(inodeId, &out);
+    if (yes) {
         return CURVEFS_ERROR::OK;
     }
 
@@ -129,24 +87,12 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     // refresh data
     REFRESH_DATA_REMOTE(out, streaming);
 
-    // put to cache
-    PUT_INODE_CACHE(inodeId, out);
-
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
                                                   InodeAttr *out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    // 1. find in icache
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    bool ok = iCache_->Get(inodeId, &inodeWrapper);
-    if (ok && NeedUseCahce(inodeId, inodeWrapper->IsDirty())) {
-        inodeWrapper->GetInodeAttr(out);
-        return CURVEFS_ERROR::OK;
-    }
-
-    // 2. get form metaserver
     std::set<uint64_t> inodeIds;
     std::list<InodeAttr> attrs;
     inodeIds.emplace(inodeId);
@@ -156,7 +102,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed"
                    << ", inodeId = " << inodeId << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
     if (attrs.size() != 1) {
@@ -173,21 +119,6 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
     std::set<uint64_t> *inodeIds,
     std::list<InodeAttr> *attrs) {
-    // get some inode attr in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
-            InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace_back(std::move(tmpAttr));
-            iter = inodeIds->erase(iter);
-            continue;
-        }
-        ++iter;
-    }
-
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -199,7 +130,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return MetaStatusCodeToCurvefsErrCode(ret);
+    return ToFSError(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
@@ -207,26 +138,6 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     std::set<uint64_t> *inodeIds,
     std::map<uint64_t, InodeAttr> *attrs) {
     NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    std::map<uint64_t, InodeAttr> cachedAttr;
-    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
-
-    // get some inode attr in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
-            InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace(*iter, std::move(tmpAttr));
-            iter = inodeIds->erase(iter);
-        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
-            attrs->emplace(*iter, cachedAttr[*iter]);
-            iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
-        }
-    }
 
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
@@ -238,12 +149,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
         return CURVEFS_ERROR::NOTEXIST;
     }
 
+    ::curve::common::Mutex mutex;
     std::shared_ptr<CountDownEvent> cond =
         std::make_shared<CountDownEvent>(inodeGroups.size());
     for (const auto& it : inodeGroups) {
         VLOG(3) << "BatchGetInodeAttrAsync Send " << it.size();
-        auto* done = new BatchGetInodeAttrAsyncDone(shared_from_this(),
-                                                    cond, parentId);
+        auto* done = new BatchGetInodeAttrAsyncDone(attrs, &mutex, cond);
         MetaStatusCode ret = metaClient_->BatchGetInodeAttrAsync(fsId_, it,
                                                                  done);
         if (MetaStatusCode::OK != ret) {
@@ -256,30 +167,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
 
     // wait for all sudrequest finished
     cond->Wait();
-
-    bool ok  = iAttrCache_->Get(parentId, attrs);
-    if (!ok) {
-        LOG(WARNING) << "get attrs form iAttrCache_ failed.";
-    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
     std::set<uint64_t> *inodeIds,
     std::list<XAttr> *xattr) {
-    // get some inode in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
-            xattr->emplace_back(inodeWrapper->GetXattr());
-            iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
-        }
-    }
-
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -290,7 +183,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return MetaStatusCodeToCurvefsErrCode(ret);
+    return ToFSError(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
@@ -301,159 +194,46 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ CreateInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-    uint64_t inodeid = inode.inodeid();
     out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
         s3ChunkInfoMetric_, option_.maxDataSize,
         option_.refreshDataIntervalSec);
+    return CURVEFS_ERROR::OK;
+}
 
-    std::shared_ptr<InodeWrapper> eliminatedOne;
-    bool eliminated = false;
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeid));
-        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
-    }
-    if (eliminated) {
-        /* iCache does not evict inodes via put interface */
-        assert(0);
+CURVEFS_ERROR InodeCacheManagerImpl::CreateManageInode(
+    const InodeParam &param,
+    std::shared_ptr<InodeWrapper> &out) {
+    Inode inode;
+    MetaStatusCode ret = metaClient_->CreateManageInode(param, &inode);
+    if (ret != MetaStatusCode::OK) {
+        LOG(ERROR) << "metaClient_ CreateManageInode failed, MetaStatusCode = "
+                   << ret << ", MetaStatusCode_Name = "
+                   << MetaStatusCode_Name(ret);
+        return ToFSError(ret);
     }
+    out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
+        s3ChunkInfoMetric_, option_.maxDataSize,
+        option_.refreshDataIntervalSec);
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::DeleteInode(uint64_t inodeId) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    iCache_->Remove(inodeId);
     MetaStatusCode ret = metaClient_->DeleteInode(fsId_, inodeId);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeId = " << inodeId;
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-
-    curve::common::LockGuard lg2(dirtyMapMutex_);
-    dirtyMap_.erase(inodeId);
     return CURVEFS_ERROR::OK;
 }
 
-void InodeCacheManagerImpl::AddInodeAttrs(
-    uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
-    iAttrCache_->Set(parentId, inodeAttrs);
-}
-
-void InodeCacheManagerImpl::ClearInodeCache(uint64_t inodeId) {
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeId));
-        iCache_->Remove(inodeId);
-    }
-    curve::common::LockGuard lg2(dirtyMapMutex_);
-    dirtyMap_.erase(inodeId);
-}
-
 void InodeCacheManagerImpl::ShipToFlush(
-    const std::shared_ptr<InodeWrapper> &inodeWrapper) {
-    curve::common::LockGuard lg(dirtyMapMutex_);
-    dirtyMap_.emplace(inodeWrapper->GetInodeId(), inodeWrapper);
-}
-
-void InodeCacheManagerImpl::FlushAll() {
-    while (!dirtyMap_.empty()) {
-        FlushInodeOnce();
-    }
-}
-
-void InodeCacheManagerImpl::FlushInodeOnce() {
-    std::map<uint64_t, std::shared_ptr<InodeWrapper>> temp_;
-    {
-        curve::common::LockGuard lg(dirtyMapMutex_);
-        temp_.swap(dirtyMap_);
-    }
-    for (auto it = temp_.begin(); it != temp_.end(); it++) {
-        curve::common::UniqueLock ulk = it->second->GetUniqueLock();
-        it->second->Async(nullptr, true);
-    }
-}
-
-void InodeCacheManagerImpl::ReleaseCache(uint64_t parentId) {
-    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    iAttrCache_->Release(parentId);
-}
-
-void InodeCacheManagerImpl::FlushInodeBackground() {
-    LOG(INFO) << "flush thread is start.";
-    while (!isStop_.load()) {
-        if (iCache_->Size() > maxCacheSize_) {
-            TrimIcache(iCache_->Size() - maxCacheSize_);
-        }
-        FlushInodeOnce();
-        sleeper_.wait_for(std::chrono::seconds(flushPeriodSec_));
-    }
-    LOG(INFO) << "flush thread is stop.";
-}
-
-namespace {
-// Wether an inode is dirty or not.
-// if |needLock| is true, we acquire the lock firstly and do the test,
-// otherwise, we assume the lock is already held.
-bool IsDirtyInode(InodeWrapper *ino, bool needLock) {
-    auto check = [ino]() {
-        return ino->IsDirty() || !ino->S3ChunkInfoEmptyNolock() ||
-               ino->GetMutableExtentCacheLocked()->HasDirtyExtents();
-    };
-
-    if (needLock) {
-        auto lk = ino->GetUniqueLock();
-        return check();
-    }
-
-    return check();
-}
-}  // namespace
-
-void InodeCacheManagerImpl::RemoveICache(
-    const std::shared_ptr<InodeWrapper> &inode) {
-    if (!IsDirtyInode(inode.get(), true)) {
-        uint64_t inodeId = inode->GetInodeId();
-        NameLockGuard lock(nameLock_, std::to_string(inodeId));
-        ::curve::common::UniqueLock lgGuard = inode->GetUniqueLock();
-        iCache_->Remove(inodeId);
-    }
-}
-
-void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    uint64_t inodeId;
-    VLOG(3) << "TrimIcache trimSize " << trimSize;
-    while (trimSize > 0) {
-        bool ok = iCache_->GetLast(&inodeId, &inodeWrapper);
-        if (ok) {
-            NameLockGuard lock(nameLock_, std::to_string(inodeId));
-            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-            if (IsDirtyInode(inodeWrapper.get(), false)) {
-                VLOG(9) << "TrimIcache sync dirty inode " << inodeId;
-                dirtyMapMutex_.lock();
-                dirtyMap_.erase(inodeId);
-                dirtyMapMutex_.unlock();
-                auto *done =
-                    new TrimICacheAsyncDone(inodeWrapper, shared_from_this());
-                inodeWrapper->Async(done);
-            } else {
-                VLOG(9) << "TrimIcache remove inode " << inodeId
-                        << " from iCache";
-                iCache_->Remove(inodeId);
-            }
-            trimSize--;
-            // remove the attr of the inode in iattrcache
-            auto parents = inodeWrapper->GetParentLocked();
-            for (uint64_t parent : parents) {
-                iAttrCache_->Remove(parent, inodeId);
-            }
-        } else {
-            LOG(ERROR) << "icache size " << iCache_->Size();
-            assert(0);
-        }
-    }
+    const std::shared_ptr<InodeWrapper>& inode) {
+    deferSync_->Push(inode);
 }
 
 CURVEFS_ERROR
@@ -490,33 +270,5 @@ InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
     return rc;
 }
 
-void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
-    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
-    curve::common::LockGuard lg(openInodesMutex_);
-    openedInodes_.emplace(inodeId);
-}
-
-void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
-    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
-    curve::common::LockGuard lg(openInodesMutex_);
-    auto iter = openedInodes_.find(inodeId);
-    if (iter != openedInodes_.end()) {
-        openedInodes_.erase(iter);
-    }
-}
-
-bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
-    curve::common::LockGuard lg(openInodesMutex_);
-    auto iter = openedInodes_.find(inodeId);
-    return iter != openedInodes_.end();
-}
-
-bool InodeCacheManagerImpl::NeedUseCahce(uint64_t inodeId, bool IsDirty) {
-    if (!FLAGS_enableCto || OpenInodeCached(inodeId) || IsDirty) {
-        return true;
-    }
-    return false;
-}
-
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_cache_manager.h

@@ -38,12 +38,14 @@
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "src/common/concurrent/name_lock.h"
 #include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
 
 using ::curve::common::LRUCache;
 using ::curve::common::CacheMetrics;
@@ -64,57 +66,8 @@ using rpcclient::BatchGetInodeAttrDone;
 using curve::common::CountDownEvent;
 using metric::S3ChunkInfoMetric;
 using common::RefreshDataOption;
-
-class InodeAttrCache {
- public:
-    InodeAttrCache() {}
-    ~InodeAttrCache() {}
-
-    bool Get(uint64_t parentId, std::map<uint64_t, InodeAttr> *imap) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            imap->insert(iter->second.begin(), iter->second.end());
-            return true;
-        }
-        return false;
-    }
-
-    void Set(uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        VLOG(1) << "parentId = " << parentId
-                << ", iAttrCache set size = " << inodeAttrs.size();
-        auto& inner = iAttrCache_[parentId];
-        for (const auto &it : inodeAttrs) {
-            inner.emplace(it.inodeid(), it);
-        }
-    }
-
-    void Release(uint64_t parentId) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto size = iAttrCache_.size();
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            iAttrCache_.erase(iter);
-        }
-        VLOG(1) << "inodeId = " << parentId
-                << ", inode attr cache release, before = "
-                << size << ", after = " << iAttrCache_.size();
-    }
-
-    void Remove(uint64_t parentId, uint64_t inodeId) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            iter->second.erase(inodeId);
-        }
-    }
-
- private:
-    // inodeAttr cache; <parentId, <inodeId, inodeAttr>>
-    std::map<uint64_t, std::map<uint64_t, InodeAttr>> iAttrCache_;
-    curve::common::Mutex iAttrCacheMutex_;
-};
+using ::curvefs::client::filesystem::OpenFiles;
+using ::curvefs::client::filesystem::DeferSync;
 
 class InodeCacheManager {
  public:
@@ -126,13 +79,9 @@ class InodeCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-                               uint32_t flushPeriodSec,
-                               RefreshDataOption option) = 0;
-
-    virtual void Run() = 0;
-
-    virtual void Stop() = 0;
+    virtual CURVEFS_ERROR Init(RefreshDataOption option,
+                               std::shared_ptr<OpenFiles> openFiles,
+                               std::shared_ptr<DeferSync> deferSync) = 0;
 
     virtual CURVEFS_ERROR
     GetInode(uint64_t inodeId,
@@ -155,26 +104,14 @@ class InodeCacheManager {
     virtual CURVEFS_ERROR CreateInode(const InodeParam &param,
         std::shared_ptr<InodeWrapper> &out) = 0;   // NOLINT
 
+    virtual CURVEFS_ERROR CreateManageInode(const InodeParam &param,
+        std::shared_ptr<InodeWrapper> &out) = 0;   // NOLINT
+
     virtual CURVEFS_ERROR DeleteInode(uint64_t inodeId) = 0;
 
-    virtual void AddInodeAttrs(uint64_t parentId,
-        const RepeatedPtrField<InodeAttr>& inodeAttrs) = 0;
-
-    virtual void ClearInodeCache(uint64_t inodeId) = 0;
-
     virtual void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) = 0;
 
-    virtual void FlushAll() = 0;
-
-    virtual void FlushInodeOnce() = 0;
-
-    virtual void ReleaseCache(uint64_t parentId) = 0;
-
-    virtual void AddOpenedInode(uint64_t inodeId) = 0;
-
-    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
-
  protected:
     uint32_t fsId_;
 };
@@ -183,55 +120,22 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     public std::enable_shared_from_this<InodeCacheManagerImpl> {
  public:
     InodeCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()),
-        iCache_(nullptr),
-        iAttrCache_(nullptr),
-        isStop_(true) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
 
     explicit InodeCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient),
-        iCache_(nullptr),
-        iAttrCache_(nullptr) {}
+      : metaClient_(metaClient) {}
 
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-                       uint32_t flushPeriodSec,
-                       RefreshDataOption option) override {
-        if (enableCacheMetrics) {
-            iCache_ = std::make_shared<
-                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0,
-                    std::make_shared<CacheMetrics>("icache"));
-        } else {
-            iCache_ = std::make_shared<
-                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0);
-        }
-        maxCacheSize_ = cacheSize;
+    CURVEFS_ERROR Init(RefreshDataOption option,
+                       std::shared_ptr<OpenFiles> openFiles,
+                       std::shared_ptr<DeferSync> deferSync) override {
         option_ = option;
-        flushPeriodSec_ = flushPeriodSec;
-        iAttrCache_ = std::make_shared<InodeAttrCache>();
         s3ChunkInfoMetric_ = std::make_shared<S3ChunkInfoMetric>();
+        openFiles_ =  openFiles;
+        deferSync_ = deferSync;
         return CURVEFS_ERROR::OK;
     }
 
-    void Run() {
-        isStop_.exchange(false);
-        flushThread_ =
-            Thread(&InodeCacheManagerImpl::FlushInodeBackground, this);
-        LOG(INFO) << "Start inodeManager flush thread ok.";
-    }
-
-    void Stop() {
-        isStop_.exchange(true);
-        LOG(INFO) << "stop inodeManager flush thread ...";
-        sleeper_.interrupt();
-        flushThread_.join();
-    }
-
-    bool IsDirtyMapExist(uint64_t inodeId) {
-        curve::common::LockGuard lg(dirtyMapMutex_);
-        return dirtyMap_.count(inodeId) > 0;
-    }
-
     CURVEFS_ERROR GetInode(uint64_t inodeId,
                            std::shared_ptr<InodeWrapper> &out) override;
 
@@ -250,73 +154,42 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     CURVEFS_ERROR CreateInode(const InodeParam &param,
         std::shared_ptr<InodeWrapper> &out) override;
 
+    CURVEFS_ERROR CreateManageInode(const InodeParam &param,
+        std::shared_ptr<InodeWrapper> &out) override;
+
     CURVEFS_ERROR DeleteInode(uint64_t inodeId) override;
 
-    void AddInodeAttrs(uint64_t parentId,
-        const RepeatedPtrField<InodeAttr>& inodeAttrs) override;
-
-    void ClearInodeCache(uint64_t inodeId) override;
-
     void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) override;
 
-    void FlushAll() override;
-
-    void FlushInodeOnce() override;
-
-    void ReleaseCache(uint64_t parentId) override;
-
-    void RemoveICache(const std::shared_ptr<InodeWrapper> &inode);
-
-    void AddOpenedInode(uint64_t inodeId) override;
-
-    void RemoveOpenedInode(uint64_t inodeId) override;
-
-    bool NeedUseCahce(uint64_t inodeId, bool IsDirty);
-
  private:
-    virtual void FlushInodeBackground();
-    void TrimIcache(uint64_t trimSize);
     CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
                               bool streaming = true);
-    bool OpenInodeCached(uint64_t inodeId);
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    std::shared_ptr<LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>> iCache_;
     std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric_;
 
-    std::shared_ptr<InodeAttrCache> iAttrCache_;
+    std::shared_ptr<OpenFiles> openFiles_;
 
-    // dirty map, key is inodeid
-    std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
-    curve::common::Mutex dirtyMapMutex_;
-
-    // record opened inode
-    std::multiset<uint64_t> openedInodes_;
-    curve::common::Mutex openInodesMutex_;
+    std::shared_ptr<DeferSync> deferSync_;
 
     curve::common::GenericNameLock<Mutex> nameLock_;
 
     curve::common::GenericNameLock<Mutex> asyncNameLock_;
 
-    uint64_t maxCacheSize_;
     RefreshDataOption option_;
-    uint32_t flushPeriodSec_;
-    Thread flushThread_;
-    InterruptibleSleeper sleeper_;
-    Atomic<bool> isStop_;
 };
 
 class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
  public:
-    BatchGetInodeAttrAsyncDone(
-        const std::shared_ptr<InodeCacheManager> &inodeCacheManager,
-        std::shared_ptr<CountDownEvent> cond,
-        uint64_t parentId):
-        inodeCacheManager_(inodeCacheManager),
-        cond_(cond),
-        parentId_(parentId) {}
+    BatchGetInodeAttrAsyncDone(std::map<uint64_t, InodeAttr>* attrs,
+                               ::curve::common::Mutex* mutex,
+                               std::shared_ptr<CountDownEvent> cond):
+        attrs_(attrs),
+        mutex_(mutex),
+        cond_(cond) {}
+
     ~BatchGetInodeAttrAsyncDone() {}
 
     void Run() override {
@@ -324,22 +197,25 @@ class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
         MetaStatusCode ret = GetStatusCode();
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "BatchGetInodeAttrAsync failed, "
-                       << "parentId = " << parentId_
                        << ", MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret);
         } else {
             auto inodeAttrs = GetInodeAttrs();
             VLOG(3) << "BatchGetInodeAttrAsyncDone update inodeAttrCache"
                     << " size = " << inodeAttrs.size();
-            inodeCacheManager_->AddInodeAttrs(parentId_, inodeAttrs);
+
+            curve::common::LockGuard lk(*mutex_);
+            for (const auto& attr : inodeAttrs) {
+                attrs_->emplace(attr.inodeid(), attr);
+            }
         }
         cond_->Signal();
     };
 
  private:
-    std::shared_ptr<InodeCacheManager> inodeCacheManager_;
+    ::curve::common::Mutex* mutex_;
+    std::map<uint64_t, InodeAttr>* attrs_;
     std::shared_ptr<CountDownEvent> cond_;
-    uint64_t parentId_;
 };
 
 }  // namespace client

curvefs/src/client/inode_wrapper.cpp

@@ -33,7 +33,6 @@
 
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/async_request_closure.h"
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
@@ -47,6 +46,7 @@ namespace client {
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::DataIndices;
+using ::curvefs::client::filesystem::ToFSError;
 
 bvar::Adder<int64_t> g_alive_inode_count{"alive_inode_count"};
 
@@ -107,6 +107,7 @@ class UpdateInodeAsyncDone : public MetaServerClientDone {
                        << ", inodeid: " << inodeWrapper_->GetInodeId();
             inodeWrapper_->MarkInodeError();
         }
+        inodeWrapper_->ClearDirty();
         inodeWrapper_->ReleaseSyncingInode();
 
         if (parent_ != nullptr) {
@@ -192,7 +193,7 @@ CURVEFS_ERROR InodeWrapper::SyncAttr(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
 
         dirty_ = false;
@@ -212,7 +213,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3ChunkInfo(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         ClearS3ChunkInfoAdd();
     }
@@ -226,7 +227,6 @@ void InodeWrapper::AsyncFlushAttr(MetaServerClientDone* done,
         metaClient_->UpdateInodeWithOutNlinkAsync(
             inode_.fsid(), inode_.inodeid(), dirtyAttr_,
             new UpdateInodeAsyncDone(shared_from_this(), done));
-        dirty_ = false;
         dirtyAttr_.Clear();
         return;
     }
@@ -294,14 +294,14 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
                    << "MetaStatusCode: " << ret
                    << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                    << ", inodeid: " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     auto before = s3ChunkInfoSize_;
     inode_.mutable_s3chunkinfomap()->swap(s3ChunkInfoMap);
     UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize() - before);
     ClearS3ChunkInfoAdd();
     UpdateMaxS3ChunkInfoSize();
-    lastRefreshTime_ = ::curve::common::TimeUtility::GetTimeofDaySec();
+    lastRefreshTime_ = TimeUtility::GetTimeofDaySec();
     return CURVEFS_ERROR::OK;
 }
 
@@ -328,7 +328,7 @@ CURVEFS_ERROR InodeWrapper::Link(uint64_t parent) {
                    <<", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
     dirty_ = false;
@@ -396,7 +396,7 @@ CURVEFS_ERROR InodeWrapper::UnLink(uint64_t parent) {
                        << ", MetaStatusCode = " << ret
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", inodeid = " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         dirty_ = false;
         dirtyAttr_.Clear();
@@ -427,7 +427,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParent(
                    << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     dirty_ = false;
     dirtyAttr_.Clear();
@@ -457,7 +457,7 @@ CURVEFS_ERROR InodeWrapper::Sync(bool internal) {
 }
 
 void InodeWrapper::Async(MetaServerClientDone *done, bool internal) {
-    VLOG(3) << "async inode: " << inode_.ShortDebugString();
+    VLOG(9) << "async inode: " << inode_.ShortDebugString();
 
     switch (inode_.type()) {
         case FsFileType::TYPE_S3:
@@ -489,7 +489,6 @@ void InodeWrapper::AsyncFlushAttrAndExtents(MetaServerClientDone *done,
             new UpdateInodeAttrAndExtentClosure{shared_from_this(), done},
             std::move(indices));
 
-        dirty_ = false;
         dirtyAttr_.Clear();
         return;
     }
@@ -515,7 +514,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         ClearS3ChunkInfoAdd();
         dirty_ = false;
@@ -543,6 +542,7 @@ class UpdateInodeAsyncS3Done : public MetaServerClientDone {
             inodeWrapper_->MarkInodeError();
         }
         VLOG(9) << "inode " << inodeWrapper_->GetInodeId() << " async success.";
+        inodeWrapper_->ClearDirty();
         inodeWrapper_->ReleaseSyncingInode();
         inodeWrapper_->ReleaseSyncingS3ChunkInfo();
 
@@ -559,6 +559,7 @@ class UpdateInodeAsyncS3Done : public MetaServerClientDone {
 }  // namespace
 
 void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
+    (void)internal;
     if (dirty_ || !s3ChunkInfoAdd_.empty()) {
         LockSyncingInode();
         LockSyncingS3ChunkInfo();
@@ -570,7 +571,6 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
             inode_.fsid(), inode_.inodeid(), dirtyAttr_,
             new UpdateInodeAsyncS3Done{shared_from_this(), done},
             std::move(indices));
-        dirty_ = false;
         dirtyAttr_.Clear();
         ClearS3ChunkInfoAdd();
         return;
@@ -597,7 +597,7 @@ CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
         LOG(ERROR) << "GetVolumeExtent failed, inodeid: " << inode_.inodeid();
     }
 
-    return MetaStatusCodeToCurvefsErrCode(st);
+    return ToFSError(st);
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshNlink() {
@@ -610,7 +610,8 @@ CURVEFS_ERROR InodeWrapper::RefreshNlink() {
         return CURVEFS_ERROR::INTERNAL;
     }
     inode_.set_nlink(attr.nlink());
-    VLOG(3) << "RefreshNlink from metaserver, newnlink: " << attr.nlink();
+    VLOG(3) << "RefreshNlink from metaserver, newnlink: " << attr.nlink()
+            << ", inodeid: " << inode_.inodeid();
     return CURVEFS_ERROR::OK;
 }
 

curvefs/src/client/inode_wrapper.h

@@ -34,12 +34,12 @@
 
 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/timeutility.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Inode;
 using ::curvefs::metaserver::S3ChunkInfoList;
@@ -53,20 +53,19 @@ constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;
 
 using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 enum class InodeStatus {
     kNormal = 0,
     kError = -1,
 };
 
-// TODO(xuchaojie) : get from conf maybe?
-const uint32_t kOptimalIOBlockSize = 0x10000u;
-
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;
 using metric::S3ChunkInfoMetric;
 using common::NlinkChange;
+using curve::common::TimeUtility;
 
 std::ostream &operator<<(std::ostream &os, const struct stat &attr);
 void AppendS3ChunkInfoToMap(uint64_t chunkIndex, const S3ChunkInfo &info,
@@ -79,18 +78,19 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     InodeWrapper(Inode inode,
                  std::shared_ptr<MetaServerClient> metaClient,
                  std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric = nullptr,
-                 uint64_t maxDataSize = ULONG_MAX,
+                 int64_t maxDataSize = LONG_MAX,
                  uint32_t refreshDataInterval = UINT_MAX)
         : inode_(std::move(inode)),
           status_(InodeStatus::kNormal),
           baseMaxDataSize_(maxDataSize),
           maxDataSize_(maxDataSize),
           refreshDataInterval_(refreshDataInterval),
-          lastRefreshTime_(::curve::common::TimeUtility::GetTimeofDaySec()),
+          lastRefreshTime_(TimeUtility::GetTimeofDaySec()),
           s3ChunkInfoAddSize_(0),
           metaClient_(std::move(metaClient)),
           s3ChunkInfoMetric_(std::move(s3ChunkInfoMetric)),
-          dirty_(false) {
+          dirty_(false),
+          time_(TimeUtility::GetTimeofDaySec()) {
         UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize());
         g_alive_inode_count << 1;
     }
@@ -339,6 +339,10 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         return false;
     }
 
+    uint32_t GetCachedTime() const {
+        return time_;
+    }
+
  private:
     CURVEFS_ERROR SyncS3ChunkInfo(bool internal = false);
 
@@ -394,8 +398,8 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     InodeAttr dirtyAttr_;
 
     InodeStatus status_;
-    uint64_t baseMaxDataSize_;
-    uint64_t maxDataSize_;
+    int64_t baseMaxDataSize_;
+    int64_t maxDataSize_;
     uint32_t refreshDataInterval_;
     uint64_t lastRefreshTime_;
 
@@ -413,6 +417,9 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
 
     mutable ::curve::common::Mutex syncingVolumeExtentsMtx_;
     ExtentCache extentCache_;
+
+    // timestamp when put in cache
+    uint64_t time_;
 };
 
 }  // namespace client

curvefs/src/client/kvclient/kvclient_manager.cpp

@@ -21,6 +21,7 @@
  */
 
 #include "curvefs/src/client/kvclient/kvclient_manager.h"
+#include <memory>
 #include "src/client/client_metric.h"
 #include "src/common/concurrent/count_down_event.h"
 
@@ -31,59 +32,48 @@ using curvefs::client::metric::KVClientMetric;
 namespace curvefs {
 namespace client {
 
-KVClientManager *g_kvClientManager = nullptr;
-KVClientMetric *g_kvClientMetric = nullptr;
-
-#define ONRETURN(TYPE, RES, KEY, ERRORLOG)                                     \
+#define ONRETURN(TYPE, RES)                                                    \
     if (RES) {                                                                 \
-        g_kvClientMetric->kvClient##TYPE.qps.count << 1;                       \
-        VLOG(9) << "##TYPE key = " << KEY << " OK";                            \
+        kvClientMetric_.kvClient##TYPE.qps.count << 1;                        \
     } else {                                                                   \
-        g_kvClientMetric->kvClient##TYPE.eps.count << 1;                       \
-        LOG(ERROR) << "##TYPE key = " << KEY << " error = " << ERRORLOG;       \
-    }
+        kvClientMetric_.kvClient##TYPE.eps.count << 1;                        \
+    }                                                                          \
 
 bool KVClientManager::Init(const KVClientManagerOpt &config,
                            const std::shared_ptr<KVClient> &kvclient) {
     client_ = kvclient;
-    g_kvClientMetric = new KVClientMetric();
     return threadPool_.Start(config.setThreadPooln) == 0;
 }
 
 void KVClientManager::Uninit() {
     client_->UnInit();
     threadPool_.Stop();
-    delete g_kvClientMetric;
 }
 
 void KVClientManager::Set(std::shared_ptr<SetKVCacheTask> task) {
     threadPool_.Enqueue([task, this]() {
-        LatencyGuard guard(&g_kvClientMetric->kvClientSet.latency);
+        LatencyGuard guard(&kvClientMetric_.kvClientSet.latency);
 
         std::string error_log;
         auto res =
             client_->Set(task->key, task->value, task->length, &error_log);
-        ONRETURN(Set, res, task->key, error_log);
+        ONRETURN(Set, res);
 
         task->done(task);
     });
 }
 
-bool KVClientManager::Get(std::shared_ptr<GetKvCacheContext> task) {
-    assert(nullptr != task->value);
-    return Get(task->key, task->value, task->offset, task->length);
-}
+void KVClientManager::Get(std::shared_ptr<GetKVCacheTask> task) {
+    threadPool_.Enqueue([task, this]() {
+        LatencyGuard guard(&kvClientMetric_.kvClientGet.latency);
 
+        std::string error_log;
+        task->res = client_->Get(task->key, task->value, task->offset,
+                                task->length, &error_log);
+        ONRETURN(Get, task->res);
 
-bool KVClientManager::Get(const std::string &key, char *value, uint64_t offset,
-                          uint64_t length) {
-    LatencyGuard guard(&g_kvClientMetric->kvClientGet.latency);
-
-    assert(nullptr != value);
-    std::string error_log;
-    auto res = client_->Get(key, value, offset, length, &error_log);
-    ONRETURN(Get, res, key, error_log);
-    return res;
+        task->done(task);
+    });
 }
 
 }  // namespace client

curvefs/src/client/kvclient/kvclient_manager.h

@@ -44,14 +44,14 @@ namespace client {
 
 class KVClientManager;
 class SetKVCacheTask;
+class GetKVCacheTask;
 using curve::common::TaskThreadPool;
 using curvefs::client::common::KVClientManagerOpt;
 
-extern KVClientManager *g_kvClientManager;
-extern KVClientMetric *g_kvClientMetric;
-
 typedef std::function<void(const std::shared_ptr<SetKVCacheTask> &)>
     SetKVCacheDone;
+typedef std::function<void(const std::shared_ptr<GetKVCacheTask> &)>
+    GetKVCacheDone;
 
 struct SetKVCacheTask {
     std::string key;
@@ -59,17 +59,23 @@ struct SetKVCacheTask {
     uint64_t length;
     SetKVCacheDone done;
     SetKVCacheTask() = default;
-    SetKVCacheTask(const std::string &k, const char *val, const uint64_t len)
-        : key(k), value(val), length(len) {}
+    SetKVCacheTask(
+        const std::string &k, const char *val, const uint64_t len,
+        SetKVCacheDone done = [](const std::shared_ptr<SetKVCacheTask> &) {})
+        : key(k), value(val), length(len), done(std::move(done)) {}
 };
 
-struct GetKvCacheContext {
+struct GetKVCacheTask {
     const std::string &key;
     char *value;
     uint64_t offset;
     uint64_t length;
-    GetKvCacheContext(const std::string &k, char *v, uint64_t off, uint64_t len)
-        : key(k), value(v), offset(off), length(len) {}
+    bool res;
+    GetKVCacheDone done;
+    GetKVCacheTask(const std::string &k, char *v, uint64_t off, uint64_t len)
+        : key(k), value(v), offset(off), length(len), res(false) {
+        done = [](const std::shared_ptr<GetKVCacheTask> &) {};
+    }
 };
 
 class KVClientManager {
@@ -87,10 +93,9 @@ class KVClientManager {
      */
     void Set(std::shared_ptr<SetKVCacheTask> task);
 
-    bool Get(const std::string &key, char *value, uint64_t offset,
-             uint64_t length);
+    void Get(std::shared_ptr<GetKVCacheTask> task);
 
-    bool Get(std::shared_ptr<GetKvCacheContext> task);
+    KVClientMetric *GetClientMetricForTesting() { return &kvClientMetric_; }
 
  private:
     void Uninit();
@@ -98,6 +103,7 @@ class KVClientManager {
  private:
     TaskThreadPool<bthread::Mutex, bthread::ConditionVariable> threadPool_;
     std::shared_ptr<KVClient> client_;
+    KVClientMetric kvClientMetric_;
 };
 
 }  // namespace client

curvefs/src/client/kvclient/memcache_client.h

@@ -92,12 +92,9 @@ class MemCachedClient : public KVClient {
                 return false;
             }
         }
-
         memcached_behavior_set(client_, MEMCACHED_BEHAVIOR_DISTRIBUTION,
                                MEMCACHED_DISTRIBUTION_CONSISTENT);
         memcached_behavior_set(client_, MEMCACHED_BEHAVIOR_RETRY_TIMEOUT, 5);
-        memcached_behavior_set(client_,
-                               MEMCACHED_BEHAVIOR_REMOVE_FAILED_SERVERS, 1);
 
         return PushServer();
     }
@@ -117,9 +114,13 @@ class MemCachedClient : public KVClient {
         auto res = memcached_set(tcli, key.c_str(), key.length(), value,
                                  value_len, 0, 0);
         if (MEMCACHED_SUCCESS == res) {
+            VLOG(9) << "Set key = " << key << " OK";
             return true;
         }
         *errorlog = ResError(res);
+        memcached_free(tcli);
+        tcli = nullptr;
+        LOG(ERROR) << "Set key = " << key << " error = " << *errorlog;
         return false;
     }
 
@@ -135,13 +136,23 @@ class MemCachedClient : public KVClient {
         memcached_return_t ue;
         char *res = memcached_get(tcli, key.c_str(), key.length(),
                                   &value_length, &flags, &ue);
-        if (res != nullptr && value) {
+        if (MEMCACHED_SUCCESS == ue && res != nullptr && value &&
+            value_length >= length) {
+            VLOG(9) << "Get key = " << key << " OK";
             memcpy(value, res + offset, length);
             free(res);
             return true;
         }
 
         *errorlog = ResError(ue);
+        if (ue != MEMCACHED_NOTFOUND) {
+          LOG(ERROR) << "Get key = " << key << " error = " << *errorlog
+                     << ", get_value_len = " << value_length
+                     << ", expect_value_len = " << length;
+          memcached_free(tcli);
+          tcli = nullptr;
+        }
+
         return false;
     }
 

curvefs/src/client/lease/lease_excutor.cpp

@@ -41,7 +41,7 @@ LeaseExecutor::~LeaseExecutor() {
 bool LeaseExecutor::Start() {
     if (opt_.leaseTimeUs <= 0 || opt_.refreshTimesPerLease <= 0) {
         LOG(ERROR) << "LeaseExecutor start fail. Invalid param in leaseopt, "
-                      "leasTimesUs = "
+                      "leaseTimeUs = "
                    << opt_.leaseTimeUs
                    << ", refreshTimePerLease = " << opt_.refreshTimesPerLease;
         return false;
@@ -78,7 +78,8 @@ bool LeaseExecutor::RefreshLease() {
     // refresh from mds
     std::vector<PartitionTxId> latestTxIdList;
     FSStatusCode ret = mdsCli_->RefreshSession(txIds, &latestTxIdList,
-                                               fsName_, mountpoint_);
+                                               fsName_, mountpoint_,
+                                               enableSumInDir_);
     if (ret != FSStatusCode::OK) {
         LOG(ERROR) << "LeaseExecutor refresh session fail, ret = " << ret
                    << ", errorName = " << FSStatusCode_Name(ret);