curvefs/client: fix s3 object will not be removed

signed-off-by: hzwuhongsong hzwuhongsong@corp.netease.com

.bazelrc

@@ -1,10 +1,5 @@
-build --verbose_failures
-
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
-build --incompatible_blacklisted_protos_requires_proto_info=false
-build --copt=-fdiagnostics-color=always
-
-run --copt=-fdiagnostics-color=always
+build --incompatible_blacklisted_protos_requires_proto_info=false

.clang-format

@@ -1,147 +0,0 @@
----
-Language:        Cpp
-AccessModifierOffset: -3
-AlignAfterOpenBracket: Align
-AlignConsecutiveMacros: true
-AlignConsecutiveAssignments: false
-AlignConsecutiveBitFields: false
-AlignConsecutiveDeclarations: false
-AlignEscapedNewlines: Right
-AlignOperands:   Align
-AlignTrailingComments: true
-AllowAllArgumentsOnNextLine: true
-AllowAllConstructorInitializersOnNextLine: true
-AllowAllParametersOfDeclarationOnNextLine: true
-AllowShortEnumsOnASingleLine: true
-AllowShortBlocksOnASingleLine: Never
-AllowShortCaseLabelsOnASingleLine: false
-AllowShortFunctionsOnASingleLine: All
-AllowShortLambdasOnASingleLine: All
-AllowShortIfStatementsOnASingleLine: Never
-AllowShortLoopsOnASingleLine: false
-AlwaysBreakAfterDefinitionReturnType: None
-AlwaysBreakAfterReturnType: None
-AlwaysBreakBeforeMultilineStrings: false
-AlwaysBreakTemplateDeclarations: MultiLine
-BinPackArguments: true
-BinPackParameters: true
-BraceWrapping:
-  AfterCaseLabel:  false
-  AfterClass:      false
-  AfterControlStatement: Never
-  AfterEnum:       false
-  AfterFunction:   false
-  AfterNamespace:  false
-  AfterObjCDeclaration: false
-  AfterStruct:     false
-  AfterUnion:      false
-  AfterExternBlock: false
-  BeforeCatch:     false
-  BeforeElse:      false
-  BeforeLambdaBody: false
-  BeforeWhile:     false
-  IndentBraces:    false
-  SplitEmptyFunction: true
-  SplitEmptyRecord: true
-  SplitEmptyNamespace: true
-BreakBeforeBinaryOperators: None
-BreakBeforeBraces: Attach
-BreakBeforeInheritanceComma: false
-BreakInheritanceList: BeforeColon
-BreakBeforeTernaryOperators: true
-BreakConstructorInitializersBeforeComma: false
-BreakConstructorInitializers: BeforeColon
-BreakAfterJavaFieldAnnotations: false
-BreakStringLiterals: true
-ColumnLimit:     80
-CommentPragmas:  '^ IWYU pragma:'
-CompactNamespaces: false
-ConstructorInitializerAllOnOneLineOrOnePerLine: false
-ConstructorInitializerIndentWidth: 4
-ContinuationIndentWidth: 4
-Cpp11BracedListStyle: true
-DeriveLineEnding: true
-DerivePointerAlignment: false
-DisableFormat:   false
-ExperimentalAutoDetectBinPacking: false
-FixNamespaceComments: true
-ForEachMacros:
-  - foreach
-  - Q_FOREACH
-  - BOOST_FOREACH
-IncludeBlocks:   Preserve
-IncludeCategories:
-  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
-    Priority:        2
-    SortPriority:    0
-  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
-    Priority:        3
-    SortPriority:    0
-  - Regex:           '.*'
-    Priority:        1
-    SortPriority:    0
-IncludeIsMainRegex: '(Test)?$'
-IncludeIsMainSourceRegex: ''
-IndentCaseLabels: false
-IndentCaseBlocks: false
-IndentGotoLabels: true
-IndentPPDirectives: None
-IndentExternBlock: AfterExternBlock
-IndentWidth:     4
-IndentWrappedFunctionNames: false
-InsertTrailingCommas: None
-JavaScriptQuotes: Leave
-JavaScriptWrapImports: true
-KeepEmptyLinesAtTheStartOfBlocks: true
-MacroBlockBegin: ''
-MacroBlockEnd:   ''
-MaxEmptyLinesToKeep: 2
-NamespaceIndentation: None
-ObjCBinPackProtocolList: Auto
-ObjCBlockIndentWidth: 2
-ObjCBreakBeforeNestedBlockParam: true
-ObjCSpaceAfterProperty: false
-ObjCSpaceBeforeProtocolList: true
-PenaltyBreakAssignment: 2
-PenaltyBreakBeforeFirstCallParameter: 19
-PenaltyBreakComment: 300
-PenaltyBreakFirstLessLess: 120
-PenaltyBreakString: 1000
-PenaltyBreakTemplateDeclaration: 10
-PenaltyExcessCharacter: 1000000
-PenaltyReturnTypeOnItsOwnLine: 60
-PointerAlignment: Right
-ReflowComments:  true
-SortIncludes:    false
-SortUsingDeclarations: true
-SpaceAfterCStyleCast: false
-SpaceAfterLogicalNot: false
-SpaceAfterTemplateKeyword: true
-SpaceBeforeAssignmentOperators: true
-SpaceBeforeCpp11BracedList: false
-SpaceBeforeCtorInitializerColon: true
-SpaceBeforeInheritanceColon: true
-SpaceBeforeParens: ControlStatements
-SpaceBeforeRangeBasedForLoopColon: true
-SpaceInEmptyBlock: false
-SpaceInEmptyParentheses: false
-SpacesBeforeTrailingComments: 2
-SpacesInAngles:  false
-SpacesInConditionalStatement: false
-SpacesInContainerLiterals: true
-SpacesInCStyleCastParentheses: false
-SpacesInParentheses: false
-SpacesInSquareBrackets: false
-SpaceBeforeSquareBrackets: false
-Standard:        Cpp11
-StatementMacros:
-  - Q_UNUSED
-  - QT_REQUIRE_VERSION
-TabWidth:        4
-UseCRLF:         false
-UseTab:          Never
-WhitespaceSensitiveMacros:
-  - STRINGIZE
-  - PP_STRINGIZE
-  - BOOST_PP_STRINGIZE
-...

.clangd

@@ -1,12 +0,0 @@
-Diagnostics:
-  ClangTidy:
-    Add: [performance-*, modernize-*, readability-*,bugprone-*,]
-    Remove: [modernize-use-trailing-return-type]
-    CheckOptions:
-      readability-identifier-naming.VariableCase: camelBack
-  UnusedIncludes: Strict
-
-InlayHints:
-  Enabled: No
-  ParameterNames: No
-  DeducedTypes: No

.gitignore

@@ -78,7 +78,6 @@ runlog/
 .gdb_history
 
 .clang-format
-.clangd
 !curve-snapshotcloneserver-nginx/app/lib
 !nebd/nebd-package/usr/bin
 
@@ -91,7 +90,6 @@ test/client/configs/*
 projects/*
 docker/curvebs
 docker/base/*
-docker/debian*/*so*
 !docker/base/Dockerfile
 !docker/base/Makefile
 
@@ -109,7 +107,6 @@ curvefs/monitor/prometheus/target.json
 curvefs/docker/*/curvefs
 curvefs/docker/curvefs
 curvefs/docker/base/*
-curvefs/docker/debian*/*so*
 !curvefs/docker/base/Dockerfile
 !curvefs/docker/base/Makefile
 curvefs/BUILD_MODE
@@ -147,22 +144,3 @@ tools-v2/proto/curvefs/*
 tools-v2/*/*.test
 tools-v2/__debug_bin
 tools-v2/vendor/
-
-.test
-.note
-.playground
-.dumpfile
-metastore_test.dat
-GPATH
-GRTAGS
-GTAGS
-core.*
-
-test/integration/*.conf
-test/integration/client/config/client.conf*
-test/integration/snapshotcloneserver/config/*.conf
-
-.pre-commit-config.yaml
-
-*.deb
-*.whl

.obm.cfg

@@ -1,2 +0,0 @@
-container_name: curve-build-playground-master
-container_image: opencurvedocker/curve-base:build-debian9

CHANGELOG-2.5.md

@@ -1,49 +0,0 @@
-# CHANGELOG of v2.5
-
-Previous change logs can be found at [CHANGELOG-2.4](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.4.md)
-
-
-
-## New Features
-
-- Memcache support
-    - [curvefs/client: support memcached cluster](https://github.com/opencurve/curve/pull/2096) @ilixiaocui
-    - [curvefs/mds: support memcache cluster](https://github.com/opencurve/curve/pull/2108) @Cyber-SiKu
-    - [curvefs/client: add global cache client like memcached](https://github.com/opencurve/curve/pull/2102) @fansehep
-    - [curvefs/client: fix client core dump error](https://github.com/opencurve/curve/pull/2157) @ilixiaocui
-    - [add ut](https://github.com/opencurve/curve/pull/2164) @Cyber-SiKu
-
-- curvefs new tools 
-    - [add delete impl](https://github.com/opencurve/curve/pull/2088) @shentupenghui
-    - [feat: [tools-v2] bs list dir](https://github.com/opencurve/curve/pull/2082) @Sindweller
-    - [fix missing docs links](https://github.com/opencurve/curve/pull/2082) @zyb521
-    - [curve/toos-v2: add list client #2037](https://github.com/opencurve/curve/pull/2076) @tsonglew
-    - [support create volume directory by curve_ops_tool](https://github.com/opencurve/curve/pull/2078) @aspirer
-
-- [install: added playground.](https://github.com/opencurve/curve/pull/2053) @Wine93
-
-## Optimization
-
-- [update the braft to vesion v1.1.2](https://github.com/opencurve/curve/pull/2091) @tangwz
-
-- [curvefs/client: local cache policy optimization](https://github.com/opencurve/curve/pull/2064) @Tangruilin
-
-
-- [sync by threadpool](https://github.com/opencurve/curve/pull/1912) @fansehep
-
-- Merge block storage and file storage compilation scripts
-    - [Merge block storage and file storage compilation scripts
-](https://github.com/opencurve/curve/pull/2089) @linshiyx
-    - [Add help_msg in Makefile](https://github.com/opencurve/curve/pull/2133) @linshiyx
-    - [fix image.sh](https://github.com/opencurve/curve/pull/2124) @SeanHai
-
-- [fix bs list zone typo](https://github.com/opencurve/curve/pull/2066) @tsonglew 
-
-- [spell optimize](https://github.com/opencurve/curve/pull/2059) @shentupenghui
-
-
-## Bug Fix
-
-- aws s3 sdk revert
-    - [revert new s3 sdk due to critical bug](https://github.com/opencurve/curve/pull/2149) @h0hmj
-    - [curvefs/client: change the default value of s3](https://github.com/opencurve/curve/pull/2158) @wuhongsong

Community_Guidelines.md

@@ -1,4 +1,4 @@
-[中文版](developers_guide_cn.md)
+[中文版](Community_Guidelines_cn.md)
 
 
 ## Overview
@@ -18,15 +18,15 @@ The premise of participating in an open source project is to understand it, espe
 
 Through the study of the above materials, I believe that you already have an overall understanding of the Curve project, and there may also be some curiosity and doubts. At this time, you can deploy a Curve experience environment, which is conducive to a more intuitive perception of the Curve system. If you encounter a problem or want a new feature, you can track the relevant code. In the process, it is easy to understand the relevant modules. This is lots of contributors completed their first contributions.
 
-The Curve community has multiple [communication channels](#Communication), and there will be an online community meeting every two weeks. The regular meeting will synchronize the recent progress of the Curve project and answer your questions. You are also welcome to participate in the Curve community regular meeting. We will communicate at the meeting, so that we can quickly answer your questions, improve our understanding of the Curve project and synchronize what we are doing at this stage.
+The Curve community has multiple [communication channels](#Comminication), and there will be an online community meeting every two weeks. The regular meeting will synchronize the recent progress of the Curve project and answer your questions. You are also welcome to participate in the Curve community regular meeting. We will communicate at the meeting, so that we can quickly answer your questions, improve our understanding of the Curve project and synchronize what we are doing at this stage.
 
 ## How to participate in Curve
 
 After you have a certain understanding of the Curve project, if you are interested, you can choose the starting point to participate in the Curve. You can choose from the following aspects:
 
-- Start by selecting a interested one from the [issues](https://github.com/opencurve/curve/issues) of the Curve project. You can pay attention to issues with the [good first issue](https://github.com/opencurve/curve/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) tag which we have assessed as relatively good starters.
+- Start by selecting a interested one from the [Issues](https://github.com/opencurve/curve/issues) of the Curve project. You can pay attention to issues with the good_first_issue tag which we have assessed as relatively good starters.
 - Based on a certain understanding of the Curve project, you can also choose from [Roadmap](https://github.com/opencurve/curve/wiki/Roadmap).
-- Selecting form [issues](https://github.com/opencurve/curveadm/issues) and [Roadmap](https://github.com/opencurve/curveadm/issues/92) of  Curve Operation and maintenance tools [Curveadm](https://github.com/opencurve/curveadm). Here are [quick start guidelines](https://github.com/opencurve/curveadm/wiki/develop)。
+- Selecting form [Issues](https://github.com/opencurve/curveadm/issues) and [Roadmap](https://github.com/opencurve/curveadm/issues/92) of  Curve Operation and maintenance tools [Curveadm](https://github.com/opencurve/curveadm). Here are [quick start guidelines](https://github.com/opencurve/curveadm/wiki/develop)。
 - In addition to the existing issues, you are also welcome to submit issues that you have discovered or new features you hope for and resolve them.
 - You can pay attention to the *TODO* in the existing Curve code, most of which are code logic optimization and features to be supplemented, choose the ones you are interested in and raise relevant issues to follow up and try to solve.
 
@@ -87,12 +87,6 @@ If the work to be done is more complicated, you need to write a detailed design
 It is also recommended to communicate with us before starting the plan writing. If you are not sure about the feasibility of the plan, you can provide a simple plan description first, and then refine and develop it after our evaluation.
 
 
-## How to start a document contribution
-
-The curve docs path is in [curve/docs/](https://github.com/opencurve/curve/tree/master/docs), please use markdown format about the docs contribution except ppt.
-
-Document contribution don't need to trigger CI. please add ```[skipci]``` in your github pr title.
-
 ## How to submit a PR
 
 Once you've finished writing the code, you can submit a PR. Of course, if the development has not been completed, you can submit PR first in some cases. For example, if you want to let the community take a look at the general solution, you can raise the price of PR after completing the code framework.
@@ -101,46 +95,15 @@ Once you've finished writing the code, you can submit a PR. Of course, if the de
 
 [Build and run Curve tests](https://github.com/opencurve/curve/blob/master/docs/en/build_and_run_en.md#test-case-compilation-and-execution)
 
-Please use clangd for your code completions. More details is in [curve_clangd.md](https://github.com/opencurve/curve/blob/master/docs/cn/clangd.md).
-
-Curve Ci use ```cpplint``` check what your changed.
-
-- install ``cpplint``` (need root)
-  ```bash
-  $ pip install cpplint
-  ```
-
-- check your changed local
-  ```bash
-  $ cpplint --filter=-build/c++11 --quiet --recursive your_path
-  ```
-
 For PR we have the following requirements:
 
-- The Curve coding standard strictly follows the [Google C++ Open Source Project Coding Guide](https://google.github.io/styleguide/cppguide.html), but we use 4 spaces to indent, Clang-format will more helpful for you. Of course, CI will check what your changed. 
+- The CURVE coding standard strictly follows the [Google C++ Open Source Project Coding Guide](https://google.github.io/styleguide/cppguide.html), please follow this guide as well Submit your code.
 - The code must have test cases, excluding documentation, unit tests (incremental lines cover more than 80%, and incremental branches cover more than 70%); integration tests (merge statistics with unit tests, and meet the unit test coverage requirements).
 - Please fill in the description of the PR as detailed as possible,  associate with the relevant issues, and the PR commit message can clearly see the resolved issues. After submitting to the Curve master branch, Curve CI will be triggered automatically. It is necessary to ensure that the CI is passed, and the Jenkins username and password of the CI is netease/netease, if the CI fails to run, you can log in to the Jenkins platform to view the reason for the failure.
 - After the CI is passed, the review can start, and each PR needs to get at least two LGTMs of Committer/Maintainer before merging.
 - PR code requires a certain amount of comments to make the code easy to understand, and all comments and review comments and replies are required to be in English.
 
-- Please make sure your changed can pass the compile locally.
-
-- We usually merge only one commit after you rebase.
-
-- Trigger CI please comment ```cicheck```.
-  
-  CI checkpoints:
-    
-    1. The changed code style check(cpplint).
-    2. Branch test coverage testing.
-    3. Unit tests(If failed, run the failed tests locally).
-    4. Chaos tests.
-
-Repush will trigger CI, If github page have no reaction. Please wait.
-
-If CI is not stabled, repeatedly comment ```cicheck``` will trigger CI again.
-
-## Communication
+## Comminication
 
 At present, the Curve community has multiple communication channels, please choose an appropriate and efficient communication method according to your needs:
 
@@ -151,4 +114,4 @@ At present, the Curve community has multiple communication channels, please choo
 - **slack**: cloud-native.slack.com，channel #project_curve
 - **Curve User Group**: In order to facilitate instant communication with everyone, the Curve User Group is currently a WeChat group. Due to the large number of people in the group, it is necessary to add OpenCurve_bot WeChat first, and then invite into the group. In the user group, everyone can freely communicate about Curve and storage-related topics, and get immediate feedback on problems.
 
-<img src="docs/images/curve-wechat.jpeg" style="zoom: 75%;" />
+<img src="docs/images/curve-wechat.jpeg" style="zoom: 75%;" />

Community_Guidelines_cn.md

@@ -1,4 +1,4 @@
-[English version](developers_guide.md)
+[English version](Community_Guidelines.md)
 
 
 ## 概述
@@ -24,9 +24,9 @@ Curve 社区有多个[沟通渠道](#社区交流)，每两周还会有线上的
 
 在对 Curve 项目有一定了解后，如果感兴趣就可以选择入手点进行 Curve 开源项目的参与了，可以从以下几个方面进行选择：
 
-- 从 Curve 项目的 [issue](https://github.com/opencurve/curve/issues) 中选择感兴趣的问题入手，可以特别关注带有 [good first issue](https://github.com/opencurve/curve/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc+label%3A%22good+first+issue%22) tag 的 issue，这些是我们经过评估认为是相对好入门的一些问题。
+- 从 Curve 项目的 [Issue](https://github.com/opencurve/curve/issues) 中选择感兴趣的问题入手，可以特别关注带有 good_first_issue tag 的 issue，这些是我们经过评估认为是相对好入门的一些问题。
 - 在对 Curve 项目有一定了解的基础上也可以从 [Roadmap](https://github.com/opencurve/curve/wiki/Roadmap_CN) 中进行选择。
-- 从 Curve 项目的运维工具 [Curveadm](https://github.com/opencurve/curveadm) 的 [issue](https://github.com/opencurve/curveadm/issues) 和 [Roadmap](https://github.com/opencurve/curveadm/issues/92) 中进行选择，开发方式可参见 [快速上手 CurveAdm 开发](https://github.com/opencurve/curveadm/wiki/develop)。
+- 从 Curve 项目的运维工具 [Curveadm](https://github.com/opencurve/curveadm) 的 [Issue](https://github.com/opencurve/curveadm/issues) 和 [Roadmap](https://github.com/opencurve/curveadm/issues/92) 中进行选择，开发方式可参见[快速上手 CurveAdm 开发](https://github.com/opencurve/curveadm/wiki/develop)。
 - 除了已有的 issue，也欢迎将自己发现的问题或希望的新特性提出 issue 并进行解决。
 - 可以关注现有 Curve 代码中的 *TODO* ，大部分为代码逻辑优化和待补充特性，选择感兴趣的提出相关 issue 跟进并尝试解决。
 
@@ -45,11 +45,6 @@ Curve 社区有多个[沟通渠道](#社区交流)，每两周还会有线上的
 
 在开始方案撰写之前同样建议先与我们进行沟通，对方案的可行性不太确定的可以先提供简单的方案描述，经过我们评估后再进行细化和开发。
 
-## 如何进行文档贡献
-
-Curve 的文档位于 [curve/docs/](https://github.com/opencurve/curve/tree/master/docs), 贡献请使用 markdown 格式(除了ppt).
-
-文档贡献无需触发 CI, 请在 github pull request 标题头部加上 ```[skipci]```.
 
 ## 如何提交PR
 
@@ -59,23 +54,11 @@ Curve 的文档位于 [curve/docs/](https://github.com/opencurve/curve/tree/mast
 
 [Curve 测试用例编译及执行](https://github.com/opencurve/curve/blob/master/docs/cn/build_and_run.md#测试用例编译及执行)
 
-代码补全: 请使用 clangd 作为代码补全, 请见 [curve_clangd.md](https://github.com/opencurve/curve/blob/master/docs/cn/clangd.md).
-
-Curve CI 使用```cpplint```检查更改的代码,
-- 安装```cpplint``` (需要root):
-  ```bash
-  $ pip install cpplint
-  ```
-- 本地进行代码检查:
-  ```bash
-  $ cpplint --filter=-build/c++11 --quiet --recursive your_path
-  ```
-
 对于 PR 我们有如下要求：
 
-- Curve编码规范严格按照[Google C++开源项目编码指南](https://zh-google-styleguide.readthedocs.io/en/latest/google-cpp-styleguide/contents/)来进行代码编写，但使用 4 空格进行缩进, 可使用 clang-format 进行格式化, CI 会检查相关更改代码是否符合规则.
+- CURVE编码规范严格按照[Google C++开源项目编码指南](https://zh-google-styleguide.readthedocs.io/en/latest/google-cpp-styleguide/contents/)来进行代码编写，请您也遵循这一指南来提交您的代码。
 - 代码必须有测试，文档除外，单元测试（增量行覆盖80%以上，增量分支覆盖70%以上）；集成测试（与单元测试合并统计，满足单元测试覆盖率要求即可）
-- 请尽可能详细的填写 PR 的描述，关联相关问题的 issue，PR commit message 能清晰看出解决的问题，提交到 Curve master 分支之后会自动触发Curve CI，需保证 CI 通过，CI 的 Jenkins 用户名密码为 netease/netease，如遇到 CI 运行失败可以登录 Jenkins 平台查看失败原因。
+- 请尽可能详细的填写 PR 的描述，关联相关问题的 issuse，PR commit message 能清晰看出解决的问题，提交到 Curve master 分支之后会自动触发Curve CI，需保证 CI 通过，CI 的 Jenkins 用户名密码为 netease/netease，如遇到 CI 运行失败可以登录 Jenkins 平台查看失败原因。
 - CI 通过之后可开始进行 review，每个 PR 在合并之前都需要至少得到两个 Committer/Maintainer 的 LGTM。
 - PR 代码需要一定量的注释来使代码容易理解，且所有注释和 review 意见和回复均要求使用英语。
 
@@ -119,25 +102,6 @@ type 可以是以下类型之一：
 
 footer 是可选的，用来记录对应因为这些更改而可以关闭的 issue，如 `Close #12345`
 
-提交 PR 之前, 请确保你的更改可以成功在本地编译.
-
-与大多数项目一样, review 通过之后请将你诸多commits rebase 成为一个 commit.
-
-触发 CI 请 comment ```cicheck```.
-  
-若发现测试错误, 可查看 CI 对应错误测试.
-
-CI 检查点有:
-  
-  1. 更改代码风格检查(cpplint)
-  2. 分支覆盖率检查(可能会失败)
-  3. 单元测试 (若失败请先尝试在本地运行对应失败测试)
-  4. 混沌测试
-  
-重新 push 会触发 CI, 若暂时无反应, 请耐心等待, 测试处于排队中.
-
-若 CI 不稳定, 可重复 comment ```cicheck``` 以重新触发 CI.
-
 ## 社区交流
 
 目前 Curve 社区有多个沟通交流的渠道，请根据自己的需求选择合适高效的沟通方式：
@@ -149,4 +113,4 @@ CI 检查点有:
 - **slack**: cloud-native.slack.com，channel #project_curve
 - **Curve User Group**: 为了便于大家即时的沟通，Curve User Group 目前为微信群，由于群人数过多，需要先添加OpenCurve_bot微信，再邀请进群。在用户群里大家可以自由的沟通关于 Curve 和存储相关的话题，对于存在问题也可以较为即时的得到反馈。
 
-<img src="docs/images/curve-wechat.jpeg" style="zoom: 75%;" />
+<img src="docs/images/curve-wechat.jpeg" style="zoom: 75%;" />

Makefile

@@ -1,6 +1,6 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build dep install image playground check test
+.PHONY: list build dep install image
 
 stor?=""
 prefix?= "$(PWD)/projects"
@@ -70,12 +70,3 @@ install:
 
 image:
 	@bash util/image.sh $(stor) $(tag) $(os)
-
-playground:
-	@bash util/playground.sh
-
-check:
-	@bash util/check.sh $(stor)
-
-test:
-	@bash util/test.sh $(stor) $(only)

README.md

@@ -4,9 +4,7 @@
 
 <div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
 
-**A cloud-native distributed storage system**
-
-**A sandbox project hosted by the CNCF Foundation**
+**A could-native distributed storage system**
 
 #### English | [简体中文](README_cn.md)
 ### 📄 [Documents](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [Official Website](https://www.opencurve.io/Curve/HOME) || 🏠 [Forum](https://ask.opencurve.io/t/topic/7)
@@ -44,24 +42,11 @@
   - [LICENSE](#license)
   - [Release Cycle](#release-cycle)
   - [Branch](#branch)
-  - [Contributors](#contributors)
-  - [Star History](#star-history)
   - [Feedback & Contact](#feedback--contact)
 
 ##  About Curve
 
-**Curve** is a modern storage system developed by netease, currently supporting file storage(CurveFS) and block storage(CurveBS). Now it's hosted at [CNCF](https://www.cncf.io/) as a sandbox project.
-
-The core application scenarios of CurveBS mainly include:
-- the performance, mixed, capacity cloud disk or persistent volume of virtual machine/container, and remote disks of physical machines
-- high-performance separation of storage and computation architecture: high-performance and low latency architecture based on RDMA+SPDK, supporting the separation deployment structure of various databases such as MySQL and Kafka
-
-
-The core application scenarios of CurveFS mainly include:
-- the cost-effective storage in AI training scene
-- the hot and cold data automation layered storage in big data scenarios
-- the cost-effective shared file storage on the public cloud: It can be used for business scenarios such as AI, big data, file sharing
-- Hybrid storage: Hot data is stored in the local IDC, cold data is stored in public cloud
+**Curve** is a mordern storage system developed by netease, currently supporting file storage(CurveFS) and block storage(CurveBS). Now it's hosted at [CNCF](https://www.cncf.io/) as a sandbox project.
 
 <details>
   <summary><b><font=5>High Performance | More stable | Easy Operation | Cloud Native</b></font></summary>
@@ -126,7 +111,7 @@ The core application scenarios of CurveFS mainly include:
 
 </details>
 <details>
-  <summary><b><font=5>Docking Kubernetes</b></font></summary>
+  <summary><b><font=5>Docking Kubernates</b></font></summary>
 
 - Use [Curve CSI Driver](https://github.com/opencurve/curve-csi), The plugin implements the Container Storage Interface(CSI) between Container Orchestrator(CO) and Curve cluster. It allows dynamically provisioning curve volumes and attaching them to workloads.
 - For details of the documentation, see [CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md).
@@ -158,7 +143,7 @@ Curve supports deployment in private and public cloud environments, and can also
 <div align=center> <image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
 <div align=left>
 
-One of them, CurveFS shared file storage system, can be elastically scaled to public cloud storage, which can provide users with greater capacity elasticity, lower cost, and better performance experience.
+One of them, CurveFS shared file storage system, can be elasticly scaled to public cloud storage, which can provide users with greater capacity elasticity, lower cost, and better performance experience.
 
 </details>
 
@@ -214,8 +199,6 @@ Fio Curve engine is added, you can clone https://github.com/opencurve/fio and co
 $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
 ```
 
-If you have any questions during performance testing, please check the [Curve block storage performance tuning guide](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md).
-
 ##  CurveFS quick start
 Please use [CurveAdm](https://github.com/opencurve/curveadm/wiki) tool to deploy CurveFS，see [CurveFS Deployment Process](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), and the [CurveFS Command Instructions](curvefs/src/tools#readme).
 
@@ -225,13 +208,12 @@ Please refer to the [Test environment configuration](docs/cn/测试环境配置
 
 ## Practical
 - [CurveBS+NFS Build NFS Server](docs/practical/curvebs_nfs.md)
-- [CurveFS+MinIO S3 Gateway](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
 
 ## Governance
 See [Governance](https://github.com/opencurve/community/blob/master/GOVERNANCE.md).
 
 ##  Contribute us
-Participation in the Curve project is described in the [Curve Developers Guidelines](developers_guide.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
+Participation in the Curve project is described in the [Curve Open Source Community Guidelines](Community_Guidelines.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
 We welcome your contribution!
 
 ## Code of Conduct
@@ -248,19 +230,6 @@ Curve is distributed under the [**Apache 2.0 LICENSE**](LICENSE).
 ## Branch
 All the developments will be done under master branch. If there's any new version to establish, a new branch release-x.y will be pulled from the master, and the new version will be released from this branch.
 
-## Contributors
-This project exists thanks to all the people who contribute.
-
-<a href="https://github.com/opencurve/curve/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=opencurve/curve" />
-</a>
-
-Made with [contrib.rocks](https://contrib.rocks).
-
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=opencurve/curve&type=Date)](https://star-history.com/#opencurve/curve&Date)
-
 ## Feedback & Contact
 - [Github Issues](https://github.com/openCURVE/CURVE/issues)：You are sincerely welcomed to issue any bugs you came across or any suggestions through Github issues. If you have any question you can refer to our FAQ or join our user group for more details.
 - [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：Frequently asked question in our user group, and we'll keep working on it.

README_cn.md

@@ -4,9 +4,7 @@
 
 <div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
 
-**云原生高性能分布式存储系统**
-
-**CNCF基金会的沙箱托管项目**
+**A could-native distributed storage system**
 
 #### [English](README.md) | 简体中文
 ### 📄 [文档](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [官网](https://www.opencurve.io/Curve/HOME) || 🏠 [论坛](https://ask.opencurve.io/t/topic/7)
@@ -50,18 +48,6 @@
 ##  关于 Curve
 **Curve** 是网易主导自研的现代化存储系统, 目前支持文件存储(CurveFS)和块存储(CurveBS)。现作为沙箱项目托管于[CNCF](https://www.cncf.io/)。
 
-CurveBS的核心应用场景主要包括：
-- 虚拟机/容器的性能型、混合型、容量型云盘或持久化卷，以及物理机的远程存储盘
-- 高性能存算分离架构：基于RDMA+SPDK的高性能低时延架构，支撑MySQL、kafka等各类数据库、中间件的存算分离部署架构，提升实例交付效率和资源利用率
-
-
-CurveFS的核心应用场景主要包括：
-- AI训练（含机器学习等）场景下的高性价比存储
-- 大数据场景下的冷热数据自动化分层存储
-- 公有云上高性价比的共享文件存储：可用于AI、大数据、文件共享等业务场景
-- 混合云存储：热数据存储在本地IDC，冷数据存储在公有云
-
-
 <details>
   <summary><b><font=5>高性能 | 更稳定 | 易运维 | 云原生</b></font></summary>
 
@@ -126,7 +112,7 @@ CurveFS的核心应用场景主要包括：
 
 </details>
 <details>
-  <summary><b><font=5>对接 Kubernetes</b></font></summary>
+  <summary><b><font=5>对接 Kubernates</b></font></summary>
 
 - 使用 [Curve CSI Driver](https://github.com/opencurve/curve-csi) 插件在 Container Orchestrator (CO) 与 Curve 集群中实现了 Container Storage Interface(CSI)。
 - 文档详见[CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md)。
@@ -210,8 +196,6 @@ fio的Curve块存储引擎代码已经上传到 https://github.com/opencurve/fio
 $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
 ```
 
-在性能测试过程中有任何问题，请查看[Curve块存储性能调优指南](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md)
-
 ##  CurveFS 快速体验
 请使用 [CurveAdm](https://github.com/opencurve/curveadm/wiki) 工具进行 CurveFS 的部署，具体流程见：[CurveFS部署流程](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), 以及[CurveFS命令行工具说明](curvefs/src/tools#readme)。
 
@@ -223,11 +207,10 @@ $ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest
 
 ##  贡献我们
 
-参与 Curve 项目开发详见[Curve 开发者指南](developers_guide_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
+参与 Curve 项目开发详见[Curve 开源社区指南](Community_Guidelines_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
 
 ## 最佳实践
 - [CurveBS+NFS搭建NFS存储](docs/practical/curvebs_nfs.md)
-- [CurveFS+S3网关部署实践](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
 
 ##  行为守则
 Curve 的行为守则遵循[CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md)。

WORKSPACE

@@ -75,14 +75,11 @@ bind(
 )
 
 #import the  gtest files.
-http_archive(
+new_git_repository(
     name = "com_google_googletest",
-    urls = [
-        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
-        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
-    ],
-    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
-    strip_prefix = "googletest-release-1.12.1",
+    build_file = "//:thirdparties/gmock.BUILD",
+    remote = "https://github.com/google/googletest",
+    tag = "release-1.8.0",
 )
 
 bind(
@@ -231,24 +228,6 @@ http_archive(
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
 
-# fmt
-http_archive(
-  name = "fmt",
-  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
-  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
-  strip_prefix = "fmt-9.1.0",
-  build_file = "//:thirdparties/fmt.BUILD",
-)
-
-# spdlog
-http_archive(
-  name = "spdlog",
-  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
-  strip_prefix = "spdlog-1.11.0",
-  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
-  build_file = "//:thirdparties/spdlog.BUILD",
-)
-
 # Bazel platform rules.
 http_archive(
     name = "platforms",
@@ -269,14 +248,14 @@ new_local_repository(
 http_archive(
     name = "hedron_compile_commands",
 
-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
+    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. 
     # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
     urls = [
         "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
         "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
     ],
     strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
+    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." 
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()

buildfs.sh

@@ -7,22 +7,12 @@ then
     exit
 fi
 
-if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
-then
-    bazelflags=''
-else
-    bazelflags='--copt -faligned-new'
-fi
-
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi
 
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
---define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
--DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
-
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
 if [ $? -ne 0 ]
 then
     echo "build curvefs failed"
@@ -44,5 +34,4 @@ then
     echo "mds_test failed"
     exit
 fi
-fi
-echo "end compile"
+fi

conf/chunkserver.conf

@@ -9,20 +9,13 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
-# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
-# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
-# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
-# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
-global.block_size=4096
-
 # clone chunk允许的最长location长度
 global.location_limit=3000
+# minimum alignment for io request
+global.min_io_alignment=512
 
 #
 # MDS settings
@@ -115,7 +108,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=true
+copyset.enable_odsync_when_open_chunkfile=false
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/chunkserver.conf.example

@@ -13,12 +13,9 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
-# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
-# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
-# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
-# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
-global.block_size=4096
 global.location_limit=3000
+# minimum alignment for io request
+global.min_io_alignment=512
 
 #
 # MDS settings
@@ -111,7 +108,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=true
+copyset.enable_odsync_when_open_chunkfile=false
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/client.conf

@@ -168,6 +168,13 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
 
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
+
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@@ -181,3 +188,4 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
+

conf/cs_client.conf

@@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

conf/mds.conf

@@ -156,7 +156,7 @@ mds.topology.CreateCopysetRpcRetryTimes=20
 # 请求chunkserver上创建copyset重试间隔
 mds.topology.CreateCopysetRpcRetrySleepTimeMs=1000
 # Topology模块刷新metric时间间隔
-mds.topology.UpdateMetricIntervalSec=10
+mds.topology.UpdateMetricIntervalSec=60
 #和mds.chunkserver.failure.tolerance设置有关,一个zone 标准配置20台节点，如果允许3台节点failover,
 #那么剩余17台机器需要承载原先20台机器的空间,17/20=0.85,即使用量超过这个值即不再往这个池分配,
 #具体分为来两种情况, 当不使用chunkfilepool,物理池限制使用百分比,当使用 chunkfilepool 进行chunkfilepool分配时需预留failover空间,
@@ -197,8 +197,6 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
-# smallest read/write unit for volume, support |512| and |4096|
-mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -239,16 +237,3 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
-
-#
-## poolset rules
-#
-# for backward compatibility, rules are applied for select poolset when creating file
-#
-# for example
-#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
-#
-# when creating file reqeust doesn't have poolset, above rules are used to select poolset
-# - if filename is /dir1/file, then poolset1 is select
-# - if filename is /dir1/sub/file, then sub is select
-mds.poolset.rules=

conf/py_client.conf

@@ -153,3 +153,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

conf/snap_client.conf

@@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

curve-ansible/roles/generate_config/defaults/main.yml

@@ -164,6 +164,7 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
+chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -243,6 +244,8 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
+client_alignment_common: 512
+client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -14,6 +14,8 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
+# minimum alignment for io request
+global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -175,3 +175,10 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume={{ client_alignment_common }}
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume={{ client_alignment_clone }}

curvefs/conf/client.conf

@@ -77,7 +77,16 @@ rpc.healthCheckIntervalSec=0
 
 #### fuseClient
 # TODO(xuchaojie): add unit
+fuseClient.attrTimeOut=1.0
+fuseClient.entryTimeOut=1.0
 fuseClient.listDentryLimit=65536
+fuseClient.flushPeriodSec=5
+fuseClient.maxNameLength=255
+fuseClient.iCacheLruSize=65536
+fuseClient.dCacheLruSize=1000000
+fuseClient.enableICacheMetrics=true
+fuseClient.enableDCacheMetrics=true
+fuseClient.cto=true
 fuseClient.downloadMaxRetryTimes=3
 
 ### kvcache opt
@@ -104,61 +113,6 @@ fuseClient.maxDataSize=1024
 fuseClient.refreshDataIntervalSec=30
 fuseClient.warmupThreadsNum=10
 
-# the write throttle bps of fuseClient, default no limit
-fuseClient.throttle.avgWriteBytes=0
-# the write burst bps of fuseClient, default no limit
-fuseClient.throttle.burstWriteBytes=0
-# the times that write burst bps can continue, default 180s
-fuseClient.throttle.burstWriteBytesSecs=180
-
-# the write throttle iops of fuseClient, default no limit
-fuseClient.throttle.avgWriteIops=0
-# the write burst iops of fuseClient, default no limit
-fuseClient.throttle.burstWriteIops=0
-# the times that write burst Iops can continue, default 180s
-fuseClient.throttle.burstWriteIopsSecs=180
-
-# the read throttle bps of fuseClient, default no limit
-fuseClient.throttle.avgReadBytes=0
-# the read burst bps of fuseClient, default no limit
-fuseClient.throttle.burstReadBytes=0
-# the times that read burst bps can continue, default 180s
-fuseClient.throttle.burstReadBytesSecs=180
-
-# the read throttle iops of fuseClient, default no limit
-fuseClient.throttle.avgReadIops=0
-# the read burst Iops of fuseClient, default no limit
-fuseClient.throttle.burstReadIops=0
-# the times that read burst Iops can continue, default 180s
-fuseClient.throttle.burstReadIopsSecs=180
-
-#### filesystem metadata
-# {
-# fs.disableXattr:
-#   if you want to get better metadata performance,
-#   you can mount fs with |fs.disableXattr| is true
-#
-# fs.lookupCache.negativeTimeoutSec:
-#   entry which not found will be cached if |timeout| > 0
-fs.cto=true
-fs.maxNameLength=255
-fs.disableXattr=false
-fs.accessLogging=true
-fs.kernelCache.attrTimeoutSec=3600
-fs.kernelCache.dirAttrTimeoutSec=3600
-fs.kernelCache.entryTimeoutSec=3600
-fs.kernelCache.dirEntryTimeoutSec=3600
-fs.lookupCache.negativeTimeoutSec=0
-fs.lookupCache.minUses=1
-fs.lookupCache.lruSize=100000
-fs.dirCache.lruSize=5000000
-fs.openFile.lruSize=65536
-fs.attrWatcher.lruSize=5000000
-fs.rpc.listDentryLimit=65536
-fs.deferSync.delay=3
-fs.deferSync.deferDirMtime=false
-# }
-
 #### volume
 volume.bigFileSize=1048576
 volume.volBlockSize=4096
@@ -180,6 +134,8 @@ volume.blockGroup.allocateOnce=4
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false
+# the max size that fuse send
+s3.fuseMaxSize=131072
 s3.pageSize=65536
 # prefetch blocks that disk cache use
 s3.prefetchBlocks=1
@@ -194,11 +150,8 @@ s3.baseSleepUs=500
 s3.threadScheduleInterval=3
 # data cache flush wait time
 s3.cacheFlushIntervalSec=5
-# write cache < 8,388,608 (8MB) is not allowed
 s3.writeCacheMaxByte=838860800
 s3.readCacheMaxByte=209715200
-# file cache read thread num
-s3.readCacheThreads=5
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=False

curvefs/conf/mds.conf

@@ -6,9 +6,6 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
 mds.loglevel=0
-# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
-# Default value is -1 which disables the feature.
-mds.server.idleTimeoutSec=-1
 
 #
 # space options

curvefs/conf/metaserver.conf

@@ -128,25 +128,16 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000
 
-#
-# Concurrent apply queue
-### concurrent apply queue options for each copyset
-### concurrent apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
+### apply queue options for each copyset
+### apply queue is used to isolate raft threads, each worker has its own queue
+### when a task can be applied it's been pushed into a corresponding worker queue by certain rules
+# number of apply queue workers for each, each worker will start a indepent thread
+applyqueue.worker_count=3
 
-# worker_count: number of apply queue workers for each, each worker will start a indepent thread
-# queue_depth: apply queue depth for each copyset
+# apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-# write apply queue workers count
-applyqueue.write_worker_count=3
-# write apply queue depth
-applyqueue.write_queue_depth=1
-# read apply queue workers count
-applyqueue.read_worker_count=2
-# read apply queue depth
-applyqueue.read_queue_depth=1
-
+applyqueue.queue_depth=1
 
 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@@ -154,10 +145,6 @@ applyqueue.read_queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto
 
-# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
-# Default value is -1 which disables the feature.
-server.idleTimeoutSec=-1
-
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need
@@ -280,43 +267,3 @@ storage.rocksdb.perf_sampling_ratio=0
 # we will sending its with rpc streaming instead of
 # padding its into inode (default: 25000, about 25000 * 41 (byte) = 1MB)
 storage.s3_meta_inside_inode.limit_size=25000
-
-# recycle options
-# metaserver scan recycle period, default 1h
-recycle.manager.scanPeriodSec=3600
-# metaserver recycle cleaner scan list dentry limit, default 1000
-recycle.cleaner.scanLimit=1000
-
-#### excutorOpt
-# excutorOpt rpc with metaserver
-# rpc retry times with metaserver
-excutorOpt.maxRetry=4294967295
-# internal rpc retry times with metaserver
-excutorOpt.maxInternalRetry = 3
-# Retry sleep time between failed RPCs
-excutorOpt.retryIntervalUS=100000
-# RPC timeout for communicating with metaserver
-excutorOpt.rpcTimeoutMS=1000
-# RPC stream idle timeout
-excutorOpt.rpcStreamIdleTimeoutMS=500
-# The maximum timeout RPC time of the retry request.
-# The timeout time will follow the exponential backoff policy.
-# Because the timeout occurs when the network is congested, the RPC timeout needs to be increased
-excutorOpt.maxRPCTimeoutMS=8000
-# Maximum sleep time between retry requests.
-# when the network is congested or the metaserver is overloaded,
-# it is necessary to increase the sleep time
-excutorOpt.maxRetrySleepIntervalUS=8000000
-excutorOpt.minRetryTimesForceTimeoutBackoff=5
-excutorOpt.maxRetryTimesBeforeConsiderSuspend=20
-# batch limit of get inode attr and xattr
-excutorOpt.batchInodeAttrLimit=10000
-excutorOpt.enableMultiMountPointRename=true
-
-#### metaCacheOpt
-# Gets the number of retries for the leader
-metaCacheOpt.metacacheGetLeaderRetry=3
-# Need to sleep for a period of time before each get leader retry
-metaCacheOpt.metacacheRPCRetryIntervalUS=100000
-# RPC timeout of get leader
-metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000

curvefs/conf/tools.conf

@@ -24,7 +24,6 @@ volumeName=volume
 volumeUser=user
 volumePassword=password
 volumeBlockGroupSize=134217728
-volumeCluster=127.0.0.1:6666,127.0.0.1:6667,127.0.0.1:6668
 # support |AtStart| and |AtEnd|
 volumeBitmapLocation=AtStart
 # s3
@@ -35,12 +34,5 @@ s3.bucket_name=bucket
 s3.blocksize=4194304
 s3.chunksize=67108864
 s3.useVirtualAddressing=false
-# s3 objectPrefix, if set 0, means no prefix, if set 1, means inode prefix
-# if set 2 and other values mean hash prefix
-s3.objectPrefix=0
 # statistic info in xattr, hardlink will not be supported when enable
 enableSumInDir=true
-
-# fs recycle, if set 0, disable fs recycle, delete files directly,
-# if set not 0, enable fs recycle, delete files after a period of time
-recycleTimeHour=0

curvefs/docker/debian10/Dockerfile

@@ -0,0 +1,11 @@
+FROM opencurvedocker/curve-base:debian10
+ENV TZ=Asia/Shanghai
+RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
+COPY curvefs /curvefs
+COPY entrypoint.sh /
+COPY curvefs/tools/sbin/curvefs_tool /usr/bin
+COPY libmemcached.so /usr/lib/
+COPY libmemcached.so.11 /usr/lib/
+COPY libhashkit.so.2 /usr/lib
+RUN chmod a+x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian10/base/Dockerfile

@@ -0,0 +1,39 @@
+FROM opencurvedocker/curve-base:debian10
+
+RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        gcc \
+        gdb \
+        make \
+        openssl \
+        net-tools \
+        libcurl3-gnutls \
+        perl \
+        linux-perf \
+        vim \
+        curl \
+        cron \
+        procps \
+        lsof \
+        nginx \
+        less \
+        fuse3 \
+        libnl-3-200 \
+        libnl-genl-3-200 \
+        libjemalloc2 \
+        libsnappy1v5 \
+        liblz4-1 \
+        zlib1g \
+    && ln -sf /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
+
+COPY libetcdclient.so /usr/lib/

curvefs/docker/debian10/base/Makefile

@@ -0,0 +1,4 @@
+.PHONY: build
+
+build:
+	docker build -t opencurvedocker/curve-base:debian10 .

curvefs/docker/debian10/compile/Dockerfile

@@ -0,0 +1,52 @@
+FROM opencurvedocker/curve-base:build-debian10
+
+RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        wget \
+        git \
+        gcc \
+        g++ \
+        clang \
+        libssl-dev \
+        libnl-genl-3-dev \
+        libcurl4-gnutls-dev \
+        uuid-dev \
+        libfiu-dev \
+        libfuse3-dev \
+        zlib1g-dev \
+        make \
+        openjdk-11-jdk \
+        sudo \
+        zlib1g-dev \
+        libsnappy-dev \
+        liblz4-dev \
+        cmake \
+        ca-certificates \
+        curl \
+        gnupg \
+        lsb-release \
+        curl \
+        git \
+    && mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get update \
+    && apt-get install -y docker-ce docker-ce-cli containerd.io \
+    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
+    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
+    && ln -sf /usr/bin/bazelisk /usr/bin/bazel \
+    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
+    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
+    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
+    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk

curvefs/docker/debian10/compile/Makefile

@@ -0,0 +1,4 @@
+.PHONY: build
+
+build:
+	docker build -t opencurvedocker/curve-base:build-debian10 .

curvefs/docker/debian10/entrypoint.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
+
+############################  GLOBAL VARIABLES
+g_role=""
+g_args=""
+g_prefix=""
+g_binary=""
+g_start_args=""
+
+############################  BASIC FUNCTIONS
+function msg() {
+    printf '%b' "$1" >&2
+}
+
+function success() {
+    msg "\33[32m[✔]\33[0m ${1}${2}"
+}
+
+function die() {
+    msg "\33[31m[✘]\33[0m ${1}${2}"
+    exit 1
+}
+
+############################ FUNCTIONS
+function usage () {
+    cat << _EOC_
+Usage:
+    entrypoint.sh --role=ROLE
+    entrypoint.sh --role=ROLE --args=ARGS
+
+Examples:
+    entrypoint.sh --role=etcd
+    entrypoint.sh --role=client --args="-o default_permissions"
+_EOC_
+}
+
+function get_options() {
+    local long_opts="role:,args:,help"
+    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
+    eval set -- "${args}"
+    while true
+    do
+        case "$1" in
+            -r|--role)
+                g_role=$2
+                shift 2
+                ;;
+            -a|--args)
+                g_args=$2
+                shift 2
+                ;;
+            -h)
+                usage
+                exit 1
+                ;;
+            --)
+                shift
+                break
+                ;;
+            *)
+                exit 1
+                ;;
+        esac
+    done
+}
+
+function prepare() {
+    g_prefix="/curvefs/$g_role"
+    conf_path="$g_prefix/conf/$g_role.conf"
+
+    case $g_role in
+        etcd)
+            g_binary="$g_prefix/sbin/etcd"
+            g_start_args="--config-file $conf_path"
+            ;;
+        mds)
+            g_binary="$g_prefix/sbin/curvefs-mds"
+            g_start_args="--confPath $conf_path"
+            ;;
+        metaserver)
+            g_binary="$g_prefix/sbin/curvefs-metaserver"
+            g_start_args="--confPath $conf_path"
+            ;;
+        client)
+            g_binary="$g_prefix/sbin/curve-fuse"
+            g_start_args="--confPath $conf_path"
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+
+    if [ "$g_args" != "" ]; then
+        g_start_args=$g_args
+    fi
+}
+
+function create_directory() {
+    chmod 700 "$g_prefix/data"
+    if [ "$g_role" == "etcd" ]; then
+        mkdir -p "$g_prefix/data/wal"
+    elif [ "$g_role" == "metaserver" ]; then
+        mkdir -p "$g_prefix/data/storage"
+    elif [ "$g_role" == "client" ]; then
+        mkdir -p "$g_prefix/mnt"
+    fi
+}
+
+function main() {
+    get_options "$@"
+
+    prepare
+    create_directory
+    [[ $(command -v crontab) ]] && cron
+    if [ $g_role == "etcd" ]; then
+        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
+    else
+        exec $g_binary $g_start_args
+    fi
+}
+
+############################  MAIN()
+main "$@"

curvefs/docker/debian11/Dockerfile

@@ -0,0 +1,11 @@
+FROM opencurvedocker/curve-base:debian11
+ENV TZ=Asia/Shanghai
+RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
+COPY curvefs /curvefs
+COPY entrypoint.sh /
+COPY curvefs/tools/sbin/curvefs_tool /usr/bin
+COPY libmemcached.so /usr/lib/
+COPY libmemcached.so.11 /usr/lib/
+COPY libhashkit.so.2 /usr/lib
+RUN chmod a+x /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian11/base/Dockerfile

@@ -0,0 +1,39 @@
+FROM opencurvedocker/curve-base:debian11
+
+RUN echo "deb http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        gcc \
+        gdb \
+        make \
+        openssl \
+        net-tools \
+        libcurl3-gnutls \
+        perl \
+        linux-perf \
+        vim \
+        curl \
+        cron \
+        procps \
+        lsof \
+        nginx \
+        less \
+        fuse3 \
+        libnl-3-200 \
+        libnl-genl-3-200 \
+        libjemalloc2 \
+        libsnappy1v5 \
+        liblz4-1 \
+        zlib1g
+    && ln -sf /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
+
+COPY libetcdclient.so /usr/lib/

curvefs/docker/debian11/base/Makefile

@@ -0,0 +1,4 @@
+.PHONY: build
+
+build:
+	docker build -t opencurvedocker/curve-base:debian11 .

curvefs/docker/debian11/compile/Dockerfile

@@ -0,0 +1,51 @@
+FROM opencurvedocker/curve-base:build-debian11 
+
+RUN echo "deb http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        wget \
+        git \
+        gcc \
+        g++ \
+        clang \
+        libssl-dev \
+        libnl-genl-3-dev \
+        libcurl4-gnutls-dev \
+        uuid-dev \
+        libfiu-dev \
+        libfuse3-dev \
+        zlib1g-dev \
+        make \
+        openjdk-11-jdk \
+        sudo \
+        libsnappy-dev \
+        liblz4-dev \
+        cmake \
+        ca-certificates \
+        curl \
+        gnupg \
+        lsb-release \
+        curl \
+        git \
+    && mkdir -p /etc/apt/keyrings \
+    # && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get update \
+    && apt-get install -y docker-ce docker-ce-cli containerd.io \
+    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
+    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
+    && ln -sf /usr/bin/bazelisk /usr/bin/bazel \
+    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
+    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
+    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
+    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk

curvefs/docker/debian11/compile/Makefile

@@ -0,0 +1,4 @@
+.PHONY: build
+
+build:
+	docker build -t opencurvedocker/curve-base:build-debian11 .

curvefs/docker/debian11/entrypoint.sh

@@ -0,0 +1,126 @@
+#!/usr/bin/env bash
+
+# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
+
+############################  GLOBAL VARIABLES
+g_role=""
+g_args=""
+g_prefix=""
+g_binary=""
+g_start_args=""
+
+############################  BASIC FUNCTIONS
+function msg() {
+    printf '%b' "$1" >&2
+}
+
+function success() {
+    msg "\33[32m[✔]\33[0m ${1}${2}"
+}
+
+function die() {
+    msg "\33[31m[✘]\33[0m ${1}${2}"
+    exit 1
+}
+
+############################ FUNCTIONS
+function usage () {
+    cat << _EOC_
+Usage:
+    entrypoint.sh --role=ROLE
+    entrypoint.sh --role=ROLE --args=ARGS
+
+Examples:
+    entrypoint.sh --role=etcd
+    entrypoint.sh --role=client --args="-o default_permissions"
+_EOC_
+}
+
+function get_options() {
+    local long_opts="role:,args:,help"
+    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
+    eval set -- "${args}"
+    while true
+    do
+        case "$1" in
+            -r|--role)
+                g_role=$2
+                shift 2
+                ;;
+            -a|--args)
+                g_args=$2
+                shift 2
+                ;;
+            -h)
+                usage
+                exit 1
+                ;;
+            --)
+                shift
+                break
+                ;;
+            *)
+                exit 1
+                ;;
+        esac
+    done
+}
+
+function prepare() {
+    g_prefix="/curvefs/$g_role"
+    conf_path="$g_prefix/conf/$g_role.conf"
+
+    case $g_role in
+        etcd)
+            g_binary="$g_prefix/sbin/etcd"
+            g_start_args="--config-file $conf_path"
+            ;;
+        mds)
+            g_binary="$g_prefix/sbin/curvefs-mds"
+            g_start_args="--confPath $conf_path"
+            ;;
+        metaserver)
+            g_binary="$g_prefix/sbin/curvefs-metaserver"
+            g_start_args="--confPath $conf_path"
+            ;;
+        client)
+            g_binary="$g_prefix/sbin/curve-fuse"
+            g_start_args="--confPath $conf_path"
+            ;;
+        *)
+            usage
+            exit 1
+            ;;
+    esac
+
+    if [ "$g_args" != "" ]; then
+        g_start_args=$g_args
+    fi
+}
+
+function create_directory() {
+    chmod 700 "$g_prefix/data"
+    if [ "$g_role" == "etcd" ]; then
+        mkdir -p "$g_prefix/data/wal"
+    elif [ "$g_role" == "metaserver" ]; then
+        mkdir -p "$g_prefix/data/storage"
+    elif [ "$g_role" == "client" ]; then
+        mkdir -p "$g_prefix/mnt"
+    fi
+}
+
+function main() {
+    get_options "$@"
+
+    prepare
+    create_directory
+    [[ $(command -v crontab) ]] && cron
+    if [ $g_role == "etcd" ]; then
+        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
+    else
+        exec $g_binary $g_start_args
+    fi
+}
+
+############################  MAIN()
+main "$@"

curvefs/docker/debian9/Dockerfile

@@ -1,8 +1,11 @@
 FROM opencurvedocker/curve-base:debian9
-COPY entrypoint.sh /
+ENV TZ=Asia/Shanghai
+RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
 COPY curvefs /curvefs
-COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 /usr/lib/
-RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \
-    && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \
-    && cp /curvefs/tools-v2/sbin/curve /usr/bin/
+COPY entrypoint.sh /
+COPY curvefs/tools/sbin/curvefs_tool /usr/bin
+COPY libmemcached.so /usr/lib/
+COPY libmemcached.so.11 /usr/lib/
+COPY libhashkit.so.2 /usr/lib
+RUN chmod a+x /entrypoint.sh
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian9/base/Dockerfile

@@ -0,0 +1,38 @@
+FROM opencurvedocker/curve-base:debian9
+
+RUN echo "deb http://mirrors.163.com/debian/ stretch main\n" \
+         "deb http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
+         "deb-src http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
+         "deb http://mirrors.163.com/debian-security/ stretch/updates main non-free contrib\n" \
+         "deb http://httpredir.debian.org/debian stretch-backports main contrib non-free\n" \
+    > /etc/apt/sources.list \
+    && apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        gcc \
+        gdb \
+        make \
+        openssl \
+        net-tools \
+        libcurl3-gnutls \
+        podlators-perl \
+        linux-perf \
+        vim \
+        curl \
+        cron \
+        procps \
+        lsof \
+        nginx \
+        less \
+        zlib1g \
+        libsnappy1v5 \
+        liblz4-1 \
+        python \
+        wget \
+        python3 \
+    && wget https://curve-build.nos-eastchina1.126.net/curve-base.tar.gz \
+    && tar -xzvf curve-base.tar.gz
+COPY fusermount3 /usr/local/bin
+COPY libetcdclient.so /usr/lib/
+COPY libfuse3.so libfuse3.so.3 libfuse3.so.3.10.5 libnl-3.so.200 libnl-genl-3.so.200 /usr/lib/x86_64-linux-gnu/
+COPY libjemalloc.so libjemalloc.so.1 libjemalloc.so.2 /usr/local/lib/

curvefs/docker/debian9/base/Makefile

@@ -0,0 +1,4 @@
+.PHONY: build
+
+build:
+	docker build -t opencurvedocker/curve-base:debian9 .

curvefs/docker/debian9/compile/Dockerfile

@@ -0,0 +1,50 @@
+FROM opencurvedocker/curve-base:build-debian9 
+
+RUN apt-get clean \
+    && apt-get -y update \
+    && apt-get -y install \
+        g++ \
+        wget \
+        clang \
+        git \
+        libnl-genl-3-dev \
+        libssl-dev \
+        uuid-dev \
+        libcurl4-gnutls-dev \
+        libfiu-dev \
+        openjdk-8-jdk \
+        sudo \
+        python-dev \
+        python-setuptools \
+        python-wheel \
+        python3-dev \
+        python3-setuptools \
+        python3-wheel \
+        zlib1g-dev \
+        libsnappy-dev \
+        liblz4-dev \
+        cmake \
+        ca-certificates \
+        curl \
+        gnupg \
+        lsb-release \
+        curl \
+        apt-transport-https \
+        git \
+    && mkdir -p /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+    && echo  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+  $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get update \
+    && apt-get install -y docker-ce docker-ce-cli containerd.io \
+    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
+    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
+    && ln -sf /usr/bin/bazelisk /usr/bin/bazel \
+    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
+    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
+    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
+    && wget https://github.com/bazelbuild/bazel/releases/download/0.17.2/bazel-0.17.2-linux-x86_64 \
+    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin \
+    && mv bazel-0.17.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel \
+    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk
+COPY fuse3/* /usr/include/fuse3/

curvefs/docker/debian9/compile/Makefile

@@ -0,0 +1,4 @@
+.PHONY: build
+
+build:
+	docker build -t opencurvedocker/curve-base:build-debian9 .

curvefs/docker/debian9/entrypoint.sh

@@ -8,7 +8,6 @@ g_args=""
 g_prefix=""
 g_binary=""
 g_start_args=""
-g_preexec="/curvefs/tools-v2/sbin/daemon"
 
 ############################  BASIC FUNCTIONS
 function msg() {
@@ -120,7 +119,6 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
-    [[ ! -z $g_preexec ]] && $g_preexec &
     if [ $g_role == "etcd" ]; then
         exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
     elif [ $g_role == "monitor" ]; then

curvefs/proto/common.proto

@@ -58,7 +58,6 @@ message S3Info {
     required string bucketname = 4;
     required uint64 blockSize = 5;
     required uint64 chunkSize = 6;
-    optional uint32 objectPrefix = 7;
 }
 
 enum PartitionStatus {
@@ -84,7 +83,6 @@ message PartitionInfo {
     optional uint64 inodeNum = 10;  // hearbeat upload this value to mds topo, topo/metaserver needn't persist this value
     optional uint64 dentryNum = 11;  // hearbeat upload this value to mds topo, topo/metaserver needn't persist this value
     map<int32, uint64> fileType2inodeNum = 12;
-    optional bool manageFlag = 13; // if a partition has recyclebin inode, set this flag true
 }
 
 message Peer {

curvefs/proto/mds.proto

@@ -56,10 +56,6 @@ enum FSStatusCode {
     VOLUME_INFO_ERROR = 31;
     MOUNT_POINT_CONFLICT = 32;
     FSNAME_INVALID = 33;
-    INSERT_DENTRY_FAIL = 34;
-    INSERT_MANAGE_INODE_FAIL = 35;
-    DELETE_DENTRY_FAIL = 36;
-    UPDATE_FS_FAIL = 37;
 }
 
 // fs interface
@@ -101,7 +97,6 @@ message FsInfo {
     required string owner = 12;
     optional uint64 txSequence = 13;
     optional string txOwner = 14;
-    optional uint32 recycleTimeHour =15;
 }
 
 message GetFsInfoResponse {
@@ -117,7 +112,6 @@ message CreateFsRequest {
     required bool enableSumInDir = 5;
     required string owner = 6;
     required uint64 capacity = 7;
-    optional uint32 recycleTimeHour = 8;
 }
 
 message CreateFsResponse {
@@ -191,7 +185,6 @@ message RefreshSessionRequest {
 message RefreshSessionResponse {
     required FSStatusCode statusCode = 1;
     repeated topology.PartitionTxId latestTxIdList = 2;
-    optional bool enableSumInDir = 3;
 }
 
 message DLockValue {

curvefs/proto/metaserver.proto

@@ -272,28 +272,6 @@ message CreateRootInodeResponse {
     optional uint64 appliedIndex = 2;
 }
 
-enum ManageInodeType {
-    TYPE_NOT_MANAGE = 0;
-    TYPE_RECYCLE = 1;
-};
-
-message CreateManageInodeRequest {
-    required uint32 poolId = 1;
-    required uint32 copysetId = 2;
-    required uint32 partitionId = 3;
-    required uint32 fsId = 4;
-    required uint32 uid = 5;
-    required uint32 gid = 6;
-    required uint32 mode = 7;
-    required ManageInodeType manageType = 8;
-}
-
-message CreateManageInodeResponse {
-    required MetaStatusCode statusCode = 1;
-    optional Inode inode = 2;
-    optional uint64 appliedIndex = 3;
-}
-
 message UpdateInodeRequest {
     required uint32 poolId = 1;
     required uint32 copysetId = 2;
@@ -496,7 +474,6 @@ service MetaServerService {
     rpc DeleteInode(DeleteInodeRequest) returns (DeleteInodeResponse);
     rpc CreateRootInode(CreateRootInodeRequest) returns
                                             (CreateRootInodeResponse);
-    rpc CreateManageInode(CreateManageInodeRequest) returns (CreateManageInodeResponse);
     rpc GetOrModifyS3ChunkInfo(GetOrModifyS3ChunkInfoRequest) returns (GetOrModifyS3ChunkInfoResponse);
     rpc BatchGetInodeAttr(BatchGetInodeAttrRequest) returns (BatchGetInodeAttrResponse);
     rpc BatchGetXAttr(BatchGetXAttrRequest) returns (BatchGetXAttrResponse);

curvefs/src/client/BUILD

@@ -43,10 +43,6 @@ cc_library(
             "s3/*.h",
             "volume/*.cpp",
             "volume/*.h",
-            "filesystem/*.cpp",
-            "filesystem/*.h",
-            "warmup/*.h",
-            "warmup/*.cpp",
         ],
         exclude = ["main.cpp"],
     ),
@@ -73,14 +69,10 @@ cc_library(
         "//curvefs/src/volume",
         "//curvefs/src/common:metric_utils",
         "//curvefs/src/common:dynamic_vlog",
-        "//curvefs/src/common:threading",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
-        "@com_google_absl//absl/synchronization",
-        "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",
         "@com_google_googletest//:gtest_prod",
-        "@spdlog//:spdlog",
     ],
 )

curvefs/src/client/async_request_closure.cpp

@@ -28,7 +28,7 @@
 #include <mutex>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 namespace curvefs {
@@ -63,7 +63,7 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
         cond_.wait(lk);
     }
 
-    return ToFSError(GetStatusCode());
+    return MetaStatusCodeToCurvefsErrCode(GetStatusCode());
 }
 
 void UpdateVolumeExtentClosure::Run() {
@@ -101,7 +101,7 @@ void UpdateInodeAttrAndExtentClosure::Run() {
                    << ", inode: " << inode_->GetInodeId();
         inode_->MarkInodeError();
     }
-    inode_->ClearDirty();
+
     inode_->syncingVolumeExtentsMtx_.unlock();
     inode_->ReleaseSyncingInode();
 

curvefs/src/client/async_request_closure.h

@@ -28,15 +28,12 @@
 
 #include <memory>
 
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
-#include "curvefs/src/client/filesystem/error.h"
 
 namespace curvefs {
 namespace client {
 
-using ::curvefs::client::filesystem::CURVEFS_ERROR;
-using ::curvefs::client::filesystem::ToFSError;
-
 class InodeWrapper;
 
 namespace internal {

curvefs/src/client/client_operator.cpp

@@ -24,14 +24,13 @@
 
 #include "src/common/uuid.h"
 #include "curvefs/src/client/client_operator.h"
-#include "curvefs/src/client/filesystem/error.h"
+
 namespace curvefs {
 namespace client {
 
 using ::curve::common::UUIDGenerator;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::mds::topology::PartitionTxId;
-using ::curvefs::client::filesystem::ToFSError;
 
 #define LOG_ERROR(action, rc) \
     LOG(ERROR) << action << " failed, retCode = " << rc \
@@ -96,7 +95,7 @@ CURVEFS_ERROR RenameOperator::GetTxId(uint32_t fsId,
     if (rc != MetaStatusCode::OK) {
         LOG_ERROR("GetTxId", rc);
     }
-    return ToFSError(rc);
+    return MetaStatusCodeToCurvefsErrCode(rc);
 }
 
 CURVEFS_ERROR RenameOperator::GetLatestTxIdWithLock() {
@@ -205,7 +204,7 @@ CURVEFS_ERROR RenameOperator::PrepareRenameTx(
         LOG_ERROR("PrepareRenameTx", rc);
     }
 
-    return ToFSError(rc);
+    return MetaStatusCodeToCurvefsErrCode(rc);
 }
 
 CURVEFS_ERROR RenameOperator::PrepareTx() {
@@ -397,28 +396,9 @@ CURVEFS_ERROR RenameOperator::UpdateInodeParent() {
     return rc;
 }
 
-CURVEFS_ERROR RenameOperator::UpdateInodeCtime() {
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    auto rc = inodeManager_->GetInode(srcDentry_.inodeid(), inodeWrapper);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG_ERROR("GetInode", rc);
-        return rc;
-    }
-
-    curve::common::UniqueLock lk = inodeWrapper->GetUniqueLock();
-    inodeWrapper->UpdateTimestampLocked(kChangeTime);
-
-    rc = inodeWrapper->SyncAttr();
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG_ERROR("UpdateInodeCtime", rc);
-        return rc;
-    }
-
-    LOG(INFO) << "UpdateInodeCtime inodeid = " << srcDentry_.inodeid();
-    return rc;
-}
-
 void RenameOperator::UpdateCache() {
+    dentryManager_->DeleteCache(parentId_, name_);
+    dentryManager_->InsertOrReplaceCache(newDentry_);
     SetTxId(srcPartitionId_, srcTxId_ + 1);
     SetTxId(dstPartitionId_, dstTxId_ + 1);
 }

curvefs/src/client/client_operator.h

@@ -59,7 +59,6 @@ class RenameOperator {
     CURVEFS_ERROR UnlinkSrcParentInode();
     void UnlinkOldInode();
     CURVEFS_ERROR UpdateInodeParent();
-    CURVEFS_ERROR UpdateInodeCtime();
     void UpdateCache();
 
     void GetOldInode(uint64_t *oldInodeId, int64_t *oldInodeSize,

curvefs/src/client/common/common.cpp

@@ -22,8 +22,6 @@
  */
 
 #include "curvefs/src/client/common/common.h"
-#include "src/common/string_util.h"
-
 namespace curvefs {
 namespace client {
 namespace common {
@@ -79,7 +77,6 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) {
 }
 
 const char kCurveFsWarmupOpAdd[] = "add";
-const char kCurveFsWarmupOpQuery[] = "query";
 const char kCurveFsWarmupTypeList[] = "list";
 const char kCurveFsWarmupTypeSingle[] = "single";
 
@@ -88,9 +85,6 @@ WarmupOpType GetWarmupOpType(const std::string& op) {
     if (op == kCurveFsWarmupOpAdd) {
         ret = WarmupOpType::kWarmupOpAdd;
     }
-    if (op == kCurveFsWarmupOpQuery) {
-        ret = WarmupOpType::kWarmupOpQuery;
-    }
     return ret;
 }
 
@@ -105,59 +99,6 @@ WarmupType GetWarmupType(const std::string& type) {
     return ret;
 }
 
-const char kCurveFsWarmupStorageDisk[] = "disk";
-const char kCurveFsWarmupStorageKvclient[] = "kvclient";
-
-WarmupStorageType GetWarmupStorageType(const std::string &type) {
-    auto ret = WarmupStorageType::kWarmupStorageTypeUnknown;
-    if (type == kCurveFsWarmupStorageDisk) {
-        ret = WarmupStorageType::kWarmupStorageTypeDisk;
-    } else if (type == kCurveFsWarmupStorageKvclient) {
-        ret = WarmupStorageType::kWarmupStorageTypeKvClient;
-    }
-    return ret;
-}
-
-using ::curve::common::StringToUll;
-
-// if direction is true means '+', false means '-'
-// is direction is true, add second to first
-// if direction is false, sub second from first
-bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction) {
-    uint64_t firstNum = 0;
-    uint64_t secondNum = second;
-    if (StringToUll(*first, &firstNum)) {
-        if (direction) {
-            *first = std::to_string(firstNum + secondNum);
-        } else {
-            if (firstNum < secondNum) {
-                *first = std::to_string(0);
-                LOG(WARNING) << "AddUllStringToFirst failed when minus,"
-                             << " first = " << firstNum
-                             << ", second = " << secondNum;
-                return false;
-            }
-            *first = std::to_string(firstNum - secondNum);
-        }
-    } else {
-        LOG(ERROR) << "StringToUll failed, first = " << *first
-                   << ", second = " << second;
-        return false;
-    }
-    return true;
-}
-
-bool AddUllStringToFirst(uint64_t *first, const std::string &second) {
-    uint64_t secondNum = 0;
-    if (StringToUll(second, &secondNum)) {
-        *first += secondNum;
-        return true;
-    }
-
-    LOG(ERROR) << "StringToUll failed, second = " << second;
-    return false;
-}
-
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/common/common.h

@@ -59,7 +59,6 @@ enum class MetaServerOpType {
     GetOrModifyS3ChunkInfo,
     GetVolumeExtent,
     UpdateVolumeExtent,
-    CreateManageInode,
 };
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);
@@ -69,13 +68,9 @@ const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024;
 
 const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op";
 
-
-constexpr int kWarmupOpNum = 4;
-
 enum class WarmupOpType {
     kWarmupOpUnknown = 0,
     kWarmupOpAdd = 1,
-    kWarmupOpQuery = 2,
 };
 
 WarmupOpType GetWarmupOpType(const std::string& op);
@@ -86,15 +81,7 @@ enum class WarmupType {
     kWarmupTypeSingle = 2,
 };
 
-WarmupType GetWarmupType(const std::string &type);
-
-enum class WarmupStorageType {
-    kWarmupStorageTypeUnknown = 0,
-    kWarmupStorageTypeDisk = 1,
-    kWarmupStorageTypeKvClient = 2,
-};
-
-WarmupStorageType GetWarmupStorageType(const std::string &type);
+WarmupType GetWarmupType(const std::string& type);
 
 enum class FileHandle : uint64_t {
     kDefaultValue = 0,
@@ -106,11 +93,6 @@ enum class NlinkChange : int32_t {
     kSubOne = -1,
 };
 
-// if direction is true means '+', false means '-'
-// is direction is true, add second to first
-// if direction is false, sub second from first
-bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction);
-bool AddUllStringToFirst(uint64_t *first, const std::string &second);
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/common/config.cpp

@@ -50,56 +50,6 @@ DEFINE_bool(enableCto, true, "acheieve cto consistency");
 DEFINE_bool(useFakeS3, false,
             "Use fake s3 to inject more metadata for testing metaserver");
 DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing");
-DEFINE_bool(access_logging, true, "enable access log");
-
-/**
- * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true
- * for dynamic parameter configuration
- */
-static bool pass_uint64(const char *, uint64_t) { return true; }
-
-DEFINE_uint64(fuseClientAvgWriteBytes, 0,
-              "the write throttle bps of fuse client");
-DEFINE_validator(fuseClientAvgWriteBytes, &pass_uint64);
-DEFINE_uint64(fuseClientBurstWriteBytes, 0,
-              "the write burst bps of fuse client");
-DEFINE_validator(fuseClientBurstWriteBytes, &pass_uint64);
-DEFINE_uint64(fuseClientBurstWriteBytesSecs, 180,
-              "the times that write burst bps can continue");
-DEFINE_validator(fuseClientBurstWriteBytesSecs, &pass_uint64);
-
-
-DEFINE_uint64(fuseClientAvgWriteIops, 0,
-              "the write throttle iops of fuse client");
-DEFINE_validator(fuseClientAvgWriteIops, &pass_uint64);
-DEFINE_uint64(fuseClientBurstWriteIops, 0,
-              "the write burst iops of fuse client");
-DEFINE_validator(fuseClientBurstWriteIops, &pass_uint64);
-DEFINE_uint64(fuseClientBurstWriteIopsSecs, 180,
-              "the times that write burst iops can continue");
-DEFINE_validator(fuseClientBurstWriteIopsSecs, &pass_uint64);
-
-
-DEFINE_uint64(fuseClientAvgReadBytes, 0,
-              "the Read throttle bps of fuse client");
-DEFINE_validator(fuseClientAvgReadBytes, &pass_uint64);
-DEFINE_uint64(fuseClientBurstReadBytes, 0,
-              "the Read burst bps of fuse client");
-DEFINE_validator(fuseClientBurstReadBytes, &pass_uint64);
-DEFINE_uint64(fuseClientBurstReadBytesSecs, 180,
-              "the times that Read burst bps can continue");
-DEFINE_validator(fuseClientBurstReadBytesSecs, &pass_uint64);
-
-
-DEFINE_uint64(fuseClientAvgReadIops, 0,
-              "the Read throttle iops of fuse client");
-DEFINE_validator(fuseClientAvgReadIops, &pass_uint64);
-DEFINE_uint64(fuseClientBurstReadIops, 0,
-              "the Read burst iops of fuse client");
-DEFINE_validator(fuseClientBurstReadIops, &pass_uint64);
-DEFINE_uint64(fuseClientBurstReadIopsSecs, 180,
-              "the times that Read burst iops can continue");
-DEFINE_validator(fuseClientBurstReadIopsSecs, &pass_uint64);
 
 void InitMdsOption(Configuration *conf, MdsOption *mdsOpt) {
     conf->GetValueFatalIfFail("mdsOpt.mdsMaxRetryMS", &mdsOpt->mdsMaxRetryMS);
@@ -207,6 +157,8 @@ void InitDiskCacheOption(Configuration *conf,
 
 void InitS3Option(Configuration *conf, S3Option *s3Opt) {
     conf->GetValueFatalIfFail("s3.fakeS3", &FLAGS_useFakeS3);
+    conf->GetValueFatalIfFail("s3.fuseMaxSize",
+                              &s3Opt->s3ClientAdaptorOpt.fuseMaxSize);
     conf->GetValueFatalIfFail("s3.pageSize",
                               &s3Opt->s3ClientAdaptorOpt.pageSize);
     conf->GetValueFatalIfFail("s3.prefetchBlocks",
@@ -223,8 +175,6 @@ void InitS3Option(Configuration *conf, S3Option *s3Opt) {
                               &s3Opt->s3ClientAdaptorOpt.writeCacheMaxByte);
     conf->GetValueFatalIfFail("s3.readCacheMaxByte",
                               &s3Opt->s3ClientAdaptorOpt.readCacheMaxByte);
-    conf->GetValueFatalIfFail("s3.readCacheThreads",
-                              &s3Opt->s3ClientAdaptorOpt.readCacheThreads);
     conf->GetValueFatalIfFail("s3.nearfullRatio",
                               &s3Opt->s3ClientAdaptorOpt.nearfullRatio);
     conf->GetValueFatalIfFail("s3.baseSleepUs",
@@ -293,55 +243,6 @@ void InitKVClientManagerOpt(Configuration *conf,
                               &config->getThreadPooln);
 }
 
-void InitFileSystemOption(Configuration* c, FileSystemOption* option) {
-    c->GetValueFatalIfFail("fs.cto", &option->cto);
-    c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto);
-    c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr);
-    c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength);
-    c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging);
-    {  // kernel cache option
-        auto o = &option->kernelCacheOption;
-        c->GetValueFatalIfFail("fs.kernelCache.attrTimeoutSec",
-                               &o->attrTimeoutSec);
-        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeoutSec",
-                               &o->dirAttrTimeoutSec);
-        c->GetValueFatalIfFail("fs.kernelCache.entryTimeoutSec",
-                               &o->entryTimeoutSec);
-        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeoutSec",
-                               &o->dirEntryTimeoutSec);
-    }
-    {  // lookup cache option
-        auto o = &option->lookupCacheOption;
-        c->GetValueFatalIfFail("fs.lookupCache.lruSize",
-                               &o->lruSize);
-        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeoutSec",
-                               &o->negativeTimeoutSec);
-        c->GetValueFatalIfFail("fs.lookupCache.minUses",
-                               &o->minUses);
-    }
-    {  // dir cache option
-        auto o = &option->dirCacheOption;
-        c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize);
-    }
-    {  // open file option
-        auto o = &option->openFilesOption;
-        c->GetValueFatalIfFail("fs.openFile.lruSize", &o->lruSize);
-    }
-    {  // attr watcher option
-        auto o = &option->attrWatcherOption;
-        c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize);
-    }
-    {  // rpc option
-        auto o = &option->rpcOption;
-        c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit);
-    }
-    {  // defer sync option
-        auto o = &option->deferSyncOption;
-        c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay);
-        c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime);
-    }
-}
-
 void SetBrpcOpt(Configuration *conf) {
     curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy;
     dummy.Load(conf, "defer_close_second", "rpc.defer.close.second",
@@ -362,16 +263,34 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
     InitLeaseOpt(conf, &clientOption->leaseOpt);
     InitRefreshDataOpt(conf, &clientOption->refreshDataOption);
     InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt);
-    InitFileSystemOption(conf, &clientOption->fileSystemOption);
 
+    conf->GetValueFatalIfFail("fuseClient.attrTimeOut",
+                              &clientOption->attrTimeOut);
+    conf->GetValueFatalIfFail("fuseClient.entryTimeOut",
+                              &clientOption->entryTimeOut);
     conf->GetValueFatalIfFail("fuseClient.listDentryLimit",
                               &clientOption->listDentryLimit);
     conf->GetValueFatalIfFail("fuseClient.listDentryThreads",
                               &clientOption->listDentryThreads);
+    conf->GetValueFatalIfFail("fuseClient.flushPeriodSec",
+                              &clientOption->flushPeriodSec);
+    conf->GetValueFatalIfFail("fuseClient.maxNameLength",
+                              &clientOption->maxNameLength);
+    conf->GetValueFatalIfFail("fuseClient.iCacheLruSize",
+                              &clientOption->iCacheLruSize);
+    conf->GetValueFatalIfFail("fuseClient.dCacheLruSize",
+                              &clientOption->dCacheLruSize);
+    conf->GetValueFatalIfFail("fuseClient.enableICacheMetrics",
+                              &clientOption->enableICacheMetrics);
+    conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
+                              &clientOption->enableDCacheMetrics);
     conf->GetValueFatalIfFail("client.dummyServer.startPort",
                               &clientOption->dummyServerStartPort);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                               &clientOption->enableMultiMountPointRename);
+    conf->GetValueFatalIfFail("fuseClient.disableXattr",
+                              &clientOption->disableXattr);
+    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
     conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
                               &clientOption->downloadMaxRetryTimes);
     conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
@@ -381,33 +300,12 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
         << "Not found `fuseClient.enableSplice` in conf, use default value `"
         << std::boolalpha << clientOption->enableFuseSplice << '`';
 
-    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteBytes",
-                              &FLAGS_fuseClientAvgWriteBytes);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytes",
-                              &FLAGS_fuseClientBurstWriteBytes);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytesSecs",
-                              &FLAGS_fuseClientBurstWriteBytesSecs);
+    // if enableCto, attr and entry cache must invalid
+    if (FLAGS_enableCto) {
+        clientOption->attrTimeOut = 0;
+        clientOption->entryTimeOut = 0;
+    }
 
-    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteIops",
-                              &FLAGS_fuseClientAvgWriteIops);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIops",
-                              &FLAGS_fuseClientBurstWriteIops);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIopsSecs",
-                              &FLAGS_fuseClientBurstWriteIopsSecs);
-
-    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadBytes",
-                              &FLAGS_fuseClientAvgReadBytes);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytes",
-                              &FLAGS_fuseClientBurstReadBytes);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytesSecs",
-                              &FLAGS_fuseClientBurstReadBytesSecs);
-
-    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadIops",
-                              &FLAGS_fuseClientAvgReadIops);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIops",
-                              &FLAGS_fuseClientBurstReadIops);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIopsSecs",
-                              &FLAGS_fuseClientBurstReadIopsSecs);
     SetBrpcOpt(conf);
 }
 
@@ -415,7 +313,6 @@ void SetFuseClientS3Option(FuseClientOption *clientOption,
     const S3InfoOption &fsS3Opt) {
     clientOption->s3Opt.s3ClientAdaptorOpt.blockSize = fsS3Opt.blockSize;
     clientOption->s3Opt.s3ClientAdaptorOpt.chunkSize = fsS3Opt.chunkSize;
-    clientOption->s3Opt.s3ClientAdaptorOpt.objectPrefix = fsS3Opt.objectPrefix;
     clientOption->s3Opt.s3AdaptrOpt.s3Address = fsS3Opt.s3Address;
     clientOption->s3Opt.s3AdaptrOpt.ak = fsS3Opt.ak;
     clientOption->s3Opt.s3AdaptrOpt.sk = fsS3Opt.sk;
@@ -430,7 +327,6 @@ void S3Info2FsS3Option(const curvefs::common::S3Info& s3,
     fsS3Opt->bucketName = s3.bucketname();
     fsS3Opt->blockSize = s3.blocksize();
     fsS3Opt->chunkSize = s3.chunksize();
-    fsS3Opt->objectPrefix = s3.has_objectprefix() ? s3.objectprefix() : 0;
 }
 
 }  // namespace common

curvefs/src/client/common/config.h

@@ -32,13 +32,15 @@
 #include "src/common/configuration.h"
 #include "src/common/s3_adapter.h"
 
-namespace curvefs {
-namespace client {
-namespace common {
 using ::curve::common::Configuration;
 using ::curve::common::S3AdapterOption;
 using ::curvefs::client::common::DiskCacheType;
 using ::curve::common::S3InfoOption;
+
+namespace curvefs {
+namespace client {
+namespace common {
+
 using MdsOption = ::curve::client::MetaServerOption;
 
 struct BlockDeviceClientOptions {
@@ -104,7 +106,7 @@ struct DiskCacheOption {
     // the max time system command can run
     uint32_t cmdTimeoutSec;
     // threads for disk cache
-    uint32_t threads = 10;
+    uint32_t threads;
     // the write throttle bps of disk cache
     uint64_t avgFlushBytes;
     // the write burst bps of disk cache
@@ -122,6 +124,7 @@ struct DiskCacheOption {
 struct S3ClientAdaptorOption {
     uint64_t blockSize;
     uint64_t chunkSize;
+    uint32_t fuseMaxSize;
     uint64_t pageSize;
     uint32_t prefetchBlocks;
     uint32_t prefetchExecQueueNum;
@@ -130,12 +133,10 @@ struct S3ClientAdaptorOption {
     uint32_t flushIntervalSec;
     uint64_t writeCacheMaxByte;
     uint64_t readCacheMaxByte;
-    uint32_t readCacheThreads;
     uint32_t nearfullRatio;
     uint32_t baseSleepUs;
     uint32_t maxReadRetryIntervalMs;
     uint32_t readRetryIntervalMs;
-    uint32_t objectPrefix;
     DiskCacheOption diskCacheOpt;
 };
 
@@ -174,59 +175,6 @@ struct RefreshDataOption {
     uint64_t maxDataSize = 1024;
     uint32_t refreshDataIntervalSec = 30;
 };
-
-// { filesystem option
-struct KernelCacheOption {
-    uint32_t entryTimeoutSec;
-    uint32_t dirEntryTimeoutSec;
-    uint32_t attrTimeoutSec;
-    uint32_t dirAttrTimeoutSec;
-};
-
-struct LookupCacheOption {
-    uint64_t lruSize;
-    uint32_t negativeTimeoutSec;
-    uint32_t minUses;
-};
-
-struct DirCacheOption {
-    uint64_t lruSize;
-    uint32_t timeoutSec;
-};
-
-struct AttrWatcherOption {
-    uint64_t lruSize;
-};
-
-struct OpenFilesOption {
-    uint64_t lruSize;
-    uint32_t deferSyncSecond;
-};
-
-struct RPCOption {
-    uint32_t listDentryLimit;
-};
-
-struct DeferSyncOption {
-    uint32_t delay;
-    bool deferDirMtime;
-};
-
-struct FileSystemOption {
-    bool cto;
-    bool disableXattr;
-    uint32_t maxNameLength;
-    uint32_t blockSize = 0x10000u;
-    KernelCacheOption kernelCacheOption;
-    LookupCacheOption lookupCacheOption;
-    DirCacheOption dirCacheOption;
-    OpenFilesOption openFilesOption;
-    AttrWatcherOption attrWatcherOption;
-    RPCOption rpcOption;
-    DeferSyncOption deferSyncOption;
-};
-// }
-
 struct FuseClientOption {
     MdsOption mdsOpt;
     MetaCacheOpt metaCacheOpt;
@@ -240,15 +188,23 @@ struct FuseClientOption {
     LeaseOpt leaseOpt;
     RefreshDataOption refreshDataOption;
     KVClientManagerOpt kvClientManagerOpt;
-    FileSystemOption fileSystemOption;
 
+    double attrTimeOut;
+    double entryTimeOut;
     uint32_t listDentryLimit;
     uint32_t listDentryThreads;
+    uint32_t flushPeriodSec;
+    uint32_t maxNameLength;
+    uint64_t iCacheLruSize;
+    uint64_t dCacheLruSize;
+    bool enableICacheMetrics;
+    bool enableDCacheMetrics;
     uint32_t dummyServerStartPort;
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;
+    bool disableXattr = false;
     uint32_t downloadMaxRetryTimes;
-    uint32_t warmupThreadsNum = 10;
+    uint32_t warmupThreadsNum;
 };
 
 void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption);

curvefs/src/client/curve_fuse_op.h

@@ -37,9 +37,10 @@
 extern "C" {
 #endif
 
-int InitLog(const char *confPath, const char *argv0);
+int InitGlog(const char *confPath, const char *argv0);
 
-int InitFuseClient(const struct MountOption *mountOption);
+int InitFuseClient(const char *confPath, const char *fsName,
+        const char* fsType, const char* mdsAddr);
 
 void UnInitFuseClient();
 

curvefs/src/client/dentry_cache_manager.cpp

@@ -44,13 +44,31 @@ namespace client {
 
 using curve::common::WriteLockGuard;
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
-using ::curvefs::client::filesystem::ToFSError;
+
+void DentryCacheManagerImpl::InsertOrReplaceCache(const Dentry &dentry) {
+    std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name());
+    if (!curvefs::client::common::FLAGS_enableCto) {
+        NameLockGuard lock(nameLock_, key);
+        dCache_->Put(key, dentry);
+    }
+}
+
+void DentryCacheManagerImpl::DeleteCache(uint64_t parentId,
+                                         const std::string &name) {
+    std::string key = GetDentryCacheKey(parentId, name);
+    NameLockGuard lock(nameLock_, key);
+    dCache_->Remove(key);
+}
 
 CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
                                                 const std::string &name,
                                                 Dentry *out) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
+    bool ok = dCache_->Get(key, out);
+    if (ok) {
+        return CURVEFS_ERROR::OK;
+    }
 
     MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out);
     if (ret != MetaStatusCode::OK) {
@@ -58,7 +76,10 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
             << "metaClient_ GetDentry failed, MetaStatusCode = " << ret
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
             << ", parent = " << parent << ", name = " << name;
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
+    }
+    if (!curvefs::client::common::FLAGS_enableCto) {
+        dCache_->Put(key, *out);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -73,9 +94,12 @@ CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) {
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << dentry.parentinodeid()
                    << ", name = " << dentry.name();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
 
+    if (!curvefs::client::common::FLAGS_enableCto) {
+        dCache_->Put(key, dentry);
+    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -84,13 +108,14 @@ CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent,
                                                    FsFileType type) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
+    dCache_->Remove(key);
 
     MetaStatusCode ret = metaClient_->DeleteDentry(fsId_, parent, name, type);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << parent << ", name = " << name;
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -124,7 +149,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", parent = " << parent << ", last = " << last
                        << ", count = " << limit << ", onlyDir = " << onlyDir;
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
 
         if (!onlyDir) {

curvefs/src/client/dentry_cache_manager.h

@@ -33,18 +33,20 @@
 #include <utility>
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
+#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
+#include "src/common/lru_cache.h"
 #include "src/common/concurrent/name_lock.h"
-#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Dentry;
+using ::curve::common::LRUCache;
+using ::curve::common::CacheMetrics;
 
 namespace curvefs {
 namespace client {
 
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
-using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 static const char* kDentryKeyDelimiter = ":";
 
@@ -57,6 +59,12 @@ class DentryCacheManager {
         fsId_ = fsId;
     }
 
+    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) = 0;
+
+    virtual void InsertOrReplaceCache(const Dentry& dentry) = 0;
+
+    virtual void DeleteCache(uint64_t parentId, const std::string& name) = 0;
+
     virtual CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) = 0;
 
@@ -77,11 +85,29 @@ class DentryCacheManager {
 class DentryCacheManagerImpl : public DentryCacheManager {
  public:
     DentryCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()),
+        dCache_(nullptr) {}
 
     explicit DentryCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient) {}
+      : metaClient_(metaClient),
+        dCache_(nullptr) {}
+
+    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) override {
+        if (enableCacheMetrics) {
+            dCache_ = std::make_shared<
+                LRUCache<std::string, Dentry>>(cacheSize,
+                    std::make_shared<CacheMetrics>("dcache"));
+        } else {
+            dCache_ = std::make_shared<
+                LRUCache<std::string, Dentry>>(cacheSize);
+        }
+        return CURVEFS_ERROR::OK;
+    }
+
+    void InsertOrReplaceCache(const Dentry& dentry) override;
+
+    void DeleteCache(uint64_t parentId, const std::string& name) override;
 
     CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) override;
@@ -102,6 +128,8 @@ class DentryCacheManagerImpl : public DentryCacheManager {
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
+    // key is parentId + name
+    std::shared_ptr<LRUCache<std::string, Dentry>> dCache_;
     curve::common::GenericNameLock<Mutex> nameLock_;
 };
 

curvefs/src/client/error_code.cpp

@@ -0,0 +1,98 @@
+/*
+ *  Copyright (c) 2021 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*
+ * Project: curve
+ * Created Date: Thur May 27 2021
+ * Author: xuchaojie
+ */
+
+#include "curvefs/src/client/error_code.h"
+
+#include <map>
+#include <string>
+
+namespace curvefs {
+namespace client {
+
+using ::curvefs::metaserver::MetaStatusCode;
+
+static const std::map<CURVEFS_ERROR, std::string> err2Msg = {
+    {CURVEFS_ERROR::OK, "OK"},
+    {CURVEFS_ERROR::INTERNAL, "internal error"},
+    {CURVEFS_ERROR::UNKNOWN, "unknown"},
+    {CURVEFS_ERROR::EXISTS, "inode or dentry already exist"},
+    {CURVEFS_ERROR::NOTEXIST, "inode or dentry not exist"},
+    {CURVEFS_ERROR::NO_SPACE, "no space to alloc"},
+    {CURVEFS_ERROR::BAD_FD, "bad fd"},
+    {CURVEFS_ERROR::INVALIDPARAM, "invalid param"},
+    {CURVEFS_ERROR::NOPERMISSION, "no permission"},
+    {CURVEFS_ERROR::NOTEMPTY, "dir not empty"},
+    {CURVEFS_ERROR::NOFLUSH, "no flush"},
+    {CURVEFS_ERROR::NOTSUPPORT, "not support"},
+    {CURVEFS_ERROR::NAMETOOLONG, "name too long"},
+    {CURVEFS_ERROR::MOUNT_POINT_EXIST, "mount point already exist"},
+    {CURVEFS_ERROR::MOUNT_FAILED, "mount failed"},
+};
+
+std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
+    os << static_cast<int>(code) << "[" << [code]() {
+        auto it = err2Msg.find(code);
+        if (it != err2Msg.end()) {
+            return it->second;
+        }
+
+        return std::string{"Unknown"};
+    }() << "]";
+
+    return os;
+}
+
+CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
+    MetaStatusCode code) {
+    CURVEFS_ERROR ret = CURVEFS_ERROR::UNKNOWN;
+    switch (code) {
+        case MetaStatusCode::OK:
+            ret = CURVEFS_ERROR::OK;
+            break;
+
+        case MetaStatusCode::NOT_FOUND:
+            ret = CURVEFS_ERROR::NOTEXIST;
+            break;
+
+        case MetaStatusCode::PARAM_ERROR:
+            ret = CURVEFS_ERROR::INVALIDPARAM;
+            break;
+
+        case MetaStatusCode::INODE_EXIST:
+        case MetaStatusCode::DENTRY_EXIST:
+            ret = CURVEFS_ERROR::EXISTS;
+            break;
+
+        case MetaStatusCode::SYM_LINK_EMPTY:
+        case MetaStatusCode::RPC_ERROR:
+            ret = CURVEFS_ERROR::INTERNAL;
+            break;
+
+        default:
+            ret = CURVEFS_ERROR::UNKNOWN;
+    }
+    return ret;
+}
+
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/error_code.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2023 NetEase Inc.
+ *  Copyright (c) 2021 NetEase Inc.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -14,27 +14,22 @@
  *  limitations under the License.
  */
 
+
 /*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
+ * Project: curve
+ * Created Date: Thur May 27 2021
+ * Author: xuchaojie
  */
 
-#include <map>
-#include <string>
-#include <iostream>
+#ifndef CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#define CURVEFS_SRC_CLIENT_ERROR_CODE_H_
 
 #include "curvefs/proto/metaserver.pb.h"
 
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
-
 namespace curvefs {
 namespace client {
-namespace filesystem {
-
-using ::curvefs::metaserver::MetaStatusCode;
 
+// notice : the error code should be negative.
 enum class CURVEFS_ERROR {
     OK = 0,
     INTERNAL = -1,
@@ -54,21 +49,15 @@ enum class CURVEFS_ERROR {
     OUT_OF_RANGE = -15,
     NODATA = -16,
     IO_ERROR = -17,
-    CACHETOOSMALL = -18,
-    STALE = -19,
-    NOSYS = -20,
 };
 
-std::string StrErr(CURVEFS_ERROR code);
-
-int SysErr(CURVEFS_ERROR code);
 
 std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code);
 
-CURVEFS_ERROR ToFSError(MetaStatusCode code);
+CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
+    ::curvefs::metaserver::MetaStatusCode code);
 
-}  // namespace filesystem
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+#endif  // CURVEFS_SRC_CLIENT_ERROR_CODE_H_

curvefs/src/client/filesystem/README.md

@@ -1,78 +0,0 @@
-CurveFS Client Metadata Cache Design
-===
-
-```
-+----------------+
-+  Kernel Cache  |
-+----------------+
-    |       ^
-(1) |       | (4)
-    v       |
-+----------------+
-+  CurveFS Fuse  |
-+----------------+
-    |       ^
-(2) |       | (3)
-    v       |
-+----------------+
-+   MetaServer   |
-+----------------+
-```
-
-(1) kernel cache timeout
----
-
-* requests are always sent to fuse layer, except `lookup` and `getattr` request which affected by caching.
-* if entry or attribute cache timeout, vfs layer will launch request to fuse layer.
-
-(2) revalidate cache
----
-
-* for `open` and `opendir` request, fuse layer should revalidate cache by comparing mtime, if modified, fuse layer should:
-    * `open`: return **ESTALE** to trigger vfs layer to invoke the `open` again with ignoring cache.
-    * `opendir`: drop all directory cache.
-* others, proxy request to metaserver directly.
-
-
-(3) retrive fresh metadata
----
-
-* for `readdir` request, fuse layer should cache direcoty entries and their attributes.
-* others, no caching.
-
-(4) reply with timeout
----
-
-* reply entry or attribute to vfs layer with corresponding cache timeout.
-* four type timeouts provided: `entryTimeout`, `dirEntryTimeout`, `attrTimeout`, `dirAtttTimeout`.
-* fuse layer should remeber the `mtime` of attribute while reply it to vfs layer.
-
-Cache Layer Level
-===
-
-```
-+-------------------------------------------------------+
-|                       Kernel Cache                    |  (L1 Cache)
-+-------------------------------------------------------+
-                            ^  (update length)
-                            |
-+--------------------------------------------------------+
-|                       Open File Cache                  |  (L2 Cache)
-|                   (only length and chunk)              |
-+--------------------------------------------------------+
-         ^                            ^
-         | (reply entry/attr)         |  (reply entry/attr)
-+-----------------+                   |
-| Directory Cache |  (L3 Cache)       |
-+-----------------+                   |
-         ^                            |
-         |                            |
-+-----------------+ +------------------------------------+
-|     readdir     | | lookup、getattr、setattr、create... |
-+-----------------+ +------------------------------------+
-         ^                            ^
-         |                            |
-+--------------------------------------------------------+
-|                        MetaServer                      |
-+--------------------------------------------------------+
-```

curvefs/src/client/filesystem/access_log.h

@@ -1,89 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-17
- * Author: Jingli Chen (Wine93)
- */
-
-#include <unistd.h>
-#include <butil/time.h>
-#include <spdlog/spdlog.h>
-#include <spdlog/sinks/basic_file_sink.h>
-#include <spdlog/sinks/daily_file_sink.h>
-
-#include <string>
-#include <memory>
-
-#include "absl/strings/str_format.h"
-#include "curvefs/src/client/common/config.h"
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
-
-namespace curvefs {
-namespace client {
-namespace common {
-
-DECLARE_bool(access_logging);
-
-}
-namespace filesystem {
-
-using ::absl::StrFormat;
-using ::curvefs::client::common::FLAGS_access_logging;
-using MessageHandler = std::function<std::string()>;
-
-static std::shared_ptr<spdlog::logger> Logger;
-
-bool InitAccessLog(const std::string& prefix) {
-    std::string filename = StrFormat("%s/access.%d.log", prefix, getpid());
-    Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0);
-    spdlog::flush_every(std::chrono::seconds(1));
-    return true;
-}
-
-struct AccessLogGuard {
-    explicit AccessLogGuard(MessageHandler handler)
-        : enable(FLAGS_access_logging),
-          handler(handler) {
-        if (!enable) {
-            return;
-        }
-
-        timer.start();
-    }
-
-    ~AccessLogGuard() {
-        if (!enable) {
-            return;
-        }
-
-        timer.stop();
-        Logger->info("{0} <{1:.6f}>", handler(), timer.u_elapsed() / 1e6);
-    }
-
-    bool enable;
-    MessageHandler handler;
-    butil::Timer timer;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_

curvefs/src/client/filesystem/attr_watcher.cpp

@@ -1,117 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#include "curvefs/src/client/filesystem/utils.h"
-#include "curvefs/src/client/filesystem/attr_watcher.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-AttrWatcher::AttrWatcher(AttrWatcherOption option,
-                         std::shared_ptr<OpenFiles> openFiles,
-                         std::shared_ptr<DirCache> dirCache)
-    : modifiedAt_(std::make_shared<LRUType>(option.lruSize)),
-      openFiles_(openFiles),
-      dirCache_(dirCache) {}
-
-void AttrWatcher::RemeberMtime(const InodeAttr& attr) {
-    WriteLockGuard lk(rwlock_);
-    modifiedAt_->Put(attr.inodeid(), AttrMtime(attr));
-}
-
-bool AttrWatcher::GetMtime(Ino ino, TimeSpec* time) {
-    ReadLockGuard lk(rwlock_);
-    return modifiedAt_->Get(ino, time);
-}
-
-void AttrWatcher::UpdateDirEntryAttr(Ino ino, const InodeAttr& attr) {
-    std::shared_ptr<DirEntryList> entries;
-    for (const auto parent : attr.parent()) {
-        bool yes = dirCache_->Get(parent, &entries);
-        if (!yes) {
-            continue;
-        }
-
-        entries->UpdateAttr(ino, attr);
-
-        VLOG(1) << "Write back attribute to dir entry cache: ino = " << ino
-                << ", attr = " << attr.ShortDebugString();
-    }
-}
-
-void AttrWatcher::UpdateDirEntryLength(Ino ino, const InodeAttr& open) {
-    std::shared_ptr<DirEntryList> entries;
-    for (const auto parent : open.parent()) {
-        bool yes = dirCache_->Get(parent, &entries);
-        if (!yes) {
-            continue;
-        }
-
-        entries->UpdateLength(ino, open);
-
-        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
-                << ", attr = " << open.ShortDebugString();
-    }
-}
-
-AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
-                                   InodeAttr* attr,
-                                   ReplyType type,
-                                   bool writeBack)
-    : watcher(watcher), attr(attr), type(type), writeBack(writeBack) {
-    InodeAttr open;
-    Ino ino = attr->inodeid();
-    bool yes = watcher->openFiles_->GetFileAttr(ino, &open);
-    if (!yes) {
-        return;
-    }
-
-    attr->set_length(open.length());
-    attr->set_mtime(open.mtime());
-    attr->set_mtime_ns(open.mtime_ns());
-    if (AttrCtime(open) > AttrCtime(*attr)) {
-        attr->set_ctime(open.ctime());
-        attr->set_ctime_ns(open.ctime_ns());
-    }
-}
-
-AttrWatcherGuard::~AttrWatcherGuard() {
-    switch (type) {
-        case ReplyType::ATTR:
-            watcher->RemeberMtime(*attr);
-            if (writeBack) {
-                watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
-            }
-            break;
-
-        case ReplyType::ONLY_LENGTH:
-            if (writeBack) {
-                watcher->UpdateDirEntryLength(attr->inodeid(), *attr);
-            }
-            break;
-    }
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/attr_watcher.h

@@ -1,105 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
-
-#include <memory>
-
-#include "src/common/lru_cache.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/openfile.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::RWLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::common::AttrWatcherOption;
-
-class AttrWatcher {
- public:
-    using LRUType = LRUCache<Ino, struct TimeSpec>;
-
- public:
-    AttrWatcher(AttrWatcherOption option,
-                std::shared_ptr<OpenFiles> openFiles,
-                std::shared_ptr<DirCache> dirCache);
-
-    void RemeberMtime(const InodeAttr& attr);
-
-    bool GetMtime(Ino ino, TimeSpec* time);
-
-    void UpdateDirEntryAttr(Ino ino, const InodeAttr& attr);
-
-    void UpdateDirEntryLength(Ino ino, const InodeAttr& open);
-
- private:
-    friend class AttrWatcherGuard;
-
- private:
-    RWLock rwlock_;
-    std::shared_ptr<LRUType> modifiedAt_;
-    std::shared_ptr<OpenFiles> openFiles_;
-    std::shared_ptr<DirCache> dirCache_;
-};
-
-
-enum class ReplyType {
-    ATTR,
-    ONLY_LENGTH
-};
-
-/*
- * each attribute reply to kernel, the watcher will:
- *  before reply:
- *    1) set attibute length if the corresponding file is opened
- *  after reply:
- *    1) remeber attribute modified time.
- *    2) write back attribute to dir entry cache if |writeBack| is true,
- *       because the dir-entry attribute maybe stale.
- */
-struct AttrWatcherGuard {
- public:
-    AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
-                     InodeAttr* attr,
-                     ReplyType type,
-                     bool writeBack);
-
-    ~AttrWatcherGuard();
-
- private:
-    std::shared_ptr<AttrWatcher> watcher;
-    InodeAttr* attr;
-    ReplyType type;
-    bool writeBack;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_

curvefs/src/client/filesystem/defer_sync.cpp

@@ -1,86 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <vector>
-#include <memory>
-
-#include "curvefs/src/client/filesystem/defer_sync.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-DeferSync::DeferSync(DeferSyncOption option)
-    : option_(option),
-      mutex_(),
-      running_(false),
-      thread_(),
-      sleeper_(),
-      inodes_() {
-}
-
-void DeferSync::Start() {
-    if (!running_.exchange(true)) {
-        thread_ = std::thread(&DeferSync::SyncTask, this);
-        LOG(INFO) << "Defer sync thread start success";
-    }
-}
-
-void DeferSync::Stop() {
-    if (running_.exchange(false)) {
-        LOG(INFO) << "Stop defer sync thread...";
-        sleeper_.interrupt();
-        thread_.join();
-        LOG(INFO) << "Defer sync thread stopped";
-    }
-}
-
-void DeferSync::SyncTask() {
-    std::vector<std::shared_ptr<InodeWrapper>> inodes;
-    for ( ;; ) {
-        bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay));
-
-        {
-            LockGuard lk(mutex_);
-            inodes.swap(inodes_);
-        }
-        for (const auto& inode : inodes) {
-            UniqueLock lk(inode->GetUniqueLock());
-            inode->Async(nullptr, true);
-        }
-        inodes.clear();
-
-        if (!running) {
-            break;
-        }
-    }
-}
-
-void DeferSync::Push(const std::shared_ptr<InodeWrapper>& inode) {
-    LockGuard lk(mutex_);
-    inodes_.emplace_back(inode);
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/defer_sync.h

@@ -1,70 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
-
-#include <atomic>
-#include <vector>
-#include <memory>
-
-#include "src/common/interruptible_sleeper.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curvefs::client::common::DeferSyncOption;
-
-using ::curve::common::Mutex;
-using ::curve::common::LockGuard;
-using ::curve::common::InterruptibleSleeper;
-
-class DeferSync {
- public:
-    explicit DeferSync(DeferSyncOption option);
-
-    void Start();
-
-    void Stop();
-
-    void Push(const std::shared_ptr<InodeWrapper>& inode);
-
- private:
-    void SyncTask();
-
- private:
-    DeferSyncOption option_;
-    Mutex mutex_;
-    std::atomic<bool> running_;
-    std::thread thread_;
-    InterruptibleSleeper sleeper_;
-    std::vector<std::shared_ptr<InodeWrapper>> inodes_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_

curvefs/src/client/filesystem/dir_cache.cpp

@@ -1,189 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#include <utility>
-
-#include "curvefs/src/client/filesystem/dir_cache.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-DirEntryList::DirEntryList()
-    : rwlock_(),
-      mtime_(),
-      entries_(),
-      attrs_() {}
-
-size_t DirEntryList::Size() {
-    ReadLockGuard lk(rwlock_);
-    return entries_.size();
-}
-
-void DirEntryList::Add(const DirEntry& dirEntry) {
-    WriteLockGuard lk(rwlock_);
-    entries_.push_back(std::move(dirEntry));
-    attrs_[dirEntry.ino] = &entries_.back();
-}
-
-bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) {
-    ReadLockGuard lk(rwlock_);
-    auto iter = attrs_.find(ino);
-    if (iter == attrs_.end()) {
-        return false;
-    }
-    *dirEntry = *iter->second;
-    return true;
-}
-
-bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) {
-    WriteLockGuard lk(rwlock_);
-    auto iter = attrs_.find(ino);
-    if (iter == attrs_.end()) {
-        return false;
-    }
-
-    DirEntry* dirEntry = iter->second;
-    dirEntry->attr = std::move(attr);
-    return true;
-}
-
-bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) {
-    WriteLockGuard lk(rwlock_);
-    auto iter = attrs_.find(ino);
-    if (iter == attrs_.end()) {
-        return false;
-    }
-
-    InodeAttr* attr = &iter->second->attr;
-    attr->set_length(open.length());
-    attr->set_mtime(open.mtime());
-    attr->set_mtime_ns(open.mtime_ns());
-    if (AttrCtime(open) > AttrCtime(*attr)) {
-        attr->set_ctime(open.ctime());
-        attr->set_ctime_ns(open.ctime_ns());
-    }
-    return true;
-}
-
-void DirEntryList::Iterate(IterateHandler handler) {
-    ReadLockGuard lk(rwlock_);
-    for (auto iter = entries_.begin(); iter != entries_.end(); iter++) {
-        handler(&(*iter));
-    }
-}
-
-void DirEntryList::Clear() {
-    WriteLockGuard lk(rwlock_);
-    entries_.clear();
-    attrs_.clear();
-}
-
-void DirEntryList::SetMtime(TimeSpec mtime) {
-    WriteLockGuard lk(rwlock_);
-    mtime_ = mtime;
-}
-
-TimeSpec DirEntryList::GetMtime() {
-    ReadLockGuard lk(rwlock_);
-    return mtime_;
-}
-
-DirCache::DirCache(DirCacheOption option)
-    : rwlock_(),
-      nentries_(0),
-      option_(option) {
-    lru_ = std::make_shared<LRUType>(0);  // control size by ourself
-    mq_ = std::make_shared<MessageQueueType>("dircache", 10000);
-    mq_->Subscribe([&](const std::shared_ptr<DirEntryList>& entries){
-        entries->Clear();
-    });
-
-    LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize;
-}
-
-void DirCache::Start() {
-    mq_->Start();
-}
-
-void DirCache::Stop() {
-    WriteLockGuard lk(rwlock_);
-    Evit(option_.lruSize);
-    mq_->Stop();
-}
-
-void DirCache::Delete(Ino parent,
-                      std::shared_ptr<DirEntryList> entries,
-                      bool evit) {
-    nentries_ -= entries->Size();
-    mq_->Publish(entries);  // clear entries in background
-    lru_->Remove(parent);
-
-    VLOG(1) << "Delete directory cache (evit=" << evit << "): "
-            << "parent = " << parent
-            << ", mtime = " << entries->GetMtime()
-            << ", size = " << entries->Size()
-            << ", nentries = " << nentries_;
-}
-
-void DirCache::Evit(size_t size) {
-    Ino parent;
-    std::shared_ptr<DirEntryList> entries;
-    while (nentries_ + size >= option_.lruSize) {
-        bool yes = lru_->GetLast(&parent, &entries);
-        if (!yes) {
-            break;
-        }
-        Delete(parent, entries, true);
-    }
-}
-
-void DirCache::Put(Ino parent, std::shared_ptr<DirEntryList> entries) {
-    WriteLockGuard lk(rwlock_);
-    Evit(entries->Size());  // it guarantee put entries success
-    lru_->Put(parent, entries);
-    nentries_ += entries->Size();
-
-    VLOG(1) << "Insert directory cache: parent = " << parent
-            << ", mtime = " << entries->GetMtime()
-            << ", size = " << entries->Size()
-            << ", nentries = " << nentries_;
-}
-
-bool DirCache::Get(Ino parent, std::shared_ptr<DirEntryList>* entries) {
-    ReadLockGuard lk(rwlock_);
-    return lru_->Get(parent, entries);
-}
-
-void DirCache::Drop(Ino parent) {
-    WriteLockGuard lk(rwlock_);
-    std::shared_ptr<DirEntryList> entries;
-    bool yes = lru_->Get(parent, &entries);
-    if (yes) {
-        Delete(parent, entries, false);
-    }
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/dir_cache.h

@@ -1,114 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
-
-#include <map>
-#include <list>
-#include <memory>
-
-#include "src/common/lru_cache.h"
-#include "src/common/concurrent/concurrent.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/message_queue.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::RWLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::common::DirCacheOption;
-
-class DirEntryList {
- public:
-    using IterateHandler = std::function<void(DirEntry* dirEntry)>;
-
- public:
-    DirEntryList();
-
-    size_t Size();
-
-    void Add(const DirEntry& dirEntry);
-
-    void Iterate(IterateHandler handler);
-
-    bool Get(Ino ino, DirEntry* dirEntry);
-
-    bool UpdateAttr(Ino ino, const InodeAttr& attr);
-
-    bool UpdateLength(Ino ino, const InodeAttr& open);
-
-    void Clear();
-
-    void SetMtime(TimeSpec mtime);
-
-    TimeSpec GetMtime();
-
- private:
-    RWLock rwlock_;
-    TimeSpec mtime_;
-    std::list<DirEntry> entries_;
-    std::map<Ino, DirEntry*> attrs_;
-};
-
-class DirCache {
- public:
-    using LRUType = LRUCache<Ino, std::shared_ptr<DirEntryList>>;
-    using MessageType = std::shared_ptr<DirEntryList>;
-    using MessageQueueType = MessageQueue<MessageType>;
-
- public:
-    explicit DirCache(DirCacheOption option);
-
-    void Start();
-
-    void Stop();
-
-    void Put(Ino parent, std::shared_ptr<DirEntryList> entries);
-
-    bool Get(Ino parent, std::shared_ptr<DirEntryList>* entries);
-
-    void Drop(Ino parent);
-
- private:
-    void Delete(Ino parent, std::shared_ptr<DirEntryList> entries, bool evit);
-
-    void Evit(size_t size);
-
- private:
-    RWLock rwlock_;
-    size_t nentries_;
-    DirCacheOption option_;
-    std::shared_ptr<LRUType> lru_;
-    std::shared_ptr<MessageQueueType> mq_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_

curvefs/src/client/filesystem/error.cpp

@@ -1,106 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <map>
-#include <utility>
-#include <ostream>
-
-#include "curvefs/src/client/filesystem/error.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-static const std::map<CURVEFS_ERROR, std::pair<int, std::string>> errors = {
-    { CURVEFS_ERROR::OK, { 0, "OK"} },
-    { CURVEFS_ERROR::INTERNAL,  { EIO, "internal error" } },
-    { CURVEFS_ERROR::UNKNOWN,  { -1, "unknown" } },
-    { CURVEFS_ERROR::EXISTS,  { EEXIST, "inode or dentry already exist" } },
-    { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } },
-    { CURVEFS_ERROR::NO_SPACE,  { ENOSPC, "no space to alloc" } },
-    { CURVEFS_ERROR::BAD_FD,  { EBADF, "bad file number" } },
-    { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } },
-    { CURVEFS_ERROR::NOPERMISSION,  { EACCES, "permission denied" } },
-    { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } },
-    { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } },
-    { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } },
-    { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } },
-    { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } },
-    { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } },
-    { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } },
-    { CURVEFS_ERROR::NODATA, { ENODATA, "no data available" } },
-    { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } },
-    { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } },
-    { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } },
-};
-
-std::string StrErr(CURVEFS_ERROR code) {
-    auto it = errors.find(code);
-    if (it != errors.end()) {
-        return it->second.second;
-    }
-    return "unknown";
-}
-
-int SysErr(CURVEFS_ERROR code) {
-    int syscode = -1;
-    auto it = errors.find(code);
-    if (it != errors.end()) {
-        syscode = it->second.first;
-    }
-    return (syscode == -1) ? EIO : syscode;
-}
-
-std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
-    os << static_cast<int>(code) << "[" << [code]() {
-        auto it = errors.find(code);
-        if (it != errors.end()) {
-            return it->second.second;
-        }
-
-        return std::string{"Unknown"};
-    }() << "]";
-
-    return os;
-}
-
-CURVEFS_ERROR ToFSError(MetaStatusCode code) {
-    static std::map<MetaStatusCode, CURVEFS_ERROR> errs = {
-        { MetaStatusCode::OK, CURVEFS_ERROR::OK },
-        { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST },
-        { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM },
-        { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS },
-        { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS },
-        { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL },
-        { MetaStatusCode::RPC_ERROR, CURVEFS_ERROR::INTERNAL },
-    };
-
-    auto it = errs.find(code);
-    if (it != errs.end()) {
-        return it->second;
-    }
-    return CURVEFS_ERROR::UNKNOWN;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/filesystem.cpp

@@ -1,353 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-08
- * Author: Jingli Chen (Wine93)
- */
-
-#include <map>
-
-#include "curvefs/src/client/filesystem/filesystem.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
-    : option_(option), member(member) {
-    deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
-    negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
-    dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
-    openFiles_ = std::make_shared<OpenFiles>(option_.openFilesOption,
-                                             deferSync_);
-    attrWatcher_ = std::make_shared<AttrWatcher>(option_.attrWatcherOption,
-                                                 openFiles_, dirCache_);
-    handlerManager_ = std::make_shared<HandlerManager>();
-    rpc_ = std::make_shared<RPCClient>(option.rpcOption, member);
-}
-
-void FileSystem::Run() {
-    deferSync_->Start();
-    dirCache_->Start();
-}
-
-void FileSystem::Destory() {
-    openFiles_->CloseAll();
-    deferSync_->Stop();
-    dirCache_->Stop();
-}
-
-void FileSystem::Attr2Stat(InodeAttr* attr, struct stat* stat) {
-    std::memset(stat, 0, sizeof(struct stat));
-    stat->st_ino = attr->inodeid();  //  inode number
-    stat->st_mode = attr->mode();  // permission mode
-    stat->st_nlink = attr->nlink();  // number of links
-    stat->st_uid = attr->uid();  // user ID of owner
-    stat->st_gid = attr->gid();  // group ID of owner
-    stat->st_size = attr->length();  // total size, in bytes
-    stat->st_rdev = attr->rdev();  // device ID (if special file)
-    stat->st_atim.tv_sec = attr->atime();  // time of last access
-    stat->st_atim.tv_nsec = attr->atime_ns();
-    stat->st_mtim.tv_sec = attr->mtime();  // time of last modification
-    stat->st_mtim.tv_nsec = attr->mtime_ns();
-    stat->st_ctim.tv_sec = attr->ctime();  // time of last status change
-    stat->st_ctim.tv_nsec = attr->ctime_ns();
-    stat->st_blksize = option_.blockSize;  // blocksize for file system I/O
-    stat->st_blocks = 0;  // number of 512B blocks allocated
-    if (IsS3File(*attr)) {
-        stat->st_blocks = (attr->length() + 511) / 512;
-    }
-}
-
-void FileSystem::Entry2Param(EntryOut* entryOut,
-                             fuse_entry_param* e) {
-    std::memset(e, 0, sizeof(fuse_entry_param));
-    e->ino = entryOut->attr.inodeid();
-    e->generation = 0;
-    Attr2Stat(&entryOut->attr, &e->attr);
-    e->entry_timeout = entryOut->entryTimeout;
-    e->attr_timeout = entryOut->attrTimeout;
-}
-
-void FileSystem::SetEntryTimeout(EntryOut* entryOut) {
-    auto option = option_.kernelCacheOption;
-    if (IsDir(entryOut->attr)) {
-        entryOut->entryTimeout = option.dirEntryTimeoutSec;
-        entryOut->attrTimeout = option.dirAttrTimeoutSec;
-    } else {
-        entryOut->entryTimeout = option.entryTimeoutSec;
-        entryOut->attrTimeout = option.attrTimeoutSec;
-    }
-}
-
-void FileSystem::SetAttrTimeout(AttrOut* attrOut) {
-    auto option = option_.kernelCacheOption;
-    if (IsDir(attrOut->attr)) {
-        attrOut->attrTimeout = option.dirAttrTimeoutSec;
-    } else {
-        attrOut->attrTimeout = option.attrTimeoutSec;
-    }
-}
-
-// fuse reply*
-void FileSystem::ReplyError(Request req, CURVEFS_ERROR code) {
-    fuse_reply_err(req, SysErr(code));
-}
-
-void FileSystem::ReplyEntry(Request req,
-                            EntryOut* entryOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
-                             ReplyType::ATTR, true);
-    fuse_entry_param e;
-    SetEntryTimeout(entryOut);
-    Entry2Param(entryOut, &e);
-    fuse_reply_entry(req, &e);
-}
-
-void FileSystem::ReplyAttr(Request req,
-                           AttrOut* attrOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr,
-                             ReplyType::ATTR, true);
-    struct stat stat;
-    SetAttrTimeout(attrOut);
-    Attr2Stat(&attrOut->attr, &stat);
-    fuse_reply_attr(req, &stat, attrOut->attrTimeout);
-}
-
-void FileSystem::ReplyReadlink(Request req, const std::string& link) {
-    fuse_reply_readlink(req, link.c_str());
-}
-
-void FileSystem::ReplyOpen(Request req, FileInfo* fi) {
-    fuse_reply_open(req, fi);
-}
-
-void FileSystem::ReplyOpen(Request req, FileOut* fileOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
-                             ReplyType::ONLY_LENGTH, true);
-    fuse_reply_open(req, fileOut->fi);
-}
-
-void FileSystem::ReplyData(Request req,
-                           struct fuse_bufvec *bufv,
-                           enum fuse_buf_copy_flags flags) {
-    fuse_reply_data(req, bufv, flags);
-}
-
-void FileSystem::ReplyWrite(Request req, FileOut* fileOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
-                             ReplyType::ONLY_LENGTH, true);
-    fuse_reply_write(req, fileOut->nwritten);
-}
-
-void FileSystem::ReplyBuffer(Request req, const char *buf, size_t size) {
-    fuse_reply_buf(req, buf, size);
-}
-
-void FileSystem::ReplyStatfs(Request req, const struct statvfs *stbuf) {
-    fuse_reply_statfs(req, stbuf);
-}
-
-void FileSystem::ReplyXattr(Request req, size_t size) {
-    fuse_reply_xattr(req, size);
-}
-
-void FileSystem::ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi) {
-    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
-                             ReplyType::ATTR, true);
-    fuse_entry_param e;
-    SetEntryTimeout(entryOut);
-    Entry2Param(entryOut, &e);
-    fuse_reply_create(req, &e, fi);
-}
-
-void FileSystem::AddDirEntry(Request req,
-                             DirBufferHead* buffer,
-                             DirEntry* dirEntry) {
-    struct stat stat;
-    std::memset(&stat, 0, sizeof(stat));
-    stat.st_ino = dirEntry->ino;
-
-    // add a directory entry to the buffer
-    size_t oldsize = buffer->size;
-    const char* name = dirEntry->name.c_str();
-    buffer->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);
-    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
-    fuse_add_direntry(req,
-                      buffer->p + oldsize,  // char* buf
-                      buffer->size - oldsize,  // size_t bufisze
-                      name, &stat, buffer->size);
-}
-
-void FileSystem::AddDirEntryPlus(Request req,
-                                 DirBufferHead* buffer,
-                                 DirEntry* dirEntry) {
-    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr,
-                             ReplyType::ATTR, false);
-    struct fuse_entry_param e;
-    EntryOut entryOut(dirEntry->attr);
-    SetEntryTimeout(&entryOut);
-    Entry2Param(&entryOut, &e);
-
-    // add a directory entry to the buffer with the attributes
-    size_t oldsize = buffer->size;
-    const char* name = dirEntry->name.c_str();
-    buffer->size += fuse_add_direntry_plus(req, NULL, 0, name, NULL, 0);
-    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
-    fuse_add_direntry_plus(req,
-                           buffer->p + oldsize,  // char* buf
-                           buffer->size - oldsize,  // size_t bufisze
-                           name, &e, buffer->size);
-}
-
-// handler*
-std::shared_ptr<FileHandler> FileSystem::NewHandler() {
-    return handlerManager_->NewHandler();
-}
-
-std::shared_ptr<FileHandler> FileSystem::FindHandler(uint64_t fh) {
-    return handlerManager_->FindHandler(fh);
-}
-
-void FileSystem::ReleaseHandler(uint64_t fh) {
-    return handlerManager_->ReleaseHandler(fh);
-}
-
-FileSystemMember FileSystem::BorrowMember() {
-    return FileSystemMember(deferSync_, openFiles_, attrWatcher_);
-}
-
-// fuse request*
-CURVEFS_ERROR FileSystem::Lookup(Request req,
-                                 Ino parent,
-                                 const std::string& name,
-                                 EntryOut* entryOut) {
-    if (name.size() > option_.maxNameLength) {
-        return CURVEFS_ERROR::NAMETOOLONG;
-    }
-
-    bool yes = negative_->Get(parent, name);
-    if (yes) {
-        return CURVEFS_ERROR::NOTEXIST;
-    }
-
-    auto rc = rpc_->Lookup(parent, name, entryOut);
-    if (rc == CURVEFS_ERROR::OK) {
-        negative_->Delete(parent, name);
-    } else if (rc == CURVEFS_ERROR::NOTEXIST) {
-        negative_->Put(parent, name);
-    }
-    return rc;
-}
-
-CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) {
-    InodeAttr attr;
-    auto rc = rpc_->GetAttr(ino, &attr);
-    if (rc == CURVEFS_ERROR::OK) {
-        *attrOut = AttrOut(attr);
-    }
-    return rc;
-}
-
-CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) {
-    InodeAttr attr;
-    CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    // revalidate directory cache
-    std::shared_ptr<DirEntryList> entries;
-    bool yes = dirCache_->Get(ino, &entries);
-    if (yes) {
-        if (entries->GetMtime() != AttrMtime(attr)) {
-            dirCache_->Drop(ino);
-        }
-    }
-
-    auto handler = NewHandler();
-    handler->mtime = AttrMtime(attr);
-    fi->fh = handler->fh;
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::ReadDir(Request req,
-                                  Ino ino,
-                                  FileInfo* fi,
-                                  std::shared_ptr<DirEntryList>* entries) {
-    bool yes = dirCache_->Get(ino, entries);
-    if (yes) {
-        return CURVEFS_ERROR::OK;
-    }
-
-    CURVEFS_ERROR rc = rpc_->ReadDir(ino, entries);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    (*entries)->SetMtime(FindHandler(fi->fh)->mtime);
-    dirCache_->Put(ino, *entries);
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) {
-    ReleaseHandler(fi->fh);
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) {
-    std::shared_ptr<InodeWrapper> inode;
-    bool yes = openFiles_->IsOpened(ino, &inode);
-    if (yes) {
-        openFiles_->Open(ino, inode);
-        // fi->keep_cache = 1;
-        return CURVEFS_ERROR::OK;
-    }
-
-    CURVEFS_ERROR rc = rpc_->Open(ino, &inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    TimeSpec mtime;
-    yes = attrWatcher_->GetMtime(ino, &mtime);
-    if (!yes) {
-        // It is rare which only arise when attribute evited for attr-watcher.
-        LOG(WARNING) << "open(" << ino << "): stale file handler"
-                     << ": attribute not found in wacther";
-        return CURVEFS_ERROR::STALE;
-    } else if (mtime != InodeMtime(inode)) {
-        LOG(WARNING) << "open(" << ino << "): stale file handler"
-                     << ", cache(" << mtime << ") vs remote("
-                     << InodeMtime(inode) << ")";
-        return CURVEFS_ERROR::STALE;
-    }
-
-    openFiles_->Open(ino, inode);
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) {
-    openFiles_->Close(ino);
-    return CURVEFS_ERROR::OK;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/filesystem.h

@@ -1,167 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-08
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
-
-#include <gtest/gtest_prod.h>
-
-#include <memory>
-#include <string>
-
-#include "curvefs/src/client/filesystem/error.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/package.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/lookup_cache.h"
-#include "curvefs/src/client/filesystem/dir_cache.h"
-#include "curvefs/src/client/filesystem/openfile.h"
-#include "curvefs/src/client/filesystem/attr_watcher.h"
-#include "curvefs/src/client/filesystem/rpc_client.h"
-#include "curvefs/src/client/filesystem/defer_sync.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curvefs::client::common::FileSystemOption;
-
-struct FileSystemMember {
-    FileSystemMember(std::shared_ptr<DeferSync> deferSync,
-                     std::shared_ptr<OpenFiles> openFiles,
-                     std::shared_ptr<AttrWatcher> attrWatcher)
-        : deferSync(deferSync),
-          openFiles(openFiles),
-          attrWatcher(attrWatcher) {}
-
-    std::shared_ptr<DeferSync> deferSync;
-    std::shared_ptr<OpenFiles> openFiles;
-    std::shared_ptr<AttrWatcher> attrWatcher;
-};
-
-class FileSystem {
- public:
-    FileSystem(struct FileSystemOption option, ExternalMember member);
-
-    void Run();
-
-    void Destory();
-
-    // fuse request
-    CURVEFS_ERROR Lookup(Request req,
-                         Ino parent,
-                         const std::string& name,
-                         EntryOut* entryOut);
-
-    CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut);
-
-    CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi);
-
-    CURVEFS_ERROR ReadDir(Request req,
-                          Ino ino,
-                          FileInfo* fi,
-                          std::shared_ptr<DirEntryList>* entries);
-
-    CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi);
-
-    CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi);
-
-    CURVEFS_ERROR Release(Request req, Ino ino);
-
-    // fuse reply: we control all replies to vfs layer in same entrance.
-    void ReplyError(Request req, CURVEFS_ERROR code);
-
-    void ReplyEntry(Request req, EntryOut* entryOut);
-
-    void ReplyAttr(Request req, AttrOut* attrOut);
-
-    void ReplyReadlink(Request req, const std::string& link);
-
-    void ReplyOpen(Request req, FileInfo *fi);
-
-    void ReplyOpen(Request req, FileOut* fileOut);
-
-    void ReplyData(Request req,
-                   struct fuse_bufvec *bufv,
-                   enum fuse_buf_copy_flags flags);
-
-    void ReplyWrite(Request req, FileOut* fileOut);
-
-    void ReplyBuffer(Request req, const char *buf, size_t size);
-
-    void ReplyStatfs(Request req, const struct statvfs *stbuf);
-
-    void ReplyXattr(Request req, size_t size);
-
-    void ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi);
-
-    void AddDirEntry(Request req,
-                     DirBufferHead* buffer,
-                     DirEntry* dirEntry);
-
-    void AddDirEntryPlus(Request req,
-                         DirBufferHead* buffer,
-                         DirEntry* dirEntry);
-
-    // utility: file handler
-    std::shared_ptr<FileHandler> NewHandler();
-
-    std::shared_ptr<FileHandler> FindHandler(uint64_t fh);
-
-    void ReleaseHandler(uint64_t fh);
-
-    // utility: others
-    FileSystemMember BorrowMember();
-
- private:
-    FRIEND_TEST(FileSystemTest, Attr2Stat);
-    FRIEND_TEST(FileSystemTest, Entry2Param);
-    FRIEND_TEST(FileSystemTest, SetEntryTimeout);
-    FRIEND_TEST(FileSystemTest, SetAttrTimeout);
-
-    // utility: convert to system type.
-    void Attr2Stat(InodeAttr* attr, struct stat* stat);
-
-    void Entry2Param(EntryOut* entryOut, fuse_entry_param* e);
-
-    // utility: set entry/attribute timeout
-    void SetEntryTimeout(EntryOut* entryOut);
-
-    void SetAttrTimeout(AttrOut* attrOut);
-
- private:
-    FileSystemOption option_;
-    ExternalMember member;
-    std::shared_ptr<DeferSync> deferSync_;
-    std::shared_ptr<LookupCache> negative_;
-    std::shared_ptr<DirCache> dirCache_;
-    std::shared_ptr<OpenFiles> openFiles_;
-    std::shared_ptr<AttrWatcher> attrWatcher_;
-    std::shared_ptr<HandlerManager> handlerManager_;
-    std::shared_ptr<RPCClient> rpc_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_

curvefs/src/client/filesystem/lookup_cache.cpp

@@ -1,104 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-31
- * Author: Jingli Chen (Wine93)
- */
-
-#include <glog/logging.h>
-
-#include <ctime>
-
-#include "absl/strings/str_format.h"
-#include "curvefs/src/client/filesystem/utils.h"
-#include "curvefs/src/client/filesystem/lookup_cache.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-#define RETURN_FALSE_IF_DISABLED() \
-    do {                           \
-        if (!enable_) {            \
-            return false;          \
-        }                          \
-    } while (0)
-
-
-LookupCache::LookupCache(LookupCacheOption option)
-    : enable_(option.negativeTimeoutSec > 0),
-      rwlock_(),
-      option_(option) {
-    lru_ = std::make_shared<LRUType>(option.lruSize);
-    if (enable_) {
-        LOG(INFO) << "Using lookup negative lru cache"
-                  << ", timeout = " << option.negativeTimeoutSec
-                  << ", capacity = " << option.lruSize;
-    }
-}
-
-std::string LookupCache::CacheKey(Ino parent, const std::string& name) {
-    return absl::StrFormat("%d:%s", parent, name);
-}
-
-bool LookupCache::Get(Ino parent, const std::string& name) {
-    RETURN_FALSE_IF_DISABLED();
-    ReadLockGuard lk(rwlock_);
-    CacheEntry entry;
-    auto key = CacheKey(parent, name);
-    bool yes = lru_->Get(key, &entry);
-    if (!yes) {
-        VLOG(1) << absl::StrFormat("Lookup cache not found: key(%d,%s)",
-                                   parent, name);
-        return false;
-    } else if (entry.uses < option_.minUses) {
-        return false;
-    } else if (entry.expireTime < Now()) {
-        return false;
-    }
-    return true;
-}
-
-bool LookupCache::Put(Ino parent, const std::string& name) {
-    RETURN_FALSE_IF_DISABLED();
-    WriteLockGuard lk(rwlock_);
-    CacheEntry entry;
-    auto key = CacheKey(parent, name);
-    bool yes = lru_->Get(key, &entry);
-    if (yes) {
-        entry.uses++;
-    } else {
-        entry.uses = 0;
-    }
-
-    entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0);
-    lru_->Put(key, entry);
-    return true;
-}
-
-bool LookupCache::Delete(Ino parent, const std::string& name) {
-    RETURN_FALSE_IF_DISABLED();
-    WriteLockGuard lk(rwlock_);
-    auto key = CacheKey(parent, name);
-    lru_->Remove(key);
-    return true;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/lookup_cache.h

@@ -1,77 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-31
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
-
-#include <memory>
-#include <string>
-
-#include "src/common/lru_cache.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::RWLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::common::LookupCacheOption;
-
-// memory cache for lookup result, now we only support cache negative result,
-// and other positive entry will be cached in kernel.
-class LookupCache {
- public:
-    struct CacheEntry {
-        uint32_t uses;
-        TimeSpec expireTime;
-    };
-
-    using LRUType = LRUCache<std::string, CacheEntry>;
-
- public:
-    explicit LookupCache(LookupCacheOption option);
-
-    bool Get(Ino parent, const std::string& name);
-
-    bool Put(Ino parent, const std::string& name);
-
-    bool Delete(Ino parent, const std::string& name);
-
- private:
-    std::string CacheKey(Ino parent, const std::string& name);
-
- private:
-    bool enable_;
-    RWLock rwlock_;
-    LookupCacheOption option_;
-    std::shared_ptr<LRUType> lru_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_

curvefs/src/client/filesystem/message_queue.h

@@ -1,118 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
-
-#include <glog/logging.h>
-
-#include <string>
-#include <atomic>
-#include <thread>
-
-#include "src/common/concurrent/task_queue.h"
-#include "curvefs/src/common/threading.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::TaskQueue;
-using ::curvefs::common::SetThreadName;
-
-template <typename MessageT>
-class MessageQueue {
- public:
-    using MessageHandler = std::function<void(const MessageT& message)>;
-
- public:
-    MessageQueue(const std::string& name, size_t bufferSize)
-        : name_(name),
-          running_(false),
-          thread_(),
-          handler_(),
-          queue_(bufferSize) {}
-
-    void Start() {
-        if (running_.exchange(true)) {
-            return;
-        }
-
-        thread_ = std::thread(&MessageQueue::Consumer, this);
-        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
-                  << "consumer thread start success";
-    }
-
-    void Stop() {
-        if (!running_.exchange(false)) {
-            return;
-        }
-
-        auto wakeup = []() {};
-        queue_.Push(wakeup);
-
-        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
-                  << "consumer thread stoping...";
-
-        thread_.join();
-
-        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
-                  << "consumer thread stopped";
-    }
-
-    void Publish(MessageT message) {
-        if (handler_ != nullptr) {
-            queue_.Push([this, message](){
-                this->handler_(message);
-            });
-        }
-    }
-
-    void Subscribe(MessageHandler handler) {
-        handler_ = handler;
-    }
-
- private:
-    void Consumer() {
-        SetThreadName(name_.c_str());
-        while (running_.load(std::memory_order_relaxed)) {
-            queue_.Pop()();
-        }
-
-        while (queue_.Size() > 0) {
-            queue_.Pop()();
-        }
-    }
-
- private:
-    std::string name_;
-    std::atomic<bool> running_;
-    std::thread thread_;
-    MessageHandler handler_;
-    TaskQueue queue_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_

curvefs/src/client/filesystem/meta.cpp

@@ -1,129 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <sys/stat.h>
-
-#include "absl/strings/str_format.h"
-#include "curvefs/src/client/filesystem/meta.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-HandlerManager::HandlerManager()
-    : mutex_(),
-      dirBuffer_(std::make_shared<DirBuffer>()),
-      handlers_() {}
-
-HandlerManager::~HandlerManager() {
-    dirBuffer_->DirBufferFreeAll();
-}
-
-std::shared_ptr<FileHandler> HandlerManager::NewHandler() {
-    UniqueLock lk(mutex_);
-    auto handler = std::make_shared<FileHandler>();
-    handler->fh = dirBuffer_->DirBufferNew();
-    handler->buffer = dirBuffer_->DirBufferGet(handler->fh);
-    handler->padding = false;
-    handlers_.emplace(handler->fh, handler);
-    return handler;
-}
-
-std::shared_ptr<FileHandler> HandlerManager::FindHandler(uint64_t fh) {
-    UniqueLock lk(mutex_);
-    auto iter = handlers_.find(fh);
-    if (iter == handlers_.end()) {
-        return nullptr;
-    }
-    return iter->second;
-}
-
-void HandlerManager::ReleaseHandler(uint64_t fh) {
-    UniqueLock lk(mutex_);
-    dirBuffer_->DirBufferRelease(fh);
-    handlers_.erase(fh);
-}
-
-std::string StrMode(uint16_t mode) {
-    static std::map<uint16_t, char> type2char = {
-        { S_IFSOCK, 's' },
-        { S_IFLNK, 'l' },
-        { S_IFREG, '-' },
-        { S_IFBLK, 'b' },
-        { S_IFDIR, 'd' },
-        { S_IFCHR, 'c' },
-        { S_IFIFO, 'f' },
-        { 0, '?' },
-    };
-
-    std::string s("?rwxrwxrwx");
-    s[0] = type2char[mode & (S_IFMT & 0xffff)];
-    if (mode & S_ISUID) {
-        s[3] = 's';
-    }
-    if (mode & S_ISGID) {
-        s[6] = 's';
-    }
-    if (mode & S_ISVTX) {
-        s[9] = 't';
-    }
-
-    for (auto i = 0; i < 9; i++) {
-        if ((mode & (1 << i)) == 0) {
-            if ((s[9-i] == 's') || (s[9-i] == 't')) {
-                s[9-i] &= 0xDF;
-            } else {
-                s[9-i] = '-';
-            }
-        }
-    }
-    return s;
-}
-
-namespace {
-
-std::string Attr2Str(const InodeAttr& attr) {
-    if (!attr.IsInitialized()) {
-        return "";
-    }
-
-    std::string smode;
-    return absl::StrFormat(" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])",
-        attr.inodeid(), StrMode(attr.mode()).c_str(), attr.mode(), attr.nlink(),
-        attr.uid(), attr.gid(),
-        attr.atime(), attr.mtime(), attr.ctime(),
-        attr.length());
-}
-
-}  // namespace
-
-std::string StrEntry(EntryOut entryOut) {
-    return Attr2Str(entryOut.attr);
-}
-
-std::string StrAttr(AttrOut attrOut) {
-    return Attr2Str(attrOut.attr);
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/meta.h

@@ -1,178 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
-
-#include <map>
-#include <memory>
-#include <string>
-#include <iostream>
-
-#include "src/common/concurrent/concurrent.h"
-#include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/fuse_common.h"
-#include "curvefs/src/client/dir_buffer.h"
-#include "curvefs/src/client/inode_wrapper.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::Mutex;
-using ::curve::common::RWLock;
-using ::curve::common::UniqueLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::metaserver::XAttr;
-using ::curvefs::metaserver::Dentry;
-using ::curvefs::metaserver::InodeAttr;
-using ::curvefs::metaserver::FsFileType;
-using ::curvefs::client::DirBufferHead;
-using ::curvefs::client::InodeWrapper;
-
-using Ino = fuse_ino_t;
-using Request = fuse_req_t;
-using FileInfo = struct fuse_file_info;
-
-struct EntryOut {
-    EntryOut() = default;
-
-    explicit EntryOut(InodeAttr attr) : attr(attr) {}
-
-    InodeAttr attr;
-    double entryTimeout;
-    double attrTimeout;
-};
-
-struct AttrOut {
-    AttrOut() = default;
-
-    explicit AttrOut(InodeAttr attr) : attr(attr) {}
-
-    InodeAttr attr;
-    double attrTimeout;
-};
-
-struct DirEntry {
-    DirEntry() = default;
-
-    DirEntry(Ino ino, const std::string& name, InodeAttr attr)
-        : ino(ino), name(name), attr(attr) {}
-
-    Ino ino;
-    std::string name;
-    InodeAttr attr;
-};
-
-struct FileOut {
-    FileOut() = default;
-
-    FileOut(FileInfo* fi, InodeAttr attr)
-        : fi(fi), attr(attr), nwritten(0) {}
-
-    FileOut(InodeAttr attr, size_t nwritten)
-        : fi(nullptr), attr(attr), nwritten(nwritten) {}
-
-    FileInfo* fi;
-    InodeAttr attr;
-    size_t nwritten;
-};
-
-struct TimeSpec {
-    TimeSpec() : seconds(0), nanoSeconds(0) {}
-
-    TimeSpec(uint64_t seconds, uint32_t nanoSeconds)
-        : seconds(seconds), nanoSeconds(nanoSeconds) {}
-
-    TimeSpec(const TimeSpec& time)
-        : seconds(time.seconds), nanoSeconds(time.nanoSeconds) {}
-
-    TimeSpec& operator=(const TimeSpec& time) = default;
-
-    TimeSpec operator+(const TimeSpec& time) const {
-        return TimeSpec(seconds + time.seconds, nanoSeconds + time.nanoSeconds);
-    }
-
-    uint64_t seconds;
-    uint32_t nanoSeconds;
-};
-
-struct FileHandler {
-    uint64_t fh;
-    DirBufferHead* buffer;
-    TimeSpec mtime;
-    bool padding;  // padding buffer
-};
-
-class HandlerManager {
- public:
-    HandlerManager();
-
-    ~HandlerManager();
-
-    std::shared_ptr<FileHandler> NewHandler();
-
-    std::shared_ptr<FileHandler> FindHandler(uint64_t id);
-
-    void ReleaseHandler(uint64_t id);
-
- private:
-    Mutex mutex_;
-    std::shared_ptr<DirBuffer> dirBuffer_;
-    std::map<uint64_t, std::shared_ptr<FileHandler>> handlers_;
-};
-
-inline bool operator==(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return (lhs.seconds == rhs.seconds) &&
-        (lhs.nanoSeconds == rhs.nanoSeconds);
-}
-
-inline bool operator!=(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return !(lhs == rhs);
-}
-
-inline bool operator<(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return (lhs.seconds < rhs.seconds) ||
-        (lhs.seconds == rhs.seconds && lhs.nanoSeconds < rhs.nanoSeconds);
-}
-
-inline bool operator>(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return (lhs.seconds > rhs.seconds) ||
-        (lhs.seconds == rhs.seconds && lhs.nanoSeconds > rhs.nanoSeconds);
-}
-
-inline std::ostream &operator<<(std::ostream &os, const TimeSpec& time) {
-    return os << time.seconds << "." << time.nanoSeconds;
-}
-
-std::string StrMode(uint16_t mode);
-
-std::string StrEntry(EntryOut entryOut);
-
-std::string StrAttr(AttrOut attrOut);
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_

curvefs/src/client/filesystem/openfile.cpp

@@ -1,141 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#include "curvefs/src/client/filesystem/openfile.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-OpenFiles::OpenFiles(OpenFilesOption option,
-                     std::shared_ptr<DeferSync> deferSync)
-    : rwlock_(),
-      option_(option),
-      deferSync_(deferSync),
-      files_(std::make_shared<LRUType>(option.lruSize)) {
-    LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize;
-}
-
-/*
- * Delete(...) does:
- *   1) publish to message queue which will flush file to server with async
- *   2) delete file from lru cache
- */
-void OpenFiles::Delete(Ino ino,
-                       const std::shared_ptr<OpenFile>& file,
-                       bool flush) {
-    if (flush) {
-        deferSync_->Push(file->inode);
-    }
-    files_->Remove(ino);
-
-    VLOG(1) << "Delete open file cache: ino = " << ino
-            << ", refs = " << file->refs
-            << ", mtime = " << InodeMtime(file->inode);
-}
-
-void OpenFiles::Evit(size_t size) {
-    Ino ino;
-    std::shared_ptr<OpenFile> file;
-    while (files_->Size() + size >= option_.lruSize) {
-        bool yes = files_->GetLast(&ino, &file);
-        if (!yes) {
-            break;
-        }
-        Delete(ino, file, true);
-    }
-}
-
-void OpenFiles::Open(Ino ino, std::shared_ptr<InodeWrapper> inode) {
-    WriteLockGuard lk(rwlock_);
-
-    Evit(1);
-
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (yes) {
-        file->refs++;
-        return;
-    }
-
-    file = std::make_shared<OpenFile>(inode);
-    file->refs++;
-    files_->Put(ino, file);
-
-    VLOG(1) << "Insert open file cache: ino = " << ino
-            << ", refs = " << file->refs
-            << ", mtime = " << InodeMtime(file->inode);
-}
-
-bool OpenFiles::IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
-    ReadLockGuard lk(rwlock_);
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (!yes) {
-        return false;
-    }
-    *inode = file->inode;
-    return true;
-}
-
-void OpenFiles::Close(Ino ino) {
-    WriteLockGuard lk(rwlock_);
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (!yes) {
-        return;
-    }
-
-    if (file->refs > 0) {
-        file->refs--;
-    }
-
-    if (file->refs == 0) {
-        Delete(ino, file, false);  // file already flushed before close
-    }
-}
-
-/*
- * CloseAll() does:
- *   flush all file to server and delete from LRU cache
- */
-void OpenFiles::CloseAll() {
-    WriteLockGuard lk(rwlock_);
-    Evit(option_.lruSize);
-}
-
-bool OpenFiles::GetFileAttr(Ino ino, InodeAttr* attr) {
-    ReadLockGuard lk(rwlock_);
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (!yes) {
-        return false;
-    }
-
-    file->inode->GetInodeAttr(attr);
-    return true;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/openfile.h

@@ -1,86 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
-
-#include <memory>
-
-#include "src/common/lru_cache.h"
-#include "curvefs/src/client/inode_wrapper.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/defer_sync.h"
-#include "curvefs/src/client/filesystem/dir_cache.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::InodeWrapper;
-using ::curvefs::client::common::OpenFilesOption;
-
-struct OpenFile {
-    explicit OpenFile(std::shared_ptr<InodeWrapper> inode)
-        : inode(inode), refs(0) {}
-
-    std::shared_ptr<InodeWrapper> inode;
-    uint64_t refs;
-};
-
-class OpenFiles {
- public:
-    using LRUType = LRUCache<Ino, std::shared_ptr<OpenFile>>;
-
- public:
-    explicit OpenFiles(OpenFilesOption option,
-                       std::shared_ptr<DeferSync> deferSync);
-
-    void Open(Ino ino, std::shared_ptr<InodeWrapper> inode);
-
-    bool IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode);
-
-    void Close(Ino ino);
-
-    void CloseAll();
-
-    bool GetFileAttr(Ino ino, InodeAttr* inode);
-
- private:
-    void Delete(Ino ino, const std::shared_ptr<OpenFile>& file, bool flush);
-
-    void Evit(size_t size);
-
- private:
-    RWLock rwlock_;
-    OpenFilesOption option_;
-    std::shared_ptr<DeferSync> deferSync_;
-    std::shared_ptr<LRUType> files_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_

curvefs/src/client/filesystem/package.h

@@ -1,51 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
-
-#include <memory>
-
-#include "curvefs/src/client/dentry_cache_manager.h"
-#include "curvefs/src/client/inode_cache_manager.h"
-#include "curvefs/src/client/xattr_manager.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-struct ExternalMember {  // external member depended by FileSystem
-    ExternalMember() = delete;
-    ExternalMember(std::shared_ptr<DentryCacheManager> dentryManager,
-                   std::shared_ptr<InodeCacheManager> inodeManager)
-        : dentryManager(dentryManager),
-          inodeManager(inodeManager) {}
-
-    std::shared_ptr<DentryCacheManager> dentryManager;
-    std::shared_ptr<InodeCacheManager> inodeManager;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_

curvefs/src/client/filesystem/rpc_client.cpp

@@ -1,128 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#include <map>
-#include <set>
-#include <list>
-#include <string>
-#include <memory>
-#include <utility>
-
-#include "curvefs/src/client/filesystem/rpc_client.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-RPCClient::RPCClient(RPCOption option,
-                     ExternalMember member)
-    : option_(option),
-      inodeManager_(member.inodeManager),
-      dentryManager_(member.dentryManager) {}
-
-CURVEFS_ERROR RPCClient::GetAttr(Ino ino, InodeAttr* attr) {
-    CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(ino, attr);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(getattr::GetInodeAttr) failed, retCode = " << rc
-                   << ", ino = " << ino;
-    }
-    return rc;
-}
-
-CURVEFS_ERROR RPCClient::Lookup(Ino parent,
-                                const std::string& name,
-                                EntryOut* entryOut) {
-    Dentry dentry;
-    CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry);
-    if (rc != CURVEFS_ERROR::OK) {
-        if (rc != CURVEFS_ERROR::NOTEXIST) {
-            LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc
-                       << ", parent = " << parent << ", name = " << name;
-        }
-        return rc;
-    }
-
-    Ino ino = dentry.inodeid();
-    rc = inodeManager_->GetInodeAttr(ino, &entryOut->attr);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(lookup::GetInodeAttr) failed, retCode = " << rc
-                   << ", ino = " << ino;
-    }
-    return rc;
-}
-
-CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
-                                 std::shared_ptr<DirEntryList>* entries) {
-    uint32_t limit = option_.listDentryLimit;
-
-    std::list<Dentry> dentries;
-    CURVEFS_ERROR rc = dentryManager_->ListDentry(ino, &dentries, limit);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc
-                   << ", ino = " << ino;
-        return rc;
-    }
-
-    std::set<uint64_t> inos;
-    std::map<uint64_t, InodeAttr> attrs;
-    std::for_each(dentries.begin(), dentries.end(), [&](Dentry& dentry){
-        inos.emplace(dentry.inodeid());
-    });
-    rc = inodeManager_->BatchGetInodeAttrAsync(ino, &inos, &attrs);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(readdir::BatchGetInodeAttrAsync) failed"
-                   << ", retCode = " << rc << ", ino = " << ino;
-        return rc;
-    }
-
-    DirEntry dirEntry;
-    for (const auto& dentry : dentries) {
-        Ino ino = dentry.inodeid();
-        auto iter = attrs.find(ino);
-        if (iter == attrs.end()) {
-            LOG(WARNING) << "rpc(readdir::BatchGetInodeAttrAsync) "
-                         << "missing attribute, ino = " << ino;
-            continue;
-        }
-
-        // NOTE: we can't use std::move() for attribute for hard link
-        // which will sharing inode attribute.
-        dirEntry.ino = ino;
-        dirEntry.name = std::move(dentry.name());
-        dirEntry.attr = iter->second;
-        (*entries)->Add(dirEntry);
-    }
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR RPCClient::Open(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
-    CURVEFS_ERROR rc = inodeManager_->GetInode(ino, *inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(open/GetInode) failed" << ", retCode = " << rc
-                   << ", ino = " << ino;
-    }
-    return rc;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/rpc_client.h

@@ -1,65 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
-
-#include <memory>
-#include <string>
-
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/package.h"
-#include "curvefs/src/client/filesystem/dir_cache.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curvefs::client::common::RPCOption;
-
-class RPCClient {
- public:
-    RPCClient(RPCOption option,
-              ExternalMember member);
-
-    CURVEFS_ERROR GetAttr(Ino ino, InodeAttr* attr);
-
-    CURVEFS_ERROR Lookup(Ino parent,
-                         const std::string& name,
-                         EntryOut* entryOut);
-
-    CURVEFS_ERROR ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries);
-
-    CURVEFS_ERROR Open(Ino ino, std::shared_ptr<InodeWrapper>* inode);
-
- private:
-    RPCOption option_;
-    std::shared_ptr<InodeCacheManager> inodeManager_;
-    std::shared_ptr<DentryCacheManager> dentryManager_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_

curvefs/src/client/filesystem/utils.cpp

@@ -1,69 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <glog/logging.h>
-
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-bool IsDir(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_DIRECTORY;
-}
-
-bool IsS3File(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_S3;
-}
-
-bool IsVolmeFile(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_FILE;
-}
-
-bool IsSymLink(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_SYM_LINK;
-}
-
-TimeSpec AttrMtime(const InodeAttr& attr) {
-    return TimeSpec(attr.mtime(), attr.mtime_ns());
-}
-
-TimeSpec AttrCtime(const InodeAttr& attr) {
-    return TimeSpec(attr.ctime(), attr.ctime_ns());
-}
-
-TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode) {
-    InodeAttr attr;
-    inode->GetInodeAttr(&attr);
-    return AttrMtime(attr);
-}
-
-TimeSpec Now() {
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    return TimeSpec(now.tv_sec, now.tv_nsec);
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/utils.h

@@ -1,59 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
-
-#include <memory>
-
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/package.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-// directory
-bool IsDir(const InodeAttr& attr);
-
-// file which data is stored in s3
-bool IsS3File(const InodeAttr& attr);
-
-// file which data is stored in volume
-bool IsVolmeFile(const InodeAttr& attr);
-
-// symbol link
-bool IsSymLink(const InodeAttr& attr);
-
-struct TimeSpec AttrMtime(const InodeAttr& attr);
-
-struct TimeSpec AttrCtime(const InodeAttr& attr);
-
-struct TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode);
-
-struct TimeSpec Now();
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_

curvefs/src/client/fuse_client.h

@@ -25,7 +25,6 @@
 
 #include <unistd.h>
 #include <sys/stat.h>
-#include <bthread/unstable.h>
 
 #include <map>
 #include <memory>
@@ -33,7 +32,6 @@
 #include <list>
 #include <utility>
 #include <vector>
-#include <atomic>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/mds.pb.h"
@@ -54,46 +52,37 @@
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/lease/lease_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
-#include "curvefs/src/client/warmup/warmup_manager.h"
-#include "src/common/throttle.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/filesystem.h"
 
 #define DirectIOAlignment 512
+#define WARMUP_CHECKINTERVAL_US 1000*1000
 
 using ::curve::common::Atomic;
 using ::curve::common::InterruptibleSleeper;
 using ::curve::common::Thread;
 using ::curvefs::common::FSType;
 using ::curvefs::metaserver::DentryFlag;
-using ::curvefs::metaserver::ManageInodeType;
 using ::curvefs::client::metric::FSMetric;
 
 namespace curvefs {
 namespace client {
 
-
-namespace warmup {
-class WarmupManager;
-}
-
 using common::FuseClientOption;
 using rpcclient::MDSBaseClient;
 using rpcclient::MdsClient;
 using rpcclient::MdsClientImpl;
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
-using ::curvefs::client::filesystem::FileSystem;
-using ::curvefs::client::filesystem::EntryOut;
-using ::curvefs::client::filesystem::AttrOut;
-using ::curvefs::client::filesystem::FileOut;
 
 using curvefs::common::is_aligned;
 
 const uint32_t kMaxHostNameLength = 255u;
 
 using mds::Mountpoint;
-
+typedef struct WarmUpFileContext {
+    fuse_ino_t inode;
+    uint64_t fileLen;
+    bool exist;
+} WarmUpFileContext_t;
 class FuseClient {
  public:
     FuseClient()
@@ -101,30 +90,31 @@ class FuseClient {
         metaClient_(std::make_shared<MetaServerClientImpl>()),
         inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
         dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
+        dirBuf_(std::make_shared<DirBuffer>()),
         fsInfo_(nullptr),
-        init_(false),
-        enableSumInDir_(false),
-        warmupManager_(nullptr),
         mdsBase_(nullptr),
-        isStop_(true) {}
+        isStop_(true),
+        init_(false),
+        mounted_(false),
+        enableSumInDir_(false) {}
 
     virtual ~FuseClient() {}
 
     FuseClient(const std::shared_ptr<MdsClient> &mdsClient,
         const std::shared_ptr<MetaServerClient> &metaClient,
         const std::shared_ptr<InodeCacheManager> &inodeManager,
-        const std::shared_ptr<DentryCacheManager> &dentryManager,
-        const std::shared_ptr<warmup::WarmupManager> &warmupManager)
+        const std::shared_ptr<DentryCacheManager> &dentryManager)
           : mdsClient_(mdsClient),
             metaClient_(metaClient),
             inodeManager_(inodeManager),
             dentryManager_(dentryManager),
+            dirBuf_(std::make_shared<DirBuffer>()),
             fsInfo_(nullptr),
-            init_(false),
-            enableSumInDir_(false),
-            warmupManager_(warmupManager),
             mdsBase_(nullptr),
-            isStop_(true) {}
+            isStop_(true),
+            init_(false),
+            mounted_(false),
+            enableSumInDir_(false) {}
 
     virtual CURVEFS_ERROR Init(const FuseClientOption &option);
 
@@ -142,42 +132,31 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                       const char* buf, size_t size, off_t off,
                                       struct fuse_file_info* fi,
-                                      FileOut* fileOut) = 0;
+                                      size_t* wSize) = 0;
 
     virtual CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                      size_t size, off_t off,
                                      struct fuse_file_info* fi, char* buffer,
                                      size_t* rSize) = 0;
 
-    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       EntryOut* entryOut);
+    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
+                                       const char* name, fuse_entry_param* e);
 
-    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     struct fuse_file_info* fi,
-                                     FileOut* fileOut);
+    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
+                                     struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       mode_t mode,
+    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
+                                       const char* name, mode_t mode,
                                        struct fuse_file_info* fi,
-                                       EntryOut* entryOut) = 0;
+                                       fuse_entry_param* e) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
-                                      fuse_ino_t parent,
-                                      const char* name,
-                                      mode_t mode,
-                                      dev_t rdev,
-                                      EntryOut* entryOut) = 0;
+    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+                                      const char* name, mode_t mode, dev_t rdev,
+                                      fuse_entry_param* e) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req,
-                                      fuse_ino_t parent,
-                                      const char* name,
-                                      mode_t mode,
-                                      EntryOut* entryOut);
+    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
+                                      const char* name, mode_t mode,
+                                      fuse_entry_param* e);
 
     virtual CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                        const char* name) = 0;
@@ -185,38 +164,30 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
                                       const char* name);
 
-    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req,
-                                        fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req,
-                                           fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
                                            struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReadDir(fuse_req_t req,
-                                        fuse_ino_t ino,
-                                        size_t size,
-                                        off_t off,
-                                        struct fuse_file_info* fi,
-                                        char** bufferOut,
-                                        size_t* rSize,
-                                        bool plus);
+    virtual CURVEFS_ERROR FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
+                                            size_t size, off_t off,
+                                            struct fuse_file_info* fi,
+                                            char** buffer, size_t* rSize,
+                                            bool cacheDir);
 
-    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       fuse_ino_t newparent,
-                                       const char* newname,
-                                       unsigned int flags);
+    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req, fuse_ino_t parent,
+                                       const char* name, fuse_ino_t newparent,
+                                       const char* newname);
 
     virtual CURVEFS_ERROR FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info* fi,
-                                        struct AttrOut* out);
+                                        struct stat* attr);
 
     virtual CURVEFS_ERROR FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct stat* attr, int to_set,
                                         struct fuse_file_info* fi,
-                                        struct AttrOut* out);
+                                        struct stat* attrOut);
 
     virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, std::string* value,
@@ -229,17 +200,13 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 
-    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req,
-                                        const char* link,
-                                        fuse_ino_t parent,
-                                        const char* name,
-                                        EntryOut* entryOut);
+    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req, const char* link,
+                                        fuse_ino_t parent, const char* name,
+                                        fuse_entry_param* e);
 
-    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     fuse_ino_t newparent,
-                                     const char* newname,
-                                     EntryOut* entryOut) = 0;
+    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char* newname,
+                                     fuse_entry_param* e) = 0;
 
     virtual CURVEFS_ERROR FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string* linkStr);
@@ -252,16 +219,11 @@ class FuseClient {
                                       struct fuse_file_info* fi) = 0;
     virtual CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                       struct fuse_file_info *fi) {
-        (void)req;
-        (void)ino;
-        (void)fi;
         return CURVEFS_ERROR::OK;
     }
 
     virtual CURVEFS_ERROR FuseOpStatFs(fuse_req_t req, fuse_ino_t ino,
                                        struct statvfs* stbuf) {
-        (void)req;
-        (void)ino;
         // TODO(chengyi01,wuhanqing): implement in s3 and volume client
         stbuf->f_frsize = stbuf->f_bsize = fsInfo_->blocksize();
         stbuf->f_blocks = 10UL << 30;
@@ -273,30 +235,31 @@ class FuseClient {
         stbuf->f_fsid = fsInfo_->fsid();
 
         stbuf->f_flag = 0;
-        stbuf->f_namemax = option_.fileSystemOption.maxNameLength;
+        stbuf->f_namemax = option_.maxNameLength;
         return CURVEFS_ERROR::OK;
     }
 
-    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
-
     void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
         fsInfo_ = fsInfo;
         init_ = true;
     }
 
     void SetMounted(bool mounted) {
-        if (warmupManager_ != nullptr) {
-            warmupManager_->SetMounted(mounted);
-        }
+        mounted_ = mounted;
+    }
+
+    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable> &
+      GetTaskFetchPool() {
+        return taskFetchMetaPool_;
     }
 
     std::shared_ptr<FsInfo> GetFsInfo() {
         return fsInfo_;
     }
 
-    std::shared_ptr<FileSystem> GetFileSystem() {
-        return fs_;
-    }
+    virtual void FlushInode();
+
+    virtual void FlushInodeAll();
 
     virtual void FlushAll();
 
@@ -304,77 +267,47 @@ class FuseClient {
     void SetEnableSumInDir(bool enable) {
         enableSumInDir_ = enable;
     }
-
-    bool PutWarmFilelistTask(fuse_ino_t key, common::WarmupStorageType type) {
-        if (fsInfo_->fstype() == FSType::TYPE_S3) {
-            return warmupManager_->AddWarmupFilelist(key, type);
-        }  // only support s3
-        return true;
+    void GetReadAheadFiles(std::list<fuse_ino_t>* readAheadFiles) {
+        std::unique_lock<std::mutex> lck(fetchMtx_);
+        *readAheadFiles = std::move(readAheadFiles_);
     }
 
-    bool PutWarmFileTask(fuse_ino_t key, const std::string &path,
-                         common::WarmupStorageType type) {
-        if (fsInfo_->fstype() == FSType::TYPE_S3) {
-            return warmupManager_->AddWarmupFile(key, path, type);
-        }  // only support s3
-        return true;
+    void GetWarmUpFile(WarmUpFileContext_t* warmUpFile) {
+        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
+        *warmUpFile = std::move(warmUpFile_);
+        warmUpFile_.exist = false;
+        return;
+    }
+    void SetWarmUpFile(WarmUpFileContext_t warmUpFile) {
+        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
+        warmUpFile_ = warmUpFile;
+        warmUpFile_.exist = true;
+    }
+    bool hasWarmUpTask() {
+        std::unique_lock<std::mutex> lck(warmUpFileMtx_);
+        return warmUpFile_.exist;
     }
 
-    bool GetWarmupProgress(fuse_ino_t key, warmup::WarmupProgress *progress) {
-        if (fsInfo_->fstype() == FSType::TYPE_S3) {
-            return warmupManager_->QueryWarmupProgress(key, progress);
-        }
-        return false;
+    void FetchDentryEnqueue(std::string file);
+
+    void PutWarmTask(const std::string& warmUpTask) {
+        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
+        warmUpTasks_.push_back(warmUpTask);
+        WarmUpRun();
     }
 
-    CURVEFS_ERROR SetMountStatus(const struct MountOption *mountOption);
-
-    void Add(bool isRead, size_t size) { throttle_.Add(isRead, size); }
-
-    void InitQosParam();
-
  protected:
-    CURVEFS_ERROR MakeNode(fuse_req_t req,
-                           fuse_ino_t parent,
-                           const char* name,
-                           mode_t mode,
-                           FsFileType type,
-                           dev_t rdev,
-                           bool internal,
-                           std::shared_ptr<InodeWrapper>& InodeWrapper);  // NOLINT
+    CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
+                           mode_t mode, FsFileType type, dev_t rdev,
+                           fuse_entry_param* e);
 
     CURVEFS_ERROR RemoveNode(fuse_req_t req, fuse_ino_t parent,
                              const char* name, FsFileType type);
 
-    CURVEFS_ERROR CreateManageNode(fuse_req_t req,
-                                   uint64_t parent,
-                                   const char* name,
-                                   mode_t mode,
-                                   ManageInodeType manageType,
-                                   EntryOut* entryOut);
-
-    CURVEFS_ERROR GetOrCreateRecycleDir(fuse_req_t req, Dentry *out);
-
-    CURVEFS_ERROR DeleteNode(fuse_ino_t ino, fuse_ino_t parent,
-                             const char* name, FsFileType type);
-
-    CURVEFS_ERROR MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
-                                fuse_ino_t parent, const char* name,
-                                FsFileType type);
-
-    bool ShouldMoveToRecycle(fuse_ino_t parent);
-
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
-                             fuse_ino_t ino,
-                             fuse_ino_t newparent,
-                             const char* newname,
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                             fuse_ino_t newparent, const char* newname,
                              FsFileType type,
-                             EntryOut* entryOut);
-
-    CURVEFS_ERROR HandleOpenFlags(fuse_req_t req,
-                                  fuse_ino_t ino,
-                                  struct fuse_file_info* fi,
-                                  FileOut* fileOut);
+                             fuse_entry_param* e);
 
     int SetHostPortInMountPoint(Mountpoint* out) {
         char hostname[kMaxHostNameLength];
@@ -389,24 +322,54 @@ class FuseClient {
         return 0;
     }
 
+    void splitStr(const std::string& srcStr, const std::string& delimiter,
+      std::vector<std::string>* splitPath) {
+        char* pToken = nullptr;
+        char* pSave = nullptr;
+        pToken = strtok_r(const_cast<char*>(srcStr.c_str()),
+          const_cast<char*>(delimiter.c_str()), &pSave);
+        if (nullptr == pToken) {
+            VLOG(6) << "del lookpath end";
+            return;
+        }
+        splitPath->push_back(pToken);
+        while (true) {
+            pToken = strtok_r(NULL, const_cast<char*>(
+              delimiter.c_str()), &pSave);
+            if (nullptr == pToken) {
+                VLOG(6) << "del lookpath end";
+                break;
+            }
+            VLOG(9) << "del pToken is:" << pToken
+                    << "pSave:" << pSave;
+            splitPath->push_back(pToken);
+        }
+    }
 
  private:
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
+
     virtual void FlushData() = 0;
 
     CURVEFS_ERROR UpdateParentMCTimeAndNlink(
         fuse_ino_t parent, FsFileType type,  NlinkChange nlink);
 
-    std::string GenerateNewRecycleName(fuse_ino_t ino,
-            fuse_ino_t parent, const char* name) {
-        std::string newName(name);
-        newName = std::to_string(parent) + "_" + std::to_string(ino)
-                    + "_" + newName;
-        if (newName.length() > option_.fileSystemOption.maxNameLength) {
-            newName = newName.substr(0, option_.fileSystemOption.maxNameLength);
-        }
+    void WarmUpTask();
 
-        return newName;
+    void WarmUpRun() {
+        std::lock_guard<std::mutex> lk(mtx_);
+        runned_ = true;
+        cond_.notify_one();
     }
+    void WaitWarmUp() {
+        std::unique_lock<std::mutex> lk(mtx_);
+        cond_.wait(lk, [this]() { return runned_; });
+        runned_ = false;
+    }
+    std::mutex mtx_;
+    std::condition_variable cond_;
+    bool runned_ = false;
+
 
  protected:
     // mds client
@@ -426,6 +389,9 @@ class FuseClient {
 
     std::shared_ptr<LeaseExecutor> leaseExecutor_;
 
+    // dir buffer
+    std::shared_ptr<DirBuffer> dirBuf_;
+
     // filesystem info
     std::shared_ptr<FsInfo> fsInfo_;
 
@@ -434,18 +400,15 @@ class FuseClient {
     // init flags
     bool init_;
 
+    std::atomic<bool>  mounted_;
+
     // enable record summary info in dir inode xattr
-    std::atomic<bool> enableSumInDir_;
+    bool enableSumInDir_;
 
     std::shared_ptr<FSMetric> fsMetric_;
 
     Mountpoint mountpoint_;
 
-    // warmup manager
-    std::shared_ptr<warmup::WarmupManager> warmupManager_;
-
-    std::shared_ptr<FileSystem> fs_;
-
  private:
     MDSBaseClient* mdsBase_;
 
@@ -453,9 +416,39 @@ class FuseClient {
 
     curve::common::Mutex renameMutex_;
 
-    Throttle throttle_;
+    Thread bgCmdTaskThread_;
+    std::atomic<bool> bgCmdStop_;
+    std::mutex cmdMtx_;
 
-    bthread_timer_t throttleTimer_;
+    void FetchChildDentryEnqueue(fuse_ino_t  ino);
+    void FetchChildDentry(fuse_ino_t ino);
+    void FetchDentry(fuse_ino_t ino, std::string file);
+    void LookPath(std::string file);
+    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
+        taskFetchMetaPool_;
+
+    // need warmup files
+    std::list<fuse_ino_t> readAheadFiles_;
+    std::mutex fetchMtx_;
+
+    //  one warmup file provided by the user
+    WarmUpFileContext_t warmUpFile_;
+    std::mutex warmUpFileMtx_;
+
+    std::list<std::string> warmUpTasks_;  // todo: need size control ?
+    std::mutex warmUpTaskMtx_;
+
+    void GetwarmTask(std::string *warmUpTask) {
+        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
+        if (warmUpTasks_.empty())
+            return;
+        *warmUpTask = std::move(warmUpTasks_.front());
+        warmUpTasks_.pop_front();
+    }
+    bool hasWarmTask() {
+        std::unique_lock<std::mutex> lck(warmUpTaskMtx_);
+        return !warmUpTasks_.empty();
+    }
 };
 
 }  // namespace client

curvefs/src/client/fuse_common.h

@@ -34,9 +34,9 @@ extern "C" {
 #endif
 
 struct MountOption {
-    const char* mountPoint;
-    const char* fsName;
-    const char* fsType;
+    char* mountPoint;
+    char* fsName;
+    char* fsType;
     char* conf;
     char* mdsAddr;
 };

curvefs/src/client/fuse_s3_client.cpp

@@ -48,11 +48,13 @@ using curvefs::mds::topology::MemcacheServerInfo;
 
 CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
     FuseClientOption opt(option);
+    initbgFetchThread_ = false;
 
     CURVEFS_ERROR ret = FuseClient::Init(opt);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
+    downloadMaxRetryTimes_ = option.downloadMaxRetryTimes;
 
     // init kvcache
     if (FLAGS_supportKVcache && !InitKVCache(option.kvClientManagerOpt)) {
@@ -67,20 +69,10 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
 
     auto s3Client = std::make_shared<S3ClientImpl>();
     s3Client->Init(opt.s3Opt.s3AdaptrOpt);
-
-    const uint64_t writeCacheMaxByte =
-        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
-    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
-        LOG(ERROR) << "writeCacheMaxByte is too small"
-                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
-                      ", writeCacheMaxByte = " << writeCacheMaxByte;
-        return CURVEFS_ERROR::CACHETOOSMALL;
-    }
-
     auto fsCacheManager = std::make_shared<FsCacheManager>(
         dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte, writeCacheMaxByte,
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheThreads, kvClientManager_);
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
+        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte);
     if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
         DiskCacheType::Disable) {
         auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
@@ -94,12 +86,17 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
             diskCacheManager, s3DiskCacheClient);
         ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
                                inodeManager_, mdsClient_, fsCacheManager,
-                               diskCacheManagerImpl, kvClientManager_, true);
+                               diskCacheManagerImpl, true);
     } else {
         ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
                                inodeManager_, mdsClient_, fsCacheManager,
-                               nullptr, kvClientManager_, true);
+                               nullptr, true);
     }
+    isWarmUping_.store(false);
+    bgFetchStop_.store(false, std::memory_order_release);
+    bgFetchThread_ = Thread(&FuseS3Client::BackGroundFetch, this);
+    initbgFetchThread_ = true;
+    GetTaskFetchPool();
     return ret;
 }
 
@@ -122,21 +119,304 @@ bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
         return false;
     }
 
-    kvClientManager_ = std::make_shared<KVClientManager>();
-    if (!kvClientManager_->Init(opt, memcacheClient)) {
+    g_kvClientManager = new KVClientManager();
+    if (!g_kvClientManager->Init(opt, memcacheClient)) {
         LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache
                    << ", but init kvClientManager fail";
         return false;
     }
 
-    if (warmupManager_ != nullptr) {
-        warmupManager_->SetKVClientManager(kvClientManager_);
-    }
-
     return true;
 }
 
+void FuseS3Client::GetWarmUpFileList(const WarmUpFileContext_t&warmUpFile,
+  std::vector<std::string>& warmUpFilelist) {
+        struct fuse_file_info fi{};
+        fi.flags &= ~O_DIRECT;
+        size_t rSize = 0;
+        std::unique_ptr<char[]> data(new char[warmUpFile.fileLen+1]);
+        std::memset(data.get(), 0, warmUpFile.fileLen);
+        data[warmUpFile.fileLen] = '\n';
+        FuseOpRead(nullptr, warmUpFile.inode,
+            warmUpFile.fileLen, 0, &fi, data.get(), &rSize);
+        std::string file = data.get();
+        VLOG(9) << "file is: " << file;
+        // remove enter, newline, blank
+        std::string blanks("\r\n ");
+        file.erase(0, file.find_first_not_of(blanks));
+        file.erase(file.find_last_not_of(blanks) + 1);
+        VLOG(9) << "after del file is: " << file;
+        splitStr(file, "\n", &warmUpFilelist);
+}
+
+void FuseS3Client::BackGroundFetch() {
+    while (!bgFetchStop_.load(std::memory_order_acquire)) {
+        usleep(WARMUP_CHECKINTERVAL_US);
+        if (hasWarmUpTask()) {  // new warmup task
+            WarmUpFileContext_t warmUpFile;
+            GetWarmUpFile(&warmUpFile);
+            VLOG(9) << " len is: " << warmUpFile.fileLen
+                << "ino is: " << warmUpFile.inode;
+
+            std::vector<std::string> warmUpFilelist;
+            GetWarmUpFileList(warmUpFile, warmUpFilelist);
+            for (auto filePath : warmUpFilelist) {
+                FetchDentryEnqueue(filePath);
+            }
+        }
+        {   // file need warmup
+            std::list<fuse_ino_t> readAheadFiles;
+            GetReadAheadFiles(&readAheadFiles);
+            if (!readAheadFiles.empty()) {
+                LOG(INFO) << "num of files is need loaded is: "
+                        << readAheadFiles.size();
+                for (auto iter : readAheadFiles) {
+                    VLOG(9) << "BackGroundFetch: " << iter;
+                    fetchDataEnqueue(iter);
+                }
+            }
+        }
+        {  // objs will be downloaded
+            {
+                std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                if (needWarmupObjs_.empty()) {
+                    continue;
+                }
+            }
+            if (isWarmUping_.exchange(true)) {
+                continue;
+            }
+            std::thread downloadThread =
+              std::thread(&FuseS3Client::WarmUpAllObjs, this);
+            downloadThread.detach();
+        }
+    }
+    return;
+}
+
+void FuseS3Client::fetchDataEnqueue(fuse_ino_t ino) {
+    VLOG(9) << "fetchDataEnqueue start: " << ino;
+    auto task = [this, ino]() {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                    << ", inodeid = " << ino;
+            return;
+        }
+        google::protobuf::Map<uint64_t, S3ChunkInfoList> s3ChunkInfoMap;
+        {
+            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+            s3ChunkInfoMap = *inodeWrapper->GetChunkInfoMap();
+        }
+        if (s3ChunkInfoMap.empty()) {
+            return;
+        }
+        travelChunks(ino, s3ChunkInfoMap);
+    };
+    GetTaskFetchPool().Enqueue(task);
+}
+
+// travel and download all objs belong to the chunk
+void FuseS3Client::travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo) {
+    uint64_t blockSize = s3Adaptor_->GetBlockSize();
+    uint64_t chunkSize = s3Adaptor_->GetChunkSize();
+    uint64_t offset, len, chunkid, compaction;
+    for (size_t i = 0; i < chunkInfo.s3chunks_size(); i++) {
+        auto chunkinfo = chunkInfo.mutable_s3chunks(i);
+        auto fsId = fsInfo_->fsid();
+        chunkid = chunkinfo->chunkid();
+        compaction = chunkinfo->compaction();
+        offset = chunkinfo->offset();
+        len = chunkinfo->len();
+        // the offset in the chunk
+        uint64_t chunkPos = offset % chunkSize;
+        // the offset in the block
+        uint64_t blockPos = chunkPos % blockSize;
+        // the first blockIndex
+        uint64_t blockIndexBegin = chunkPos / blockSize;
+
+        if (len < blockSize) {  // just one block
+            auto objectName = curvefs::common::s3util::GenObjName(
+                chunkid, blockIndexBegin, compaction, fsId, ino);
+            std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+            needWarmupObjs_.push_back(std::make_pair(objectName, len));
+        } else {
+            // the offset in the block
+            uint64_t blockPos = chunkPos % blockSize;
+
+            // firstly, let's get the size in the first block
+            // then, subtract the length in the first block
+            // to obtain the remaining length
+            // lastly, We need to judge the last block is full or not
+            uint64_t firstBlockSize = (blockPos != 0) ?
+              blockSize - blockPos : blockSize;
+            uint64_t leftSize = len - firstBlockSize;
+            uint32_t blockCounts = (leftSize % blockSize == 0) ?
+              (leftSize / blockSize + 1) : (leftSize / blockSize + 1 + 1);
+            // so we can get the last blockIndex
+            // because the bolck Index is cumulative
+            uint64_t blockIndexEnd = blockIndexBegin + blockCounts - 1;
+
+            // the size of the last block
+            uint64_t lastBlockSize = leftSize % blockSize;
+            // whether the first block or the last block is full or not
+            bool firstBlockFull = (blockPos == 0) ? true : false;
+            bool lastBlockFull = (lastBlockSize == 0) ? true : false;
+            // the start and end block Index that need travel
+            uint64_t travelStartIndex, travelEndIndex;
+            // if the block is full, the size is needed download
+            // of the obj is blockSize. Otherwise, the value is special.
+            if (!firstBlockFull) {
+                travelStartIndex = blockIndexBegin + 1;
+                auto objectName = curvefs::common::s3util::GenObjName(
+                  chunkid, blockIndexBegin, compaction, fsId, ino);
+                std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                needWarmupObjs_.push_back(std::make_pair(
+                  objectName, firstBlockSize));
+            } else {
+                travelStartIndex = blockIndexBegin;
+            }
+            if (!lastBlockFull) {
+                // block index is greater than or equal to 0
+                travelEndIndex = (blockIndexEnd == blockIndexBegin) ?
+                  blockIndexEnd : blockIndexEnd - 1;
+                auto objectName = curvefs::common::s3util::GenObjName(
+                  chunkid, blockIndexEnd, compaction, fsId, ino);
+                // there is no need to care about the order
+                // in which objects are downloaded
+                std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                needWarmupObjs_.push_back(
+                  std::make_pair(objectName, lastBlockSize));
+            } else {
+                travelEndIndex = blockIndexEnd;
+            }
+            VLOG(9) << "travel obj, ino: " << ino
+                << ", chunkid: " << chunkid
+                << ", blockCounts: " << blockCounts
+                << ", compaction: " << compaction
+                << ", blockSize: " << blockSize
+                << ", chunkSize: " << chunkSize
+                << ", offset: "  << offset
+                << ", blockIndexBegin: " << blockIndexBegin
+                << ", blockIndexEnd: " << blockIndexEnd
+                << ", len: " << len
+                << ", firstBlockSize: " << firstBlockSize
+                << ", lastBlockSize: " << lastBlockSize
+                << ", blockPos: " << blockPos
+                << ", chunkPos: " << chunkPos;
+            for (auto blockIndex = travelStartIndex;
+              blockIndex <= travelEndIndex ; blockIndex++) {
+                auto objectName = curvefs::common::s3util::GenObjName(
+                    chunkid, blockIndex, compaction, fsId, ino);
+                {
+                    std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+                    needWarmupObjs_.push_back(
+                      std::make_pair(objectName, blockSize));
+                }
+            }
+        }
+    }
+}
+
+// TODO(hzwuhongsong): These logics are very similar to other place,
+// try to merge it
+void FuseS3Client::WarmUpAllObjs() {
+    std::list<std::pair<std::string, uint64_t>> needWarmupObjs;
+    {
+        std::unique_lock<std::mutex> lck(warmupObjsMtx_);
+        LOG(INFO) << "num of objs need loaded is: " << needWarmupObjs_.size();
+        needWarmupObjs = std::move(needWarmupObjs_);
+    }
+    std::atomic<uint64_t> pendingReq(0);
+    curve::common::CountDownEvent cond(1);
+    // callback function
+    GetObjectAsyncCallBack cb =
+        [&](const S3Adapter *adapter,
+            const std::shared_ptr<GetObjectAsyncContext> &context) {
+            if (bgFetchStop_.load()) {
+                LOG(INFO) << "need stop warmup";
+                cond.Signal();
+                return;
+            }
+            if (context->retCode == 0) {
+                VLOG(9) << "Get Object success: " << context->key;
+                int ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(
+                  context->key, context->buf, context->len);
+                if (ret < 0) {
+                    LOG_EVERY_SECOND(INFO) <<
+                    "write read directly failed, key: " << context->key;
+                }
+                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
+                    VLOG(6) << "pendingReq is over";
+                    cond.Signal();
+                }
+                delete []context->buf;
+                return;
+            }
+            if (++context->retry >= downloadMaxRetryTimes_) {
+                if (pendingReq.fetch_sub(1, std::memory_order_seq_cst) == 1) {
+                    VLOG(6) << "pendingReq is over";
+                    cond.Signal();
+                }
+                LOG(WARNING) << "Up to max retry times, "
+                             << "download object failed, key: "
+                             << context->key;
+                delete []context->buf;
+                return;
+            }
+
+            LOG(WARNING) << "Get Object failed, key: " << context->key
+                         << ", offset: " << context->offset;
+            s3Adaptor_->GetS3Client()->DownloadAsync(context);
+    };
+
+    pendingReq.fetch_add(needWarmupObjs.size(), std::memory_order_seq_cst);
+    if (pendingReq.load()) {
+        VLOG(9) << "wait for pendingReq" << pendingReq.load();
+        for (auto iter : needWarmupObjs) {
+            VLOG(9) << "download start: " << iter.first;
+            std::string name = iter.first;
+            uint64_t readLen = iter.second;
+            if (s3Adaptor_->GetDiskCacheManager()->IsCached(name)) {
+                pendingReq.fetch_sub(1);
+                continue;
+            }
+            char *cacheS3 = new char[readLen];
+            memset(cacheS3, 0, readLen);
+            auto context = std::make_shared<GetObjectAsyncContext>();
+            context->key = name;
+            context->buf = cacheS3;
+            context->offset = 0;
+            context->len = readLen;
+            context->cb = cb;
+            context->retry = 0;
+            s3Adaptor_->GetS3Client()->DownloadAsync(context);
+        }
+        if (pendingReq.load())
+            cond.Wait();
+    }
+    isWarmUping_.exchange(false);
+    LOG(INFO) << "num of objs is loaded over ";
+}
+
+void FuseS3Client::travelChunks(
+    fuse_ino_t ino,
+    const google::protobuf::Map<uint64_t, S3ChunkInfoList>& s3ChunkInfoMap) {
+    VLOG(9) << "travel chunk start: " << ino
+            << ", size: " << s3ChunkInfoMap.size();
+    for (auto const& iter : s3ChunkInfoMap) {
+        VLOG(9) << "travel chunk: " << iter.first;
+        travelChunk(ino, iter.second);
+    }
+    VLOG(9) << "travel chunks end";
+}
+
 void FuseS3Client::UnInit() {
+    bgFetchStop_.store(true, std::memory_order_release);
+    if (initbgFetchThread_) {
+        bgFetchThread_.join();
+    }
     FuseClient::UnInit();
     s3Adaptor_->Stop();
     curve::common::S3Adapter::Shutdown();
@@ -155,8 +435,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
 CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                         const char *buf, size_t size, off_t off,
                                         struct fuse_file_info *fi,
-                                        FileOut* fileOut) {
-    size_t *wSize = &fileOut->nwritten;
+                                        size_t *wSize) {
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -218,8 +497,6 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
             }
         }
     }
-
-    inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
     return ret;
 }
 
@@ -227,7 +504,6 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                        size_t size, off_t off,
                                        struct fuse_file_info *fi, char *buffer,
                                        size_t *rSize) {
-    (void)req;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -246,7 +522,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
     uint64_t fileSize = inodeWrapper->GetLength();
 
     size_t len = 0;
-    if (static_cast<int64_t>(fileSize) <= off) {
+    if (fileSize <= off) {
         *rSize = 0;
         return CURVEFS_ERROR::OK;
     } else if (fileSize < off + size) {
@@ -282,56 +558,32 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                          const char *name, mode_t mode,
                                          struct fuse_file_info *fi,
-                                         EntryOut* entryOut) {
+                                         fuse_entry_param *e) {
     VLOG(1) << "FuseOpCreate, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
-
-    std::shared_ptr<InodeWrapper> inode;
     CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, inode);
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, e);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-
-    auto openFiles = fs_->BorrowMember().openFiles;
-    openFiles->Open(inode->GetInodeId(), inode);
-
-    inode->GetInodeAttr(&entryOut->attr);
-    return CURVEFS_ERROR::OK;
+    return FuseOpOpen(req, e->ino, fi);
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req,
-                                        fuse_ino_t parent,
-                                        const char* name,
-                                        mode_t mode,
-                                        dev_t rdev,
-                                        EntryOut* entryOut) {
+CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+                                        const char *name, mode_t mode,
+                                        dev_t rdev, fuse_entry_param *e) {
     VLOG(1) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-
-    std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
-                                FsFileType::TYPE_S3, rdev, false,
-                                inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    InodeAttr attr;
-    inode->GetInodeAttr(&attr);
-    *entryOut = EntryOut(attr);
-    return CURVEFS_ERROR::OK;
+    return MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, rdev, e);
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req,
-                                       fuse_ino_t ino,
-                                       fuse_ino_t newparent,
-                                       const char* newname,
-                                       EntryOut* entryOut) {
+CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char *newname,
+                                     fuse_entry_param *e) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_S3, entryOut);
+        req, ino, newparent, newname, FsFileType::TYPE_S3, e);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
@@ -343,8 +595,6 @@ CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
 CURVEFS_ERROR FuseS3Client::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                         int datasync,
                                         struct fuse_file_info *fi) {
-    (void)req;
-    (void)fi;
     VLOG(1) << "FuseOpFsync, ino: " << ino << ", datasync: " << datasync;
 
     CURVEFS_ERROR ret = s3Adaptor_->Flush(ino);
@@ -373,8 +623,6 @@ CURVEFS_ERROR FuseS3Client::Truncate(InodeWrapper *inode, uint64_t length) {
 
 CURVEFS_ERROR FuseS3Client::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
-    (void)req;
-    (void)fi;
     VLOG(1) << "FuseOpFlush, ino: " << ino;
     CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
 

curvefs/src/client/fuse_s3_client.h

@@ -32,8 +32,6 @@
 
 #include "curvefs/src/client/fuse_client.h"
 #include "curvefs/src/client/s3/client_s3_cache_manager.h"
-#include "curvefs/src/client/warmup/warmup_manager.h"
-#include "curvefs/src/volume/common.h"
 #include "src/common/s3_adapter.h"
 
 namespace curvefs {
@@ -41,35 +39,18 @@ namespace client {
 
 using curve::common::GetObjectAsyncContext;
 using curve::common::GetObjectAsyncCallBack;
-using curvefs::volume::kMiB;
-namespace warmup {
-class WarmupManager;
-class WarmupManagerS3Impl;
-}  // namespace warmup
-
 
 class FuseS3Client : public FuseClient {
  public:
     FuseS3Client()
-        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {
-        auto readFunc = [this](fuse_req_t req, fuse_ino_t ino, size_t size,
-                               off_t off, struct fuse_file_info *fi,
-                               char *buffer, size_t *rSize) {
-            return FuseOpRead(req, ino, size, off, fi, buffer, rSize);
-        };
-        warmupManager_ = std::make_shared<warmup::WarmupManagerS3Impl>(
-            metaClient_, inodeManager_, dentryManager_, fsInfo_, readFunc,
-            s3Adaptor_, nullptr);
-    }
+        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {}
 
     FuseS3Client(const std::shared_ptr<MdsClient> &mdsClient,
                  const std::shared_ptr<MetaServerClient> &metaClient,
                  const std::shared_ptr<InodeCacheManager> &inodeManager,
                  const std::shared_ptr<DentryCacheManager> &dentryManager,
-                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor,
-                 const std::shared_ptr<warmup::WarmupManager> &warmupManager)
-        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
-                     warmupManager),
+                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor)
+        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager),
           s3Adaptor_(s3Adaptor) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
@@ -81,7 +62,7 @@ class FuseS3Client : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, FileOut* fileOut) override;
+        struct fuse_file_info *fi, size_t *wSize) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
         fuse_ino_t ino, size_t size, off_t off,
@@ -89,25 +70,17 @@ class FuseS3Client : public FuseClient {
         char *buffer,
         size_t *rSize) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
-                               fuse_ino_t parent,
-                               const char* name,
-                               mode_t mode,
-                               struct fuse_file_info *fi,
-                               EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, struct fuse_file_info *fi,
+        fuse_entry_param *e) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
-                              fuse_ino_t parent,
-                              const char* name,
-                              mode_t mode,
-                              dev_t rdev,
-                              EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, dev_t rdev,
+        fuse_entry_param *e) override;
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
-                             fuse_ino_t ino,
-                             fuse_ino_t newparent,
-                             const char *newname,
-                             EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+        fuse_ino_t newparent, const char *newname,
+        fuse_entry_param *e) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
         const char *name) override;
@@ -118,19 +91,39 @@ class FuseS3Client : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
         struct fuse_file_info *fi) override;
 
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
  private:
     bool InitKVCache(const KVClientManagerOpt &opt);
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
     void FlushData() override;
+    // get the warmUp filelist
+    void GetWarmUpFileList(const WarmUpFileContext_t&,
+      std::vector<std::string>&);
+    void BackGroundFetch();
+    // put the file needed warmup to queue,
+    // then can downlaod the objs belong to it
+    void fetchDataEnqueue(fuse_ino_t ino);
+    // travel all chunks
+    void travelChunks(
+        fuse_ino_t ino,
+        const google::protobuf::Map<uint64_t, S3ChunkInfoList>& s3ChunkInfoMap);
+    // travel and download all objs belong to the chunk
+    void travelChunk(fuse_ino_t ino, S3ChunkInfoList chunkInfo);
+    // warmup all the prefetchObjs
+    void WarmUpAllObjs();
 
  private:
     // s3 adaptor
     std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
-    std::shared_ptr<KVClientManager> kvClientManager_;
 
-    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
+    bool initbgFetchThread_;
+    Thread bgFetchThread_;
+    std::atomic<bool> bgFetchStop_;
+    uint32_t downloadMaxRetryTimes_;
+    std::mutex warmupObjsMtx_;
+    std::atomic<bool> isWarmUping_;
+    std::list<std::pair<std::string, uint64_t>> needWarmupObjs_;
 };
 
 

curvefs/src/client/fuse_volume_client.cpp

@@ -32,7 +32,7 @@
 #include "absl/cleanup/cleanup.h"
 #include "absl/memory/memory.h"
 #include "curvefs/proto/mds.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/volume/default_volume_storage.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/volume/common.h"
@@ -133,11 +133,10 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
                                             size_t size,
                                             off_t off,
                                             struct fuse_file_info *fi,
-                                            FileOut* fileOut) {
+                                            size_t *wSize) {
     VLOG(9) << "write start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
-    size_t* wSize = &fileOut->nwritten;
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
               is_aligned(size, DirectIOAlignment))) {
@@ -149,7 +148,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
     butil::Timer timer;
     timer.start();
 
-    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf, fileOut);
+    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf);
     if (ret != CURVEFS_ERROR::OK) {
         if (fsMetric_) {
             fsMetric_->userWrite.eps.count << 1;
@@ -190,7 +189,6 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
                                            struct fuse_file_info *fi,
                                            char *buffer,
                                            size_t *rSize) {
-    (void)req;
     VLOG(3) << "read start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
@@ -235,57 +233,33 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
 CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                              const char *name, mode_t mode,
                                              struct fuse_file_info *fi,
-                                             EntryOut* entryOut) {
+                                             fuse_entry_param *e) {
     VLOG(3) << "FuseOpCreate, parent: " << parent
               << ", name: " << name
               << ", mode: " << mode;
-
-    std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR ret = MakeNode(
-        req, parent, name, mode, FsFileType::TYPE_FILE, 0, false, inode);
+    CURVEFS_ERROR ret =
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, e);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-
-    auto openFiles = fs_->BorrowMember().openFiles;
-    openFiles->Open(inode->GetInodeId(), inode);
-
-    inode->GetInodeAttr(&entryOut->attr);
-    return CURVEFS_ERROR::OK;
+    return FuseOpOpen(req, e->ino, fi);
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req,
-                                            fuse_ino_t parent,
-                                            const char* name,
-                                            mode_t mode,
-                                            dev_t rdev,
-                                            EntryOut* entryOut) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+                                            const char *name, mode_t mode,
+                                            dev_t rdev, fuse_entry_param *e) {
     VLOG(3) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-
-    std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
-                                FsFileType::TYPE_FILE, rdev,
-                                false, inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    InodeAttr attr;
-    inode->GetInodeAttr(&attr);
-    *entryOut = EntryOut(attr);
-    return CURVEFS_ERROR::OK;
+    return MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, rdev, e);
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req,
-                                           fuse_ino_t ino,
-                                           fuse_ino_t newparent,
-                                           const char* newname,
-                                           EntryOut* entryOut) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char *newname,
+                                     fuse_entry_param *e) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_FILE, entryOut);
+        req, ino, newparent, newname, FsFileType::TYPE_FILE, e);
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
@@ -297,8 +271,6 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
 CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                             int datasync,
                                             struct fuse_file_info *fi) {
-    (void)req;
-    (void)fi;
     VLOG(3) << "FuseOpFsync start, ino: " << ino << ", datasync: " << datasync;
 
     CURVEFS_ERROR ret = storage_->Flush(ino);
@@ -325,16 +297,12 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
 }
 
 CURVEFS_ERROR FuseVolumeClient::Truncate(InodeWrapper *inode, uint64_t length) {
-    (void)inode;
-    (void)length;
     // Todo: call volume truncate
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                             struct fuse_file_info *fi) {
-    (void)req;
-    (void)fi;
     VLOG(9) << "FuseOpFlush, ino: " << ino;
 
     CURVEFS_ERROR ret = storage_->Flush(ino);

curvefs/src/client/fuse_volume_client.h

@@ -52,8 +52,7 @@ class FuseVolumeClient : public FuseClient {
         const std::shared_ptr<InodeCacheManager> &inodeManager,
         const std::shared_ptr<DentryCacheManager> &dentryManager,
         const std::shared_ptr<BlockDeviceClient> &blockDeviceClient)
-        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
-                     nullptr),
+        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager),
           blockDeviceClient_(blockDeviceClient) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
@@ -65,32 +64,24 @@ class FuseVolumeClient : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, FileOut* fileOut) override;
+        struct fuse_file_info *fi, size_t *wSize) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
             fuse_ino_t ino, size_t size, off_t off,
             struct fuse_file_info *fi,
             char *buffer,
             size_t *rSize) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, struct fuse_file_info *fi,
+        fuse_entry_param *e) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
-                               fuse_ino_t parent,
-                               const char* name,
-                               mode_t mode,
-                               struct fuse_file_info* fi,
-                               EntryOut* entryOut) override;
-
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
-                              fuse_ino_t parent,
-                              const char* name,
-                              mode_t mode,
-                              dev_t rdev,
-                              EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, dev_t rdev,
+        fuse_entry_param *e) override;
 
     CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                             fuse_ino_t newparent,
-                             const char *newname,
-                             EntryOut* entryOut) override;
+        fuse_ino_t newparent, const char *newname,
+        fuse_entry_param *e) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                const char *name) override;
@@ -101,13 +92,13 @@ class FuseVolumeClient : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                               struct fuse_file_info *fi) override;
 
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
     void SetSpaceManagerForTesting(SpaceManager *manager);
 
     void SetVolumeStorageForTesting(VolumeStorage *storage);
 
  private:
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
     void FlushData() override;
 
  private:

curvefs/src/client/inode_cache_manager.cpp

@@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 using ::curvefs::metaserver::Inode;
@@ -46,11 +46,36 @@ DECLARE_bool(enableCto);
 namespace curvefs {
 namespace client {
 
-using ::curvefs::client::filesystem::ToFSError;
-
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
 using curvefs::client::common::FLAGS_enableCto;
 
+class TrimICacheAsyncDone : public MetaServerClientDone {
+ public:
+    explicit TrimICacheAsyncDone(
+        const std::shared_ptr<InodeWrapper> &inodeWrapper,
+        const std::shared_ptr<InodeCacheManagerImpl> &inodeCacheManager)
+        : inodeWrapper_(inodeWrapper), inodeCacheManager_(inodeCacheManager) {}
+
+    void Run() override {
+        std::unique_ptr<TrimICacheAsyncDone> self_guard(this);
+        MetaStatusCode ret = GetStatusCode();
+        if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
+            LOG(ERROR) << "metaClient_ UpdateInode failed, "
+                       << "MetaStatusCode: " << ret
+                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
+                       << ", inodeid: " << inodeWrapper_->GetInodeId();
+            inodeWrapper_->MarkInodeError();
+        }
+        VLOG(9) << "Trim inode " << inodeWrapper_->GetInodeId()
+                << " async success.";
+        inodeCacheManager_->RemoveICache(inodeWrapper_);
+    };
+
+ private:
+    std::shared_ptr<InodeWrapper> inodeWrapper_;
+    std::shared_ptr<InodeCacheManagerImpl> inodeCacheManager_;
+};
+
 #define GET_INODE_REMOTE(FSID, INODEID, OUT, STREAMING)                        \
     MetaStatusCode ret = metaClient_->GetInode(FSID, INODEID, OUT, STREAMING); \
     if (ret != MetaStatusCode::OK) {                                           \
@@ -58,7 +83,17 @@ using curvefs::client::common::FLAGS_enableCto;
             << "metaClient_ GetInode failed, MetaStatusCode = " << ret         \
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)          \
             << ", inodeid = " << INODEID;                                      \
-        return ToFSError(ret);                            \
+        return MetaStatusCodeToCurvefsErrCode(ret);                            \
+    }
+
+#define PUT_INODE_CACHE(INODEID, INODEWRAPPER)                                 \
+    std::shared_ptr<InodeWrapper> eliminatedOne;                               \
+    bool eliminated = iCache_->Put(INODEID, INODEWRAPPER, &eliminatedOne);     \
+    if (eliminated) {                                                          \
+        VLOG(3) << "GetInode eliminate one inode, ino: "                       \
+                << eliminatedOne->GetInodeId()                                 \
+                << ", iCache does not evict inodes via put interface ";        \
+        assert(0);                                                             \
     }
 
 #define REFRESH_DATA_REMOTE(OUT, STREAMING)                                    \
@@ -71,8 +106,15 @@ CURVEFS_ERROR
 InodeCacheManagerImpl::GetInode(uint64_t inodeId,
                                 std::shared_ptr<InodeWrapper> &out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    bool yes = openFiles_->IsOpened(inodeId, &out);
-    if (yes) {
+    // get inode from cache
+    bool ok = iCache_->Get(inodeId, &out);
+    if (ok && NeedUseCahce(inodeId, out->IsDirty())) {
+        curve::common::UniqueLock lgGuard = out->GetUniqueLock();
+        if (out->GetType() == FsFileType::TYPE_FILE) {
+            return CURVEFS_ERROR::OK;
+        }
+
+        REFRESH_DATA_REMOTE(out, out->NeedRefreshData());
         return CURVEFS_ERROR::OK;
     }
 
@@ -87,12 +129,24 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     // refresh data
     REFRESH_DATA_REMOTE(out, streaming);
 
+    // put to cache
+    PUT_INODE_CACHE(inodeId, out);
+
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
                                                   InodeAttr *out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
+    // 1. find in icache
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    bool ok = iCache_->Get(inodeId, &inodeWrapper);
+    if (ok && NeedUseCahce(inodeId, inodeWrapper->IsDirty())) {
+        inodeWrapper->GetInodeAttr(out);
+        return CURVEFS_ERROR::OK;
+    }
+
+    // 2. get form metaserver
     std::set<uint64_t> inodeIds;
     std::list<InodeAttr> attrs;
     inodeIds.emplace(inodeId);
@@ -102,7 +156,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed"
                    << ", inodeId = " << inodeId << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
 
     if (attrs.size() != 1) {
@@ -119,6 +173,21 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
     std::set<uint64_t> *inodeIds,
     std::list<InodeAttr> *attrs) {
+    // get some inode attr in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+            InodeAttr tmpAttr;
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace_back(std::move(tmpAttr));
+            iter = inodeIds->erase(iter);
+            continue;
+        }
+        ++iter;
+    }
+
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -130,7 +199,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return ToFSError(ret);
+    return MetaStatusCodeToCurvefsErrCode(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
@@ -138,6 +207,26 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     std::set<uint64_t> *inodeIds,
     std::map<uint64_t, InodeAttr> *attrs) {
     NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
+    std::map<uint64_t, InodeAttr> cachedAttr;
+    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
+
+    // get some inode attr in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+            InodeAttr tmpAttr;
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace(*iter, std::move(tmpAttr));
+            iter = inodeIds->erase(iter);
+        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
+            attrs->emplace(*iter, cachedAttr[*iter]);
+            iter = inodeIds->erase(iter);
+        } else {
+            ++iter;
+        }
+    }
 
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
@@ -149,12 +238,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
         return CURVEFS_ERROR::NOTEXIST;
     }
 
-    ::curve::common::Mutex mutex;
     std::shared_ptr<CountDownEvent> cond =
         std::make_shared<CountDownEvent>(inodeGroups.size());
     for (const auto& it : inodeGroups) {
         VLOG(3) << "BatchGetInodeAttrAsync Send " << it.size();
-        auto* done = new BatchGetInodeAttrAsyncDone(attrs, &mutex, cond);
+        auto* done = new BatchGetInodeAttrAsyncDone(shared_from_this(),
+                                                    cond, parentId);
         MetaStatusCode ret = metaClient_->BatchGetInodeAttrAsync(fsId_, it,
                                                                  done);
         if (MetaStatusCode::OK != ret) {
@@ -167,12 +256,30 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
 
     // wait for all sudrequest finished
     cond->Wait();
+
+    bool ok  = iAttrCache_->Get(parentId, attrs);
+    if (!ok) {
+        LOG(WARNING) << "get attrs form iAttrCache_ failed.";
+    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
     std::set<uint64_t> *inodeIds,
     std::list<XAttr> *xattr) {
+    // get some inode in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCahce(*iter, inodeWrapper->IsDirty())) {
+            xattr->emplace_back(inodeWrapper->GetXattr());
+            iter = inodeIds->erase(iter);
+        } else {
+            ++iter;
+        }
+    }
+
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -183,7 +290,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return ToFSError(ret);
+    return MetaStatusCodeToCurvefsErrCode(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
@@ -194,46 +301,159 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ CreateInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
+    uint64_t inodeid = inode.inodeid();
     out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
         s3ChunkInfoMetric_, option_.maxDataSize,
         option_.refreshDataIntervalSec);
-    return CURVEFS_ERROR::OK;
-}
 
-CURVEFS_ERROR InodeCacheManagerImpl::CreateManageInode(
-    const InodeParam &param,
-    std::shared_ptr<InodeWrapper> &out) {
-    Inode inode;
-    MetaStatusCode ret = metaClient_->CreateManageInode(param, &inode);
-    if (ret != MetaStatusCode::OK) {
-        LOG(ERROR) << "metaClient_ CreateManageInode failed, MetaStatusCode = "
-                   << ret << ", MetaStatusCode_Name = "
-                   << MetaStatusCode_Name(ret);
-        return ToFSError(ret);
+    std::shared_ptr<InodeWrapper> eliminatedOne;
+    bool eliminated = false;
+    {
+        NameLockGuard lock(nameLock_, std::to_string(inodeid));
+        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
+    }
+    if (eliminated) {
+        /* iCache does not evict inodes via put interface */
+        assert(0);
     }
-    out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
-        s3ChunkInfoMetric_, option_.maxDataSize,
-        option_.refreshDataIntervalSec);
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::DeleteInode(uint64_t inodeId) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
+    iCache_->Remove(inodeId);
     MetaStatusCode ret = metaClient_->DeleteInode(fsId_, inodeId);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeId = " << inodeId;
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
+
+    curve::common::LockGuard lg2(dirtyMapMutex_);
+    dirtyMap_.erase(inodeId);
     return CURVEFS_ERROR::OK;
 }
 
+void InodeCacheManagerImpl::AddInodeAttrs(
+    uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
+    iAttrCache_->Set(parentId, inodeAttrs);
+}
+
+void InodeCacheManagerImpl::ClearInodeCache(uint64_t inodeId) {
+    {
+        NameLockGuard lock(nameLock_, std::to_string(inodeId));
+        iCache_->Remove(inodeId);
+    }
+    curve::common::LockGuard lg2(dirtyMapMutex_);
+    dirtyMap_.erase(inodeId);
+}
+
 void InodeCacheManagerImpl::ShipToFlush(
-    const std::shared_ptr<InodeWrapper>& inode) {
-    deferSync_->Push(inode);
+    const std::shared_ptr<InodeWrapper> &inodeWrapper) {
+    curve::common::LockGuard lg(dirtyMapMutex_);
+    dirtyMap_.emplace(inodeWrapper->GetInodeId(), inodeWrapper);
+}
+
+void InodeCacheManagerImpl::FlushAll() {
+    while (!dirtyMap_.empty()) {
+        FlushInodeOnce();
+    }
+}
+
+void InodeCacheManagerImpl::FlushInodeOnce() {
+    std::map<uint64_t, std::shared_ptr<InodeWrapper>> temp_;
+    {
+        curve::common::LockGuard lg(dirtyMapMutex_);
+        temp_.swap(dirtyMap_);
+    }
+    for (auto it = temp_.begin(); it != temp_.end(); it++) {
+        curve::common::UniqueLock ulk = it->second->GetUniqueLock();
+        it->second->Async(nullptr, true);
+    }
+}
+
+void InodeCacheManagerImpl::ReleaseCache(uint64_t parentId) {
+    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
+    iAttrCache_->Release(parentId);
+}
+
+void InodeCacheManagerImpl::FlushInodeBackground() {
+    LOG(INFO) << "flush thread is start.";
+    while (!isStop_.load()) {
+        if (iCache_->Size() > maxCacheSize_) {
+            TrimIcache(iCache_->Size() - maxCacheSize_);
+        }
+        FlushInodeOnce();
+        sleeper_.wait_for(std::chrono::seconds(flushPeriodSec_));
+    }
+    LOG(INFO) << "flush thread is stop.";
+}
+
+namespace {
+// Wether an inode is dirty or not.
+// if |needLock| is true, we acquire the lock firstly and do the test,
+// otherwise, we assume the lock is already held.
+bool IsDirtyInode(InodeWrapper *ino, bool needLock) {
+    auto check = [ino]() {
+        return ino->IsDirty() || !ino->S3ChunkInfoEmptyNolock() ||
+               ino->GetMutableExtentCacheLocked()->HasDirtyExtents();
+    };
+
+    if (needLock) {
+        auto lk = ino->GetUniqueLock();
+        return check();
+    }
+
+    return check();
+}
+}  // namespace
+
+void InodeCacheManagerImpl::RemoveICache(
+    const std::shared_ptr<InodeWrapper> &inode) {
+    if (!IsDirtyInode(inode.get(), true)) {
+        uint64_t inodeId = inode->GetInodeId();
+        NameLockGuard lock(nameLock_, std::to_string(inodeId));
+        ::curve::common::UniqueLock lgGuard = inode->GetUniqueLock();
+        iCache_->Remove(inodeId);
+    }
+}
+
+void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    uint64_t inodeId;
+    VLOG(3) << "TrimIcache trimSize " << trimSize;
+    while (trimSize > 0) {
+        bool ok = iCache_->GetLast(&inodeId, &inodeWrapper);
+        if (ok) {
+            NameLockGuard lock(nameLock_, std::to_string(inodeId));
+            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+            if (IsDirtyInode(inodeWrapper.get(), false)) {
+                VLOG(9) << "TrimIcache sync dirty inode " << inodeId;
+                dirtyMapMutex_.lock();
+                dirtyMap_.erase(inodeId);
+                dirtyMapMutex_.unlock();
+                auto *done =
+                    new TrimICacheAsyncDone(inodeWrapper, shared_from_this());
+                inodeWrapper->Async(done);
+            } else {
+                VLOG(9) << "TrimIcache remove inode " << inodeId
+                        << " from iCache";
+                iCache_->Remove(inodeId);
+            }
+            trimSize--;
+            // remove the attr of the inode in iattrcache
+            auto parents = inodeWrapper->GetParentLocked();
+            for (uint64_t parent : parents) {
+                iAttrCache_->Remove(parent, inodeId);
+            }
+        } else {
+            LOG(ERROR) << "icache size " << iCache_->Size();
+            assert(0);
+        }
+    }
 }
 
 CURVEFS_ERROR
@@ -270,5 +490,33 @@ InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
     return rc;
 }
 
+void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    openedInodes_.emplace(inodeId);
+}
+
+void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    if (iter != openedInodes_.end()) {
+        openedInodes_.erase(iter);
+    }
+}
+
+bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    return iter != openedInodes_.end();
+}
+
+bool InodeCacheManagerImpl::NeedUseCahce(uint64_t inodeId, bool IsDirty) {
+    if (!FLAGS_enableCto || OpenInodeCached(inodeId) || IsDirty) {
+        return true;
+    }
+    return false;
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_cache_manager.h

@@ -38,14 +38,12 @@
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "src/common/concurrent/name_lock.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/openfile.h"
-#include "curvefs/src/client/filesystem/defer_sync.h"
 
 using ::curve::common::LRUCache;
 using ::curve::common::CacheMetrics;
@@ -66,8 +64,57 @@ using rpcclient::BatchGetInodeAttrDone;
 using curve::common::CountDownEvent;
 using metric::S3ChunkInfoMetric;
 using common::RefreshDataOption;
-using ::curvefs::client::filesystem::OpenFiles;
-using ::curvefs::client::filesystem::DeferSync;
+
+class InodeAttrCache {
+ public:
+    InodeAttrCache() {}
+    ~InodeAttrCache() {}
+
+    bool Get(uint64_t parentId, std::map<uint64_t, InodeAttr> *imap) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            imap->insert(iter->second.begin(), iter->second.end());
+            return true;
+        }
+        return false;
+    }
+
+    void Set(uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        VLOG(1) << "parentId = " << parentId
+                << ", iAttrCache set size = " << inodeAttrs.size();
+        auto& inner = iAttrCache_[parentId];
+        for (const auto &it : inodeAttrs) {
+            inner.emplace(it.inodeid(), it);
+        }
+    }
+
+    void Release(uint64_t parentId) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto size = iAttrCache_.size();
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            iAttrCache_.erase(iter);
+        }
+        VLOG(1) << "inodeId = " << parentId
+                << ", inode attr cache release, before = "
+                << size << ", after = " << iAttrCache_.size();
+    }
+
+    void Remove(uint64_t parentId, uint64_t inodeId) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            iter->second.erase(inodeId);
+        }
+    }
+
+ private:
+    // inodeAttr cache; <parentId, <inodeId, inodeAttr>>
+    std::map<uint64_t, std::map<uint64_t, InodeAttr>> iAttrCache_;
+    curve::common::Mutex iAttrCacheMutex_;
+};
 
 class InodeCacheManager {
  public:
@@ -79,9 +126,13 @@ class InodeCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(RefreshDataOption option,
-                               std::shared_ptr<OpenFiles> openFiles,
-                               std::shared_ptr<DeferSync> deferSync) = 0;
+    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+                               uint32_t flushPeriodSec,
+                               RefreshDataOption option) = 0;
+
+    virtual void Run() = 0;
+
+    virtual void Stop() = 0;
 
     virtual CURVEFS_ERROR
     GetInode(uint64_t inodeId,
@@ -104,14 +155,26 @@ class InodeCacheManager {
     virtual CURVEFS_ERROR CreateInode(const InodeParam &param,
         std::shared_ptr<InodeWrapper> &out) = 0;   // NOLINT
 
-    virtual CURVEFS_ERROR CreateManageInode(const InodeParam &param,
-        std::shared_ptr<InodeWrapper> &out) = 0;   // NOLINT
-
     virtual CURVEFS_ERROR DeleteInode(uint64_t inodeId) = 0;
 
+    virtual void AddInodeAttrs(uint64_t parentId,
+        const RepeatedPtrField<InodeAttr>& inodeAttrs) = 0;
+
+    virtual void ClearInodeCache(uint64_t inodeId) = 0;
+
     virtual void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) = 0;
 
+    virtual void FlushAll() = 0;
+
+    virtual void FlushInodeOnce() = 0;
+
+    virtual void ReleaseCache(uint64_t parentId) = 0;
+
+    virtual void AddOpenedInode(uint64_t inodeId) = 0;
+
+    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
+
  protected:
     uint32_t fsId_;
 };
@@ -120,22 +183,55 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     public std::enable_shared_from_this<InodeCacheManagerImpl> {
  public:
     InodeCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()),
+        iCache_(nullptr),
+        iAttrCache_(nullptr),
+        isStop_(true) {}
 
     explicit InodeCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient) {}
+      : metaClient_(metaClient),
+        iCache_(nullptr),
+        iAttrCache_(nullptr) {}
 
-    CURVEFS_ERROR Init(RefreshDataOption option,
-                       std::shared_ptr<OpenFiles> openFiles,
-                       std::shared_ptr<DeferSync> deferSync) override {
+    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+                       uint32_t flushPeriodSec,
+                       RefreshDataOption option) override {
+        if (enableCacheMetrics) {
+            iCache_ = std::make_shared<
+                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0,
+                    std::make_shared<CacheMetrics>("icache"));
+        } else {
+            iCache_ = std::make_shared<
+                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0);
+        }
+        maxCacheSize_ = cacheSize;
         option_ = option;
+        flushPeriodSec_ = flushPeriodSec;
+        iAttrCache_ = std::make_shared<InodeAttrCache>();
         s3ChunkInfoMetric_ = std::make_shared<S3ChunkInfoMetric>();
-        openFiles_ =  openFiles;
-        deferSync_ = deferSync;
         return CURVEFS_ERROR::OK;
     }
 
+    void Run() {
+        isStop_.exchange(false);
+        flushThread_ =
+            Thread(&InodeCacheManagerImpl::FlushInodeBackground, this);
+        LOG(INFO) << "Start inodeManager flush thread ok.";
+    }
+
+    void Stop() {
+        isStop_.exchange(true);
+        LOG(INFO) << "stop inodeManager flush thread ...";
+        sleeper_.interrupt();
+        flushThread_.join();
+    }
+
+    bool IsDirtyMapExist(uint64_t inodeId) {
+        curve::common::LockGuard lg(dirtyMapMutex_);
+        return dirtyMap_.count(inodeId) > 0;
+    }
+
     CURVEFS_ERROR GetInode(uint64_t inodeId,
                            std::shared_ptr<InodeWrapper> &out) override;
 
@@ -154,42 +250,73 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     CURVEFS_ERROR CreateInode(const InodeParam &param,
         std::shared_ptr<InodeWrapper> &out) override;
 
-    CURVEFS_ERROR CreateManageInode(const InodeParam &param,
-        std::shared_ptr<InodeWrapper> &out) override;
-
     CURVEFS_ERROR DeleteInode(uint64_t inodeId) override;
 
+    void AddInodeAttrs(uint64_t parentId,
+        const RepeatedPtrField<InodeAttr>& inodeAttrs) override;
+
+    void ClearInodeCache(uint64_t inodeId) override;
+
     void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) override;
 
+    void FlushAll() override;
+
+    void FlushInodeOnce() override;
+
+    void ReleaseCache(uint64_t parentId) override;
+
+    void RemoveICache(const std::shared_ptr<InodeWrapper> &inode);
+
+    void AddOpenedInode(uint64_t inodeId) override;
+
+    void RemoveOpenedInode(uint64_t inodeId) override;
+
+    bool NeedUseCahce(uint64_t inodeId, bool IsDirty);
+
  private:
+    virtual void FlushInodeBackground();
+    void TrimIcache(uint64_t trimSize);
     CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
                               bool streaming = true);
+    bool OpenInodeCached(uint64_t inodeId);
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
+    std::shared_ptr<LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>> iCache_;
     std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric_;
 
-    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<InodeAttrCache> iAttrCache_;
 
-    std::shared_ptr<DeferSync> deferSync_;
+    // dirty map, key is inodeid
+    std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
+    curve::common::Mutex dirtyMapMutex_;
+
+    // record opened inode
+    std::multiset<uint64_t> openedInodes_;
+    curve::common::Mutex openInodesMutex_;
 
     curve::common::GenericNameLock<Mutex> nameLock_;
 
     curve::common::GenericNameLock<Mutex> asyncNameLock_;
 
+    uint64_t maxCacheSize_;
     RefreshDataOption option_;
+    uint32_t flushPeriodSec_;
+    Thread flushThread_;
+    InterruptibleSleeper sleeper_;
+    Atomic<bool> isStop_;
 };
 
 class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
  public:
-    BatchGetInodeAttrAsyncDone(std::map<uint64_t, InodeAttr>* attrs,
-                               ::curve::common::Mutex* mutex,
-                               std::shared_ptr<CountDownEvent> cond):
-        attrs_(attrs),
-        mutex_(mutex),
-        cond_(cond) {}
-
+    BatchGetInodeAttrAsyncDone(
+        const std::shared_ptr<InodeCacheManager> &inodeCacheManager,
+        std::shared_ptr<CountDownEvent> cond,
+        uint64_t parentId):
+        inodeCacheManager_(inodeCacheManager),
+        cond_(cond),
+        parentId_(parentId) {}
     ~BatchGetInodeAttrAsyncDone() {}
 
     void Run() override {
@@ -197,25 +324,22 @@ class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
         MetaStatusCode ret = GetStatusCode();
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "BatchGetInodeAttrAsync failed, "
+                       << "parentId = " << parentId_
                        << ", MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret);
         } else {
             auto inodeAttrs = GetInodeAttrs();
             VLOG(3) << "BatchGetInodeAttrAsyncDone update inodeAttrCache"
                     << " size = " << inodeAttrs.size();
-
-            curve::common::LockGuard lk(*mutex_);
-            for (const auto& attr : inodeAttrs) {
-                attrs_->emplace(attr.inodeid(), attr);
-            }
+            inodeCacheManager_->AddInodeAttrs(parentId_, inodeAttrs);
         }
         cond_->Signal();
     };
 
  private:
-    ::curve::common::Mutex* mutex_;
-    std::map<uint64_t, InodeAttr>* attrs_;
+    std::shared_ptr<InodeCacheManager> inodeCacheManager_;
     std::shared_ptr<CountDownEvent> cond_;
+    uint64_t parentId_;
 };
 
 }  // namespace client

curvefs/src/client/inode_wrapper.cpp

@@ -33,6 +33,7 @@
 
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/async_request_closure.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
@@ -46,7 +47,6 @@ namespace client {
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::DataIndices;
-using ::curvefs::client::filesystem::ToFSError;
 
 bvar::Adder<int64_t> g_alive_inode_count{"alive_inode_count"};
 
@@ -107,7 +107,6 @@ class UpdateInodeAsyncDone : public MetaServerClientDone {
                        << ", inodeid: " << inodeWrapper_->GetInodeId();
             inodeWrapper_->MarkInodeError();
         }
-        inodeWrapper_->ClearDirty();
         inodeWrapper_->ReleaseSyncingInode();
 
         if (parent_ != nullptr) {
@@ -193,7 +192,7 @@ CURVEFS_ERROR InodeWrapper::SyncAttr(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
 
         dirty_ = false;
@@ -213,7 +212,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3ChunkInfo(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
         ClearS3ChunkInfoAdd();
     }
@@ -227,6 +226,7 @@ void InodeWrapper::AsyncFlushAttr(MetaServerClientDone* done,
         metaClient_->UpdateInodeWithOutNlinkAsync(
             inode_.fsid(), inode_.inodeid(), dirtyAttr_,
             new UpdateInodeAsyncDone(shared_from_this(), done));
+        dirty_ = false;
         dirtyAttr_.Clear();
         return;
     }
@@ -294,14 +294,14 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
                    << "MetaStatusCode: " << ret
                    << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                    << ", inodeid: " << inode_.inodeid();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
     auto before = s3ChunkInfoSize_;
     inode_.mutable_s3chunkinfomap()->swap(s3ChunkInfoMap);
     UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize() - before);
     ClearS3ChunkInfoAdd();
     UpdateMaxS3ChunkInfoSize();
-    lastRefreshTime_ = TimeUtility::GetTimeofDaySec();
+    lastRefreshTime_ = ::curve::common::TimeUtility::GetTimeofDaySec();
     return CURVEFS_ERROR::OK;
 }
 
@@ -328,7 +328,7 @@ CURVEFS_ERROR InodeWrapper::Link(uint64_t parent) {
                    <<", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
 
     dirty_ = false;
@@ -396,7 +396,7 @@ CURVEFS_ERROR InodeWrapper::UnLink(uint64_t parent) {
                        << ", MetaStatusCode = " << ret
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", inodeid = " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
         dirty_ = false;
         dirtyAttr_.Clear();
@@ -427,7 +427,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParent(
                    << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
     dirty_ = false;
     dirtyAttr_.Clear();
@@ -489,6 +489,7 @@ void InodeWrapper::AsyncFlushAttrAndExtents(MetaServerClientDone *done,
             new UpdateInodeAttrAndExtentClosure{shared_from_this(), done},
             std::move(indices));
 
+        dirty_ = false;
         dirtyAttr_.Clear();
         return;
     }
@@ -514,7 +515,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
         ClearS3ChunkInfoAdd();
         dirty_ = false;
@@ -542,7 +543,6 @@ class UpdateInodeAsyncS3Done : public MetaServerClientDone {
             inodeWrapper_->MarkInodeError();
         }
         VLOG(9) << "inode " << inodeWrapper_->GetInodeId() << " async success.";
-        inodeWrapper_->ClearDirty();
         inodeWrapper_->ReleaseSyncingInode();
         inodeWrapper_->ReleaseSyncingS3ChunkInfo();
 
@@ -559,7 +559,6 @@ class UpdateInodeAsyncS3Done : public MetaServerClientDone {
 }  // namespace
 
 void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
-    (void)internal;
     if (dirty_ || !s3ChunkInfoAdd_.empty()) {
         LockSyncingInode();
         LockSyncingS3ChunkInfo();
@@ -571,6 +570,7 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
             inode_.fsid(), inode_.inodeid(), dirtyAttr_,
             new UpdateInodeAsyncS3Done{shared_from_this(), done},
             std::move(indices));
+        dirty_ = false;
         dirtyAttr_.Clear();
         ClearS3ChunkInfoAdd();
         return;
@@ -597,7 +597,7 @@ CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
         LOG(ERROR) << "GetVolumeExtent failed, inodeid: " << inode_.inodeid();
     }
 
-    return ToFSError(st);
+    return MetaStatusCodeToCurvefsErrCode(st);
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshNlink() {
@@ -610,8 +610,7 @@ CURVEFS_ERROR InodeWrapper::RefreshNlink() {
         return CURVEFS_ERROR::INTERNAL;
     }
     inode_.set_nlink(attr.nlink());
-    VLOG(3) << "RefreshNlink from metaserver, newnlink: " << attr.nlink()
-            << ", inodeid: " << inode_.inodeid();
+    VLOG(3) << "RefreshNlink from metaserver, newnlink: " << attr.nlink();
     return CURVEFS_ERROR::OK;
 }
 

curvefs/src/client/inode_wrapper.h

@@ -34,12 +34,12 @@
 
 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/timeutility.h"
-#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Inode;
 using ::curvefs::metaserver::S3ChunkInfoList;
@@ -53,19 +53,20 @@ constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;
 
 using ::curvefs::metaserver::VolumeExtentList;
-using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 enum class InodeStatus {
     kNormal = 0,
     kError = -1,
 };
 
+// TODO(xuchaojie) : get from conf maybe?
+const uint32_t kOptimalIOBlockSize = 0x10000u;
+
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;
 using metric::S3ChunkInfoMetric;
 using common::NlinkChange;
-using curve::common::TimeUtility;
 
 std::ostream &operator<<(std::ostream &os, const struct stat &attr);
 void AppendS3ChunkInfoToMap(uint64_t chunkIndex, const S3ChunkInfo &info,
@@ -78,19 +79,18 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     InodeWrapper(Inode inode,
                  std::shared_ptr<MetaServerClient> metaClient,
                  std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric = nullptr,
-                 int64_t maxDataSize = LONG_MAX,
+                 uint64_t maxDataSize = ULONG_MAX,
                  uint32_t refreshDataInterval = UINT_MAX)
         : inode_(std::move(inode)),
           status_(InodeStatus::kNormal),
           baseMaxDataSize_(maxDataSize),
           maxDataSize_(maxDataSize),
           refreshDataInterval_(refreshDataInterval),
-          lastRefreshTime_(TimeUtility::GetTimeofDaySec()),
+          lastRefreshTime_(::curve::common::TimeUtility::GetTimeofDaySec()),
           s3ChunkInfoAddSize_(0),
           metaClient_(std::move(metaClient)),
           s3ChunkInfoMetric_(std::move(s3ChunkInfoMetric)),
-          dirty_(false),
-          time_(TimeUtility::GetTimeofDaySec()) {
+          dirty_(false) {
         UpdateS3ChunkInfoMetric(CalS3ChunkInfoSize());
         g_alive_inode_count << 1;
     }
@@ -339,10 +339,6 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
         return false;
     }
 
-    uint32_t GetCachedTime() const {
-        return time_;
-    }
-
  private:
     CURVEFS_ERROR SyncS3ChunkInfo(bool internal = false);
 
@@ -398,8 +394,8 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
     InodeAttr dirtyAttr_;
 
     InodeStatus status_;
-    int64_t baseMaxDataSize_;
-    int64_t maxDataSize_;
+    uint64_t baseMaxDataSize_;
+    uint64_t maxDataSize_;
     uint32_t refreshDataInterval_;
     uint64_t lastRefreshTime_;
 
@@ -417,9 +413,6 @@ class InodeWrapper : public std::enable_shared_from_this<InodeWrapper> {
 
     mutable ::curve::common::Mutex syncingVolumeExtentsMtx_;
     ExtentCache extentCache_;
-
-    // timestamp when put in cache
-    uint64_t time_;
 };
 
 }  // namespace client

curvefs/src/client/kvclient/kvclient_manager.cpp

@@ -21,7 +21,6 @@
  */
 
 #include "curvefs/src/client/kvclient/kvclient_manager.h"
-#include <memory>
 #include "src/client/client_metric.h"
 #include "src/common/concurrent/count_down_event.h"
 
@@ -32,27 +31,32 @@ using curvefs::client::metric::KVClientMetric;
 namespace curvefs {
 namespace client {
 
+KVClientManager *g_kvClientManager = nullptr;
+KVClientMetric *g_kvClientMetric = nullptr;
+
 #define ONRETURN(TYPE, RES)                                                    \
     if (RES) {                                                                 \
-        kvClientMetric_.kvClient##TYPE.qps.count << 1;                        \
+        g_kvClientMetric->kvClient##TYPE.qps.count << 1;                       \
     } else {                                                                   \
-        kvClientMetric_.kvClient##TYPE.eps.count << 1;                        \
-    }                                                                          \
+        g_kvClientMetric->kvClient##TYPE.eps.count << 1;                       \
+    }
 
 bool KVClientManager::Init(const KVClientManagerOpt &config,
                            const std::shared_ptr<KVClient> &kvclient) {
     client_ = kvclient;
+    g_kvClientMetric = new KVClientMetric();
     return threadPool_.Start(config.setThreadPooln) == 0;
 }
 
 void KVClientManager::Uninit() {
     client_->UnInit();
     threadPool_.Stop();
+    delete g_kvClientMetric;
 }
 
 void KVClientManager::Set(std::shared_ptr<SetKVCacheTask> task) {
     threadPool_.Enqueue([task, this]() {
-        LatencyGuard guard(&kvClientMetric_.kvClientSet.latency);
+        LatencyGuard guard(&g_kvClientMetric->kvClientSet.latency);
 
         std::string error_log;
         auto res =
@@ -65,7 +69,7 @@ void KVClientManager::Set(std::shared_ptr<SetKVCacheTask> task) {
 
 void KVClientManager::Get(std::shared_ptr<GetKVCacheTask> task) {
     threadPool_.Enqueue([task, this]() {
-        LatencyGuard guard(&kvClientMetric_.kvClientGet.latency);
+        LatencyGuard guard(&g_kvClientMetric->kvClientGet.latency);
 
         std::string error_log;
         task->res = client_->Get(task->key, task->value, task->offset,

curvefs/src/client/kvclient/kvclient_manager.h

@@ -48,6 +48,9 @@ class GetKVCacheTask;
 using curve::common::TaskThreadPool;
 using curvefs::client::common::KVClientManagerOpt;
 
+extern KVClientManager *g_kvClientManager;
+extern KVClientMetric *g_kvClientMetric;
+
 typedef std::function<void(const std::shared_ptr<SetKVCacheTask> &)>
     SetKVCacheDone;
 typedef std::function<void(const std::shared_ptr<GetKVCacheTask> &)>
@@ -59,10 +62,10 @@ struct SetKVCacheTask {
     uint64_t length;
     SetKVCacheDone done;
     SetKVCacheTask() = default;
-    SetKVCacheTask(
-        const std::string &k, const char *val, const uint64_t len,
-        SetKVCacheDone done = [](const std::shared_ptr<SetKVCacheTask> &) {})
-        : key(k), value(val), length(len), done(std::move(done)) {}
+    SetKVCacheTask(const std::string &k, const char *val, const uint64_t len)
+        : key(k), value(val), length(len) {
+        done = [](const std::shared_ptr<SetKVCacheTask> &) {};
+    }
 };
 
 struct GetKVCacheTask {
@@ -95,15 +98,12 @@ class KVClientManager {
 
     void Get(std::shared_ptr<GetKVCacheTask> task);
 
-    KVClientMetric *GetClientMetricForTesting() { return &kvClientMetric_; }
-
  private:
     void Uninit();
 
  private:
     TaskThreadPool<bthread::Mutex, bthread::ConditionVariable> threadPool_;
     std::shared_ptr<KVClient> client_;
-    KVClientMetric kvClientMetric_;
 };
 
 }  // namespace client