[feat] tools-v2: add snapshot copyset --all

Signed-off-by: montaguelhz <1443171175@qq.com>

.bazelignore

@@ -1 +0,0 @@
-thirdparties/aws/aws-sdk-cpp

.bazelrc

@@ -1,5 +1,10 @@
+build --verbose_failures
+
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
 build --incompatible_blacklisted_protos_requires_proto_info=false
+build --copt=-fdiagnostics-color=always
+
+run --copt=-fdiagnostics-color=always

.clang-format

@@ -0,0 +1,147 @@
+---
+Language:        Cpp
+AccessModifierOffset: -3
+AlignAfterOpenBracket: Align
+AlignConsecutiveMacros: true
+AlignConsecutiveAssignments: false
+AlignConsecutiveBitFields: false
+AlignConsecutiveDeclarations: false
+AlignEscapedNewlines: Right
+AlignOperands:   Align
+AlignTrailingComments: true
+AllowAllArgumentsOnNextLine: true
+AllowAllConstructorInitializersOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortEnumsOnASingleLine: true
+AllowShortBlocksOnASingleLine: Never
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: All
+AllowShortLambdasOnASingleLine: All
+AllowShortIfStatementsOnASingleLine: Never
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: None
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: MultiLine
+BinPackArguments: true
+BinPackParameters: true
+BraceWrapping:
+  AfterCaseLabel:  false
+  AfterClass:      false
+  AfterControlStatement: Never
+  AfterEnum:       false
+  AfterFunction:   false
+  AfterNamespace:  false
+  AfterObjCDeclaration: false
+  AfterStruct:     false
+  AfterUnion:      false
+  AfterExternBlock: false
+  BeforeCatch:     false
+  BeforeElse:      false
+  BeforeLambdaBody: false
+  BeforeWhile:     false
+  IndentBraces:    false
+  SplitEmptyFunction: true
+  SplitEmptyRecord: true
+  SplitEmptyNamespace: true
+BreakBeforeBinaryOperators: None
+BreakBeforeBraces: Attach
+BreakBeforeInheritanceComma: false
+BreakInheritanceList: BeforeColon
+BreakBeforeTernaryOperators: true
+BreakConstructorInitializersBeforeComma: false
+BreakConstructorInitializers: BeforeColon
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+ColumnLimit:     80
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 4
+ContinuationIndentWidth: 4
+Cpp11BracedListStyle: true
+DeriveLineEnding: true
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority:        2
+    SortPriority:    0
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
+    SortPriority:    0
+  - Regex:           '.*'
+    Priority:        1
+    SortPriority:    0
+IncludeIsMainRegex: '(Test)?$'
+IncludeIsMainSourceRegex: ''
+IndentCaseLabels: false
+IndentCaseBlocks: false
+IndentGotoLabels: true
+IndentPPDirectives: None
+IndentExternBlock: AfterExternBlock
+IndentWidth:     4
+IndentWrappedFunctionNames: false
+InsertTrailingCommas: None
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 2
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 2
+ObjCBreakBeforeNestedBlockParam: true
+ObjCSpaceAfterProperty: false
+ObjCSpaceBeforeProtocolList: true
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 60
+PointerAlignment: Right
+ReflowComments:  true
+SortIncludes:    false
+SortUsingDeclarations: true
+SpaceAfterCStyleCast: false
+SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles:  false
+SpacesInConditionalStatement: false
+SpacesInContainerLiterals: true
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+SpaceBeforeSquareBrackets: false
+Standard:        Cpp11
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        4
+UseCRLF:         false
+UseTab:          Never
+WhitespaceSensitiveMacros:
+  - STRINGIZE
+  - PP_STRINGIZE
+  - BOOST_PP_STRINGIZE
+...

.clangd

@@ -0,0 +1,12 @@
+Diagnostics:
+  ClangTidy:
+    Add: [performance-*, modernize-*, readability-*,bugprone-*,]
+    Remove: [modernize-use-trailing-return-type]
+    CheckOptions:
+      readability-identifier-naming.VariableCase: camelBack
+  UnusedIncludes: Strict
+
+InlayHints:
+  Enabled: No
+  ParameterNames: No
+  DeducedTypes: No

.github/workflows/issue-action.yml

@@ -0,0 +1,16 @@
+name: Issue assignment
+
+on:
+    issues:
+        types: [opened]
+
+jobs:
+    auto-assign:
+        runs-on: ubuntu-latest
+        steps:
+            - name: 'Auto-assign issue'
+              uses: pozil/auto-assign-issue@v1.10.1
+              with:
+                  assignees: cw123,ilixiaocui,wuhongsong,Cyber-SiKu,jolly-sy
+                  numOfAssignee: 1
+                  allowSelfAssign: true

.gitignore

@@ -78,6 +78,7 @@ runlog/
 .gdb_history
 
 .clang-format
+.clangd
 !curve-snapshotcloneserver-nginx/app/lib
 !nebd/nebd-package/usr/bin
 
@@ -90,6 +91,7 @@ test/client/configs/*
 projects/*
 docker/curvebs
 docker/base/*
+docker/debian*/*so*
 !docker/base/Dockerfile
 !docker/base/Makefile
 
@@ -107,6 +109,7 @@ curvefs/monitor/prometheus/target.json
 curvefs/docker/*/curvefs
 curvefs/docker/curvefs
 curvefs/docker/base/*
+curvefs/docker/debian*/*so*
 !curvefs/docker/base/Dockerfile
 !curvefs/docker/base/Makefile
 curvefs/BUILD_MODE
@@ -118,7 +121,13 @@ __not_found__
 thirdparties/rocksdb/lib/
 thirdparties/rocksdb/include/
 thirdparties/rocksdb/rocksdb/
+thirdparties/rocksdb/*log
 thirdparties/rocksdb/*.tar.gz
+thirdparties/aws/*.tar.gz
+thirdparties/etcdclient/tmp/
+thirdparties/etcdclient/*.h
+thirdparties/memcache/*.tar.gz
+thirdparties/memcache/libmemcached-*/
 
 /external
 /bazel-*
@@ -133,6 +142,27 @@ curvefs/docker/*/curvefs
 storage_*
 
 tools-v2/sbin/*
-tools-v2/proto/*
+tools-v2/proto/proto/*
+tools-v2/proto/curvefs/*
 tools-v2/*/*.test
 tools-v2/__debug_bin
+tools-v2/vendor/
+
+.test
+.note
+.playground
+.dumpfile
+metastore_test.dat
+GPATH
+GRTAGS
+GTAGS
+core.*
+
+test/integration/*.conf
+test/integration/client/config/client.conf*
+test/integration/snapshotcloneserver/config/*.conf
+
+.pre-commit-config.yaml
+
+*.deb
+*.whl

.gitmodules

@@ -1,6 +1,3 @@
 [submodule "nbd"]
 	path = nbd
 	url = https://github.com/opencurve/curve-nbd
-[submodule "thirdparties/aws/aws-sdk-cpp"]
-	path = thirdparties/aws/aws-sdk-cpp
-	url = https://github.com/opencurve/aws-sdk-cpp

.obm.cfg

@@ -0,0 +1,2 @@
+container_name: curve-build-playground-master
+container_image: opencurvedocker/curve-base:build-debian9

CHANGELOG-2.4.md

@@ -0,0 +1,47 @@
+# CHANGELOG of v2.4
+
+Previous change logs can be found at [CHANGELOG-2.3](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.3.md)
+
+## Notable Changes
+
+- [Update aws-sdk-cpp version to 1.9](https://github.com/opencurve/curve/pull/1780)
+
+- [add feature of warmup](https://github.com/opencurve/curve/pull/1793)
+
+
+## new features
+
+- [CurveFS: implement setxattr interface](https://github.com/opencurve/curve/pull/1935)
+
+- [update aws-sdk-cpp: add ip and custom port support](https://github.com/opencurve/curve/pull/1795)
+
+- [CurveFS client: add s3.useVirtualAddressing config, default value: false](https://github.com/opencurve/curve/pull/1253)
+
+- [Update aws-sdk-cpp version to 1.9](https://github.com/opencurve/curve/pull/1780)
+
+- [add feature of warmup](https://github.com/opencurve/curve/pull/1793)
+
+## optimization
+
+- [CurveFS: update attr and extent in single rpc](https://github.com/opencurve/curve/pull/1784)
+
+- [copyset schedule select copyset random](https://github.com/opencurve/curve/pull/1811)
+
+- [CurveFS: only update dirty inode metadata](https://github.com/opencurve/curve/pull/1853)
+
+
+## bug fix
+
+- [CurveFS:  fix RefreshInode do not refresh when inode exist](https://github.com/opencurve/curve/pull/1800)
+
+- [fix misuse Locked & Unlocked calling contract](https://github.com/opencurve/curve/pull/1279)
+
+- [make callback of s3async request async to avoid deadlock](https://github.com/opencurve/curve/pull/1854)
+
+- [CurveFS :fix update nlink error ](https://github.com/opencurve/curve/pull/1901)
+
+- [CurveFS client: fix umount bug](https://github.com/opencurve/curve/pull/1923)
+
+- [CurveFS client:  fix refresh inode will overwrite data in cache when enabel cto.](https://github.com/opencurve/curve/pull/1993)
+
+

CHANGELOG-2.5.md

@@ -0,0 +1,49 @@
+# CHANGELOG of v2.5
+
+Previous change logs can be found at [CHANGELOG-2.4](https://github.com/opencurve/curve/blob/master/CHANGELOG-2.4.md)
+
+
+
+## New Features
+
+- Memcache support
+    - [curvefs/client: support memcached cluster](https://github.com/opencurve/curve/pull/2096) @ilixiaocui
+    - [curvefs/mds: support memcache cluster](https://github.com/opencurve/curve/pull/2108) @Cyber-SiKu
+    - [curvefs/client: add global cache client like memcached](https://github.com/opencurve/curve/pull/2102) @fansehep
+    - [curvefs/client: fix client core dump error](https://github.com/opencurve/curve/pull/2157) @ilixiaocui
+    - [add ut](https://github.com/opencurve/curve/pull/2164) @Cyber-SiKu
+
+- curvefs new tools 
+    - [add delete impl](https://github.com/opencurve/curve/pull/2088) @shentupenghui
+    - [feat: [tools-v2] bs list dir](https://github.com/opencurve/curve/pull/2082) @Sindweller
+    - [fix missing docs links](https://github.com/opencurve/curve/pull/2082) @zyb521
+    - [curve/toos-v2: add list client #2037](https://github.com/opencurve/curve/pull/2076) @tsonglew
+    - [support create volume directory by curve_ops_tool](https://github.com/opencurve/curve/pull/2078) @aspirer
+
+- [install: added playground.](https://github.com/opencurve/curve/pull/2053) @Wine93
+
+## Optimization
+
+- [update the braft to vesion v1.1.2](https://github.com/opencurve/curve/pull/2091) @tangwz
+
+- [curvefs/client: local cache policy optimization](https://github.com/opencurve/curve/pull/2064) @Tangruilin
+
+
+- [sync by threadpool](https://github.com/opencurve/curve/pull/1912) @fansehep
+
+- Merge block storage and file storage compilation scripts
+    - [Merge block storage and file storage compilation scripts
+](https://github.com/opencurve/curve/pull/2089) @linshiyx
+    - [Add help_msg in Makefile](https://github.com/opencurve/curve/pull/2133) @linshiyx
+    - [fix image.sh](https://github.com/opencurve/curve/pull/2124) @SeanHai
+
+- [fix bs list zone typo](https://github.com/opencurve/curve/pull/2066) @tsonglew 
+
+- [spell optimize](https://github.com/opencurve/curve/pull/2059) @shentupenghui
+
+
+## Bug Fix
+
+- aws s3 sdk revert
+    - [revert new s3 sdk due to critical bug](https://github.com/opencurve/curve/pull/2149) @h0hmj
+    - [curvefs/client: change the default value of s3](https://github.com/opencurve/curve/pull/2158) @wuhongsong

Makefile

@@ -1,22 +1,81 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build install image
+.PHONY: list build dep install image playground check test
 
+stor?=""
 prefix?= "$(PWD)/projects"
 release?= 0
+dep?= 0
 only?= "*"
 tag?= "curvebs:unknown"
 case?= "*"
 os?= "debian9"
+ci?=0
+
+define help_msg
+## list
+Usage:
+    make list stor=bs/fs
+Examples:
+    make list stor=bs
+
+## build
+Usage:
+    make build stor=bs/fs only=TARGET dep=0/1 release=0/1 os=OS
+Examples:
+    make build stor=bs only=//src/chunkserver:chunkserver
+    make build stor=bs only=src/* dep=0
+    make build stor=fs only=test/* os=debian9
+    make build stor=fs release=1
+
+
+## dep
+Usage:
+    make dep stor=bs/fs
+Examples:
+    make dep stor=bs
+
+
+## install
+Usage:
+    make install stor=bs/fs prefix=PREFIX only=TARGET
+Examples:
+    make install stor=bs prefix=/usr/local/curvebs only=*
+    make install stor=bs prefix=/usr/local/curvebs only=chunkserver
+    make install stor=fs prefix=/usr/local/curvefs only=etcd
+
+
+## image
+Usage:
+    make image stor=bs/fs tag=TAG os=OS
+Examples:
+    make image stor=bs tag=opencurvedocker/curvebs:v1.2 os=debian9
+endef
+export help_msg
+
+help:
+	@echo "$$help_msg"
 
 list:
-	@bash util/build.sh --list
+	@bash util/build.sh --stor=$(stor) --list
 
 build:
-	@bash util/build.sh --only=$(only) --release=$(release) --os=$(os)
+	@bash util/build.sh --stor=${stor} --only=$(only) --dep=$(dep) --release=$(release) --ci=$(ci) --os=$(os)
+
+dep:
+	@bash util/build.sh --stor=$(stor) --only="" --dep=1
 
 install:
-	@bash util/install.sh --prefix=$(prefix) --only=$(only)
+	@bash util/install.sh --stor=$(stor) --prefix=$(prefix) --only=$(only)
 
 image:
-	@bash util/image.sh $(tag) $(os)
+	@bash util/image.sh $(stor) $(tag) $(os)
+
+playground:
+	@bash util/playground.sh
+
+check:
+	@bash util/check.sh $(stor)
+
+test:
+	@bash util/test.sh $(stor) $(only)

README.md

@@ -1,63 +1,180 @@
-[中文版](README_cn.md)
 
 
-<img src="docs/images/curve-logo1.png"/>
+<div align=center> <img src="docs/images/curve-logo1-nobg.png" width = 45%>
 
-# CURVE
+<div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
 
-[![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://github.com/opencurve/curve/tree/master/docs)
-[![Releases](https://img.shields.io/github/v/release/opencurve/curve?include_prereleases)](https://github.com/opencurve/curve/releases)
-[![LICENSE](https://img.shields.io/badge/licence-Apache--2.0%2FGPL-blue)](https://github.com/opencurve/curve/blob/master/LICENSE)
-[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6136/badge)](https://bestpractices.coreinfrastructure.org/projects/6136)
+**A cloud-native distributed storage system**
+
+**A sandbox project hosted by the CNCF Foundation**
+
+#### English | [简体中文](README_cn.md)
+### 📄 [Documents](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [Official Website](https://www.opencurve.io/Curve/HOME) || 🏠 [Forum](https://ask.opencurve.io/t/topic/7)
+<div align=left>
+
+<div class="column" align="middle">
+  </a>
+  <a href="https://github.com/opencurve/curve/blob/master/LICENSE">
+    <img src=https://img.shields.io/aur/license/android-studio?style=plastic alt="license"/>
+  </a>
+  <a href="https://github.com/opencurve/curve/releases">
+   <img src="https://img.shields.io/github/v/release/opencurve/curve?include_prereleases" alt="release"/>
+  </a>
+  <a href="https://bestpractices.coreinfrastructure.org/projects/6136">
+    <img src="https://bestpractices.coreinfrastructure.org/projects/6136/badge">
+  </a>
+  <a href="https://github.com/opencurve/curve/tree/master/docs">
+    <img src="https://img.shields.io/badge/docs-latest-green.svg">
+</div>
+
+✨ Contents
+========
+
+  - [About Curve](#about-curve)
+  - [Curve Architecture](#curve-architecture)
+  - [Design Documentation](#design-documentation)
+  - [CurveBS quick start](#curvebs-quick-start)
+    - [Deploy an All-in-one experience environment](#deploy-an-all-in-one-experience-environment)
+    - [FIO curve block storage engine](#fio-curve-block-storage-engine)
+  - [CurveFS quick start](#curvefs-quick-start)
+  - [Test environment configuration](#test-environment-configuration)
+  - [Governance](#governance)
+  - [Contribute us](#contribute-us)
+  - [Code of Conduct](#code-of-conduct)
+  - [LICENSE](#license)
+  - [Release Cycle](#release-cycle)
+  - [Branch](#branch)
+  - [Contributors](#contributors)
+  - [Star History](#star-history)
+  - [Feedback & Contact](#feedback--contact)
+
+##  About Curve
+
+**Curve** is a modern storage system developed by netease, currently supporting file storage(CurveFS) and block storage(CurveBS). Now it's hosted at [CNCF](https://www.cncf.io/) as a sandbox project.
+
+The core application scenarios of CurveBS mainly include:
+- the performance, mixed, capacity cloud disk or persistent volume of virtual machine/container, and remote disks of physical machines
+- high-performance separation of storage and computation architecture: high-performance and low latency architecture based on RDMA+SPDK, supporting the separation deployment structure of various databases such as MySQL and Kafka
 
 
-Curve is a high-performance, lightweight-operation, cloud-native open source distributed storage system. Can be applied to mainstream cloud-native infrastructure platforms: connect to OpenStack platform to provide high-performance block storage services for cloud VM; connect to Kubernetes to provide RWO, RWX and other types of persistent volumes; connect to PolarFS as a high-performance storage for cloud-native databases, perfectly supports the storage-computing separation architecture of cloud-native databases. Curve can also be used as a cloud storage middleware using S3-compatible object storage as a data storage engine, providing cost-effective shared file storage for public cloud users.
+The core application scenarios of CurveFS mainly include:
+- the cost-effective storage in AI training scene
+- the hot and cold data automation layered storage in big data scenarios
+- the cost-effective shared file storage on the public cloud: It can be used for business scenarios such as AI, big data, file sharing
+- Hybrid storage: Hot data is stored in the local IDC, cold data is stored in public cloud
 
-Curve has hosted by the Cloud Native Computing Foundation (CNCF) as a sandbox project.
+<details>
+  <summary><b><font=5>High Performance | More stable | Easy Operation | Cloud Native</b></font></summary>
+
+- High Performance : CurveBS vs CephBS
+
+  CurveBS: v1.2.0
+
+  CephBS: L/N
+  Performance:
+  CurveBS random read and write performance far exceeds CephBS in the block storage scenario.
+
+  Environment：3 replicas on a 6-node cluster, each node has 20xSATA SSD, 2xE5-2660 v4 and 256GB memory.
+
+  Single Vol：
+  <image src="docs/images/1-nbd-en.png">
+
+  Multi Vols：
+  <image src="docs/images/10-nbd-en.png">
+
+- More stable
+  - The stability of the common abnormal Curve is better than that of Ceph in the block storage scenario.
+    | Fault Case | One Disk Failure | Slow Disk Detect | One Server Failure | Server Suspend Animation |
+    | :----: | :----: | :----: | :----: | :----: |
+    | CephBS | jitter 7s | Continuous io jitter | jitter 7s | unrecoverable |
+    | CurveBS | jitter 4s | no effect | jitter 4s | jitter 4s |
+
+- Easy Operation
+  - We have developed [CurveAdm](https://github.com/opencurve/curveadm/wiki) to help O&M staff.
+    | tools  |CephAdm | CurveAdm|
+    | :--: | :--: |:--: |
+    | easy Installation |  ✔️ |   ✔️ |
+    | easy Deployment| ❌(slightly more steps) |  ✔️ |
+    | playground | ❌|   ✔️|
+    | Multi-Cluster Management | ❌ |   ✔️ |
+    | easy Expansion | ❌(slightly more steps)|   ✔️|
+    |easy Upgrade |  ✔️ |   ✔️|
+    |easy to stop service | ❌ |   ✔️|
+    |easy Cleaning | ❌ |   ✔️ |
+    |Deployment environment testing|  ❌ |  ✔️ |
+    |Operational audit|  ❌ |   ✔️|
+    |Peripheral component deployment| ❌ |   ✔️|
+    |easy log reporting| ❌ |   ✔️|
+    |Cluster status statistics reporting| ❌|   ✔️|
+    |Error code classification and solutions|  ❌ |   ✔️|
+  - Ops
+    CurveBS ops is more friendly than CephBS in the block storage scenario.
+    | Ops scenarios | Upgrade clients | Balance |
+    | :----: | :----: | :----: |
+    | CephBS | do not support live upgrade | via plug-in with IO influence |
+    | CurveBS | support live upgrade with second jitter | auto with no influence on IO |
+
+- Cloud Native
+  - Please see [Our understanding of cloud native](https://github.com/opencurve/curve/wiki/Roadmap).
+
+  </details>
+<details>
+  <summary><b><font=5>Docking OpenStack</b></font></summary>
+
+- Please see [Curve-cinder](https://github.com/opencurve/curve-cinder).
+
+
+</details>
+<details>
+  <summary><b><font=5>Docking Kubernetes</b></font></summary>
+
+- Use [Curve CSI Driver](https://github.com/opencurve/curve-csi), The plugin implements the Container Storage Interface(CSI) between Container Orchestrator(CO) and Curve cluster. It allows dynamically provisioning curve volumes and attaching them to workloads.
+- For details of the documentation, see [CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md).
+</details>
+<details>
+  <summary><b><font=5>Docking PolarDB | PG </b></font></summary>
+
+- It serves as the underlying storage base for [polardb for postgresql](https://github.com/ApsaraDB/PolarDB-for-PostgreSQL) in the form of storage and computation separation, providing data consistency assurance for upper layer database applications, extreme elasticity scaling, and high performance HTAP.
+
+- Deployment details can be found at [PolarDB | PG Advanced Deployment(CurveBS)](https://apsaradb.github.io/PolarDB-for-PostgreSQL/zh/deploying/storage-curvebs.html).
+
+
+</details>
+<details>
+  <summary><b><font=5> More...</b></font></summary>
+
+- Curve can also be used as cloud storage middleware using S3-compatible object storage as the data storage engine, providing cost-effective shared file storage for public cloud users.
+</details>
 
 ## Curve Architecture
-The architecture overview of Curve is as follows:
 
-<image src="docs/images/Curve-arch.png" width=70%>
+<div align=center> <image src="docs/images/Curve-arch.png" width=60%>
+<div align=left>
 
-Curve supports deployment in private cloud and public cloud environment, and can also be used in hybrid cloud. The deployment architecture in private cloud environment is as follows:
+<details>
+  <summary><b><font=4>Curve on Hybrid Cloud</b></font></summary>
 
-<image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
+Curve supports deployment in private and public cloud environments, and can also be used in a hybrid cloud:
+<div align=center> <image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
+<div align=left>
 
-The CurveFS shared file storage system can be elastically scaled to public cloud storage, which can provide users with greater capacity elasticity, lower costs, and better performance experience:
+One of them, CurveFS shared file storage system, can be elastically scaled to public cloud storage, which can provide users with greater capacity elasticity, lower cost, and better performance experience.
 
-<image src="docs/images/Curve-deploy-on-public-cloud.png" width=55%>
+</details>
 
-## Curve Block Service vs Ceph Block Device
+<div align=left>
 
-Curve: v1.2.0
+<details>
+  <summary><b><font=4>Curve on Public Cloud</b></font></summary>
 
-Ceph: L/N
-### Performance
-Curve random read and write performance far exceeds Ceph in the block storage scenario.
+In a public cloud environment, users can deploy CurveFS clusters to replace the shared file storage system provided by cloud vendors and use cloud disks for acceleration, which can greatly reduce business costs, with the following deployment architecture:
+<div align=center>
+ <image src="docs/images/Curve-deploy-on-public-cloud.png" width=55%>
 
-Environment：3 replicas on a 6-node cluster, each node has 20xSATA SSD, 2xE5-2660 v4 and 256GB memory.
+</details>
+<div align=left>
 
-Single Vol：
-<image src="docs/images/1-nbd-en.png">
-
-Multi Vols：
-<image src="docs/images/10-nbd-en.png">
-
-### Stability
-The stability of the common abnormal Curve is better than that of Ceph in the block storage scenario.
-| Fault Case | One Disk Failure | Slow Disk Detect | One Server Failure | Server Suspend Animation |
-| :----: | :----: | :----: | :----: | :----: |
-| Ceph | jitter 7s | Continuous io jitter | jitter 7s | unrecoverable |
-| Curve | jitter 4s | no effect | jitter 4s | jitter 4s |
-### Ops
-Curve ops is more friendly than Curve in the block storage scenario.
-| Ops scenarios | Upgrade clients | Balance |
-| :----: | :----: | :----: |
-| Ceph | do not support live upgrade | via plug-in with IO influence |
-| Curve | support live upgrade with second jitter | auto with no influence on IO |
-
-## Design Documentation
+##  Design Documentation
 
 - Wanna have a glance at Curve? Click here for [Intro to Curve](https://www.opencurve.io/)!
 - Want more details about CurveBS? Our documentation for every component:
@@ -71,7 +188,7 @@ Curve ops is more friendly than Curve in the block storage scenario.
   - [Client Python API](docs/en/curve-client-python-api_en.md)
 - Application based on CurveBS
   - [Work with k8s](docs/en/k8s_csi_interface_en.md)
-- Want more details about CurveFS? Our documentation for every component(Chinese version only, English version will be uploaded soon):
+- CurveFS documentations
   - [Architecture design](docs/cn/curvefs_architecture.md)
   - [Client design](docs/cn/curvefs-client-design.md)
   - [Metadata management](docs/cn/curvefs-metaserver-overview.md)
@@ -79,64 +196,76 @@ Curve ops is more friendly than Curve in the block storage scenario.
   - [Space allocation](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%E7%A9%BA%E9%97%B4%E5%88%86%E9%85%8D%E6%96%B9%E6%A1%88.pdf)
   - [more details](https://github.com/opencurve/curve-meetup-slides/tree/main/CurveFS)
 
-## Quick Start of CurveBS
+- CurveAdm
+  - [CurveAdm Doc](https://github.com/opencurve/curveadm/wiki)
 
+##  CurveBS quick start
 In order to improve the operation and maintenance convenience of Curve, we designed and developed the [CurveAdm](https://github.com/opencurve/curveadm) project, which is mainly used for deploying and managing Curve clusters. Currently, it supports the deployment of CurveBS & CurveFS (scaleout, upgrade and other functions are under development), please refer to the [CurveAdm User Manual](https://github.com/opencurve/curveadm/wiki) for related documentation, and install the CurveAdm tool according to the manual before deploying the Curve cluster.
 
-### Deploy an all-in-one environment (to try how CURVE works)
-Please refer to the CurveBS cluster deployment steps in the [CurveAdm User Manual](https://github.com/opencurve/curveadm/wiki/curvebs-cluster-deployment) , for the single-machine experience environment, please use the template about "cluster topology file for single-machine deployment".
 
+###  Deploy an All-in-one experience environment
 
-[Deploy on single machine - deprecated method](docs/en/deploy_en.md#deploy-on-single-machine)
+Please refer to the [CurveBS cluster deployment steps](https://github.com/opencurve/curveadm/wiki/curvebs-cluster-deployment) in the CurveAdm user manual. For standalone experience, please use the "Cluster Topology File - Standalone Deployment" template.
 
-### Deploy multi-machine cluster (try it in production environment)
-Please refer to the CurveBS cluster deployment steps in the [CurveAdm User Manual](https://github.com/opencurve/curveadm/wiki/curvebs-cluster-deployment) , please use the template about "cluster topology file for multi-machine deployment".
+[The command tools' instructions](docs/cn/curve_ops_tool.md)
+### FIO Curve block storage engine
+Fio Curve engine is added, you can clone https://github.com/opencurve/fio and compile the fio tool with our engine(depend on nebd lib), fio command line example:
+```bash
+$ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
+```
 
+If you have any questions during performance testing, please check the [Curve block storage performance tuning guide](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md).
 
-[Deploy on multiple machines - deprecated method](docs/en/deploy_en.md#deploy-on-multiple-machines)
+##  CurveFS quick start
+Please use [CurveAdm](https://github.com/opencurve/curveadm/wiki) tool to deploy CurveFS，see [CurveFS Deployment Process](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), and the [CurveFS Command Instructions](curvefs/src/tools#readme).
 
-### curve_ops_tool introduction
+## Test environment configuration
 
-[curve_ops_tool introduction](docs/en/curve_ops_tool_en.md)
+Please refer to the [Test environment configuration](docs/cn/测试环境配置信息.md)
 
-## Quick Start of CurveFS
-In order to improve the operation and maintenance convenience of Curve, we designed and developed the [CurveAdm](https://github.com/opencurve/curveadm) project, which is mainly used for deploying and managing Curve clusters. Currently, it supports the deployment of CurveBS & CurveFS, please refer to the [CurveAdm User Manual](https://github.com/opencurve/curveadm/wiki) for related documentation, and install the CurveAdm tool according to the manual before deploying the Curve cluster.
+## Practical
+- [CurveBS+NFS Build NFS Server](docs/practical/curvebs_nfs.md)
+- [CurveFS+MinIO S3 Gateway](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
 
-Detail for deploying CurveFS cluster: [CurveFS ​​deployment](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment)
+## Governance
+See [Governance](https://github.com/opencurve/community/blob/master/GOVERNANCE.md).
 
-### curvefs_tool introduction
+##  Contribute us
+Participation in the Curve project is described in the [Curve Developers Guidelines](developers_guide.md) and is subject to a [contributor contract](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md).
+We welcome your contribution!
 
-[curvefs_tool introduction](curvefs/src/tools#readme)
+## Code of Conduct
+Curve follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
 
+## LICENSE
+Curve is distributed under the [**Apache 2.0 LICENSE**](LICENSE).
 
-## For Developers
-
-How to participate in the Curve project development is detailed in [Curve Community Guidelines](Community_Guidelines.md)
-
-### Deploy build and development environment
-
-[development environment deployment](docs/en/build_and_run_en.md)
-
-### Compile test cases and run
-[test cases compiling and running](docs/en/build_and_run_en.md#test-case-compilation-and-execution)
-
-### FIO curve block storage engine
-Fio curve engine is added, you can clone https://github.com/opencurve/fio and compile the fio tool with our engine(depend on nebd lib), fio command line example: `./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test`
-
-## Release Cycle
+##  Release Cycle
 - CURVE release cycle：Half a year for major version, 1~2 months for minor version
 
 - Versioning format: We use a sequence of three digits and a suffix (x.y.z{-suffix}), x is the major version, y is the minor version, and z is for bugfix. The suffix is for distinguishing beta (-beta), RC (-rc) and GA version (without any suffix). Major version x will increase 1 every half year, and y will increase every 1~2 months. After a version is released, number z will increase if there's any bugfix.
 
 ## Branch
-
 All the developments will be done under master branch. If there's any new version to establish, a new branch release-x.y will be pulled from the master, and the new version will be released from this branch.
 
-## Feedback & Contact
+## Contributors
+This project exists thanks to all the people who contribute.
 
+<a href="https://github.com/opencurve/curve/graphs/contributors">
+  <img src="https://contrib.rocks/image?repo=opencurve/curve" />
+</a>
+
+Made with [contrib.rocks](https://contrib.rocks).
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=opencurve/curve&type=Date)](https://star-history.com/#opencurve/curve&Date)
+
+## Feedback & Contact
 - [Github Issues](https://github.com/openCURVE/CURVE/issues)：You are sincerely welcomed to issue any bugs you came across or any suggestions through Github issues. If you have any question you can refer to our FAQ or join our user group for more details.
 - [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：Frequently asked question in our user group, and we'll keep working on it.
 - User group：We use Wechat group currently.
 - [Double Week Meetings](https://github.com/opencurve/curve-meetup-slides/tree/main/2022): We have an online community meeting every two weeks which talk about what Curve is doing and planning to do. The time and links of the meeting are public in the user group and [Double Week Meetings](https://github.com/opencurve/curve-meetup-slides/tree/main/2022).
 
-<img src="docs/images/curve-wechat.jpeg" style="zoom: 75%;" />
+
+<img src="docs/images/curve-wechat.jpeg" style="zoom: 65%;" />

README_cn.md

@@ -1,70 +1,181 @@
-[English version](README.md)
 
 
-<img src="docs/images/curve-logo1.png"/>
+<div align=center> <img src="docs/images/curve-logo1-nobg.png" width = 45%>
 
-# Curve
+<div align=center> <image src="docs/images/cncf-icon-color.png" width = 8%>
 
-[![Jenkins Coverage](https://img.shields.io/jenkins/coverage/cobertura?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_untest_job%2F)](http://59.111.91.248:8080/job/curve_untest_job/HTML_20Report/)
-[![Robot failover](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_failover_testjob%2F&label=failover)](http://59.111.91.248:8080/job/curve_failover_testjob/)
-[![Robot interface](https://img.shields.io/jenkins/tests?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fcurve_robot_job%2F)](http://59.111.91.248:8080/job/curve_robot_job/)
-[![BUILD Status](https://img.shields.io/jenkins/build?jobUrl=http%3A%2F%2F59.111.91.248%3A8080%2Fjob%2Fopencurve_multijob%2F)](http://59.111.91.248:8080/job/opencurve_multijob/lastBuild)
-[![Docs](https://img.shields.io/badge/docs-latest-green.svg)](https://github.com/opencurve/curve/tree/master/docs)
-[![Releases](https://img.shields.io/github/v/release/opencurve/curve?include_prereleases)](https://github.com/opencurve/curve/releases)
-[![LICENSE](https://img.shields.io/badge/licence-Apache--2.0%2FGPL-blue)](https://github.com/opencurve/curve/blob/master/LICENSE)
-[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6136/badge)](https://bestpractices.coreinfrastructure.org/projects/6136)
+**云原生高性能分布式存储系统**
 
-Curve是一款高性能、易运维、云原生的开源分布式存储系统。可应用于主流的云原生基础设施平台：对接 OpenStack 平台为云主机提供高性能块存储服务；对接 Kubernetes 为其提供 RWO、RWX 等类型的持久化存储卷；对接 PolarFS 作为云原生数据库的高性能存储底座，完美支持云原生数据库的存算分离架构。Curve 亦可作为云存储中间件使用 S3 兼容的对象存储作为数据存储引擎，为公有云用户提供高性价比的共享文件存储。
+**CNCF基金会的沙箱托管项目**
 
-Curve已加入云原生计算基金会（CNCF）作为沙箱项目托管。
+#### [English](README.md) | 简体中文
+### 📄 [文档](https://github.com/opencurve/curve/tree/master/docs) || 🌐 [官网](https://www.opencurve.io/Curve/HOME) || 🏠 [论坛](https://ask.opencurve.io/t/topic/7)
+<div align=left>
 
-## Curve Architecture
-Curve总体架构图：
+<div class="column" align="middle">
+  </a>
+  <a href="https://github.com/opencurve/curve/blob/master/LICENSE">
+    <img src=https://img.shields.io/aur/license/android-studio?style=plastic alt="license"/>
+  </a>
+  <a href="https://github.com/opencurve/curve/releases">
+   <img src="https://img.shields.io/github/v/release/opencurve/curve?include_prereleases" alt="release"/>
+  </a>
+  <a href="https://bestpractices.coreinfrastructure.org/projects/6136">
+    <img src="https://bestpractices.coreinfrastructure.org/projects/6136/badge">
+  </a>
+  <a href="https://github.com/opencurve/curve/tree/master/docs">
+    <img src="https://img.shields.io/badge/docs-latest-green.svg">
+</div>
 
-<image src="docs/images/Curve-arch.png" width=70%>
+✨ 目录
+========
 
+  - [关于 Curve](#关于-curve)
+  - [Curve 架构](#curve-架构)
+  - [设计文档](#设计文档)
+  - [CurveBS 快速体验](#curvebs-快速体验)
+    - [部署All-in-one体验环境](#部署all-in-one体验环境)
+    - [FIO curve块存储引擎](#fio-curve块存储引擎)
+  - [CurveFS 快速体验](#curvefs-快速体验)
+  - [测试环境配置](#测试环境配置)
+  - [社区治理](#社区治理)
+  - [贡献我们](#贡献我们)
+  - [行为守则](#行为守则)
+  - [LICENSE](#license)
+  - [版本发布周期](#版本发布周期)
+  - [分支规则](#分支规则)
+  - [反馈及交流](#反馈及交流)
+
+
+##  关于 Curve
+**Curve** 是网易主导自研的现代化存储系统, 目前支持文件存储(CurveFS)和块存储(CurveBS)。现作为沙箱项目托管于[CNCF](https://www.cncf.io/)。
+
+CurveBS的核心应用场景主要包括：
+- 虚拟机/容器的性能型、混合型、容量型云盘或持久化卷，以及物理机的远程存储盘
+- 高性能存算分离架构：基于RDMA+SPDK的高性能低时延架构，支撑MySQL、kafka等各类数据库、中间件的存算分离部署架构，提升实例交付效率和资源利用率
+
+
+CurveFS的核心应用场景主要包括：
+- AI训练（含机器学习等）场景下的高性价比存储
+- 大数据场景下的冷热数据自动化分层存储
+- 公有云上高性价比的共享文件存储：可用于AI、大数据、文件共享等业务场景
+- 混合云存储：热数据存储在本地IDC，冷数据存储在公有云
+
+
+<details>
+  <summary><b><font=5>高性能 | 更稳定 | 易运维 | 云原生</b></font></summary>
+
+- 高性能 : CurveBS vs CephBS
+
+  CurveBS: v1.2.0
+
+  CephBS: L/N
+  性能:
+  块存储场景下，CurveBS随机读写性能远优于CephBS。
+  测试环境：6台服务器*20块SATA SSD，E5-2660 v4，256G，3副本，使用nbd场景。
+
+  单卷场景：
+  <image src="docs/images/1-nbd.jpg">
+
+  多卷场景：
+  <image src="docs/images/10-nbd.jpg">
+
+- 更稳定
+  - 块存储场景下，常见异常CurveBS的稳定性优于CephBS。
+
+    | 异常场景 | 单盘故障 | 慢盘 | 机器宕机 | 机器卡住 |
+    | :----: | :----: | :----: | :----: | :----: |
+    | CephBS | 抖动7s | 持续io抖动 | 抖动7s | 不可恢复 |
+    | CurveBS | 抖动4s | 无影响 | 抖动4s | 抖动4s |
+
+- 易运维
+  - 我们开发了 [CurveAdm](https://github.com/opencurve/curveadm/wiki)来帮助运维人员。
+
+    | 工具 |CephAdm | CurveAdm|
+    | :--:  | :--: |:--: |
+    | 一键安装  |   ✔️ |   ✔️ |
+    | 一键部署  | ❌(步骤稍多) |  ✔️ |
+    | playground |  ❌|   ✔️|
+    | 多集群管理  |  ❌ |   ✔️ |
+    | 一键扩容 | ❌(步骤稍多)|   ✔️|
+    |一键升级 |   ✔️ |   ✔️|
+    |一键停服 |  ❌ |   ✔️|
+    |一键清理 |  ❌ |   ✔️ |
+    |部署环境检测|  ❌ |   ✔️ |
+    |操作审计| ❌ |   ✔️|
+    |周边组件部署| ❌ |   ✔️|
+    |一键日志上报| ❌ |   ✔️|
+    |集群状态统计上报|  ❌|   ✔️|
+    |错误码分类及解决方案|  ❌ |   ✔️|
+  - 运维
+    块存储场景下，CurveBS常见运维更友好。
+
+    | 运维场景 | 客户端升级 | 均衡 |
+    | :----: | :----: | :----: |
+    | CephBS | 不支持热升级 | 外部插件调整，影响业务IO |
+    | CurveBS | 支持热升级，秒级抖动 | 自动均衡，对业务IO无影响 |
+- 云原生
+  - 详见[我们对云原生的理解](https://github.com/opencurve/curve/wiki/Roadmap_CN)。
+
+  </details>
+<details>
+  <summary><b><font=5>对接 OpenStack</b></font></summary>
+
+- 详见 [Curve-cinder](https://github.com/opencurve/curve-cinder)。
+
+
+</details>
+<details>
+  <summary><b><font=5>对接 Kubernetes</b></font></summary>
+
+- 使用 [Curve CSI Driver](https://github.com/opencurve/curve-csi) 插件在 Container Orchestrator (CO) 与 Curve 集群中实现了 Container Storage Interface(CSI)。
+- 文档详见[CSI Curve Driver Doc](https://github.com/opencurve/curve-csi/blob/master/docs/README.md)。
+</details>
+<details>
+  <summary><b><font=5>对接 PolarDB | PG </b></font></summary>
+
+- 作为存算分离形态分布式数据库 [PolarDB | PG](https://github.com/ApsaraDB/PolarDB-for-PostgreSQL) 底层存储底座, 为上层数据库应用提供数据一致性保证, 极致弹性, 高性能HTAP。部署详见[PolarDB | PG 进阶部署(CurveBS)](https://apsaradb.github.io/PolarDB-for-PostgreSQL/zh/deploying/storage-curvebs.html)。
+
+
+</details>
+<details>
+  <summary><b><font=5> 更多...</b></font></summary>
+
+- Curve 亦可作为云存储中间件使用 S3 兼容的对象存储作为数据存储引擎，为公有云用户提供高性价比的共享文件存储。
+</details>
+
+## Curve 架构
+
+<div align=center> <image src="docs/images/Curve-arch.png" width=60%>
+<div align=left>
+
+<details>
+  <summary><b><font=4>Curve混合云支持</b></font></summary>
 
 Curve支持部署在私有云和公有云环境，也可以以混合云方式使用，私有云环境下的部署架构如下：
-<image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
+<div align=center> <image src="docs/images/Curve-deploy-on-premises-idc.png" width=60%>
+<div align=left>
 
 其中CurveFS共享文件存储系统可以弹性伸缩到公有云存储，可以为用户提供更大的容量弹性、更低的成本、更好的性能体验。
 
+</details>
+
+<div align=left>
+
+<details>
+  <summary><b><font=4>Curve公有云支持</b></font></summary>
 
 公有云环境下，用户可以部署CurveFS集群，用来替换云厂商提供的共享文件存储系统，并利用云盘进行加速，可极大的降低业务成本，其部署架构如下：
+<div align=center>
+ <image src="docs/images/Curve-deploy-on-public-cloud.png" width=55%>
 
-<image src="docs/images/Curve-deploy-on-public-cloud.png" width=55%>
+</details>
+<div align=left>
 
-## Curve Block Service vs Ceph Block Device
-
-Curve: v1.2.0
-
-Ceph: L/N
-### 性能
-块存储场景下，Curve随机读写性能远优于Ceph。
-测试环境：6台服务器*20块SATA SSD，E5-2660 v4，256G，3副本，使用nbd场景。
-
-单卷场景：
-<image src="docs/images/1-nbd.jpg">
-
-多卷场景：
-<image src="docs/images/10-nbd.jpg">
-
-### 稳定性
-块存储场景下，常见异常Curve的稳定性优于Ceph。
-| 异常场景 | 单盘故障 | 慢盘 | 机器宕机 | 机器卡住 |
-| :----: | :----: | :----: | :----: | :----: |
-| Ceph | 抖动7s | 持续io抖动 | 抖动7s | 不可恢复 |
-| Curve | 抖动4s | 无影响 | 抖动4s | 抖动4s |
-### 运维
-块存储场景下，Curve常见运维更友好。
-| 运维场景 | 客户端升级 | 均衡 |
-| :----: | :----: | :----: |
-| Ceph | 不支持热升级 | 外部插件调整，影响业务IO |
-| Curve | 支持热升级，秒级抖动 | 自动均衡，对业务IO无影响 |
 
 ## 设计文档
 
-- 通过 [Curve概述](https://opencurve.github.io/) 可以了解 Curve 架构
+- 通过 [Curve概述](https://opencurve.github.io/) 可以了解 Curve 架构。
 - CurveBS相关文档
   - [NEBD](docs/cn/nebd.md)
   - [MDS](docs/cn/mds.md)
@@ -83,62 +194,58 @@ Ceph: L/N
   - [数据缓存方案](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%94%AF%E6%8C%81S3%20%E6%95%B0%E6%8D%AE%E7%BC%93%E5%AD%98%E6%96%B9%E6%A1%88.pdf)
   - [空间分配方案](https://github.com/opencurve/curve-meetup-slides/blob/main/CurveFS/Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9F%E7%A9%BA%E9%97%B4%E5%88%86%E9%85%8D%E6%96%B9%E6%A1%88.pdf)
   - [更多](https://github.com/opencurve/curve-meetup-slides/tree/main/CurveFS)
+- CurveAdm相关文档
+  - [Wiki](https://github.com/opencurve/curveadm/wiki)
 
-## CurveBS快速开始
+##  CurveBS 快速体验
 为了提升 Curve 的运维便利性，我们设计开发了 [CurveAdm](https://github.com/opencurve/curveadm) 项目，其主要用于部署和管理 Curve 集群，目前已支持部署CurveBS & CurveFS（扩容、版本升级等更多功能正在开发中），相关使用文档请参考 [CurveAdm用户手册](https://github.com/opencurve/curveadm/wiki)，并根据手册首先安装CurveAdm工具之后再进行Curve集群的部署。
 
-### 部署All-in-one体验环境
+###  部署All-in-one体验环境
 请参考CurveAdm用户手册中[CurveBS集群部署步骤](https://github.com/opencurve/curveadm/wiki/curvebs-cluster-deployment)，单机体验环境请使用“集群拓扑文件-单机部署”模板。
 
+curve 提供了命令行工具以查看集群状态和进行基本集群操作:[命令行工具说明](docs/cn/curve_ops_tool.md)
+### FIO Curve块存储引擎
+fio的Curve块存储引擎代码已经上传到 https://github.com/opencurve/fio ，请自行编译测试（依赖nebd库），fio命令行示例：
+```bash
+$ ./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test
+```
 
-[单机部署 - 即将废弃方式](docs/cn/deploy.md#%E5%8D%95%E6%9C%BA%E9%83%A8%E7%BD%B2)
+在性能测试过程中有任何问题，请查看[Curve块存储性能调优指南](docs/cn/Curve%E5%9D%97%E5%AD%98%E5%82%A8%E6%80%A7%E8%83%BD%E8%B0%83%E4%BC%98%E6%8C%87%E5%8D%97.md)
 
-### 部署多机集群
-请参考CurveAdm用户手册中[CurveBS集群部署步骤](https://github.com/opencurve/curveadm/wiki/curvebs-cluster-deployment)，请使用“集群拓扑文件-多机部署”模板。
+##  CurveFS 快速体验
+请使用 [CurveAdm](https://github.com/opencurve/curveadm/wiki) 工具进行 CurveFS 的部署，具体流程见：[CurveFS部署流程](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment), 以及[CurveFS命令行工具说明](curvefs/src/tools#readme)。
 
+## 测试环境配置
 
-[多机部署 - 即将废弃方式](docs/cn/deploy.md#%E5%A4%9A%E6%9C%BA%E9%83%A8%E7%BD%B2)
+请参考 [测试环境配置](docs/cn/测试环境配置信息.md)
+##  社区治理
+请参考[社区治理](https://github.com/opencurve/community/blob/master/GOVERNANCE.md)。
 
-### 命令行工具说明
+##  贡献我们
 
-[命令行工具说明](docs/cn/curve_ops_tool.md)
+参与 Curve 项目开发详见[Curve 开发者指南](developers_guide_cn.md)并且请遵循[贡献者准则](https://github.com/opencurve/curve/blob/master/CODE_OF_CONDUCT.md), 我们期待您的贡献!
 
-## CurveFS快速开始
-为了提升 Curve 的运维便利性，我们设计开发了 [CurveAdm](https://github.com/opencurve/curveadm) 项目，其主要用于部署和管理 Curve 集群，目前已支持部署CurveBS & CurveFS，相关使用文档请参考 [CurveAdm用户手册](https://github.com/opencurve/curveadm/wiki)，并根据手册首先安装CurveAdm工具之后再进行Curve集群的部署。
+## 最佳实践
+- [CurveBS+NFS搭建NFS存储](docs/practical/curvebs_nfs.md)
+- [CurveFS+S3网关部署实践](https://github.com/opencurve/curve-meetup-slides/blob/main/PrePaper/2023/%E6%94%AF%E6%8C%81POSIX%E5%92%8CS3%E7%BB%9F%E4%B8%80%E5%91%BD%E5%90%8D%E7%A9%BA%E9%97%B4%E2%80%94%E2%80%94Curve%E6%96%87%E4%BB%B6%E7%B3%BB%E7%BB%9FS3%E7%BD%91%E5%85%B3%E9%83%A8%E7%BD%B2%E5%AE%9E%E8%B7%B5.md)
 
+##  行为守则
+Curve 的行为守则遵循[CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md)。
 
-具体流程见：[CurveFS部署流程](https://github.com/opencurve/curveadm/wiki/curvefs-cluster-deployment)
+## LICENSE
+Curve 在 [Apache 2.0](LICENSE) 协议下进行分发。
 
-### 命令行工具说明
-
-[命令行工具说明](curvefs/src/tools#readme)
-
-
-## 参与开发
-
-如何参与 Curve 项目开发详见[Curve 开源社区指南](Community_Guidelines_cn.md)
-
-### 部署编译开发环境
-
-[编译开发环境搭建](docs/cn/build_and_run.md)
-
-### 测试用例编译及运行
-[测试用例编译及运行](docs/cn/build_and_run.md#%E6%B5%8B%E8%AF%95%E7%94%A8%E4%BE%8B%E7%BC%96%E8%AF%91%E5%8F%8A%E6%89%A7%E8%A1%8C)
-
-### FIO curve块存储引擎
-fio的curve块存储引擎代码已经上传到 https://github.com/opencurve/fio ，请自行编译测试（依赖nebd库），fio命令行示例：`./fio --thread --rw=randwrite --bs=4k --ioengine=nebd --nebd=cbd:pool//pfstest_test_ --iodepth=10 --runtime=120 --numjobs=10 --time_based --group_reporting --name=curve-fio-test`
-
-## 版本发布周期
+##  版本发布周期
 - CURVE版本发布周期：大版本半年，小版本1~2个月
 - 版本号规则：采用3段式版本号，x.y.z{-后缀}，x是大版本，y是小版本，z是bugfix，后缀用来区beta版本(-beta)、rc版本(-rc)、和稳定版本(没有后缀)。每半年的大版本是指x增加1，每1~2个月的小版本是y增加1。正式版本发布之后，如果有bugfix是z增加1。
 
-## 分支规则
+##  分支规则
 所有的开发都在master分支开发，如果需要发布版本，从master拉取新的分支**release-x.y**。版本发布从release-x.y分支发布。
 
-## 反馈及交流
+##  反馈及交流
 
-- [Github Issues](https://github.com/openCURVE/CURVE/issues)：欢迎提交BUG、建议，使用中如遇到问题可参考FAQ或加入我们的User group进行咨询
-- [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：主要根据User group中常见问题整理，还在逐步完善中
+- [Github Issues](https://github.com/openCURVE/CURVE/issues)：欢迎提交BUG、建议，使用中如遇到问题可参考FAQ或加入我们的User group进行咨询。
+- [FAQ](https://github.com/openCURVE/CURVE/wiki/CURVE-FAQ)：主要根据User group中常见问题整理，还在逐步完善中。
 - User group：当前为微信群，由于群人数过多，需要先添加以下个人微信，再邀请进群。
 
 <img src="docs/images/curve-wechat.jpeg" style="zoom: 75%;" />
@@ -147,4 +254,3 @@ fio的curve块存储引擎代码已经上传到 https://github.com/opencurve/fio
 
 
 
-

WORKSPACE

@@ -34,7 +34,7 @@ bazel_skylib_workspace()
 git_repository(
     name = "com_github_baidu_braft",
     remote = "https://github.com/baidu/braft",
-    commit = "e255c0e4b18d1a8a5d484d4b647f41ff1385ef1e",
+    commit = "d12de388c97998f5ccd5cb97ed0da728815ef438",
 )
 
 bind(
@@ -75,11 +75,14 @@ bind(
 )
 
 #import the  gtest files.
-new_git_repository(
+http_archive(
     name = "com_google_googletest",
-    build_file = "//:thirdparties/gmock.BUILD",
-    remote = "https://github.com/google/googletest",
-    tag = "release-1.8.0",
+    urls = [
+        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
+        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
+    ],
+    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
+    strip_prefix = "googletest-release-1.12.1",
 )
 
 bind(
@@ -116,7 +119,7 @@ bind(
 
 http_archive(
     name = "com_github_google_leveldb",
-    build_file = "@com_github_apache_brpc//:leveldb.BUILD",
+    build_file = "@com_github_brpc_brpc//:leveldb.BUILD",
     strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6",
     urls = ["https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz"],
 )
@@ -127,7 +130,7 @@ bind(
 )
 
 git_repository(
-    name = "com_github_apache_brpc",
+    name = "com_github_brpc_brpc",
     remote = "https://github.com/apache/incubator-brpc",
     commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0",
     patches = ["//:thirdparties/brpc/brpc.patch"],
@@ -136,22 +139,22 @@ git_repository(
 
 bind(
     name = "brpc",
-    actual = "@com_github_apache_brpc//:brpc",
+    actual = "@com_github_brpc_brpc//:brpc",
 )
 
 bind(
     name = "butil",
-    actual = "@com_github_apache_brpc//:butil",
+    actual = "@com_github_brpc_brpc//:butil",
 )
 
 bind(
     name = "bthread",
-    actual = "@com_github_apache_brpc//:bthread",
+    actual = "@com_github_brpc_brpc//:bthread",
 )
 
 bind(
     name = "bvar",
-    actual = "@com_github_apache_brpc//:bvar",
+    actual = "@com_github_brpc_brpc//:bvar",
 )
 
 # jsoncpp
@@ -174,12 +177,44 @@ new_local_repository(
 )
 
 new_local_repository(
-    name = "aws",
-    build_file = "//:thirdparties/aws/aws.BUILD",
-    path = "thirdparties/aws/aws-sdk-cpp",
+    name = "libmemcached",
+    build_file = "//:thirdparties/memcache/memcache.BUILD",
+    path = "thirdparties/memcache/libmemcached-1.1.2",
 )
 
 
+http_archive(
+    name = "aws",
+    urls = ["https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz"],
+    sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf",
+    strip_prefix = "aws-sdk-cpp-1.7.340",
+    build_file = "//:thirdparties/aws/aws.BUILD",
+)
+
+http_archive(
+    name = "aws_c_common",
+    urls = ["https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz"],
+    sha256 = "01c2a58553a37b3aa5914d9e0bf7bf14507ff4937bc5872a678892ca20fcae1f",
+    strip_prefix = "aws-c-common-0.4.29",
+    build_file = "//:thirdparties/aws/aws-c-common.BUILD",
+)
+
+http_archive(
+    name = "aws_c_event_stream",
+    urls = ["https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz"],
+    sha256 = "31d880d1c868d3f3df1e1f4b45e56ac73724a4dc3449d04d47fc0746f6f077b6",
+    strip_prefix = "aws-c-event-stream-0.1.4",
+    build_file = "//:thirdparties/aws/aws-c-event-stream.BUILD",
+)
+
+http_archive(
+    name = "aws_checksums",
+    urls = ["https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz"],
+    sha256 = "6e6bed6f75cf54006b6bafb01b3b96df19605572131a2260fddaf0e87949ced0",
+    strip_prefix = "aws-checksums-0.1.5",
+    build_file = "//:thirdparties/aws/aws-checksums.BUILD",
+)
+
 # C++ rules for Bazel.
 http_archive(
     name = "rules_cc",
@@ -196,6 +231,24 @@ http_archive(
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
 
+# fmt
+http_archive(
+  name = "fmt",
+  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
+  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
+  strip_prefix = "fmt-9.1.0",
+  build_file = "//:thirdparties/fmt.BUILD",
+)
+
+# spdlog
+http_archive(
+  name = "spdlog",
+  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
+  strip_prefix = "spdlog-1.11.0",
+  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
+  build_file = "//:thirdparties/spdlog.BUILD",
+)
+
 # Bazel platform rules.
 http_archive(
     name = "platforms",
@@ -216,14 +269,14 @@ new_local_repository(
 http_archive(
     name = "hedron_compile_commands",
 
-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. 
+    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
     # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
     urls = [
         "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
         "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
     ],
     strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." 
+    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()

buildfs.sh

@@ -7,12 +7,22 @@ then
     exit
 fi
 
+if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
+then
+    bazelflags=''
+else
+    bazelflags='--copt -faligned-new'
+fi
+
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi
 
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
+--define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
+-DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
+
 if [ $? -ne 0 ]
 then
     echo "build curvefs failed"
@@ -34,4 +44,5 @@ then
     echo "mds_test failed"
     exit
 fi
-fi
+fi
+echo "end compile"

conf/chunkserver.conf

@@ -9,13 +9,20 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.block_size=4096
+
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -88,6 +95,8 @@ copyset.raft_snapshot_uri=curve://./0/copysets  # __CURVEADM_TEMPLATE__ curve://
 copyset.recycler_uri=local://./0/recycler  # __CURVEADM_TEMPLATE__ local://${prefix}/data/recycler __CURVEADM_TEMPLATE__
 # chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
 copyset.load_concurrency=10
+# chunkserver use how many threads to use copyset complete sync. 
+copyset.sync_concurrency=20
 # 检查copyset是否加载完成出现异常时的最大重试次数
 copyset.check_retrytimes=3
 # 当前peer的applied_index与leader上的committed_index差距小于该值
@@ -106,9 +115,13 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
-# sync timer timeout interval
-copyset.synctimer_interval_ms=30000
+copyset.enable_odsync_when_open_chunkfile=true
+# sync trigger seconds
+copyset.sync_trigger_seconds=25
+# sync chunk limit default = 2MB
+copyset.sync_chunk_limits=2097152
+# 30s if the sum of write > sync_threshold, let the sync_chunk_limits doubled.
+copyset.sync_threshold=65536
 # check syncing interval
 copyset.check_syncing_interval_ms=500
 
@@ -194,8 +207,10 @@ chunkfilepool.clean.throttle_iops=500
 #
 # WAL file pool
 #
-# walpool是否共用chunkfilepool，如果为true，则以下配置无效
+# walpool是否共用chunkfilepool，如果为true，从第三条开始配置无效
 walfilepool.use_chunk_file_pool=true
+# WALpool和ChunkFilePool共用时启用，在容量分配时会预留walpool的空间
+walfilepool.use_chunk_file_pool_reserve=15
 # 是否开启从walfilepool获取chunk，一般是true
 walfilepool.enable_get_segment_from_pool=true
 # walpool目录

conf/chunkserver.conf.example

@@ -13,9 +13,12 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -88,6 +91,8 @@ copyset.raft_snapshot_uri=curve://./0/copysets
 copyset.recycler_uri=local://./0/recycler
 # chunkserver启动时，copyset并发加载的阈值,为0则表示不做限制
 copyset.load_concurrency=10
+# chunkserver use how many threads to use copyset complete sync. 
+copyset.sync_concurrency=20
 # 检查copyset是否加载完成出现异常时的最大重试次数
 copyset.check_retrytimes=3
 # 当前peer的applied_index与leader上的committed_index差距小于该值
@@ -106,9 +111,13 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
-# sync timer timeout interval
-copyset.synctimer_interval_ms=30000
+copyset.enable_odsync_when_open_chunkfile=true
+# sync trigger seconds
+copyset.sync_trigger_seconds=25
+# sync chunk limit default = 2MB
+copyset.sync_chunk_limits=2097152
+# 30s if the sum of write > sync_threshold, let the sync_chunk_limits doubled.
+copyset.sync_threshold=65536
 # check syncing interval
 copyset.check_syncing_interval_ms=500
 
@@ -194,8 +203,10 @@ chunkfilepool.clean.throttle_iops=500
 #
 # WAL file pool
 #
-# walpool是否共用chunkfilepool，如果为true，则以下配置无效
+# walpool是否共用chunkfilepool，如果为true，从第三条开始配置无效
 walfilepool.use_chunk_file_pool=true
+# WALpool和ChunkFilePool共用时启用，在容量分配时会预留walpool的空间
+walfilepool.use_chunk_file_pool_reserve=15
 # 是否开启从walfilepool获取chunk，一般是true
 walfilepool.enable_get_segment_from_pool=true
 # walpool目录

conf/client.conf

@@ -168,13 +168,6 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
 
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
-
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@@ -188,4 +181,3 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
-

conf/cs_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/mds.conf

@@ -156,8 +156,10 @@ mds.topology.CreateCopysetRpcRetryTimes=20
 # 请求chunkserver上创建copyset重试间隔
 mds.topology.CreateCopysetRpcRetrySleepTimeMs=1000
 # Topology模块刷新metric时间间隔
-mds.topology.UpdateMetricIntervalSec=60
-# 物理池使用百分比，即使用量超过这个值即不再往这个池分配
+mds.topology.UpdateMetricIntervalSec=10
+#和mds.chunkserver.failure.tolerance设置有关,一个zone 标准配置20台节点，如果允许3台节点failover,
+#那么剩余17台机器需要承载原先20台机器的空间,17/20=0.85,即使用量超过这个值即不再往这个池分配,
+#具体分为来两种情况, 当不使用chunkfilepool,物理池限制使用百分比,当使用 chunkfilepool 进行chunkfilepool分配时需预留failover空间,
 mds.topology.PoolUsagePercentLimit=85
 # 多pool选pool策略 0:Random, 1:Weight
 mds.topology.choosePoolPolicy=0
@@ -195,6 +197,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -235,3 +239,16 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
+
+#
+## poolset rules
+#
+# for backward compatibility, rules are applied for select poolset when creating file
+#
+# for example
+#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
+#
+# when creating file reqeust doesn't have poolset, above rules are used to select poolset
+# - if filename is /dir1/file, then poolset1 is select
+# - if filename is /dir1/sub/file, then sub is select
+mds.poolset.rules=

conf/py_client.conf

@@ -153,10 +153,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/s3.conf

@@ -7,19 +7,20 @@ s3.endpoint=
 # reserved for backward compatible
 s3.snapshot_bucket_name=
 s3.bucket_name=
-s3.ak=
-s3.sk=
+s3.ak=fake
+s3.sk=fake
+s3.region=us-east-1
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=false
 s3.user_agent_conf=S3 Browser
-s3.max_connections=32
-s3.connect_timeout=60000
-s3.request_timeout=10000
+s3.maxConnections=32
+s3.connectTimeout=60000
+s3.requestTimeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.loglevel=4
+s3.logLevel=4
 s3.logPrefix=/data/log/curve/aws_
-s3.async_thread_num=64
+s3.asyncThreadNum=64
 # throttle
 s3.throttle.iopsTotalLimit=5000
 s3.throttle.iopsReadLimit=5000

conf/snap_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

curve-ansible/README.md

@@ -1,3 +1,5 @@
+**curve-ansile项目当前已经不进行维护，请使用最新工具curveadm进行部署：https://github.com/opencurve/curveadm**
+
 curve-ansible是用ansible编写的curve高性能存储系统远程部署升级工具，可以很方便的在一台主控机上做到部署升级集群所有节点。
 
 ##  一、软件依赖

curve-ansible/roles/generate_config/defaults/main.yml

@@ -164,7 +164,6 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
-chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -244,8 +243,6 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
-client_alignment_common: 512
-client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -14,8 +14,6 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
-# minimum alignment for io request
-global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -175,10 +175,3 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume={{ client_alignment_common }}
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume={{ client_alignment_clone }}

curve-ansible/roles/generate_config/templates/s3.conf.j2

@@ -11,13 +11,13 @@ s3.sk={{ s3_sk }}
 s3.http_scheme={{ s3_http_scheme }}
 s3.verify_SSL={{ s3_verify_ssl }}
 s3.user_agent_conf={{ s3_user_agent_conf }}
-s3.max_connections={{ s3_max_connections }}
-s3.connect_timeout={{ s3_connect_timeout }}
-s3.request_timeout={{ s3_request_timeout }}
+s3.maxConnections={{ s3_max_connections }}
+s3.connectTimeout={{ s3_connect_timeout }}
+s3.requestTimeout={{ s3_request_timeout }}
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.loglevel={{ s3_loglevel }}
+s3.logLevel={{ s3_loglevel }}
 s3.logPrefix={{ s3_logPrefix }}
-s3.async_thread_num={{ s3_async_thread_num }}
+s3.asyncThreadNum={{ s3_async_thread_num }}
 # throttle
 s3.throttle.iopsTotalLimit={{ s3_throttle_iopsTotalLimit }}
 s3.throttle.iopsReadLimit={{ s3_throttle_iopsReadLimit }}

curvefs/Makefile

@@ -1,23 +0,0 @@
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-.PHONY: build dep install image
-
-prefix?= "$(PWD)/devops/projects"
-only?= "*"
-dep?= 0
-release?= 0
-os?= "debian9"
-build_rocksdb?= 0
-tag?= "curvefs:unknown"
-
-build:
-	@bash util/build.sh --only=$(only) --dep=$(dep) --release=$(release) --os=$(os) --build_rocksdb=$(build_rocksdb)
-
-dep:
-	@bash util/build.sh --only="" --dep=1 --build_rocksdb=$(build_rocksdb)
-
-install:
-	@bash util/install.sh --prefix=$(prefix) --only=$(only)
-
-image:
-	@bash util/image.sh $(tag) $(os)

curvefs/conf/client.conf

@@ -17,6 +17,7 @@ mdsOpt.rpcRetryOpt.normalRetryTimesBeforeTriggerWait=3
 mdsOpt.rpcRetryOpt.waitSleepMs=1000
 mdsOpt.rpcRetryOpt.addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702  # __ANSIBLE_TEMPLATE__ {{ groups.mds | join_peer(hostvars, "mds_listen_port") }} __ANSIBLE_TEMPLATE__
 
+
 #
 # lease options
 #
@@ -32,38 +33,38 @@ metaCacheOpt.metacacheRPCRetryIntervalUS=100000
 # RPC timeout of get leader
 metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000
 
-#### excutorOpt
-# excutorOpt rpc with metaserver
+#### executorOpt
+# executorOpt rpc with metaserver
 # rpc retry times with metaserver
-excutorOpt.maxRetry=4294967295
+executorOpt.maxRetry=4294967295
 # internal rpc retry times with metaserver
-excutorOpt.maxInternalRetry = 3
+executorOpt.maxInternalRetry=3
 # Retry sleep time between failed RPCs
-excutorOpt.retryIntervalUS=100000
+executorOpt.retryIntervalUS=100000
 # RPC timeout for communicating with metaserver
-excutorOpt.rpcTimeoutMS=1000
+executorOpt.rpcTimeoutMS=1000
 # RPC stream idle timeout
-excutorOpt.rpcStreamIdleTimeoutMS=500
+executorOpt.rpcStreamIdleTimeoutMS=500
 # The maximum timeout RPC time of the retry request.
 # The timeout time will follow the exponential backoff policy.
 # Because the timeout occurs when the network is congested, the RPC timeout needs to be increased
-excutorOpt.maxRPCTimeoutMS=8000
+executorOpt.maxRPCTimeoutMS=8000
 # Maximum sleep time between retry requests.
 # when the network is congested or the metaserver is overloaded,
 # it is necessary to increase the sleep time
-excutorOpt.maxRetrySleepIntervalUS=8000000
-excutorOpt.minRetryTimesForceTimeoutBackoff=5
-excutorOpt.maxRetryTimesBeforeConsiderSuspend=20
+executorOpt.maxRetrySleepIntervalUS=8000000
+executorOpt.minRetryTimesForceTimeoutBackoff=5
+executorOpt.maxRetryTimesBeforeConsiderSuspend=20
 # batch limit of get inode attr and xattr
-excutorOpt.batchInodeAttrLimit=10000
+executorOpt.batchInodeAttrLimit=10000
 
 #### spaceserver
-spaceserver.spaceaddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
-spaceserver.rpcTimeoutMs=1000
+spaceServer.spaceAddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
+spaceServer.rpcTimeoutMs=1000
 
 #### bdev
 # curve client's config file
-bdev.confpath=/etc/curve/client.conf
+bdev.confPath=/etc/curve/client.conf
 
 #### extentManager
 extentManager.preAllocSize=65536
@@ -76,22 +77,20 @@ rpc.healthCheckIntervalSec=0
 
 #### fuseClient
 # TODO(xuchaojie): add unit
-fuseClient.attrTimeOut=1.0
-fuseClient.entryTimeOut=1.0
 fuseClient.listDentryLimit=65536
-fuseClient.flushPeriodSec=5
-fuseClient.maxNameLength=255
-fuseClient.iCacheLruSize=65536
-fuseClient.dCacheLruSize=65536
-fuseClient.enableICacheMetrics=true
-fuseClient.enableDCacheMetrics=true
-fuseClient.cto=false
+fuseClient.downloadMaxRetryTimes=3
+
+### kvcache opt
+fuseClient.supportKVcache=false
+fuseClient.setThreadPool=4
+fuseClient.getThreadPool=4
+
 # you shoudle enable it when mount one filesystem to multi mountpoints,
 # it gurantee the consistent of file after rename, otherwise you should
 # disable it for performance.
 fuseClient.enableMultiMountPointRename=true
 # splice will bring higher performance in some cases
-# but there're might be a kernel issue that will cause kernel panic when enabling it
+# but there might be a kernel issue that will cause kernel panic when enabling it
 # see https://lore.kernel.org/all/CAAmZXrsGg2xsP1CK+cbuEMumtrqdvD-NKnWzhNcvn71RV3c1yw@mail.gmail.com/
 # until this issue has been fixed, splice should be disabled
 fuseClient.enableSplice=false
@@ -103,59 +102,116 @@ fuseClient.disableXattr=false
 fuseClient.maxDataSize=1024
 # default refresh data interval 30s
 fuseClient.refreshDataIntervalSec=30
+fuseClient.warmupThreadsNum=10
+
+# the write throttle bps of fuseClient, default no limit
+fuseClient.throttle.avgWriteBytes=0
+# the write burst bps of fuseClient, default no limit
+fuseClient.throttle.burstWriteBytes=0
+# the times that write burst bps can continue, default 180s
+fuseClient.throttle.burstWriteBytesSecs=180
+
+# the write throttle iops of fuseClient, default no limit
+fuseClient.throttle.avgWriteIops=0
+# the write burst iops of fuseClient, default no limit
+fuseClient.throttle.burstWriteIops=0
+# the times that write burst Iops can continue, default 180s
+fuseClient.throttle.burstWriteIopsSecs=180
+
+# the read throttle bps of fuseClient, default no limit
+fuseClient.throttle.avgReadBytes=0
+# the read burst bps of fuseClient, default no limit
+fuseClient.throttle.burstReadBytes=0
+# the times that read burst bps can continue, default 180s
+fuseClient.throttle.burstReadBytesSecs=180
+
+# the read throttle iops of fuseClient, default no limit
+fuseClient.throttle.avgReadIops=0
+# the read burst Iops of fuseClient, default no limit
+fuseClient.throttle.burstReadIops=0
+# the times that read burst Iops can continue, default 180s
+fuseClient.throttle.burstReadIopsSecs=180
+
+#### filesystem metadata
+# {
+# fs.disableXattr:
+#   if you want to get better metadata performance,
+#   you can mount fs with |fs.disableXattr| is true
+#
+# fs.lookupCache.negativeTimeoutSec:
+#   entry which not found will be cached if |timeout| > 0
+fs.cto=true
+fs.maxNameLength=255
+fs.disableXattr=false
+fs.accessLogging=true
+fs.kernelCache.attrTimeoutSec=3600
+fs.kernelCache.dirAttrTimeoutSec=3600
+fs.kernelCache.entryTimeoutSec=3600
+fs.kernelCache.dirEntryTimeoutSec=3600
+fs.lookupCache.negativeTimeoutSec=0
+fs.lookupCache.minUses=1
+fs.lookupCache.lruSize=100000
+fs.dirCache.lruSize=5000000
+fs.openFile.lruSize=65536
+fs.attrWatcher.lruSize=5000000
+fs.rpc.listDentryLimit=65536
+fs.deferSync.delay=3
+fs.deferSync.deferDirMtime=false
+# }
 
 #### volume
 volume.bigFileSize=1048576
 volume.volBlockSize=4096
 volume.fsBlockSize=4096
 
-# alloator type, supported {bitmap}
+# allocator type, supported {bitmap}
 volume.allocator.type=bitmap
 
 ## for bitmap allocator
 # size of each bit, default is 4MiB
-volume.bitmapallocator.size_per_bit=4194304
+volume.bitmapAllocator.sizePerBit=4194304
 
 # small allocation proportion [0-1]
-volume.bitmapallocator.small_alloc_proportion=0.2
+volume.bitmapAllocator.smallAllocProportion=0.2
 
 # number of block groups that allocated once
-volume.blockgroup.allocate_once=4
+volume.blockGroup.allocateOnce=4
 
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false
-# the max size that fuse send
-s3.fuseMaxSize=131072
-s3.pagesize=65536
+s3.pageSize=65536
 # prefetch blocks that disk cache use
 s3.prefetchBlocks=1
 # prefetch threads
 s3.prefetchExecQueueNum=1
 # start sleep when mem cache use ratio is greater than nearfullRatio,
-# sleep time increase follow with mem cache use raito, baseSleepUs is baseline.
+# sleep time increase follow with mem cache use ratio, baseSleepUs is baseline.
 s3.nearfullRatio=70
 s3.baseSleepUs=500
 
-# TODO(huyao): use more meaningfull name
 # background thread schedule time
-s3.intervalSec=3
+s3.threadScheduleInterval=3
 # data cache flush wait time
-s3.flushIntervalSec=5
+s3.cacheFlushIntervalSec=5
+# write cache < 8,388,608 (8MB) is not allowed
 s3.writeCacheMaxByte=838860800
 s3.readCacheMaxByte=209715200
+# file cache read thread num
+s3.readCacheThreads=5
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=False
-s3.max_connections=32
-s3.connect_timeout=60000
-s3.request_timeout=10000
+s3.region=us-east-1
+s3.maxConnections=500
+s3.connectTimeout=60000
+s3.requestTimeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.loglevel=4
+s3.logLevel=4
 s3.logPrefix=/data/logs/curvefs/aws_ # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__
-s3.async_thread_num=30
+s3.asyncThreadNum=500
 # limit all inflight async requests' bytes, |0| means not limited
-s3.max_async_request_inflight_bytes=104857600
+s3.maxAsyncRequestInflightBytes=104857600
 s3.chunkFlushThreads=5
 # throttle
 s3.throttle.iopsTotalLimit=0
@@ -172,7 +228,9 @@ s3.maxReadRetryIntervalMs = 1000
 s3.readRetryIntervalMs = 100
 # TODO(hongsong): limit bytes、iops/bps
 #### disk cache options
-# 0:not enable disk cache 1:onlyread 2:read/write
+# 0:not enable disk cache
+# 1:readonly
+# 2:read/write
 diskCache.diskCacheType=2  # __ANSIBLE_TEMPLATE__ {{ client_disk_cache_type | default('2') }} __ANSIBLE_TEMPLATE__
 # the file system writes files use flush or not
 diskCache.forceFlush=true
@@ -212,4 +270,4 @@ client.common.logDir=/data/logs/curvefs  # __CURVEADM_TEMPLATE__ /curvefs/client
 # we have loglevel: {0,3,6,9}
 # as the number increases, it becomes more and more detailed
 client.loglevel=0
-client.dummyserver.startport=9000
+client.dummyServer.startPort=9000

curvefs/conf/mds.conf

@@ -6,6 +6,9 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
 mds.loglevel=0
+# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+mds.server.idleTimeoutSec=-1
 
 #
 # space options
@@ -116,15 +119,16 @@ mds.fsmanager.client.timeoutSec=20
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=False
-s3.max_connections=32
-s3.connect_timeout=60000
-s3.request_timeout=10000
+s3.region=us-east-1
+s3.maxConnections=32
+s3.connectTimeout=60000
+s3.requestTimeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.loglevel=4
+s3.logLevel=4
 s3.logPrefix=/data/logs/curvefs/aws_  # __CURVEADM_TEMPLATE__ /curvefs/client/logs/aws_ __CURVEADM_TEMPLATE__
-s3.async_thread_num=30
+s3.asyncThreadNum=30
 # limit all inflight async requests' bytes, |0| means not limited
-s3.max_async_request_inflight_bytes=104857600
+s3.maxAsyncRequestInflightBytes=104857600
 # throttle
 s3.throttle.iopsTotalLimit=0
 s3.throttle.iopsReadLimit=0

curvefs/conf/metaserver.conf

@@ -12,13 +12,14 @@ s3.enableDeleteObjects=False
 # http = 0, https = 1
 s3.http_scheme=0
 s3.verify_SSL=False
-s3.max_connections=32
-s3.connect_timeout=60000
-s3.request_timeout=10000
+s3.region=us-east-1
+s3.maxConnections=32
+s3.connectTimeout=60000
+s3.requestTimeout=10000
 # Off = 0,Fatal = 1,Error = 2,Warn = 3,Info = 4,Debug = 5,Trace = 6
-s3.loglevel=4
+s3.logLevel=4
 s3.logPrefix=/tmp/curvefs/metaserver/aws_
-s3.async_thread_num=10
+s3.asyncThreadNum=10
 # throttle
 s3.throttle.iopsTotalLimit=0
 s3.throttle.iopsReadLimit=0
@@ -127,16 +128,25 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000
 
-### apply queue options for each copyset
-### apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding worker queue by certain rules
-# number of apply queue workers for each, each worker will start a indepent thread
-applyqueue.worker_count=3
+#
+# Concurrent apply queue
+### concurrent apply queue options for each copyset
+### concurrent apply queue is used to isolate raft threads, each worker has its own queue
+### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
 
-# apply queue depth for each copyset
+# worker_count: number of apply queue workers for each, each worker will start a indepent thread
+# queue_depth: apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-applyqueue.queue_depth=1
+# write apply queue workers count
+applyqueue.write_worker_count=3
+# write apply queue depth
+applyqueue.write_queue_depth=1
+# read apply queue workers count
+applyqueue.read_worker_count=2
+# read apply queue depth
+applyqueue.read_queue_depth=1
+
 
 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@@ -144,6 +154,10 @@ applyqueue.queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto
 
+# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+server.idleTimeoutSec=-1
+
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need
@@ -266,3 +280,43 @@ storage.rocksdb.perf_sampling_ratio=0
 # we will sending its with rpc streaming instead of
 # padding its into inode (default: 25000, about 25000 * 41 (byte) = 1MB)
 storage.s3_meta_inside_inode.limit_size=25000
+
+# recycle options
+# metaserver scan recycle period, default 1h
+recycle.manager.scanPeriodSec=3600
+# metaserver recycle cleaner scan list dentry limit, default 1000
+recycle.cleaner.scanLimit=1000
+
+#### excutorOpt
+# excutorOpt rpc with metaserver
+# rpc retry times with metaserver
+excutorOpt.maxRetry=4294967295
+# internal rpc retry times with metaserver
+excutorOpt.maxInternalRetry = 3
+# Retry sleep time between failed RPCs
+excutorOpt.retryIntervalUS=100000
+# RPC timeout for communicating with metaserver
+excutorOpt.rpcTimeoutMS=1000
+# RPC stream idle timeout
+excutorOpt.rpcStreamIdleTimeoutMS=500
+# The maximum timeout RPC time of the retry request.
+# The timeout time will follow the exponential backoff policy.
+# Because the timeout occurs when the network is congested, the RPC timeout needs to be increased
+excutorOpt.maxRPCTimeoutMS=8000
+# Maximum sleep time between retry requests.
+# when the network is congested or the metaserver is overloaded,
+# it is necessary to increase the sleep time
+excutorOpt.maxRetrySleepIntervalUS=8000000
+excutorOpt.minRetryTimesForceTimeoutBackoff=5
+excutorOpt.maxRetryTimesBeforeConsiderSuspend=20
+# batch limit of get inode attr and xattr
+excutorOpt.batchInodeAttrLimit=10000
+excutorOpt.enableMultiMountPointRename=true
+
+#### metaCacheOpt
+# Gets the number of retries for the leader
+metaCacheOpt.metacacheGetLeaderRetry=3
+# Need to sleep for a period of time before each get leader retry
+metaCacheOpt.metacacheRPCRetryIntervalUS=100000
+# RPC timeout of get leader
+metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000

curvefs/conf/tools.conf

@@ -24,6 +24,7 @@ volumeName=volume
 volumeUser=user
 volumePassword=password
 volumeBlockGroupSize=134217728
+volumeCluster=127.0.0.1:6666,127.0.0.1:6667,127.0.0.1:6668
 # support |AtStart| and |AtEnd|
 volumeBitmapLocation=AtStart
 # s3
@@ -34,5 +35,12 @@ s3.bucket_name=bucket
 s3.blocksize=4194304
 s3.chunksize=67108864
 s3.useVirtualAddressing=false
+# s3 objectPrefix, if set 0, means no prefix, if set 1, means inode prefix
+# if set 2 and other values mean hash prefix
+s3.objectPrefix=0
 # statistic info in xattr, hardlink will not be supported when enable
 enableSumInDir=true
+
+# fs recycle, if set 0, disable fs recycle, delete files directly,
+# if set not 0, enable fs recycle, delete files after a period of time
+recycleTimeHour=0

curvefs/docker/debian10/Dockerfile

@@ -1,10 +0,0 @@
-FROM opencurvedocker/curve-base:debian10
-ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
-COPY curvefs /curvefs
-COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libaws-cpp-sdk-core.so /usr/lib/
-COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
-RUN chmod a+x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian10/base/Dockerfile

@@ -1,36 +0,0 @@
-FROM debian:10
-
-RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        gcc \
-        gdb \
-        make \
-        openssl \
-        net-tools \
-        libcurl3-gnutls \
-        perl \
-        linux-perf \
-        vim \
-        curl \
-        cron \
-        procps \
-        lsof \
-        nginx \
-        less \
-        fuse3 \
-        libnl-3-200 \
-        libnl-genl-3-200 \
-        libjemalloc2 \
-    && ln -s /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
-
-COPY libetcdclient.so /usr/lib/

curvefs/docker/debian10/base/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:debian10 .

curvefs/docker/debian10/compile/Dockerfile

@@ -1,36 +0,0 @@
-FROM debian:10
-
-RUN echo "deb http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ buster-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ buster/updates main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        wget \
-        git \
-        gcc \
-        g++ \
-        clang \
-        libssl-dev \
-        libnl-genl-3-dev \
-        libcurl4-gnutls-dev \
-        uuid-dev \
-        libfiu-dev \
-        libfuse3-dev \
-        zlib1g-dev \
-        make \
-        openjdk-11-jdk \
-        sudo \
-    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
-    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
-    && ln -s /usr/bin/bazelisk /usr/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
-    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
-    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk

curvefs/docker/debian10/compile/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:build-debian10 .

curvefs/docker/debian10/entrypoint.sh

@@ -1,126 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-############################  GLOBAL VARIABLES
-g_role=""
-g_args=""
-g_prefix=""
-g_binary=""
-g_start_args=""
-
-############################  BASIC FUNCTIONS
-function msg() {
-    printf '%b' "$1" >&2
-}
-
-function success() {
-    msg "\33[32m[✔]\33[0m ${1}${2}"
-}
-
-function die() {
-    msg "\33[31m[✘]\33[0m ${1}${2}"
-    exit 1
-}
-
-############################ FUNCTIONS
-function usage () {
-    cat << _EOC_
-Usage:
-    entrypoint.sh --role=ROLE
-    entrypoint.sh --role=ROLE --args=ARGS
-
-Examples:
-    entrypoint.sh --role=etcd
-    entrypoint.sh --role=client --args="-o default_permissions"
-_EOC_
-}
-
-function get_options() {
-    local long_opts="role:,args:,help"
-    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
-    eval set -- "${args}"
-    while true
-    do
-        case "$1" in
-            -r|--role)
-                g_role=$2
-                shift 2
-                ;;
-            -a|--args)
-                g_args=$2
-                shift 2
-                ;;
-            -h)
-                usage
-                exit 1
-                ;;
-            --)
-                shift
-                break
-                ;;
-            *)
-                exit 1
-                ;;
-        esac
-    done
-}
-
-function prepare() {
-    g_prefix="/curvefs/$g_role"
-    conf_path="$g_prefix/conf/$g_role.conf"
-
-    case $g_role in
-        etcd)
-            g_binary="$g_prefix/sbin/etcd"
-            g_start_args="--config-file $conf_path"
-            ;;
-        mds)
-            g_binary="$g_prefix/sbin/curvefs-mds"
-            g_start_args="--confPath $conf_path"
-            ;;
-        metaserver)
-            g_binary="$g_prefix/sbin/curvefs-metaserver"
-            g_start_args="--confPath $conf_path"
-            ;;
-        client)
-            g_binary="$g_prefix/sbin/curve-fuse"
-            g_start_args="--confPath $conf_path"
-            ;;
-        *)
-            usage
-            exit 1
-            ;;
-    esac
-
-    if [ "$g_args" != "" ]; then
-        g_start_args=$g_args
-    fi
-}
-
-function create_directory() {
-    chmod 700 "$g_prefix/data"
-    if [ "$g_role" == "etcd" ]; then
-        mkdir -p "$g_prefix/data/wal"
-    elif [ "$g_role" == "metaserver" ]; then
-        mkdir -p "$g_prefix/data/storage"
-    elif [ "$g_role" == "client" ]; then
-        mkdir -p "$g_prefix/mnt"
-    fi
-}
-
-function main() {
-    get_options "$@"
-
-    prepare
-    create_directory
-    [[ $(command -v crontab) ]] && cron
-    if [ $g_role == "etcd" ]; then
-        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
-    else
-        exec $g_binary $g_start_args
-    fi
-}
-
-############################  MAIN()
-main "$@"

curvefs/docker/debian11/Dockerfile

@@ -1,10 +0,0 @@
-FROM opencurvedocker/curve-base:debian11
-ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
-COPY curvefs /curvefs
-COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libaws-cpp-sdk-core.so /usr/lib/
-COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
-RUN chmod a+x /entrypoint.sh
-ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian11/base/Dockerfile

@@ -1,36 +0,0 @@
-FROM debian:11
-
-RUN echo "deb http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        gcc \
-        gdb \
-        make \
-        openssl \
-        net-tools \
-        libcurl3-gnutls \
-        perl \
-        linux-perf \
-        vim \
-        curl \
-        cron \
-        procps \
-        lsof \
-        nginx \
-        less \
-        fuse3 \
-        libnl-3-200 \
-        libnl-genl-3-200 \
-        libjemalloc2 \
-    && ln -s /usr/lib/x86_64-linux-gnu/libjemalloc.so.2 /usr/local/lib/libjemalloc.so
-
-COPY libetcdclient.so /usr/lib/

curvefs/docker/debian11/base/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:debian11 .

curvefs/docker/debian11/compile/Dockerfile

@@ -1,36 +0,0 @@
-FROM debian:11
-
-RUN echo "deb http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ bullseye-backports main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian-security/ stable-security main non-free contrib\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        wget \
-        git \
-        gcc \
-        g++ \
-        clang \
-        libssl-dev \
-        libnl-genl-3-dev \
-        libcurl4-gnutls-dev \
-        uuid-dev \
-        libfiu-dev \
-        libfuse3-dev \
-        zlib1g-dev \
-        make \
-        openjdk-11-jdk \
-        sudo \
-    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
-    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
-    && ln -s /usr/bin/bazelisk /usr/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
-    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
-    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk

curvefs/docker/debian11/compile/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:build-debian11 .

curvefs/docker/debian11/entrypoint.sh

@@ -1,126 +0,0 @@
-#!/usr/bin/env bash
-
-# Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
-
-############################  GLOBAL VARIABLES
-g_role=""
-g_args=""
-g_prefix=""
-g_binary=""
-g_start_args=""
-
-############################  BASIC FUNCTIONS
-function msg() {
-    printf '%b' "$1" >&2
-}
-
-function success() {
-    msg "\33[32m[✔]\33[0m ${1}${2}"
-}
-
-function die() {
-    msg "\33[31m[✘]\33[0m ${1}${2}"
-    exit 1
-}
-
-############################ FUNCTIONS
-function usage () {
-    cat << _EOC_
-Usage:
-    entrypoint.sh --role=ROLE
-    entrypoint.sh --role=ROLE --args=ARGS
-
-Examples:
-    entrypoint.sh --role=etcd
-    entrypoint.sh --role=client --args="-o default_permissions"
-_EOC_
-}
-
-function get_options() {
-    local long_opts="role:,args:,help"
-    local args=`getopt -o ra --long $long_opts -n "$0" -- "$@"`
-    eval set -- "${args}"
-    while true
-    do
-        case "$1" in
-            -r|--role)
-                g_role=$2
-                shift 2
-                ;;
-            -a|--args)
-                g_args=$2
-                shift 2
-                ;;
-            -h)
-                usage
-                exit 1
-                ;;
-            --)
-                shift
-                break
-                ;;
-            *)
-                exit 1
-                ;;
-        esac
-    done
-}
-
-function prepare() {
-    g_prefix="/curvefs/$g_role"
-    conf_path="$g_prefix/conf/$g_role.conf"
-
-    case $g_role in
-        etcd)
-            g_binary="$g_prefix/sbin/etcd"
-            g_start_args="--config-file $conf_path"
-            ;;
-        mds)
-            g_binary="$g_prefix/sbin/curvefs-mds"
-            g_start_args="--confPath $conf_path"
-            ;;
-        metaserver)
-            g_binary="$g_prefix/sbin/curvefs-metaserver"
-            g_start_args="--confPath $conf_path"
-            ;;
-        client)
-            g_binary="$g_prefix/sbin/curve-fuse"
-            g_start_args="--confPath $conf_path"
-            ;;
-        *)
-            usage
-            exit 1
-            ;;
-    esac
-
-    if [ "$g_args" != "" ]; then
-        g_start_args=$g_args
-    fi
-}
-
-function create_directory() {
-    chmod 700 "$g_prefix/data"
-    if [ "$g_role" == "etcd" ]; then
-        mkdir -p "$g_prefix/data/wal"
-    elif [ "$g_role" == "metaserver" ]; then
-        mkdir -p "$g_prefix/data/storage"
-    elif [ "$g_role" == "client" ]; then
-        mkdir -p "$g_prefix/mnt"
-    fi
-}
-
-function main() {
-    get_options "$@"
-
-    prepare
-    create_directory
-    [[ $(command -v crontab) ]] && cron
-    if [ $g_role == "etcd" ]; then
-        exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
-    else
-        exec $g_binary $g_start_args
-    fi
-}
-
-############################  MAIN()
-main "$@"

curvefs/docker/debian9/Dockerfile

@@ -1,10 +1,8 @@
 FROM opencurvedocker/curve-base:debian9
-ENV TZ=Asia/Shanghai
-RUN mkdir -p /curvefs /etc/curvefs /core /etc/curve
-COPY curvefs /curvefs
 COPY entrypoint.sh /
-COPY curvefs/tools/sbin/curvefs_tool /usr/bin
-COPY libaws-cpp-sdk-core.so /usr/lib/
-COPY libaws-cpp-sdk-s3-crt.so /usr/lib/
-RUN chmod a+x /entrypoint.sh
+COPY curvefs /curvefs
+COPY libmemcached.so libmemcached.so.11 libhashkit.so.2 /usr/lib/
+RUN mkdir -p /etc/curvefs /core /etc/curve && chmod a+x /entrypoint.sh \
+    && cp /curvefs/tools/sbin/curvefs_tool /usr/bin \
+    && cp /curvefs/tools-v2/sbin/curve /usr/bin/
 ENTRYPOINT ["/entrypoint.sh"]

curvefs/docker/debian9/base/Dockerfile

@@ -1,38 +0,0 @@
-FROM opencurvedocker/curve-base:debian9
-
-RUN echo "deb http://mirrors.163.com/debian/ stretch main\n" \
-         "deb http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
-         "deb-src http://mirrors.163.com/debian/ stretch-updates main non-free contrib\n" \
-         "deb http://mirrors.163.com/debian-security/ stretch/updates main non-free contrib\n" \
-         "deb http://httpredir.debian.org/debian stretch-backports main contrib non-free\n" \
-    > /etc/apt/sources.list \
-    && apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        gcc \
-        gdb \
-        make \
-        openssl \
-        net-tools \
-        libcurl3-gnutls \
-        podlators-perl \
-        linux-perf \
-        vim \
-        curl \
-        cron \
-        procps \
-        lsof \
-        nginx \
-        less \
-        zlib1g-dev \
-        libsnappy-dev \
-        liblz4-dev \
-        python \
-        wget \
-        python3 \
-    && wget https://curve-build.nos-eastchina1.126.net/curve-base.tar.gz \
-    && tar -xzvf curve-base.tar.gz
-COPY fusermount3 /usr/local/bin
-COPY libetcdclient.so /usr/lib/
-COPY libfuse3.so libfuse3.so.3 libfuse3.so.3.10.5 libnl-3.so.200 libnl-genl-3.so.200 /usr/lib/x86_64-linux-gnu/
-COPY libjemalloc.so libjemalloc.so.1 libjemalloc.so.2 /usr/local/lib/

curvefs/docker/debian9/base/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:debian9 .

curvefs/docker/debian9/compile/Dockerfile

@@ -1,34 +0,0 @@
-FROM opencurvedocker/curve-base:debian9
-
-RUN apt-get clean \
-    && apt-get -y update \
-    && apt-get -y install \
-        g++ \
-        wget \
-        clang \
-        git \
-        libnl-genl-3-dev \
-        libssl-dev \
-        uuid-dev \
-        zlib1g-dev \
-        libcurl4-gnutls-dev \
-        libfiu-dev \
-        openjdk-8-jdk \
-        sudo \
-        python-dev \
-        python-setuptools \
-        python-wheel \
-        python3-dev \
-        python3-setuptools \
-        python3-wheel \
-    && wget https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 \
-    && mv bazelisk-linux-amd64 /usr/bin/bazelisk \
-    && ln -s /usr/bin/bazelisk /usr/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/4.2.2/bazel-4.2.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin \
-    && mv bazel-4.2.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel \
-    && wget https://github.com/bazelbuild/bazel/releases/download/0.17.2/bazel-0.17.2-linux-x86_64 \
-    && mkdir -p /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin \
-    && mv bazel-0.17.2-linux-x86_64 /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel \
-    && chmod +x /root/.cache/bazelisk/downloads/bazelbuild/bazel-4.2.2-linux-x86_64/bin/bazel /root/.cache/bazelisk/downloads/bazelbuild/bazel-0.17.2-linux-x86_64/bin/bazel /usr/bin/bazel /usr/bin/bazelisk
-COPY fuse3/* /usr/include/fuse3/

curvefs/docker/debian9/compile/Makefile

@@ -1,4 +0,0 @@
-.PHONY: build
-
-build:
-	docker build -t opencurvedocker/curve-base:build-debian9 .

curvefs/docker/debian9/entrypoint.sh

@@ -8,6 +8,7 @@ g_args=""
 g_prefix=""
 g_binary=""
 g_start_args=""
+g_preexec="/curvefs/tools-v2/sbin/daemon"
 
 ############################  BASIC FUNCTIONS
 function msg() {
@@ -119,6 +120,7 @@ function main() {
     prepare
     create_directory
     [[ $(command -v crontab) ]] && cron
+    [[ ! -z $g_preexec ]] && $g_preexec &
     if [ $g_role == "etcd" ]; then
         exec $g_binary $g_start_args >>$g_prefix/logs/etcd.log 2>&1
     elif [ $g_role == "monitor" ]; then

curvefs/monitor/grafana/provisioning/dashboards/metaserver.json

@@ -22,8 +22,8 @@
   "editable": true,
   "gnetId": null,
   "graphTooltip": 0,
-  "id": 7,
-  "iteration": 1657537822267,
+  "id": 5,
+  "iteration": 1660902423620,
   "links": [],
   "panels": [
     {
@@ -611,7 +611,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_inode_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -704,7 +704,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_dentry_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -797,7 +797,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_inode_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -890,7 +890,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_dentry_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -983,7 +983,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_inode_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1076,7 +1076,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_dentry_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1169,7 +1169,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_batch_get_inode_attr_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_batch_get_inode_attr\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1262,7 +1262,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_list_dentry_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_list_dentry\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1355,7 +1355,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_update_inode_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_update_inode\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1448,7 +1448,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_batch_get_xattr_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_batch_get_xattr\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1541,7 +1541,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_prepare_rename_tx_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_prepare_rename_tx\", job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1634,7 +1634,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_or_modify_s3_chunk_info_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_or_modify_s3_chunk_info\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1727,7 +1727,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_partition_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_create_partition\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1820,7 +1820,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_partition_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_delete_partition\",job=\"metaserver\", instance=~\"$instance\", quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -1913,7 +1913,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_volume_extent_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_get_volume_extent\",job=\"metaserver\", instance=~\"$instance\",quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -2006,7 +2006,7 @@
           "targets": [
             {
               "exemplar": true,
-              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_update_volume_extent_[[quantile:regex]]\",job=\"metaserver\", instance=~\"$instance\"}",
+              "expr": "{__name__=~\".*curvefs_metaserver_meta_server_service_update_volume_extent\",job=\"metaserver\", instance=~\"$instance\",quantile=~\"[[service_quantile]]\"}",
               "interval": "",
               "legendFormat": "",
               "refId": "A"
@@ -4658,8 +4658,12 @@
         "allValue": null,
         "current": {
           "selected": false,
-          "text": "All",
-          "value": "$__all"
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
         },
         "datasource": null,
         "definition": "{__name__=~\"bthread_count\", job=\"metaserver\"}",
@@ -4685,8 +4689,12 @@
         "allValue": null,
         "current": {
           "selected": false,
-          "text": "All",
-          "value": "$__all"
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
         },
         "datasource": null,
         "definition": "{__name__=~\"op_apply_pool_.*_copyset.*create_inode_count\", job=\"metaserver\"}",
@@ -4712,8 +4720,12 @@
         "allValue": null,
         "current": {
           "selected": false,
-          "text": "All",
-          "value": "$__all"
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
         },
         "datasource": null,
         "definition": "{__name__=~\".*copyset_.*_create_inode_count\", job=\"metaserver\"}",
@@ -4738,12 +4750,12 @@
       {
         "allValue": "",
         "current": {
-          "selected": true,
+          "selected": false,
           "text": [
-            "latency_99"
+            "latency"
           ],
           "value": [
-            "latency_99"
+            "latency"
           ]
         },
         "datasource": null,
@@ -4770,8 +4782,12 @@
         "allValue": null,
         "current": {
           "selected": false,
-          "text": "All",
-          "value": "$__all"
+          "text": [
+            "All"
+          ],
+          "value": [
+            "$__all"
+          ]
         },
         "datasource": null,
         "definition": "{__name__=~\".*curvefs_metaserver_meta_server_service_.*_qps\", job=\"metaserver\"}",
@@ -4792,6 +4808,37 @@
         "skipUrlSync": false,
         "sort": 1,
         "type": "query"
+      },
+      {
+        "allValue": "",
+        "current": {
+          "selected": false,
+          "text": [
+            "1"
+          ],
+          "value": [
+            "1"
+          ]
+        },
+        "datasource": null,
+        "definition": "{__name__=~\".*curvefs_metaserver_meta_server_service.*\", job=\"metaserver\"}",
+        "description": null,
+        "error": null,
+        "hide": 0,
+        "includeAll": true,
+        "label": "service_quantile",
+        "multi": true,
+        "name": "service_quantile",
+        "options": [],
+        "query": {
+          "query": "{__name__=~\".*curvefs_metaserver_meta_server_service.*\", job=\"metaserver\"}",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 2,
+        "regex": "/quantile=\"(.*)\"}/",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
       }
     ]
   },
@@ -4802,6 +4849,6 @@
   "timepicker": {},
   "timezone": "",
   "title": "metaserver",
-  "uid": "-sypic6nk",
-  "version": 59
+  "uid": "d12ivuiVz",
+  "version": 8
 }

curvefs/proto/common.proto

@@ -58,6 +58,7 @@ message S3Info {
     required string bucketname = 4;
     required uint64 blockSize = 5;
     required uint64 chunkSize = 6;
+    optional uint32 objectPrefix = 7;
 }
 
 enum PartitionStatus {
@@ -83,6 +84,7 @@ message PartitionInfo {
     optional uint64 inodeNum = 10;  // hearbeat upload this value to mds topo, topo/metaserver needn't persist this value
     optional uint64 dentryNum = 11;  // hearbeat upload this value to mds topo, topo/metaserver needn't persist this value
     map<int32, uint64> fileType2inodeNum = 12;
+    optional bool manageFlag = 13; // if a partition has recyclebin inode, set this flag true
 }
 
 message Peer {

curvefs/proto/mds.proto

@@ -55,6 +55,11 @@ enum FSStatusCode {
     COMMIT_TX_SEQUENCE_MISMATCH = 30;
     VOLUME_INFO_ERROR = 31;
     MOUNT_POINT_CONFLICT = 32;
+    FSNAME_INVALID = 33;
+    INSERT_DENTRY_FAIL = 34;
+    INSERT_MANAGE_INODE_FAIL = 35;
+    DELETE_DENTRY_FAIL = 36;
+    UPDATE_FS_FAIL = 37;
 }
 
 // fs interface
@@ -96,6 +101,7 @@ message FsInfo {
     required string owner = 12;
     optional uint64 txSequence = 13;
     optional string txOwner = 14;
+    optional uint32 recycleTimeHour =15;
 }
 
 message GetFsInfoResponse {
@@ -111,6 +117,7 @@ message CreateFsRequest {
     required bool enableSumInDir = 5;
     required string owner = 6;
     required uint64 capacity = 7;
+    optional uint32 recycleTimeHour = 8;
 }
 
 message CreateFsResponse {
@@ -184,6 +191,7 @@ message RefreshSessionRequest {
 message RefreshSessionResponse {
     required FSStatusCode statusCode = 1;
     repeated topology.PartitionTxId latestTxIdList = 2;
+    optional bool enableSumInDir = 3;
 }
 
 message DLockValue {

curvefs/proto/metaserver.proto

@@ -218,7 +218,7 @@ message Inode {
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index
     optional uint32 dtime = 19;
     optional uint32 openmpcount = 20; // openmpcount mount points had the file open
-    map<string, string> xattr = 21;
+    map<string, bytes> xattr = 21;
     repeated uint64 parent = 22;
 }
 
@@ -242,6 +242,12 @@ message CreateInodeRequest {
     required uint64 parent = 10;
     optional uint64 rdev = 11;
     optional string symlink = 12;   // TYPE_SYM_LINK only
+    optional Time create = 13;
+}
+
+message Time {
+    required uint64 sec = 1;
+    required int64 nsec = 2;
 }
 
 message CreateInodeResponse {
@@ -258,6 +264,7 @@ message CreateRootInodeRequest {
     required uint32 uid = 5;
     required uint32 gid = 6;
     required uint32 mode = 7;
+    optional Time create = 8;
 }
 
 message CreateRootInodeResponse {
@@ -265,10 +272,26 @@ message CreateRootInodeResponse {
     optional uint64 appliedIndex = 2;
 }
 
-enum InodeOpenStatusChange {
-    OPEN = 1;
-    CLOSE = 2;
-    NOCHANGE = 3;
+enum ManageInodeType {
+    TYPE_NOT_MANAGE = 0;
+    TYPE_RECYCLE = 1;
+};
+
+message CreateManageInodeRequest {
+    required uint32 poolId = 1;
+    required uint32 copysetId = 2;
+    required uint32 partitionId = 3;
+    required uint32 fsId = 4;
+    required uint32 uid = 5;
+    required uint32 gid = 6;
+    required uint32 mode = 7;
+    required ManageInodeType manageType = 8;
+}
+
+message CreateManageInodeResponse {
+    required MetaStatusCode statusCode = 1;
+    optional Inode inode = 2;
+    optional uint64 appliedIndex = 3;
 }
 
 message UpdateInodeRequest {
@@ -290,8 +313,8 @@ message UpdateInodeRequest {
     // field 16 is left for compatibility
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 17;
     optional uint32 nlink = 18;
-    optional InodeOpenStatusChange inodeOpenstatusChange = 19;
-    map<string, string> xattr = 20;
+    // field 19 is left for compatibility
+    map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
     optional VolumeExtentList volumeExtents = 23;
@@ -384,7 +407,7 @@ message InodeAttr {
     optional uint64 rdev = 16;
     optional uint32 dtime = 17;
     optional uint32 openmpcount = 18;
-    map<string, string> xattr = 19;
+    map<string, bytes> xattr = 19;
     repeated uint64 parent = 20;
 }
 
@@ -415,7 +438,7 @@ message BatchGetXAttrRequest {
 message XAttr {
     required uint64 inodeId = 1;
     required uint32 fsId = 2;
-    map<string, string> xAttrInfos = 3;
+    map<string, bytes> xAttrInfos = 3;
 }
 
 message BatchGetXAttrResponse {
@@ -473,6 +496,7 @@ service MetaServerService {
     rpc DeleteInode(DeleteInodeRequest) returns (DeleteInodeResponse);
     rpc CreateRootInode(CreateRootInodeRequest) returns
                                             (CreateRootInodeResponse);
+    rpc CreateManageInode(CreateManageInodeRequest) returns (CreateManageInodeResponse);
     rpc GetOrModifyS3ChunkInfo(GetOrModifyS3ChunkInfoRequest) returns (GetOrModifyS3ChunkInfoResponse);
     rpc BatchGetInodeAttr(BatchGetInodeAttrRequest) returns (BatchGetInodeAttrResponse);
     rpc BatchGetXAttr(BatchGetXAttrRequest) returns (BatchGetXAttrResponse);

curvefs/proto/topology.proto

@@ -47,6 +47,7 @@ enum TopoStatusCode {
     TOPO_METASERVER_EXIST = 22;
     TOPO_GET_AVAILABLE_COPYSET_ERROR = 23;
     TOPO_DELETE_PARTITION_ON_METASERVER_FAIL = 24;
+    TOPO_MEMCACHECLUSTER_NOT_FOUND = 25;
 }
 
 enum OnlineState {
@@ -458,6 +459,42 @@ message ListTopologyResponse {
     required ListMetaServerResponse metaservers = 5;
 }
 
+message MemcacheServerInfo {
+    required string ip = 1;
+    required uint32 port = 2;
+}
+
+message RegistMemcacheClusterRequest {
+    repeated MemcacheServerInfo servers = 1;
+}
+
+message RegistMemcacheClusterResponse {
+    required TopoStatusCode statusCode = 1;
+    optional uint32 clusterId = 2;
+}
+
+message ListMemcacheClusterRequest {
+}
+
+message MemcacheClusterInfo {
+    required uint32 clusterId = 1;
+    repeated MemcacheServerInfo servers = 2;
+}
+
+message ListMemcacheClusterResponse {
+    required TopoStatusCode statusCode = 1;
+    repeated MemcacheClusterInfo memcacheClusters = 2;
+}
+
+message AllocOrGetMemcacheClusterRequest {
+    required uint32 fsId = 1;
+}
+
+message AllocOrGetMemcacheClusterResponse {
+    required TopoStatusCode statusCode = 1;
+    optional MemcacheClusterInfo cluster = 2;
+}
+
 service TopologyService {
     rpc RegistMetaServer(MetaServerRegistRequest)       returns (MetaServerRegistResponse);
     rpc ListMetaServer(ListMetaServerRequest)           returns (ListMetaServerResponse);
@@ -493,4 +530,8 @@ service TopologyService {
     rpc StatMetadataUsage(StatMetadataUsageRequest) returns (StatMetadataUsageResponse);
 
     rpc ListTopology(ListTopologyRequest) returns (ListTopologyResponse);
+
+    rpc RegistMemcacheCluster(RegistMemcacheClusterRequest) returns (RegistMemcacheClusterResponse);
+    rpc ListMemcacheCluster(ListMemcacheClusterRequest) returns (ListMemcacheClusterResponse);
+    rpc AllocOrGetMemcacheCluster(AllocOrGetMemcacheClusterRequest) returns (AllocOrGetMemcacheClusterResponse);
 }

curvefs/src/client/BUILD

@@ -43,6 +43,10 @@ cc_library(
             "s3/*.h",
             "volume/*.cpp",
             "volume/*.h",
+            "filesystem/*.cpp",
+            "filesystem/*.h",
+            "warmup/*.h",
+            "warmup/*.cpp",
         ],
         exclude = ["main.cpp"],
     ),
@@ -59,6 +63,7 @@ cc_library(
         "//curvefs/src/client/rpcclient",
         "//curvefs/src/common:curvefs_common",
         "//curvefs/src/client/lease:curvefs_lease",
+        "//curvefs/src/client/kvclient:memcached_client_lib",
         "//external:brpc",
         "//external:gflags",
         "//external:glog",
@@ -68,9 +73,14 @@ cc_library(
         "//curvefs/src/volume",
         "//curvefs/src/common:metric_utils",
         "//curvefs/src/common:dynamic_vlog",
+        "//curvefs/src/common:threading",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/synchronization",
+        "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",
+        "@com_google_googletest//:gtest_prod",
+        "@spdlog//:spdlog",
     ],
 )

curvefs/src/client/async_request_closure.cpp

@@ -28,7 +28,7 @@
 #include <mutex>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 namespace curvefs {
@@ -63,7 +63,7 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
         cond_.wait(lk);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(GetStatusCode());
+    return ToFSError(GetStatusCode());
 }
 
 void UpdateVolumeExtentClosure::Run() {
@@ -101,7 +101,7 @@ void UpdateInodeAttrAndExtentClosure::Run() {
                    << ", inode: " << inode_->GetInodeId();
         inode_->MarkInodeError();
     }
-
+    inode_->ClearDirty();
     inode_->syncingVolumeExtentsMtx_.unlock();
     inode_->ReleaseSyncingInode();
 

curvefs/src/client/async_request_closure.h

@@ -28,12 +28,15 @@
 
 #include <memory>
 
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
+using ::curvefs::client::filesystem::ToFSError;
+
 class InodeWrapper;
 
 namespace internal {

curvefs/src/client/client_operator.cpp

@@ -24,13 +24,14 @@
 
 #include "src/common/uuid.h"
 #include "curvefs/src/client/client_operator.h"
-
+#include "curvefs/src/client/filesystem/error.h"
 namespace curvefs {
 namespace client {
 
 using ::curve::common::UUIDGenerator;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::mds::topology::PartitionTxId;
+using ::curvefs::client::filesystem::ToFSError;
 
 #define LOG_ERROR(action, rc) \
     LOG(ERROR) << action << " failed, retCode = " << rc \
@@ -95,7 +96,7 @@ CURVEFS_ERROR RenameOperator::GetTxId(uint32_t fsId,
     if (rc != MetaStatusCode::OK) {
         LOG_ERROR("GetTxId", rc);
     }
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::GetLatestTxIdWithLock() {
@@ -183,11 +184,11 @@ CURVEFS_ERROR RenameOperator::Precheck() {
 // record old inode info if overwrite
 CURVEFS_ERROR RenameOperator::RecordOldInodeInfo() {
     if (oldInodeId_ != 0) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        auto rc = inodeManager_->GetInode(oldInodeId_, inodeWrapper);
+        InodeAttr attr;
+        CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(oldInodeId_, &attr);
         if (rc == CURVEFS_ERROR::OK) {
-            oldInodeSize_ = inodeWrapper->GetLength();
-            oldInodeType_ = inodeWrapper->GetType();
+            oldInodeSize_ = attr.length();
+            oldInodeType_ = attr.type();
         } else {
             LOG_ERROR("GetInode", rc);
             return CURVEFS_ERROR::NOTEXIST;
@@ -204,7 +205,7 @@ CURVEFS_ERROR RenameOperator::PrepareRenameTx(
         LOG_ERROR("PrepareRenameTx", rc);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::PrepareTx() {
@@ -279,7 +280,7 @@ CURVEFS_ERROR RenameOperator::LinkInode(uint64_t inodeId, uint64_t parent) {
         return rc;
     }
 
-    rc = inodeWrapper->LinkLocked(parent);
+    rc = inodeWrapper->Link(parent);
     if (rc != CURVEFS_ERROR::OK) {
         LOG_ERROR("Link", rc);
         return rc;
@@ -297,7 +298,7 @@ CURVEFS_ERROR RenameOperator::UnLinkInode(uint64_t inodeId, uint64_t parent) {
         return rc;
     }
 
-    rc = inodeWrapper->UnLinkLocked(parent);
+    rc = inodeWrapper->UnLink(parent);
     if (rc != CURVEFS_ERROR::OK) {
         LOG_ERROR("UnLink", rc);
         return rc;
@@ -316,10 +317,7 @@ CURVEFS_ERROR RenameOperator::UpdateMCTime(uint64_t inodeId) {
     }
 
     curve::common::UniqueLock lk = inodeWrapper->GetUniqueLock();
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    inodeWrapper->SetMTime(now.tv_sec, now.tv_nsec);
-    inodeWrapper->SetCTime(now.tv_sec, now.tv_nsec);
+    inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
 
     rc = inodeWrapper->SyncAttr();
     if (rc != CURVEFS_ERROR::OK) {
@@ -388,7 +386,7 @@ CURVEFS_ERROR RenameOperator::UpdateInodeParent() {
         return rc;
     }
 
-    rc = inodeWrapper->UpdateParentLocked(parentId_, newParentId_);
+    rc = inodeWrapper->UpdateParent(parentId_, newParentId_);
     if (rc != CURVEFS_ERROR::OK) {
         LOG_ERROR("UpdateInodeParent", rc);
         return rc;
@@ -399,9 +397,28 @@ CURVEFS_ERROR RenameOperator::UpdateInodeParent() {
     return rc;
 }
 
+CURVEFS_ERROR RenameOperator::UpdateInodeCtime() {
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    auto rc = inodeManager_->GetInode(srcDentry_.inodeid(), inodeWrapper);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG_ERROR("GetInode", rc);
+        return rc;
+    }
+
+    curve::common::UniqueLock lk = inodeWrapper->GetUniqueLock();
+    inodeWrapper->UpdateTimestampLocked(kChangeTime);
+
+    rc = inodeWrapper->SyncAttr();
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG_ERROR("UpdateInodeCtime", rc);
+        return rc;
+    }
+
+    LOG(INFO) << "UpdateInodeCtime inodeid = " << srcDentry_.inodeid();
+    return rc;
+}
+
 void RenameOperator::UpdateCache() {
-    dentryManager_->DeleteCache(parentId_, name_);
-    dentryManager_->InsertOrReplaceCache(newDentry_);
     SetTxId(srcPartitionId_, srcTxId_ + 1);
     SetTxId(dstPartitionId_, dstTxId_ + 1);
 }

curvefs/src/client/client_operator.h

@@ -59,6 +59,7 @@ class RenameOperator {
     CURVEFS_ERROR UnlinkSrcParentInode();
     void UnlinkOldInode();
     CURVEFS_ERROR UpdateInodeParent();
+    CURVEFS_ERROR UpdateInodeCtime();
     void UpdateCache();
 
     void GetOldInode(uint64_t *oldInodeId, int64_t *oldInodeSize,
@@ -68,9 +69,9 @@ class RenameOperator {
         *oldInodeType = oldInodeType_;
     }
 
- private:
     std::string DebugString();
 
+ private:
     CURVEFS_ERROR CheckOverwrite();
 
     CURVEFS_ERROR GetLatestTxIdWithLock();

curvefs/src/client/common/common.cpp

@@ -22,6 +22,8 @@
  */
 
 #include "curvefs/src/client/common/common.h"
+#include "src/common/string_util.h"
+
 namespace curvefs {
 namespace client {
 namespace common {
@@ -76,6 +78,86 @@ std::ostream &operator<<(std::ostream &os, MetaServerOpType optype) {
     return os;
 }
 
+const char kCurveFsWarmupOpAdd[] = "add";
+const char kCurveFsWarmupOpQuery[] = "query";
+const char kCurveFsWarmupTypeList[] = "list";
+const char kCurveFsWarmupTypeSingle[] = "single";
+
+WarmupOpType GetWarmupOpType(const std::string& op) {
+    auto ret = WarmupOpType::kWarmupOpUnknown;
+    if (op == kCurveFsWarmupOpAdd) {
+        ret = WarmupOpType::kWarmupOpAdd;
+    }
+    if (op == kCurveFsWarmupOpQuery) {
+        ret = WarmupOpType::kWarmupOpQuery;
+    }
+    return ret;
+}
+
+WarmupType GetWarmupType(const std::string& type) {
+    auto ret =
+        WarmupType::kWarmupTypeUnknown;
+    if (type == kCurveFsWarmupTypeList) {
+        ret = WarmupType::kWarmupTypeList;
+    } else if (type == kCurveFsWarmupTypeSingle) {
+        ret = WarmupType::kWarmupTypeSingle;
+    }
+    return ret;
+}
+
+const char kCurveFsWarmupStorageDisk[] = "disk";
+const char kCurveFsWarmupStorageKvclient[] = "kvclient";
+
+WarmupStorageType GetWarmupStorageType(const std::string &type) {
+    auto ret = WarmupStorageType::kWarmupStorageTypeUnknown;
+    if (type == kCurveFsWarmupStorageDisk) {
+        ret = WarmupStorageType::kWarmupStorageTypeDisk;
+    } else if (type == kCurveFsWarmupStorageKvclient) {
+        ret = WarmupStorageType::kWarmupStorageTypeKvClient;
+    }
+    return ret;
+}
+
+using ::curve::common::StringToUll;
+
+// if direction is true means '+', false means '-'
+// is direction is true, add second to first
+// if direction is false, sub second from first
+bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction) {
+    uint64_t firstNum = 0;
+    uint64_t secondNum = second;
+    if (StringToUll(*first, &firstNum)) {
+        if (direction) {
+            *first = std::to_string(firstNum + secondNum);
+        } else {
+            if (firstNum < secondNum) {
+                *first = std::to_string(0);
+                LOG(WARNING) << "AddUllStringToFirst failed when minus,"
+                             << " first = " << firstNum
+                             << ", second = " << secondNum;
+                return false;
+            }
+            *first = std::to_string(firstNum - secondNum);
+        }
+    } else {
+        LOG(ERROR) << "StringToUll failed, first = " << *first
+                   << ", second = " << second;
+        return false;
+    }
+    return true;
+}
+
+bool AddUllStringToFirst(uint64_t *first, const std::string &second) {
+    uint64_t secondNum = 0;
+    if (StringToUll(second, &secondNum)) {
+        *first += secondNum;
+        return true;
+    }
+
+    LOG(ERROR) << "StringToUll failed, second = " << second;
+    return false;
+}
+
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/common/common.h

@@ -27,6 +27,7 @@
 
 #include <bthread/mutex.h>
 #include <ostream>
+#include <string>
 
 namespace curvefs {
 namespace client {
@@ -58,12 +59,58 @@ enum class MetaServerOpType {
     GetOrModifyS3ChunkInfo,
     GetVolumeExtent,
     UpdateVolumeExtent,
+    CreateManageInode,
 };
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);
 
-const uint32_t MAXXATTRLENGTH = 256;
+const uint32_t MAX_XATTR_NAME_LENGTH = 255;
+const uint32_t MAX_XATTR_VALUE_LENGTH = 64 * 1024;
 
+const char kCurveFsWarmupXAttr[] = "curvefs.warmup.op";
+
+
+constexpr int kWarmupOpNum = 4;
+
+enum class WarmupOpType {
+    kWarmupOpUnknown = 0,
+    kWarmupOpAdd = 1,
+    kWarmupOpQuery = 2,
+};
+
+WarmupOpType GetWarmupOpType(const std::string& op);
+
+enum class WarmupType {
+    kWarmupTypeUnknown = 0,
+    kWarmupTypeList = 1,
+    kWarmupTypeSingle = 2,
+};
+
+WarmupType GetWarmupType(const std::string &type);
+
+enum class WarmupStorageType {
+    kWarmupStorageTypeUnknown = 0,
+    kWarmupStorageTypeDisk = 1,
+    kWarmupStorageTypeKvClient = 2,
+};
+
+WarmupStorageType GetWarmupStorageType(const std::string &type);
+
+enum class FileHandle : uint64_t {
+    kDefaultValue = 0,
+    kKeepCache = 1,
+};
+
+enum class NlinkChange : int32_t {
+    kAddOne = 1,
+    kSubOne = -1,
+};
+
+// if direction is true means '+', false means '-'
+// is direction is true, add second to first
+// if direction is false, sub second from first
+bool AddUllStringToFirst(std::string *first, uint64_t second, bool direction);
+bool AddUllStringToFirst(uint64_t *first, const std::string &second);
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/common/config.cpp

@@ -49,6 +49,57 @@ namespace common {
 DEFINE_bool(enableCto, true, "acheieve cto consistency");
 DEFINE_bool(useFakeS3, false,
             "Use fake s3 to inject more metadata for testing metaserver");
+DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing");
+DEFINE_bool(access_logging, true, "enable access log");
+
+/**
+ * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true
+ * for dynamic parameter configuration
+ */
+static bool pass_uint64(const char *, uint64_t) { return true; }
+
+DEFINE_uint64(fuseClientAvgWriteBytes, 0,
+              "the write throttle bps of fuse client");
+DEFINE_validator(fuseClientAvgWriteBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteBytes, 0,
+              "the write burst bps of fuse client");
+DEFINE_validator(fuseClientBurstWriteBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteBytesSecs, 180,
+              "the times that write burst bps can continue");
+DEFINE_validator(fuseClientBurstWriteBytesSecs, &pass_uint64);
+
+
+DEFINE_uint64(fuseClientAvgWriteIops, 0,
+              "the write throttle iops of fuse client");
+DEFINE_validator(fuseClientAvgWriteIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteIops, 0,
+              "the write burst iops of fuse client");
+DEFINE_validator(fuseClientBurstWriteIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstWriteIopsSecs, 180,
+              "the times that write burst iops can continue");
+DEFINE_validator(fuseClientBurstWriteIopsSecs, &pass_uint64);
+
+
+DEFINE_uint64(fuseClientAvgReadBytes, 0,
+              "the Read throttle bps of fuse client");
+DEFINE_validator(fuseClientAvgReadBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadBytes, 0,
+              "the Read burst bps of fuse client");
+DEFINE_validator(fuseClientBurstReadBytes, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadBytesSecs, 180,
+              "the times that Read burst bps can continue");
+DEFINE_validator(fuseClientBurstReadBytesSecs, &pass_uint64);
+
+
+DEFINE_uint64(fuseClientAvgReadIops, 0,
+              "the Read throttle iops of fuse client");
+DEFINE_validator(fuseClientAvgReadIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadIops, 0,
+              "the Read burst iops of fuse client");
+DEFINE_validator(fuseClientBurstReadIops, &pass_uint64);
+DEFINE_uint64(fuseClientBurstReadIopsSecs, 180,
+              "the times that Read burst iops can continue");
+DEFINE_validator(fuseClientBurstReadIopsSecs, &pass_uint64);
 
 void InitMdsOption(Configuration *conf, MdsOption *mdsOpt) {
     conf->GetValueFatalIfFail("mdsOpt.mdsMaxRetryMS", &mdsOpt->mdsMaxRetryMS);
@@ -85,26 +136,26 @@ void InitMetaCacheOption(Configuration *conf, MetaCacheOpt *opts) {
 
 void InitExcutorOption(Configuration *conf, ExcutorOpt *opts, bool internal) {
     if (internal) {
-        conf->GetValueFatalIfFail("excutorOpt.maxInternalRetry",
+        conf->GetValueFatalIfFail("executorOpt.maxInternalRetry",
                                   &opts->maxRetry);
     } else {
-        conf->GetValueFatalIfFail("excutorOpt.maxRetry", &opts->maxRetry);
+        conf->GetValueFatalIfFail("executorOpt.maxRetry", &opts->maxRetry);
     }
 
-    conf->GetValueFatalIfFail("excutorOpt.retryIntervalUS",
+    conf->GetValueFatalIfFail("executorOpt.retryIntervalUS",
                               &opts->retryIntervalUS);
-    conf->GetValueFatalIfFail("excutorOpt.rpcTimeoutMS", &opts->rpcTimeoutMS);
-    conf->GetValueFatalIfFail("excutorOpt.rpcStreamIdleTimeoutMS",
+    conf->GetValueFatalIfFail("executorOpt.rpcTimeoutMS", &opts->rpcTimeoutMS);
+    conf->GetValueFatalIfFail("executorOpt.rpcStreamIdleTimeoutMS",
                               &opts->rpcStreamIdleTimeoutMS);
-    conf->GetValueFatalIfFail("excutorOpt.maxRPCTimeoutMS",
+    conf->GetValueFatalIfFail("executorOpt.maxRPCTimeoutMS",
                               &opts->maxRPCTimeoutMS);
-    conf->GetValueFatalIfFail("excutorOpt.maxRetrySleepIntervalUS",
+    conf->GetValueFatalIfFail("executorOpt.maxRetrySleepIntervalUS",
                               &opts->maxRetrySleepIntervalUS);
-    conf->GetValueFatalIfFail("excutorOpt.minRetryTimesForceTimeoutBackoff",
+    conf->GetValueFatalIfFail("executorOpt.minRetryTimesForceTimeoutBackoff",
                               &opts->minRetryTimesForceTimeoutBackoff);
-    conf->GetValueFatalIfFail("excutorOpt.maxRetryTimesBeforeConsiderSuspend",
+    conf->GetValueFatalIfFail("executorOpt.maxRetryTimesBeforeConsiderSuspend",
                               &opts->maxRetryTimesBeforeConsiderSuspend);
-    conf->GetValueFatalIfFail("excutorOpt.batchInodeAttrLimit",
+    conf->GetValueFatalIfFail("executorOpt.batchInodeAttrLimit",
                               &opts->batchInodeAttrLimit);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                               &opts->enableRenameParallel);
@@ -112,7 +163,7 @@ void InitExcutorOption(Configuration *conf, ExcutorOpt *opts, bool internal) {
 
 void InitBlockDeviceOption(Configuration *conf,
                            BlockDeviceClientOptions *bdevOpt) {
-    conf->GetValueFatalIfFail("bdev.confpath", &bdevOpt->configPath);
+    conf->GetValueFatalIfFail("bdev.confPath", &bdevOpt->configPath);
 }
 
 void InitDiskCacheOption(Configuration *conf,
@@ -156,17 +207,15 @@ void InitDiskCacheOption(Configuration *conf,
 
 void InitS3Option(Configuration *conf, S3Option *s3Opt) {
     conf->GetValueFatalIfFail("s3.fakeS3", &FLAGS_useFakeS3);
-    conf->GetValueFatalIfFail("s3.fuseMaxSize",
-                              &s3Opt->s3ClientAdaptorOpt.fuseMaxSize);
-    conf->GetValueFatalIfFail("s3.pagesize",
+    conf->GetValueFatalIfFail("s3.pageSize",
                               &s3Opt->s3ClientAdaptorOpt.pageSize);
     conf->GetValueFatalIfFail("s3.prefetchBlocks",
                               &s3Opt->s3ClientAdaptorOpt.prefetchBlocks);
     conf->GetValueFatalIfFail("s3.prefetchExecQueueNum",
                               &s3Opt->s3ClientAdaptorOpt.prefetchExecQueueNum);
-    conf->GetValueFatalIfFail("s3.intervalSec",
+    conf->GetValueFatalIfFail("s3.threadScheduleInterval",
                               &s3Opt->s3ClientAdaptorOpt.intervalSec);
-    conf->GetValueFatalIfFail("s3.flushIntervalSec",
+    conf->GetValueFatalIfFail("s3.cacheFlushIntervalSec",
                               &s3Opt->s3ClientAdaptorOpt.flushIntervalSec);
     conf->GetValueFatalIfFail("s3.chunkFlushThreads",
                               &s3Opt->s3ClientAdaptorOpt.chunkFlushThreads);
@@ -174,6 +223,8 @@ void InitS3Option(Configuration *conf, S3Option *s3Opt) {
                               &s3Opt->s3ClientAdaptorOpt.writeCacheMaxByte);
     conf->GetValueFatalIfFail("s3.readCacheMaxByte",
                               &s3Opt->s3ClientAdaptorOpt.readCacheMaxByte);
+    conf->GetValueFatalIfFail("s3.readCacheThreads",
+                              &s3Opt->s3ClientAdaptorOpt.readCacheThreads);
     conf->GetValueFatalIfFail("s3.nearfullRatio",
                               &s3Opt->s3ClientAdaptorOpt.nearfullRatio);
     conf->GetValueFatalIfFail("s3.baseSleepUs",
@@ -196,15 +247,15 @@ void InitVolumeOption(Configuration *conf, VolumeOption *volumeOpt) {
                               &volumeOpt->allocatorOption.type);
 
     conf->GetValueFatalIfFail(
-        "volume.blockgroup.allocate_once",
+        "volume.blockGroup.allocateOnce",
         &volumeOpt->allocatorOption.blockGroupOption.allocateOnce);
 
     if (volumeOpt->allocatorOption.type == "bitmap") {
         conf->GetValueFatalIfFail(
-            "volume.bitmapallocator.size_per_bit",
+            "volume.bitmapAllocator.sizePerBit",
             &volumeOpt->allocatorOption.bitmapAllocatorOption.sizePerBit);
         conf->GetValueFatalIfFail(
-            "volume.bitmapallocator.small_alloc_proportion",
+            "volume.bitmapAllocator.smallAllocProportion",
             &volumeOpt->allocatorOption.bitmapAllocatorOption
                  .smallAllocProportion);
     } else {
@@ -232,6 +283,65 @@ void InitRefreshDataOpt(Configuration *conf,
                               &opt->refreshDataIntervalSec);
 }
 
+void InitKVClientManagerOpt(Configuration *conf,
+                               KVClientManagerOpt *config) {
+    conf->GetValueFatalIfFail("fuseClient.supportKVcache",
+                              &FLAGS_supportKVcache);
+    conf->GetValueFatalIfFail("fuseClient.setThreadPool",
+                              &config->setThreadPooln);
+    conf->GetValueFatalIfFail("fuseClient.getThreadPool",
+                              &config->getThreadPooln);
+}
+
+void InitFileSystemOption(Configuration* c, FileSystemOption* option) {
+    c->GetValueFatalIfFail("fs.cto", &option->cto);
+    c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto);
+    c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr);
+    c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength);
+    c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging);
+    {  // kernel cache option
+        auto o = &option->kernelCacheOption;
+        c->GetValueFatalIfFail("fs.kernelCache.attrTimeoutSec",
+                               &o->attrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeoutSec",
+                               &o->dirAttrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.entryTimeoutSec",
+                               &o->entryTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeoutSec",
+                               &o->dirEntryTimeoutSec);
+    }
+    {  // lookup cache option
+        auto o = &option->lookupCacheOption;
+        c->GetValueFatalIfFail("fs.lookupCache.lruSize",
+                               &o->lruSize);
+        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeoutSec",
+                               &o->negativeTimeoutSec);
+        c->GetValueFatalIfFail("fs.lookupCache.minUses",
+                               &o->minUses);
+    }
+    {  // dir cache option
+        auto o = &option->dirCacheOption;
+        c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize);
+    }
+    {  // open file option
+        auto o = &option->openFilesOption;
+        c->GetValueFatalIfFail("fs.openFile.lruSize", &o->lruSize);
+    }
+    {  // attr watcher option
+        auto o = &option->attrWatcherOption;
+        c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize);
+    }
+    {  // rpc option
+        auto o = &option->rpcOption;
+        c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit);
+    }
+    {  // defer sync option
+        auto o = &option->deferSyncOption;
+        c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay);
+        c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime);
+    }
+}
+
 void SetBrpcOpt(Configuration *conf) {
     curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy;
     dummy.Load(conf, "defer_close_second", "rpc.defer.close.second",
@@ -251,46 +361,53 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
     InitVolumeOption(conf, &clientOption->volumeOpt);
     InitLeaseOpt(conf, &clientOption->leaseOpt);
     InitRefreshDataOpt(conf, &clientOption->refreshDataOption);
+    InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt);
+    InitFileSystemOption(conf, &clientOption->fileSystemOption);
 
-    conf->GetValueFatalIfFail("fuseClient.attrTimeOut",
-                              &clientOption->attrTimeOut);
-    conf->GetValueFatalIfFail("fuseClient.entryTimeOut",
-                              &clientOption->entryTimeOut);
     conf->GetValueFatalIfFail("fuseClient.listDentryLimit",
                               &clientOption->listDentryLimit);
     conf->GetValueFatalIfFail("fuseClient.listDentryThreads",
                               &clientOption->listDentryThreads);
-    conf->GetValueFatalIfFail("fuseClient.flushPeriodSec",
-                              &clientOption->flushPeriodSec);
-    conf->GetValueFatalIfFail("fuseClient.maxNameLength",
-                              &clientOption->maxNameLength);
-    conf->GetValueFatalIfFail("fuseClient.iCacheLruSize",
-                              &clientOption->iCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.dCacheLruSize",
-                              &clientOption->dCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.enableICacheMetrics",
-                              &clientOption->enableICacheMetrics);
-    conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
-                              &clientOption->enableDCacheMetrics);
-    conf->GetValueFatalIfFail("client.dummyserver.startport",
+    conf->GetValueFatalIfFail("client.dummyServer.startPort",
                               &clientOption->dummyServerStartPort);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                               &clientOption->enableMultiMountPointRename);
-    conf->GetValueFatalIfFail("fuseClient.disableXattr",
-                              &clientOption->disableXattr);
-    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
-
+    conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
+                              &clientOption->downloadMaxRetryTimes);
+    conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
+                              &clientOption->warmupThreadsNum);
     LOG_IF(WARNING, conf->GetBoolValue("fuseClient.enableSplice",
                                        &clientOption->enableFuseSplice))
         << "Not found `fuseClient.enableSplice` in conf, use default value `"
         << std::boolalpha << clientOption->enableFuseSplice << '`';
 
-    // if enableCto, attr and entry cache must invalid
-    if (FLAGS_enableCto) {
-        clientOption->attrTimeOut = 0;
-        clientOption->entryTimeOut = 0;
-    }
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteBytes",
+                              &FLAGS_fuseClientAvgWriteBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytes",
+                              &FLAGS_fuseClientBurstWriteBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytesSecs",
+                              &FLAGS_fuseClientBurstWriteBytesSecs);
 
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteIops",
+                              &FLAGS_fuseClientAvgWriteIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIops",
+                              &FLAGS_fuseClientBurstWriteIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIopsSecs",
+                              &FLAGS_fuseClientBurstWriteIopsSecs);
+
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadBytes",
+                              &FLAGS_fuseClientAvgReadBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytes",
+                              &FLAGS_fuseClientBurstReadBytes);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytesSecs",
+                              &FLAGS_fuseClientBurstReadBytesSecs);
+
+    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadIops",
+                              &FLAGS_fuseClientAvgReadIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIops",
+                              &FLAGS_fuseClientBurstReadIops);
+    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIopsSecs",
+                              &FLAGS_fuseClientBurstReadIopsSecs);
     SetBrpcOpt(conf);
 }
 
@@ -298,6 +415,7 @@ void SetFuseClientS3Option(FuseClientOption *clientOption,
     const S3InfoOption &fsS3Opt) {
     clientOption->s3Opt.s3ClientAdaptorOpt.blockSize = fsS3Opt.blockSize;
     clientOption->s3Opt.s3ClientAdaptorOpt.chunkSize = fsS3Opt.chunkSize;
+    clientOption->s3Opt.s3ClientAdaptorOpt.objectPrefix = fsS3Opt.objectPrefix;
     clientOption->s3Opt.s3AdaptrOpt.s3Address = fsS3Opt.s3Address;
     clientOption->s3Opt.s3AdaptrOpt.ak = fsS3Opt.ak;
     clientOption->s3Opt.s3AdaptrOpt.sk = fsS3Opt.sk;
@@ -312,6 +430,7 @@ void S3Info2FsS3Option(const curvefs::common::S3Info& s3,
     fsS3Opt->bucketName = s3.bucketname();
     fsS3Opt->blockSize = s3.blocksize();
     fsS3Opt->chunkSize = s3.chunksize();
+    fsS3Opt->objectPrefix = s3.has_objectprefix() ? s3.objectprefix() : 0;
 }
 
 }  // namespace common

curvefs/src/client/common/config.h

@@ -32,15 +32,13 @@
 #include "src/common/configuration.h"
 #include "src/common/s3_adapter.h"
 
+namespace curvefs {
+namespace client {
+namespace common {
 using ::curve::common::Configuration;
 using ::curve::common::S3AdapterOption;
 using ::curvefs::client::common::DiskCacheType;
 using ::curve::common::S3InfoOption;
-
-namespace curvefs {
-namespace client {
-namespace common {
-
 using MdsOption = ::curve::client::MetaServerOption;
 
 struct BlockDeviceClientOptions {
@@ -80,6 +78,11 @@ struct SpaceAllocServerOption {
     uint64_t rpcTimeoutMs;
 };
 
+struct KVClientManagerOpt {
+    int setThreadPooln = 4;
+    int getThreadPooln = 4;
+};
+
 struct DiskCacheOption {
     DiskCacheType diskCacheType;
     // cache disk dir
@@ -101,7 +104,7 @@ struct DiskCacheOption {
     // the max time system command can run
     uint32_t cmdTimeoutSec;
     // threads for disk cache
-    uint32_t threads;
+    uint32_t threads = 10;
     // the write throttle bps of disk cache
     uint64_t avgFlushBytes;
     // the write burst bps of disk cache
@@ -119,7 +122,6 @@ struct DiskCacheOption {
 struct S3ClientAdaptorOption {
     uint64_t blockSize;
     uint64_t chunkSize;
-    uint32_t fuseMaxSize;
     uint64_t pageSize;
     uint32_t prefetchBlocks;
     uint32_t prefetchExecQueueNum;
@@ -128,10 +130,12 @@ struct S3ClientAdaptorOption {
     uint32_t flushIntervalSec;
     uint64_t writeCacheMaxByte;
     uint64_t readCacheMaxByte;
+    uint32_t readCacheThreads;
     uint32_t nearfullRatio;
     uint32_t baseSleepUs;
     uint32_t maxReadRetryIntervalMs;
     uint32_t readRetryIntervalMs;
+    uint32_t objectPrefix;
     DiskCacheOption diskCacheOpt;
 };
 
@@ -170,6 +174,59 @@ struct RefreshDataOption {
     uint64_t maxDataSize = 1024;
     uint32_t refreshDataIntervalSec = 30;
 };
+
+// { filesystem option
+struct KernelCacheOption {
+    uint32_t entryTimeoutSec;
+    uint32_t dirEntryTimeoutSec;
+    uint32_t attrTimeoutSec;
+    uint32_t dirAttrTimeoutSec;
+};
+
+struct LookupCacheOption {
+    uint64_t lruSize;
+    uint32_t negativeTimeoutSec;
+    uint32_t minUses;
+};
+
+struct DirCacheOption {
+    uint64_t lruSize;
+    uint32_t timeoutSec;
+};
+
+struct AttrWatcherOption {
+    uint64_t lruSize;
+};
+
+struct OpenFilesOption {
+    uint64_t lruSize;
+    uint32_t deferSyncSecond;
+};
+
+struct RPCOption {
+    uint32_t listDentryLimit;
+};
+
+struct DeferSyncOption {
+    uint32_t delay;
+    bool deferDirMtime;
+};
+
+struct FileSystemOption {
+    bool cto;
+    bool disableXattr;
+    uint32_t maxNameLength;
+    uint32_t blockSize = 0x10000u;
+    KernelCacheOption kernelCacheOption;
+    LookupCacheOption lookupCacheOption;
+    DirCacheOption dirCacheOption;
+    OpenFilesOption openFilesOption;
+    AttrWatcherOption attrWatcherOption;
+    RPCOption rpcOption;
+    DeferSyncOption deferSyncOption;
+};
+// }
+
 struct FuseClientOption {
     MdsOption mdsOpt;
     MetaCacheOpt metaCacheOpt;
@@ -182,21 +239,16 @@ struct FuseClientOption {
     VolumeOption volumeOpt;
     LeaseOpt leaseOpt;
     RefreshDataOption refreshDataOption;
+    KVClientManagerOpt kvClientManagerOpt;
+    FileSystemOption fileSystemOption;
 
-    double attrTimeOut;
-    double entryTimeOut;
     uint32_t listDentryLimit;
     uint32_t listDentryThreads;
-    uint32_t flushPeriodSec;
-    uint32_t maxNameLength;
-    uint64_t iCacheLruSize;
-    uint64_t dCacheLruSize;
-    bool enableICacheMetrics;
-    bool enableDCacheMetrics;
     uint32_t dummyServerStartPort;
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;
-    bool disableXattr = false;
+    uint32_t downloadMaxRetryTimes;
+    uint32_t warmupThreadsNum = 10;
 };
 
 void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption);

curvefs/src/client/curve_fuse_op.h

@@ -37,10 +37,9 @@
 extern "C" {
 #endif
 
-int InitGlog(const char *confPath, const char *argv0);
+int InitLog(const char *confPath, const char *argv0);
 
-int InitFuseClient(const char *confPath, const char *fsName,
-        const char* fsType, const char* mdsAddr);
+int InitFuseClient(const struct MountOption *mountOption);
 
 void UnInitFuseClient();
 

curvefs/src/client/dentry_cache_manager.cpp

@@ -44,31 +44,13 @@ namespace client {
 
 using curve::common::WriteLockGuard;
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
-
-void DentryCacheManagerImpl::InsertOrReplaceCache(const Dentry &dentry) {
-    std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name());
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        NameLockGuard lock(nameLock_, key);
-        dCache_->Put(key, dentry);
-    }
-}
-
-void DentryCacheManagerImpl::DeleteCache(uint64_t parentId,
-                                         const std::string &name) {
-    std::string key = GetDentryCacheKey(parentId, name);
-    NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
-}
+using ::curvefs::client::filesystem::ToFSError;
 
 CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
                                                 const std::string &name,
                                                 Dentry *out) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    bool ok = dCache_->Get(key, out);
-    if (ok) {
-        return CURVEFS_ERROR::OK;
-    }
 
     MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out);
     if (ret != MetaStatusCode::OK) {
@@ -76,11 +58,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
             << "metaClient_ GetDentry failed, MetaStatusCode = " << ret
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
             << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
-    }
-
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, *out);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -95,12 +73,9 @@ CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) {
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << dentry.parentinodeid()
                    << ", name = " << dentry.name();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, dentry);
-    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -109,14 +84,13 @@ CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent,
                                                    FsFileType type) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
 
     MetaStatusCode ret = metaClient_->DeleteDentry(fsId_, parent, name, type);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -150,7 +124,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", parent = " << parent << ", last = " << last
                        << ", count = " << limit << ", onlyDir = " << onlyDir;
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
 
         if (!onlyDir) {

curvefs/src/client/dentry_cache_manager.h

@@ -33,20 +33,18 @@
 #include <utility>
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
-#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
-#include "src/common/lru_cache.h"
 #include "src/common/concurrent/name_lock.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Dentry;
-using ::curve::common::LRUCache;
-using ::curve::common::CacheMetrics;
 
 namespace curvefs {
 namespace client {
 
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 static const char* kDentryKeyDelimiter = ":";
 
@@ -59,12 +57,6 @@ class DentryCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) = 0;
-
-    virtual void InsertOrReplaceCache(const Dentry& dentry) = 0;
-
-    virtual void DeleteCache(uint64_t parentId, const std::string& name) = 0;
-
     virtual CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) = 0;
 
@@ -85,29 +77,11 @@ class DentryCacheManager {
 class DentryCacheManagerImpl : public DentryCacheManager {
  public:
     DentryCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()),
-        dCache_(nullptr) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
 
     explicit DentryCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient),
-        dCache_(nullptr) {}
-
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics) override {
-        if (enableCacheMetrics) {
-            dCache_ = std::make_shared<
-                LRUCache<std::string, Dentry>>(cacheSize,
-                    std::make_shared<CacheMetrics>("dcache"));
-        } else {
-            dCache_ = std::make_shared<
-                LRUCache<std::string, Dentry>>(cacheSize);
-        }
-        return CURVEFS_ERROR::OK;
-    }
-
-    void InsertOrReplaceCache(const Dentry& dentry) override;
-
-    void DeleteCache(uint64_t parentId, const std::string& name) override;
+      : metaClient_(metaClient) {}
 
     CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) override;
@@ -128,8 +102,6 @@ class DentryCacheManagerImpl : public DentryCacheManager {
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    // key is parentId + name
-    std::shared_ptr<LRUCache<std::string, Dentry>> dCache_;
     curve::common::GenericNameLock<Mutex> nameLock_;
 };
 

curvefs/src/client/error_code.cpp

@@ -1,98 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-/*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
- */
-
-#include "curvefs/src/client/error_code.h"
-
-#include <map>
-#include <string>
-
-namespace curvefs {
-namespace client {
-
-using ::curvefs::metaserver::MetaStatusCode;
-
-static const std::map<CURVEFS_ERROR, std::string> err2Msg = {
-    {CURVEFS_ERROR::OK, "OK"},
-    {CURVEFS_ERROR::INTERNAL, "internal error"},
-    {CURVEFS_ERROR::UNKNOWN, "unknown"},
-    {CURVEFS_ERROR::EXISTS, "inode or dentry already exist"},
-    {CURVEFS_ERROR::NOTEXIST, "inode or dentry not exist"},
-    {CURVEFS_ERROR::NO_SPACE, "no space to alloc"},
-    {CURVEFS_ERROR::BAD_FD, "bad fd"},
-    {CURVEFS_ERROR::INVALIDPARAM, "invalid param"},
-    {CURVEFS_ERROR::NOPERMISSION, "no permission"},
-    {CURVEFS_ERROR::NOTEMPTY, "dir not empty"},
-    {CURVEFS_ERROR::NOFLUSH, "no flush"},
-    {CURVEFS_ERROR::NOTSUPPORT, "not support"},
-    {CURVEFS_ERROR::NAMETOOLONG, "name too long"},
-    {CURVEFS_ERROR::MOUNT_POINT_EXIST, "mount point already exist"},
-    {CURVEFS_ERROR::MOUNT_FAILED, "mount failed"},
-};
-
-std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
-    os << static_cast<int>(code) << "[" << [code]() {
-        auto it = err2Msg.find(code);
-        if (it != err2Msg.end()) {
-            return it->second;
-        }
-
-        return std::string{"Unknown"};
-    }() << "]";
-
-    return os;
-}
-
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    MetaStatusCode code) {
-    CURVEFS_ERROR ret = CURVEFS_ERROR::UNKNOWN;
-    switch (code) {
-        case MetaStatusCode::OK:
-            ret = CURVEFS_ERROR::OK;
-            break;
-
-        case MetaStatusCode::NOT_FOUND:
-            ret = CURVEFS_ERROR::NOTEXIST;
-            break;
-
-        case MetaStatusCode::PARAM_ERROR:
-            ret = CURVEFS_ERROR::INVALIDPARAM;
-            break;
-
-        case MetaStatusCode::INODE_EXIST:
-        case MetaStatusCode::DENTRY_EXIST:
-            ret = CURVEFS_ERROR::EXISTS;
-            break;
-
-        case MetaStatusCode::SYM_LINK_EMPTY:
-        case MetaStatusCode::RPC_ERROR:
-            ret = CURVEFS_ERROR::INTERNAL;
-            break;
-
-        default:
-            ret = CURVEFS_ERROR::UNKNOWN;
-    }
-    return ret;
-}
-
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/README.md

@@ -0,0 +1,78 @@
+CurveFS Client Metadata Cache Design
+===
+
+```
++----------------+
++  Kernel Cache  |
++----------------+
+    |       ^
+(1) |       | (4)
+    v       |
++----------------+
++  CurveFS Fuse  |
++----------------+
+    |       ^
+(2) |       | (3)
+    v       |
++----------------+
++   MetaServer   |
++----------------+
+```
+
+(1) kernel cache timeout
+---
+
+* requests are always sent to fuse layer, except `lookup` and `getattr` request which affected by caching.
+* if entry or attribute cache timeout, vfs layer will launch request to fuse layer.
+
+(2) revalidate cache
+---
+
+* for `open` and `opendir` request, fuse layer should revalidate cache by comparing mtime, if modified, fuse layer should:
+    * `open`: return **ESTALE** to trigger vfs layer to invoke the `open` again with ignoring cache.
+    * `opendir`: drop all directory cache.
+* others, proxy request to metaserver directly.
+
+
+(3) retrive fresh metadata
+---
+
+* for `readdir` request, fuse layer should cache direcoty entries and their attributes.
+* others, no caching.
+
+(4) reply with timeout
+---
+
+* reply entry or attribute to vfs layer with corresponding cache timeout.
+* four type timeouts provided: `entryTimeout`, `dirEntryTimeout`, `attrTimeout`, `dirAtttTimeout`.
+* fuse layer should remeber the `mtime` of attribute while reply it to vfs layer.
+
+Cache Layer Level
+===
+
+```
++-------------------------------------------------------+
+|                       Kernel Cache                    |  (L1 Cache)
++-------------------------------------------------------+
+                            ^  (update length)
+                            |
++--------------------------------------------------------+
+|                       Open File Cache                  |  (L2 Cache)
+|                   (only length and chunk)              |
++--------------------------------------------------------+
+         ^                            ^
+         | (reply entry/attr)         |  (reply entry/attr)
++-----------------+                   |
+| Directory Cache |  (L3 Cache)       |
++-----------------+                   |
+         ^                            |
+         |                            |
++-----------------+ +------------------------------------+
+|     readdir     | | lookup、getattr、setattr、create... |
++-----------------+ +------------------------------------+
+         ^                            ^
+         |                            |
++--------------------------------------------------------+
+|                        MetaServer                      |
++--------------------------------------------------------+
+```

curvefs/src/client/filesystem/access_log.h

@@ -0,0 +1,89 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-17
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <unistd.h>
+#include <butil/time.h>
+#include <spdlog/spdlog.h>
+#include <spdlog/sinks/basic_file_sink.h>
+#include <spdlog/sinks/daily_file_sink.h>
+
+#include <string>
+#include <memory>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/common/config.h"
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+
+namespace curvefs {
+namespace client {
+namespace common {
+
+DECLARE_bool(access_logging);
+
+}
+namespace filesystem {
+
+using ::absl::StrFormat;
+using ::curvefs::client::common::FLAGS_access_logging;
+using MessageHandler = std::function<std::string()>;
+
+static std::shared_ptr<spdlog::logger> Logger;
+
+bool InitAccessLog(const std::string& prefix) {
+    std::string filename = StrFormat("%s/access.%d.log", prefix, getpid());
+    Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0);
+    spdlog::flush_every(std::chrono::seconds(1));
+    return true;
+}
+
+struct AccessLogGuard {
+    explicit AccessLogGuard(MessageHandler handler)
+        : enable(FLAGS_access_logging),
+          handler(handler) {
+        if (!enable) {
+            return;
+        }
+
+        timer.start();
+    }
+
+    ~AccessLogGuard() {
+        if (!enable) {
+            return;
+        }
+
+        timer.stop();
+        Logger->info("{0} <{1:.6f}>", handler(), timer.u_elapsed() / 1e6);
+    }
+
+    bool enable;
+    MessageHandler handler;
+    butil::Timer timer;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_

curvefs/src/client/filesystem/attr_watcher.cpp

@@ -0,0 +1,117 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+AttrWatcher::AttrWatcher(AttrWatcherOption option,
+                         std::shared_ptr<OpenFiles> openFiles,
+                         std::shared_ptr<DirCache> dirCache)
+    : modifiedAt_(std::make_shared<LRUType>(option.lruSize)),
+      openFiles_(openFiles),
+      dirCache_(dirCache) {}
+
+void AttrWatcher::RemeberMtime(const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    modifiedAt_->Put(attr.inodeid(), AttrMtime(attr));
+}
+
+bool AttrWatcher::GetMtime(Ino ino, TimeSpec* time) {
+    ReadLockGuard lk(rwlock_);
+    return modifiedAt_->Get(ino, time);
+}
+
+void AttrWatcher::UpdateDirEntryAttr(Ino ino, const InodeAttr& attr) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : attr.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateAttr(ino, attr);
+
+        VLOG(1) << "Write back attribute to dir entry cache: ino = " << ino
+                << ", attr = " << attr.ShortDebugString();
+    }
+}
+
+void AttrWatcher::UpdateDirEntryLength(Ino ino, const InodeAttr& open) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : open.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateLength(ino, open);
+
+        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
+                << ", attr = " << open.ShortDebugString();
+    }
+}
+
+AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                                   InodeAttr* attr,
+                                   ReplyType type,
+                                   bool writeBack)
+    : watcher(watcher), attr(attr), type(type), writeBack(writeBack) {
+    InodeAttr open;
+    Ino ino = attr->inodeid();
+    bool yes = watcher->openFiles_->GetFileAttr(ino, &open);
+    if (!yes) {
+        return;
+    }
+
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+}
+
+AttrWatcherGuard::~AttrWatcherGuard() {
+    switch (type) {
+        case ReplyType::ATTR:
+            watcher->RemeberMtime(*attr);
+            if (writeBack) {
+                watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
+            }
+            break;
+
+        case ReplyType::ONLY_LENGTH:
+            if (writeBack) {
+                watcher->UpdateDirEntryLength(attr->inodeid(), *attr);
+            }
+            break;
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/attr_watcher.h

@@ -0,0 +1,105 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::AttrWatcherOption;
+
+class AttrWatcher {
+ public:
+    using LRUType = LRUCache<Ino, struct TimeSpec>;
+
+ public:
+    AttrWatcher(AttrWatcherOption option,
+                std::shared_ptr<OpenFiles> openFiles,
+                std::shared_ptr<DirCache> dirCache);
+
+    void RemeberMtime(const InodeAttr& attr);
+
+    bool GetMtime(Ino ino, TimeSpec* time);
+
+    void UpdateDirEntryAttr(Ino ino, const InodeAttr& attr);
+
+    void UpdateDirEntryLength(Ino ino, const InodeAttr& open);
+
+ private:
+    friend class AttrWatcherGuard;
+
+ private:
+    RWLock rwlock_;
+    std::shared_ptr<LRUType> modifiedAt_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<DirCache> dirCache_;
+};
+
+
+enum class ReplyType {
+    ATTR,
+    ONLY_LENGTH
+};
+
+/*
+ * each attribute reply to kernel, the watcher will:
+ *  before reply:
+ *    1) set attibute length if the corresponding file is opened
+ *  after reply:
+ *    1) remeber attribute modified time.
+ *    2) write back attribute to dir entry cache if |writeBack| is true,
+ *       because the dir-entry attribute maybe stale.
+ */
+struct AttrWatcherGuard {
+ public:
+    AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                     InodeAttr* attr,
+                     ReplyType type,
+                     bool writeBack);
+
+    ~AttrWatcherGuard();
+
+ private:
+    std::shared_ptr<AttrWatcher> watcher;
+    InodeAttr* attr;
+    ReplyType type;
+    bool writeBack;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_

curvefs/src/client/filesystem/defer_sync.cpp

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <vector>
+#include <memory>
+
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DeferSync::DeferSync(DeferSyncOption option)
+    : option_(option),
+      mutex_(),
+      running_(false),
+      thread_(),
+      sleeper_(),
+      inodes_() {
+}
+
+void DeferSync::Start() {
+    if (!running_.exchange(true)) {
+        thread_ = std::thread(&DeferSync::SyncTask, this);
+        LOG(INFO) << "Defer sync thread start success";
+    }
+}
+
+void DeferSync::Stop() {
+    if (running_.exchange(false)) {
+        LOG(INFO) << "Stop defer sync thread...";
+        sleeper_.interrupt();
+        thread_.join();
+        LOG(INFO) << "Defer sync thread stopped";
+    }
+}
+
+void DeferSync::SyncTask() {
+    std::vector<std::shared_ptr<InodeWrapper>> inodes;
+    for ( ;; ) {
+        bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay));
+
+        {
+            LockGuard lk(mutex_);
+            inodes.swap(inodes_);
+        }
+        for (const auto& inode : inodes) {
+            UniqueLock lk(inode->GetUniqueLock());
+            inode->Async(nullptr, true);
+        }
+        inodes.clear();
+
+        if (!running) {
+            break;
+        }
+    }
+}
+
+void DeferSync::Push(const std::shared_ptr<InodeWrapper>& inode) {
+    LockGuard lk(mutex_);
+    inodes_.emplace_back(inode);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/defer_sync.h

@@ -0,0 +1,70 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+
+#include <atomic>
+#include <vector>
+#include <memory>
+
+#include "src/common/interruptible_sleeper.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::DeferSyncOption;
+
+using ::curve::common::Mutex;
+using ::curve::common::LockGuard;
+using ::curve::common::InterruptibleSleeper;
+
+class DeferSync {
+ public:
+    explicit DeferSync(DeferSyncOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Push(const std::shared_ptr<InodeWrapper>& inode);
+
+ private:
+    void SyncTask();
+
+ private:
+    DeferSyncOption option_;
+    Mutex mutex_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    InterruptibleSleeper sleeper_;
+    std::vector<std::shared_ptr<InodeWrapper>> inodes_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_

curvefs/src/client/filesystem/dir_cache.cpp

@@ -0,0 +1,189 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <utility>
+
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DirEntryList::DirEntryList()
+    : rwlock_(),
+      mtime_(),
+      entries_(),
+      attrs_() {}
+
+size_t DirEntryList::Size() {
+    ReadLockGuard lk(rwlock_);
+    return entries_.size();
+}
+
+void DirEntryList::Add(const DirEntry& dirEntry) {
+    WriteLockGuard lk(rwlock_);
+    entries_.push_back(std::move(dirEntry));
+    attrs_[dirEntry.ino] = &entries_.back();
+}
+
+bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) {
+    ReadLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+    *dirEntry = *iter->second;
+    return true;
+}
+
+bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    DirEntry* dirEntry = iter->second;
+    dirEntry->attr = std::move(attr);
+    return true;
+}
+
+bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    InodeAttr* attr = &iter->second->attr;
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+    return true;
+}
+
+void DirEntryList::Iterate(IterateHandler handler) {
+    ReadLockGuard lk(rwlock_);
+    for (auto iter = entries_.begin(); iter != entries_.end(); iter++) {
+        handler(&(*iter));
+    }
+}
+
+void DirEntryList::Clear() {
+    WriteLockGuard lk(rwlock_);
+    entries_.clear();
+    attrs_.clear();
+}
+
+void DirEntryList::SetMtime(TimeSpec mtime) {
+    WriteLockGuard lk(rwlock_);
+    mtime_ = mtime;
+}
+
+TimeSpec DirEntryList::GetMtime() {
+    ReadLockGuard lk(rwlock_);
+    return mtime_;
+}
+
+DirCache::DirCache(DirCacheOption option)
+    : rwlock_(),
+      nentries_(0),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(0);  // control size by ourself
+    mq_ = std::make_shared<MessageQueueType>("dircache", 10000);
+    mq_->Subscribe([&](const std::shared_ptr<DirEntryList>& entries){
+        entries->Clear();
+    });
+
+    LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize;
+}
+
+void DirCache::Start() {
+    mq_->Start();
+}
+
+void DirCache::Stop() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+    mq_->Stop();
+}
+
+void DirCache::Delete(Ino parent,
+                      std::shared_ptr<DirEntryList> entries,
+                      bool evit) {
+    nentries_ -= entries->Size();
+    mq_->Publish(entries);  // clear entries in background
+    lru_->Remove(parent);
+
+    VLOG(1) << "Delete directory cache (evit=" << evit << "): "
+            << "parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
+}
+
+void DirCache::Evit(size_t size) {
+    Ino parent;
+    std::shared_ptr<DirEntryList> entries;
+    while (nentries_ + size >= option_.lruSize) {
+        bool yes = lru_->GetLast(&parent, &entries);
+        if (!yes) {
+            break;
+        }
+        Delete(parent, entries, true);
+    }
+}
+
+void DirCache::Put(Ino parent, std::shared_ptr<DirEntryList> entries) {
+    WriteLockGuard lk(rwlock_);
+    Evit(entries->Size());  // it guarantee put entries success
+    lru_->Put(parent, entries);
+    nentries_ += entries->Size();
+
+    VLOG(1) << "Insert directory cache: parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
+}
+
+bool DirCache::Get(Ino parent, std::shared_ptr<DirEntryList>* entries) {
+    ReadLockGuard lk(rwlock_);
+    return lru_->Get(parent, entries);
+}
+
+void DirCache::Drop(Ino parent) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = lru_->Get(parent, &entries);
+    if (yes) {
+        Delete(parent, entries, false);
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/dir_cache.h

@@ -0,0 +1,114 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+
+#include <map>
+#include <list>
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/message_queue.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::DirCacheOption;
+
+class DirEntryList {
+ public:
+    using IterateHandler = std::function<void(DirEntry* dirEntry)>;
+
+ public:
+    DirEntryList();
+
+    size_t Size();
+
+    void Add(const DirEntry& dirEntry);
+
+    void Iterate(IterateHandler handler);
+
+    bool Get(Ino ino, DirEntry* dirEntry);
+
+    bool UpdateAttr(Ino ino, const InodeAttr& attr);
+
+    bool UpdateLength(Ino ino, const InodeAttr& open);
+
+    void Clear();
+
+    void SetMtime(TimeSpec mtime);
+
+    TimeSpec GetMtime();
+
+ private:
+    RWLock rwlock_;
+    TimeSpec mtime_;
+    std::list<DirEntry> entries_;
+    std::map<Ino, DirEntry*> attrs_;
+};
+
+class DirCache {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<DirEntryList>>;
+    using MessageType = std::shared_ptr<DirEntryList>;
+    using MessageQueueType = MessageQueue<MessageType>;
+
+ public:
+    explicit DirCache(DirCacheOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Put(Ino parent, std::shared_ptr<DirEntryList> entries);
+
+    bool Get(Ino parent, std::shared_ptr<DirEntryList>* entries);
+
+    void Drop(Ino parent);
+
+ private:
+    void Delete(Ino parent, std::shared_ptr<DirEntryList> entries, bool evit);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    size_t nentries_;
+    DirCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+    std::shared_ptr<MessageQueueType> mq_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_

curvefs/src/client/filesystem/error.cpp

@@ -0,0 +1,106 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <utility>
+#include <ostream>
+
+#include "curvefs/src/client/filesystem/error.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+static const std::map<CURVEFS_ERROR, std::pair<int, std::string>> errors = {
+    { CURVEFS_ERROR::OK, { 0, "OK"} },
+    { CURVEFS_ERROR::INTERNAL,  { EIO, "internal error" } },
+    { CURVEFS_ERROR::UNKNOWN,  { -1, "unknown" } },
+    { CURVEFS_ERROR::EXISTS,  { EEXIST, "inode or dentry already exist" } },
+    { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } },
+    { CURVEFS_ERROR::NO_SPACE,  { ENOSPC, "no space to alloc" } },
+    { CURVEFS_ERROR::BAD_FD,  { EBADF, "bad file number" } },
+    { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } },
+    { CURVEFS_ERROR::NOPERMISSION,  { EACCES, "permission denied" } },
+    { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } },
+    { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } },
+    { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } },
+    { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } },
+    { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } },
+    { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } },
+    { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } },
+    { CURVEFS_ERROR::NODATA, { ENODATA, "no data available" } },
+    { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } },
+    { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } },
+    { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } },
+};
+
+std::string StrErr(CURVEFS_ERROR code) {
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        return it->second.second;
+    }
+    return "unknown";
+}
+
+int SysErr(CURVEFS_ERROR code) {
+    int syscode = -1;
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        syscode = it->second.first;
+    }
+    return (syscode == -1) ? EIO : syscode;
+}
+
+std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
+    os << static_cast<int>(code) << "[" << [code]() {
+        auto it = errors.find(code);
+        if (it != errors.end()) {
+            return it->second.second;
+        }
+
+        return std::string{"Unknown"};
+    }() << "]";
+
+    return os;
+}
+
+CURVEFS_ERROR ToFSError(MetaStatusCode code) {
+    static std::map<MetaStatusCode, CURVEFS_ERROR> errs = {
+        { MetaStatusCode::OK, CURVEFS_ERROR::OK },
+        { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST },
+        { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM },
+        { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL },
+        { MetaStatusCode::RPC_ERROR, CURVEFS_ERROR::INTERNAL },
+    };
+
+    auto it = errs.find(code);
+    if (it != errs.end()) {
+        return it->second;
+    }
+    return CURVEFS_ERROR::UNKNOWN;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/error.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2021 NetEase Inc.
+ *  Copyright (c) 2023 NetEase Inc.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -14,22 +14,27 @@
  *  limitations under the License.
  */
 
-
 /*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
  */
 
-#ifndef CURVEFS_SRC_CLIENT_ERROR_CODE_H_
-#define CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#include <map>
+#include <string>
+#include <iostream>
 
 #include "curvefs/proto/metaserver.pb.h"
 
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+
 namespace curvefs {
 namespace client {
+namespace filesystem {
+
+using ::curvefs::metaserver::MetaStatusCode;
 
-// notice : the error code should be negative.
 enum class CURVEFS_ERROR {
     OK = 0,
     INTERNAL = -1,
@@ -49,15 +54,21 @@ enum class CURVEFS_ERROR {
     OUT_OF_RANGE = -15,
     NODATA = -16,
     IO_ERROR = -17,
+    CACHETOOSMALL = -18,
+    STALE = -19,
+    NOSYS = -20,
 };
 
+std::string StrErr(CURVEFS_ERROR code);
+
+int SysErr(CURVEFS_ERROR code);
 
 std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code);
 
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    ::curvefs::metaserver::MetaStatusCode code);
+CURVEFS_ERROR ToFSError(MetaStatusCode code);
 
+}  // namespace filesystem
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_

curvefs/src/client/filesystem/filesystem.cpp

@@ -0,0 +1,353 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+
+#include "curvefs/src/client/filesystem/filesystem.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
+    : option_(option), member(member) {
+    deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
+    negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
+    dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
+    openFiles_ = std::make_shared<OpenFiles>(option_.openFilesOption,
+                                             deferSync_);
+    attrWatcher_ = std::make_shared<AttrWatcher>(option_.attrWatcherOption,
+                                                 openFiles_, dirCache_);
+    handlerManager_ = std::make_shared<HandlerManager>();
+    rpc_ = std::make_shared<RPCClient>(option.rpcOption, member);
+}
+
+void FileSystem::Run() {
+    deferSync_->Start();
+    dirCache_->Start();
+}
+
+void FileSystem::Destory() {
+    openFiles_->CloseAll();
+    deferSync_->Stop();
+    dirCache_->Stop();
+}
+
+void FileSystem::Attr2Stat(InodeAttr* attr, struct stat* stat) {
+    std::memset(stat, 0, sizeof(struct stat));
+    stat->st_ino = attr->inodeid();  //  inode number
+    stat->st_mode = attr->mode();  // permission mode
+    stat->st_nlink = attr->nlink();  // number of links
+    stat->st_uid = attr->uid();  // user ID of owner
+    stat->st_gid = attr->gid();  // group ID of owner
+    stat->st_size = attr->length();  // total size, in bytes
+    stat->st_rdev = attr->rdev();  // device ID (if special file)
+    stat->st_atim.tv_sec = attr->atime();  // time of last access
+    stat->st_atim.tv_nsec = attr->atime_ns();
+    stat->st_mtim.tv_sec = attr->mtime();  // time of last modification
+    stat->st_mtim.tv_nsec = attr->mtime_ns();
+    stat->st_ctim.tv_sec = attr->ctime();  // time of last status change
+    stat->st_ctim.tv_nsec = attr->ctime_ns();
+    stat->st_blksize = option_.blockSize;  // blocksize for file system I/O
+    stat->st_blocks = 0;  // number of 512B blocks allocated
+    if (IsS3File(*attr)) {
+        stat->st_blocks = (attr->length() + 511) / 512;
+    }
+}
+
+void FileSystem::Entry2Param(EntryOut* entryOut,
+                             fuse_entry_param* e) {
+    std::memset(e, 0, sizeof(fuse_entry_param));
+    e->ino = entryOut->attr.inodeid();
+    e->generation = 0;
+    Attr2Stat(&entryOut->attr, &e->attr);
+    e->entry_timeout = entryOut->entryTimeout;
+    e->attr_timeout = entryOut->attrTimeout;
+}
+
+void FileSystem::SetEntryTimeout(EntryOut* entryOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(entryOut->attr)) {
+        entryOut->entryTimeout = option.dirEntryTimeoutSec;
+        entryOut->attrTimeout = option.dirAttrTimeoutSec;
+    } else {
+        entryOut->entryTimeout = option.entryTimeoutSec;
+        entryOut->attrTimeout = option.attrTimeoutSec;
+    }
+}
+
+void FileSystem::SetAttrTimeout(AttrOut* attrOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(attrOut->attr)) {
+        attrOut->attrTimeout = option.dirAttrTimeoutSec;
+    } else {
+        attrOut->attrTimeout = option.attrTimeoutSec;
+    }
+}
+
+// fuse reply*
+void FileSystem::ReplyError(Request req, CURVEFS_ERROR code) {
+    fuse_reply_err(req, SysErr(code));
+}
+
+void FileSystem::ReplyEntry(Request req,
+                            EntryOut* entryOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_entry(req, &e);
+}
+
+void FileSystem::ReplyAttr(Request req,
+                           AttrOut* attrOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr,
+                             ReplyType::ATTR, true);
+    struct stat stat;
+    SetAttrTimeout(attrOut);
+    Attr2Stat(&attrOut->attr, &stat);
+    fuse_reply_attr(req, &stat, attrOut->attrTimeout);
+}
+
+void FileSystem::ReplyReadlink(Request req, const std::string& link) {
+    fuse_reply_readlink(req, link.c_str());
+}
+
+void FileSystem::ReplyOpen(Request req, FileInfo* fi) {
+    fuse_reply_open(req, fi);
+}
+
+void FileSystem::ReplyOpen(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_open(req, fileOut->fi);
+}
+
+void FileSystem::ReplyData(Request req,
+                           struct fuse_bufvec *bufv,
+                           enum fuse_buf_copy_flags flags) {
+    fuse_reply_data(req, bufv, flags);
+}
+
+void FileSystem::ReplyWrite(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_write(req, fileOut->nwritten);
+}
+
+void FileSystem::ReplyBuffer(Request req, const char *buf, size_t size) {
+    fuse_reply_buf(req, buf, size);
+}
+
+void FileSystem::ReplyStatfs(Request req, const struct statvfs *stbuf) {
+    fuse_reply_statfs(req, stbuf);
+}
+
+void FileSystem::ReplyXattr(Request req, size_t size) {
+    fuse_reply_xattr(req, size);
+}
+
+void FileSystem::ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_create(req, &e, fi);
+}
+
+void FileSystem::AddDirEntry(Request req,
+                             DirBufferHead* buffer,
+                             DirEntry* dirEntry) {
+    struct stat stat;
+    std::memset(&stat, 0, sizeof(stat));
+    stat.st_ino = dirEntry->ino;
+
+    // add a directory entry to the buffer
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry(req,
+                      buffer->p + oldsize,  // char* buf
+                      buffer->size - oldsize,  // size_t bufisze
+                      name, &stat, buffer->size);
+}
+
+void FileSystem::AddDirEntryPlus(Request req,
+                                 DirBufferHead* buffer,
+                                 DirEntry* dirEntry) {
+    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr,
+                             ReplyType::ATTR, false);
+    struct fuse_entry_param e;
+    EntryOut entryOut(dirEntry->attr);
+    SetEntryTimeout(&entryOut);
+    Entry2Param(&entryOut, &e);
+
+    // add a directory entry to the buffer with the attributes
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry_plus(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry_plus(req,
+                           buffer->p + oldsize,  // char* buf
+                           buffer->size - oldsize,  // size_t bufisze
+                           name, &e, buffer->size);
+}
+
+// handler*
+std::shared_ptr<FileHandler> FileSystem::NewHandler() {
+    return handlerManager_->NewHandler();
+}
+
+std::shared_ptr<FileHandler> FileSystem::FindHandler(uint64_t fh) {
+    return handlerManager_->FindHandler(fh);
+}
+
+void FileSystem::ReleaseHandler(uint64_t fh) {
+    return handlerManager_->ReleaseHandler(fh);
+}
+
+FileSystemMember FileSystem::BorrowMember() {
+    return FileSystemMember(deferSync_, openFiles_, attrWatcher_);
+}
+
+// fuse request*
+CURVEFS_ERROR FileSystem::Lookup(Request req,
+                                 Ino parent,
+                                 const std::string& name,
+                                 EntryOut* entryOut) {
+    if (name.size() > option_.maxNameLength) {
+        return CURVEFS_ERROR::NAMETOOLONG;
+    }
+
+    bool yes = negative_->Get(parent, name);
+    if (yes) {
+        return CURVEFS_ERROR::NOTEXIST;
+    }
+
+    auto rc = rpc_->Lookup(parent, name, entryOut);
+    if (rc == CURVEFS_ERROR::OK) {
+        negative_->Delete(parent, name);
+    } else if (rc == CURVEFS_ERROR::NOTEXIST) {
+        negative_->Put(parent, name);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) {
+    InodeAttr attr;
+    auto rc = rpc_->GetAttr(ino, &attr);
+    if (rc == CURVEFS_ERROR::OK) {
+        *attrOut = AttrOut(attr);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) {
+    InodeAttr attr;
+    CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    // revalidate directory cache
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = dirCache_->Get(ino, &entries);
+    if (yes) {
+        if (entries->GetMtime() != AttrMtime(attr)) {
+            dirCache_->Drop(ino);
+        }
+    }
+
+    auto handler = NewHandler();
+    handler->mtime = AttrMtime(attr);
+    fi->fh = handler->fh;
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReadDir(Request req,
+                                  Ino ino,
+                                  FileInfo* fi,
+                                  std::shared_ptr<DirEntryList>* entries) {
+    bool yes = dirCache_->Get(ino, entries);
+    if (yes) {
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->ReadDir(ino, entries);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    (*entries)->SetMtime(FindHandler(fi->fh)->mtime);
+    dirCache_->Put(ino, *entries);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) {
+    ReleaseHandler(fi->fh);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) {
+    std::shared_ptr<InodeWrapper> inode;
+    bool yes = openFiles_->IsOpened(ino, &inode);
+    if (yes) {
+        openFiles_->Open(ino, inode);
+        // fi->keep_cache = 1;
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->Open(ino, &inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    TimeSpec mtime;
+    yes = attrWatcher_->GetMtime(ino, &mtime);
+    if (!yes) {
+        // It is rare which only arise when attribute evited for attr-watcher.
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ": attribute not found in wacther";
+        return CURVEFS_ERROR::STALE;
+    } else if (mtime != InodeMtime(inode)) {
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ", cache(" << mtime << ") vs remote("
+                     << InodeMtime(inode) << ")";
+        return CURVEFS_ERROR::STALE;
+    }
+
+    openFiles_->Open(ino, inode);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) {
+    openFiles_->Close(ino);
+    return CURVEFS_ERROR::OK;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/filesystem.h

@@ -0,0 +1,167 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+
+#include <gtest/gtest_prod.h>
+
+#include <memory>
+#include <string>
+
+#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+#include "curvefs/src/client/filesystem/rpc_client.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::FileSystemOption;
+
+struct FileSystemMember {
+    FileSystemMember(std::shared_ptr<DeferSync> deferSync,
+                     std::shared_ptr<OpenFiles> openFiles,
+                     std::shared_ptr<AttrWatcher> attrWatcher)
+        : deferSync(deferSync),
+          openFiles(openFiles),
+          attrWatcher(attrWatcher) {}
+
+    std::shared_ptr<DeferSync> deferSync;
+    std::shared_ptr<OpenFiles> openFiles;
+    std::shared_ptr<AttrWatcher> attrWatcher;
+};
+
+class FileSystem {
+ public:
+    FileSystem(struct FileSystemOption option, ExternalMember member);
+
+    void Run();
+
+    void Destory();
+
+    // fuse request
+    CURVEFS_ERROR Lookup(Request req,
+                         Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut);
+
+    CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR ReadDir(Request req,
+                          Ino ino,
+                          FileInfo* fi,
+                          std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Release(Request req, Ino ino);
+
+    // fuse reply: we control all replies to vfs layer in same entrance.
+    void ReplyError(Request req, CURVEFS_ERROR code);
+
+    void ReplyEntry(Request req, EntryOut* entryOut);
+
+    void ReplyAttr(Request req, AttrOut* attrOut);
+
+    void ReplyReadlink(Request req, const std::string& link);
+
+    void ReplyOpen(Request req, FileInfo *fi);
+
+    void ReplyOpen(Request req, FileOut* fileOut);
+
+    void ReplyData(Request req,
+                   struct fuse_bufvec *bufv,
+                   enum fuse_buf_copy_flags flags);
+
+    void ReplyWrite(Request req, FileOut* fileOut);
+
+    void ReplyBuffer(Request req, const char *buf, size_t size);
+
+    void ReplyStatfs(Request req, const struct statvfs *stbuf);
+
+    void ReplyXattr(Request req, size_t size);
+
+    void ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi);
+
+    void AddDirEntry(Request req,
+                     DirBufferHead* buffer,
+                     DirEntry* dirEntry);
+
+    void AddDirEntryPlus(Request req,
+                         DirBufferHead* buffer,
+                         DirEntry* dirEntry);
+
+    // utility: file handler
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t fh);
+
+    void ReleaseHandler(uint64_t fh);
+
+    // utility: others
+    FileSystemMember BorrowMember();
+
+ private:
+    FRIEND_TEST(FileSystemTest, Attr2Stat);
+    FRIEND_TEST(FileSystemTest, Entry2Param);
+    FRIEND_TEST(FileSystemTest, SetEntryTimeout);
+    FRIEND_TEST(FileSystemTest, SetAttrTimeout);
+
+    // utility: convert to system type.
+    void Attr2Stat(InodeAttr* attr, struct stat* stat);
+
+    void Entry2Param(EntryOut* entryOut, fuse_entry_param* e);
+
+    // utility: set entry/attribute timeout
+    void SetEntryTimeout(EntryOut* entryOut);
+
+    void SetAttrTimeout(AttrOut* attrOut);
+
+ private:
+    FileSystemOption option_;
+    ExternalMember member;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<LookupCache> negative_;
+    std::shared_ptr<DirCache> dirCache_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<AttrWatcher> attrWatcher_;
+    std::shared_ptr<HandlerManager> handlerManager_;
+    std::shared_ptr<RPCClient> rpc_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_

curvefs/src/client/filesystem/lookup_cache.cpp

@@ -0,0 +1,104 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include <ctime>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+#define RETURN_FALSE_IF_DISABLED() \
+    do {                           \
+        if (!enable_) {            \
+            return false;          \
+        }                          \
+    } while (0)
+
+
+LookupCache::LookupCache(LookupCacheOption option)
+    : enable_(option.negativeTimeoutSec > 0),
+      rwlock_(),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(option.lruSize);
+    if (enable_) {
+        LOG(INFO) << "Using lookup negative lru cache"
+                  << ", timeout = " << option.negativeTimeoutSec
+                  << ", capacity = " << option.lruSize;
+    }
+}
+
+std::string LookupCache::CacheKey(Ino parent, const std::string& name) {
+    return absl::StrFormat("%d:%s", parent, name);
+}
+
+bool LookupCache::Get(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    ReadLockGuard lk(rwlock_);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
+    if (!yes) {
+        VLOG(1) << absl::StrFormat("Lookup cache not found: key(%d,%s)",
+                                   parent, name);
+        return false;
+    } else if (entry.uses < option_.minUses) {
+        return false;
+    } else if (entry.expireTime < Now()) {
+        return false;
+    }
+    return true;
+}
+
+bool LookupCache::Put(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    WriteLockGuard lk(rwlock_);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
+    if (yes) {
+        entry.uses++;
+    } else {
+        entry.uses = 0;
+    }
+
+    entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0);
+    lru_->Put(key, entry);
+    return true;
+}
+
+bool LookupCache::Delete(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    WriteLockGuard lk(rwlock_);
+    auto key = CacheKey(parent, name);
+    lru_->Remove(key);
+    return true;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/lookup_cache.h

@@ -0,0 +1,77 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+
+#include <memory>
+#include <string>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::LookupCacheOption;
+
+// memory cache for lookup result, now we only support cache negative result,
+// and other positive entry will be cached in kernel.
+class LookupCache {
+ public:
+    struct CacheEntry {
+        uint32_t uses;
+        TimeSpec expireTime;
+    };
+
+    using LRUType = LRUCache<std::string, CacheEntry>;
+
+ public:
+    explicit LookupCache(LookupCacheOption option);
+
+    bool Get(Ino parent, const std::string& name);
+
+    bool Put(Ino parent, const std::string& name);
+
+    bool Delete(Ino parent, const std::string& name);
+
+ private:
+    std::string CacheKey(Ino parent, const std::string& name);
+
+ private:
+    bool enable_;
+    RWLock rwlock_;
+    LookupCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_

curvefs/src/client/filesystem/message_queue.h

@@ -0,0 +1,118 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+
+#include <glog/logging.h>
+
+#include <string>
+#include <atomic>
+#include <thread>
+
+#include "src/common/concurrent/task_queue.h"
+#include "curvefs/src/common/threading.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::TaskQueue;
+using ::curvefs::common::SetThreadName;
+
+template <typename MessageT>
+class MessageQueue {
+ public:
+    using MessageHandler = std::function<void(const MessageT& message)>;
+
+ public:
+    MessageQueue(const std::string& name, size_t bufferSize)
+        : name_(name),
+          running_(false),
+          thread_(),
+          handler_(),
+          queue_(bufferSize) {}
+
+    void Start() {
+        if (running_.exchange(true)) {
+            return;
+        }
+
+        thread_ = std::thread(&MessageQueue::Consumer, this);
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread start success";
+    }
+
+    void Stop() {
+        if (!running_.exchange(false)) {
+            return;
+        }
+
+        auto wakeup = []() {};
+        queue_.Push(wakeup);
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stoping...";
+
+        thread_.join();
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stopped";
+    }
+
+    void Publish(MessageT message) {
+        if (handler_ != nullptr) {
+            queue_.Push([this, message](){
+                this->handler_(message);
+            });
+        }
+    }
+
+    void Subscribe(MessageHandler handler) {
+        handler_ = handler;
+    }
+
+ private:
+    void Consumer() {
+        SetThreadName(name_.c_str());
+        while (running_.load(std::memory_order_relaxed)) {
+            queue_.Pop()();
+        }
+
+        while (queue_.Size() > 0) {
+            queue_.Pop()();
+        }
+    }
+
+ private:
+    std::string name_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    MessageHandler handler_;
+    TaskQueue queue_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_

curvefs/src/client/filesystem/meta.cpp

@@ -0,0 +1,129 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <sys/stat.h>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+HandlerManager::HandlerManager()
+    : mutex_(),
+      dirBuffer_(std::make_shared<DirBuffer>()),
+      handlers_() {}
+
+HandlerManager::~HandlerManager() {
+    dirBuffer_->DirBufferFreeAll();
+}
+
+std::shared_ptr<FileHandler> HandlerManager::NewHandler() {
+    UniqueLock lk(mutex_);
+    auto handler = std::make_shared<FileHandler>();
+    handler->fh = dirBuffer_->DirBufferNew();
+    handler->buffer = dirBuffer_->DirBufferGet(handler->fh);
+    handler->padding = false;
+    handlers_.emplace(handler->fh, handler);
+    return handler;
+}
+
+std::shared_ptr<FileHandler> HandlerManager::FindHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    auto iter = handlers_.find(fh);
+    if (iter == handlers_.end()) {
+        return nullptr;
+    }
+    return iter->second;
+}
+
+void HandlerManager::ReleaseHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    dirBuffer_->DirBufferRelease(fh);
+    handlers_.erase(fh);
+}
+
+std::string StrMode(uint16_t mode) {
+    static std::map<uint16_t, char> type2char = {
+        { S_IFSOCK, 's' },
+        { S_IFLNK, 'l' },
+        { S_IFREG, '-' },
+        { S_IFBLK, 'b' },
+        { S_IFDIR, 'd' },
+        { S_IFCHR, 'c' },
+        { S_IFIFO, 'f' },
+        { 0, '?' },
+    };
+
+    std::string s("?rwxrwxrwx");
+    s[0] = type2char[mode & (S_IFMT & 0xffff)];
+    if (mode & S_ISUID) {
+        s[3] = 's';
+    }
+    if (mode & S_ISGID) {
+        s[6] = 's';
+    }
+    if (mode & S_ISVTX) {
+        s[9] = 't';
+    }
+
+    for (auto i = 0; i < 9; i++) {
+        if ((mode & (1 << i)) == 0) {
+            if ((s[9-i] == 's') || (s[9-i] == 't')) {
+                s[9-i] &= 0xDF;
+            } else {
+                s[9-i] = '-';
+            }
+        }
+    }
+    return s;
+}
+
+namespace {
+
+std::string Attr2Str(const InodeAttr& attr) {
+    if (!attr.IsInitialized()) {
+        return "";
+    }
+
+    std::string smode;
+    return absl::StrFormat(" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])",
+        attr.inodeid(), StrMode(attr.mode()).c_str(), attr.mode(), attr.nlink(),
+        attr.uid(), attr.gid(),
+        attr.atime(), attr.mtime(), attr.ctime(),
+        attr.length());
+}
+
+}  // namespace
+
+std::string StrEntry(EntryOut entryOut) {
+    return Attr2Str(entryOut.attr);
+}
+
+std::string StrAttr(AttrOut attrOut) {
+    return Attr2Str(attrOut.attr);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/meta.h

@@ -0,0 +1,178 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <iostream>
+
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/client/fuse_common.h"
+#include "curvefs/src/client/dir_buffer.h"
+#include "curvefs/src/client/inode_wrapper.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::Mutex;
+using ::curve::common::RWLock;
+using ::curve::common::UniqueLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::metaserver::XAttr;
+using ::curvefs::metaserver::Dentry;
+using ::curvefs::metaserver::InodeAttr;
+using ::curvefs::metaserver::FsFileType;
+using ::curvefs::client::DirBufferHead;
+using ::curvefs::client::InodeWrapper;
+
+using Ino = fuse_ino_t;
+using Request = fuse_req_t;
+using FileInfo = struct fuse_file_info;
+
+struct EntryOut {
+    EntryOut() = default;
+
+    explicit EntryOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double entryTimeout;
+    double attrTimeout;
+};
+
+struct AttrOut {
+    AttrOut() = default;
+
+    explicit AttrOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double attrTimeout;
+};
+
+struct DirEntry {
+    DirEntry() = default;
+
+    DirEntry(Ino ino, const std::string& name, InodeAttr attr)
+        : ino(ino), name(name), attr(attr) {}
+
+    Ino ino;
+    std::string name;
+    InodeAttr attr;
+};
+
+struct FileOut {
+    FileOut() = default;
+
+    FileOut(FileInfo* fi, InodeAttr attr)
+        : fi(fi), attr(attr), nwritten(0) {}
+
+    FileOut(InodeAttr attr, size_t nwritten)
+        : fi(nullptr), attr(attr), nwritten(nwritten) {}
+
+    FileInfo* fi;
+    InodeAttr attr;
+    size_t nwritten;
+};
+
+struct TimeSpec {
+    TimeSpec() : seconds(0), nanoSeconds(0) {}
+
+    TimeSpec(uint64_t seconds, uint32_t nanoSeconds)
+        : seconds(seconds), nanoSeconds(nanoSeconds) {}
+
+    TimeSpec(const TimeSpec& time)
+        : seconds(time.seconds), nanoSeconds(time.nanoSeconds) {}
+
+    TimeSpec& operator=(const TimeSpec& time) = default;
+
+    TimeSpec operator+(const TimeSpec& time) const {
+        return TimeSpec(seconds + time.seconds, nanoSeconds + time.nanoSeconds);
+    }
+
+    uint64_t seconds;
+    uint32_t nanoSeconds;
+};
+
+struct FileHandler {
+    uint64_t fh;
+    DirBufferHead* buffer;
+    TimeSpec mtime;
+    bool padding;  // padding buffer
+};
+
+class HandlerManager {
+ public:
+    HandlerManager();
+
+    ~HandlerManager();
+
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t id);
+
+    void ReleaseHandler(uint64_t id);
+
+ private:
+    Mutex mutex_;
+    std::shared_ptr<DirBuffer> dirBuffer_;
+    std::map<uint64_t, std::shared_ptr<FileHandler>> handlers_;
+};
+
+inline bool operator==(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds == rhs.seconds) &&
+        (lhs.nanoSeconds == rhs.nanoSeconds);
+}
+
+inline bool operator!=(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return !(lhs == rhs);
+}
+
+inline bool operator<(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds < rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds < rhs.nanoSeconds);
+}
+
+inline bool operator>(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds > rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds > rhs.nanoSeconds);
+}
+
+inline std::ostream &operator<<(std::ostream &os, const TimeSpec& time) {
+    return os << time.seconds << "." << time.nanoSeconds;
+}
+
+std::string StrMode(uint16_t mode);
+
+std::string StrEntry(EntryOut entryOut);
+
+std::string StrAttr(AttrOut attrOut);
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_

curvefs/src/client/filesystem/openfile.cpp

@@ -0,0 +1,141 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+OpenFiles::OpenFiles(OpenFilesOption option,
+                     std::shared_ptr<DeferSync> deferSync)
+    : rwlock_(),
+      option_(option),
+      deferSync_(deferSync),
+      files_(std::make_shared<LRUType>(option.lruSize)) {
+    LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize;
+}
+
+/*
+ * Delete(...) does:
+ *   1) publish to message queue which will flush file to server with async
+ *   2) delete file from lru cache
+ */
+void OpenFiles::Delete(Ino ino,
+                       const std::shared_ptr<OpenFile>& file,
+                       bool flush) {
+    if (flush) {
+        deferSync_->Push(file->inode);
+    }
+    files_->Remove(ino);
+
+    VLOG(1) << "Delete open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+void OpenFiles::Evit(size_t size) {
+    Ino ino;
+    std::shared_ptr<OpenFile> file;
+    while (files_->Size() + size >= option_.lruSize) {
+        bool yes = files_->GetLast(&ino, &file);
+        if (!yes) {
+            break;
+        }
+        Delete(ino, file, true);
+    }
+}
+
+void OpenFiles::Open(Ino ino, std::shared_ptr<InodeWrapper> inode) {
+    WriteLockGuard lk(rwlock_);
+
+    Evit(1);
+
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (yes) {
+        file->refs++;
+        return;
+    }
+
+    file = std::make_shared<OpenFile>(inode);
+    file->refs++;
+    files_->Put(ino, file);
+
+    VLOG(1) << "Insert open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+bool OpenFiles::IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+    *inode = file->inode;
+    return true;
+}
+
+void OpenFiles::Close(Ino ino) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return;
+    }
+
+    if (file->refs > 0) {
+        file->refs--;
+    }
+
+    if (file->refs == 0) {
+        Delete(ino, file, false);  // file already flushed before close
+    }
+}
+
+/*
+ * CloseAll() does:
+ *   flush all file to server and delete from LRU cache
+ */
+void OpenFiles::CloseAll() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+}
+
+bool OpenFiles::GetFileAttr(Ino ino, InodeAttr* attr) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+
+    file->inode->GetInodeAttr(attr);
+    return true;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/openfile.h

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::InodeWrapper;
+using ::curvefs::client::common::OpenFilesOption;
+
+struct OpenFile {
+    explicit OpenFile(std::shared_ptr<InodeWrapper> inode)
+        : inode(inode), refs(0) {}
+
+    std::shared_ptr<InodeWrapper> inode;
+    uint64_t refs;
+};
+
+class OpenFiles {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<OpenFile>>;
+
+ public:
+    explicit OpenFiles(OpenFilesOption option,
+                       std::shared_ptr<DeferSync> deferSync);
+
+    void Open(Ino ino, std::shared_ptr<InodeWrapper> inode);
+
+    bool IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+    void Close(Ino ino);
+
+    void CloseAll();
+
+    bool GetFileAttr(Ino ino, InodeAttr* inode);
+
+ private:
+    void Delete(Ino ino, const std::shared_ptr<OpenFile>& file, bool flush);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    OpenFilesOption option_;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<LRUType> files_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_

curvefs/src/client/filesystem/package.h

@@ -0,0 +1,51 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+
+#include <memory>
+
+#include "curvefs/src/client/dentry_cache_manager.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/xattr_manager.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+struct ExternalMember {  // external member depended by FileSystem
+    ExternalMember() = delete;
+    ExternalMember(std::shared_ptr<DentryCacheManager> dentryManager,
+                   std::shared_ptr<InodeCacheManager> inodeManager)
+        : dentryManager(dentryManager),
+          inodeManager(inodeManager) {}
+
+    std::shared_ptr<DentryCacheManager> dentryManager;
+    std::shared_ptr<InodeCacheManager> inodeManager;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_

curvefs/src/client/filesystem/rpc_client.cpp

@@ -0,0 +1,128 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <set>
+#include <list>
+#include <string>
+#include <memory>
+#include <utility>
+
+#include "curvefs/src/client/filesystem/rpc_client.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+RPCClient::RPCClient(RPCOption option,
+                     ExternalMember member)
+    : option_(option),
+      inodeManager_(member.inodeManager),
+      dentryManager_(member.dentryManager) {}
+
+CURVEFS_ERROR RPCClient::GetAttr(Ino ino, InodeAttr* attr) {
+    CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(ino, attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(getattr::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::Lookup(Ino parent,
+                                const std::string& name,
+                                EntryOut* entryOut) {
+    Dentry dentry;
+    CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry);
+    if (rc != CURVEFS_ERROR::OK) {
+        if (rc != CURVEFS_ERROR::NOTEXIST) {
+            LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc
+                       << ", parent = " << parent << ", name = " << name;
+        }
+        return rc;
+    }
+
+    Ino ino = dentry.inodeid();
+    rc = inodeManager_->GetInodeAttr(ino, &entryOut->attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(lookup::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
+                                 std::shared_ptr<DirEntryList>* entries) {
+    uint32_t limit = option_.listDentryLimit;
+
+    std::list<Dentry> dentries;
+    CURVEFS_ERROR rc = dentryManager_->ListDentry(ino, &dentries, limit);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc
+                   << ", ino = " << ino;
+        return rc;
+    }
+
+    std::set<uint64_t> inos;
+    std::map<uint64_t, InodeAttr> attrs;
+    std::for_each(dentries.begin(), dentries.end(), [&](Dentry& dentry){
+        inos.emplace(dentry.inodeid());
+    });
+    rc = inodeManager_->BatchGetInodeAttrAsync(ino, &inos, &attrs);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::BatchGetInodeAttrAsync) failed"
+                   << ", retCode = " << rc << ", ino = " << ino;
+        return rc;
+    }
+
+    DirEntry dirEntry;
+    for (const auto& dentry : dentries) {
+        Ino ino = dentry.inodeid();
+        auto iter = attrs.find(ino);
+        if (iter == attrs.end()) {
+            LOG(WARNING) << "rpc(readdir::BatchGetInodeAttrAsync) "
+                         << "missing attribute, ino = " << ino;
+            continue;
+        }
+
+        // NOTE: we can't use std::move() for attribute for hard link
+        // which will sharing inode attribute.
+        dirEntry.ino = ino;
+        dirEntry.name = std::move(dentry.name());
+        dirEntry.attr = iter->second;
+        (*entries)->Add(dirEntry);
+    }
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR RPCClient::Open(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    CURVEFS_ERROR rc = inodeManager_->GetInode(ino, *inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(open/GetInode) failed" << ", retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/rpc_client.h

@@ -0,0 +1,65 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+
+#include <memory>
+#include <string>
+
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::RPCOption;
+
+class RPCClient {
+ public:
+    RPCClient(RPCOption option,
+              ExternalMember member);
+
+    CURVEFS_ERROR GetAttr(Ino ino, InodeAttr* attr);
+
+    CURVEFS_ERROR Lookup(Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR Open(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+ private:
+    RPCOption option_;
+    std::shared_ptr<InodeCacheManager> inodeManager_;
+    std::shared_ptr<DentryCacheManager> dentryManager_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_

curvefs/src/client/filesystem/utils.cpp

@@ -0,0 +1,69 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+bool IsDir(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_DIRECTORY;
+}
+
+bool IsS3File(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_S3;
+}
+
+bool IsVolmeFile(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_FILE;
+}
+
+bool IsSymLink(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_SYM_LINK;
+}
+
+TimeSpec AttrMtime(const InodeAttr& attr) {
+    return TimeSpec(attr.mtime(), attr.mtime_ns());
+}
+
+TimeSpec AttrCtime(const InodeAttr& attr) {
+    return TimeSpec(attr.ctime(), attr.ctime_ns());
+}
+
+TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode) {
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    return AttrMtime(attr);
+}
+
+TimeSpec Now() {
+    struct timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+    return TimeSpec(now.tv_sec, now.tv_nsec);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/utils.h

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+
+#include <memory>
+
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+// directory
+bool IsDir(const InodeAttr& attr);
+
+// file which data is stored in s3
+bool IsS3File(const InodeAttr& attr);
+
+// file which data is stored in volume
+bool IsVolmeFile(const InodeAttr& attr);
+
+// symbol link
+bool IsSymLink(const InodeAttr& attr);
+
+struct TimeSpec AttrMtime(const InodeAttr& attr);
+
+struct TimeSpec AttrCtime(const InodeAttr& attr);
+
+struct TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode);
+
+struct TimeSpec Now();
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_

curvefs/src/client/fuse_client.h

@@ -25,11 +25,15 @@
 
 #include <unistd.h>
 #include <sys/stat.h>
+#include <bthread/unstable.h>
 
 #include <map>
 #include <memory>
 #include <string>
 #include <list>
+#include <utility>
+#include <vector>
+#include <atomic>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/mds.pb.h"
@@ -45,10 +49,15 @@
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/common/define.h"
+#include "curvefs/src/common/s3util.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/lease/lease_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
+#include "curvefs/src/client/warmup/warmup_manager.h"
+#include "src/common/throttle.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/filesystem.h"
 
 #define DirectIOAlignment 512
 
@@ -57,17 +66,27 @@ using ::curve::common::InterruptibleSleeper;
 using ::curve::common::Thread;
 using ::curvefs::common::FSType;
 using ::curvefs::metaserver::DentryFlag;
+using ::curvefs::metaserver::ManageInodeType;
 using ::curvefs::client::metric::FSMetric;
 
 namespace curvefs {
 namespace client {
 
+
+namespace warmup {
+class WarmupManager;
+}
+
 using common::FuseClientOption;
 using rpcclient::MDSBaseClient;
 using rpcclient::MdsClient;
 using rpcclient::MdsClientImpl;
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::FileSystem;
+using ::curvefs::client::filesystem::EntryOut;
+using ::curvefs::client::filesystem::AttrOut;
+using ::curvefs::client::filesystem::FileOut;
 
 using curvefs::common::is_aligned;
 
@@ -82,29 +101,30 @@ class FuseClient {
         metaClient_(std::make_shared<MetaServerClientImpl>()),
         inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
         dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
-        dirBuf_(std::make_shared<DirBuffer>()),
         fsInfo_(nullptr),
-        mdsBase_(nullptr),
-        isStop_(true),
         init_(false),
-        enableSumInDir_(false) {}
+        enableSumInDir_(false),
+        warmupManager_(nullptr),
+        mdsBase_(nullptr),
+        isStop_(true) {}
 
     virtual ~FuseClient() {}
 
     FuseClient(const std::shared_ptr<MdsClient> &mdsClient,
         const std::shared_ptr<MetaServerClient> &metaClient,
         const std::shared_ptr<InodeCacheManager> &inodeManager,
-        const std::shared_ptr<DentryCacheManager> &dentryManager)
+        const std::shared_ptr<DentryCacheManager> &dentryManager,
+        const std::shared_ptr<warmup::WarmupManager> &warmupManager)
           : mdsClient_(mdsClient),
             metaClient_(metaClient),
             inodeManager_(inodeManager),
             dentryManager_(dentryManager),
-            dirBuf_(std::make_shared<DirBuffer>()),
             fsInfo_(nullptr),
-            mdsBase_(nullptr),
-            isStop_(true),
             init_(false),
-            enableSumInDir_(false) {}
+            enableSumInDir_(false),
+            warmupManager_(warmupManager),
+            mdsBase_(nullptr),
+            isStop_(true) {}
 
     virtual CURVEFS_ERROR Init(const FuseClientOption &option);
 
@@ -122,31 +142,42 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                       const char* buf, size_t size, off_t off,
                                       struct fuse_file_info* fi,
-                                      size_t* wSize) = 0;
+                                      FileOut* fileOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                      size_t size, off_t off,
                                      struct fuse_file_info* fi, char* buffer,
                                      size_t* rSize) = 0;
 
-    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
-                                     struct fuse_file_info* fi);
+    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     struct fuse_file_info* fi,
+                                     FileOut* fileOut);
 
-    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, mode_t mode,
+    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       mode_t mode,
                                        struct fuse_file_info* fi,
-                                       fuse_entry_param* e) = 0;
+                                       EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode, dev_t rdev,
-                                      fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      dev_t rdev,
+                                      EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode,
-                                      fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      EntryOut* entryOut);
 
     virtual CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                        const char* name) = 0;
@@ -154,45 +185,61 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
                                       const char* name);
 
-    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req,
+                                           fuse_ino_t ino,
                                            struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
-                                            size_t size, off_t off,
-                                            struct fuse_file_info* fi,
-                                            char** buffer, size_t* rSize,
-                                            bool cacheDir);
+    virtual CURVEFS_ERROR FuseOpReadDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        size_t size,
+                                        off_t off,
+                                        struct fuse_file_info* fi,
+                                        char** bufferOut,
+                                        size_t* rSize,
+                                        bool plus);
 
-    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_ino_t newparent,
-                                       const char* newname);
+    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       unsigned int flags);
 
     virtual CURVEFS_ERROR FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info* fi,
-                                        struct stat* attr);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct stat* attr, int to_set,
                                         struct fuse_file_info* fi,
-                                        struct stat* attrOut);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
-                                         const char* name, void* value,
+                                         const char* name, std::string* value,
                                          size_t size);
 
+    virtual CURVEFS_ERROR FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
+                                         const char* name, const char* value,
+                                         size_t size, int flags);
+
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 
-    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req, const char* link,
-                                        fuse_ino_t parent, const char* name,
-                                        fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req,
+                                        const char* link,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char* newname,
-                                     fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     fuse_ino_t newparent,
+                                     const char* newname,
+                                     EntryOut* entryOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string* linkStr);
@@ -205,11 +252,16 @@ class FuseClient {
                                       struct fuse_file_info* fi) = 0;
     virtual CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                       struct fuse_file_info *fi) {
+        (void)req;
+        (void)ino;
+        (void)fi;
         return CURVEFS_ERROR::OK;
     }
 
     virtual CURVEFS_ERROR FuseOpStatFs(fuse_req_t req, fuse_ino_t ino,
                                        struct statvfs* stbuf) {
+        (void)req;
+        (void)ino;
         // TODO(chengyi01,wuhanqing): implement in s3 and volume client
         stbuf->f_frsize = stbuf->f_bsize = fsInfo_->blocksize();
         stbuf->f_blocks = 10UL << 30;
@@ -221,22 +273,30 @@ class FuseClient {
         stbuf->f_fsid = fsInfo_->fsid();
 
         stbuf->f_flag = 0;
-        stbuf->f_namemax = option_.maxNameLength;
+        stbuf->f_namemax = option_.fileSystemOption.maxNameLength;
         return CURVEFS_ERROR::OK;
     }
 
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
+
     void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
         fsInfo_ = fsInfo;
         init_ = true;
     }
 
+    void SetMounted(bool mounted) {
+        if (warmupManager_ != nullptr) {
+            warmupManager_->SetMounted(mounted);
+        }
+    }
+
     std::shared_ptr<FsInfo> GetFsInfo() {
         return fsInfo_;
     }
 
-    virtual void FlushInode();
-
-    virtual void FlushInodeAll();
+    std::shared_ptr<FileSystem> GetFileSystem() {
+        return fs_;
+    }
 
     virtual void FlushAll();
 
@@ -245,18 +305,76 @@ class FuseClient {
         enableSumInDir_ = enable;
     }
 
+    bool PutWarmFilelistTask(fuse_ino_t key, common::WarmupStorageType type) {
+        if (fsInfo_->fstype() == FSType::TYPE_S3) {
+            return warmupManager_->AddWarmupFilelist(key, type);
+        }  // only support s3
+        return true;
+    }
+
+    bool PutWarmFileTask(fuse_ino_t key, const std::string &path,
+                         common::WarmupStorageType type) {
+        if (fsInfo_->fstype() == FSType::TYPE_S3) {
+            return warmupManager_->AddWarmupFile(key, path, type);
+        }  // only support s3
+        return true;
+    }
+
+    bool GetWarmupProgress(fuse_ino_t key, warmup::WarmupProgress *progress) {
+        if (fsInfo_->fstype() == FSType::TYPE_S3) {
+            return warmupManager_->QueryWarmupProgress(key, progress);
+        }
+        return false;
+    }
+
+    CURVEFS_ERROR SetMountStatus(const struct MountOption *mountOption);
+
+    void Add(bool isRead, size_t size) { throttle_.Add(isRead, size); }
+
+    void InitQosParam();
+
  protected:
-    CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
-                           mode_t mode, FsFileType type, dev_t rdev,
-                           fuse_entry_param* e);
+    CURVEFS_ERROR MakeNode(fuse_req_t req,
+                           fuse_ino_t parent,
+                           const char* name,
+                           mode_t mode,
+                           FsFileType type,
+                           dev_t rdev,
+                           bool internal,
+                           std::shared_ptr<InodeWrapper>& InodeWrapper);  // NOLINT
 
     CURVEFS_ERROR RemoveNode(fuse_req_t req, fuse_ino_t parent,
                              const char* name, FsFileType type);
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                             fuse_ino_t newparent, const char* newname,
+    CURVEFS_ERROR CreateManageNode(fuse_req_t req,
+                                   uint64_t parent,
+                                   const char* name,
+                                   mode_t mode,
+                                   ManageInodeType manageType,
+                                   EntryOut* entryOut);
+
+    CURVEFS_ERROR GetOrCreateRecycleDir(fuse_req_t req, Dentry *out);
+
+    CURVEFS_ERROR DeleteNode(fuse_ino_t ino, fuse_ino_t parent,
+                             const char* name, FsFileType type);
+
+    CURVEFS_ERROR MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
+                                fuse_ino_t parent, const char* name,
+                                FsFileType type);
+
+    bool ShouldMoveToRecycle(fuse_ino_t parent);
+
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char* newname,
                              FsFileType type,
-                             fuse_entry_param* e);
+                             EntryOut* entryOut);
+
+    CURVEFS_ERROR HandleOpenFlags(fuse_req_t req,
+                                  fuse_ino_t ino,
+                                  struct fuse_file_info* fi,
+                                  FileOut* fileOut);
 
     int SetHostPortInMountPoint(Mountpoint* out) {
         char hostname[kMaxHostNameLength];
@@ -271,13 +389,24 @@ class FuseClient {
         return 0;
     }
 
- private:
-    virtual CURVEFS_ERROR Truncate(Inode* inode, uint64_t length) = 0;
 
+ private:
     virtual void FlushData() = 0;
 
-    CURVEFS_ERROR UpdateParentInodeMCTimeAndInvalidNlink(
-        fuse_ino_t parent, FsFileType type);
+    CURVEFS_ERROR UpdateParentMCTimeAndNlink(
+        fuse_ino_t parent, FsFileType type,  NlinkChange nlink);
+
+    std::string GenerateNewRecycleName(fuse_ino_t ino,
+            fuse_ino_t parent, const char* name) {
+        std::string newName(name);
+        newName = std::to_string(parent) + "_" + std::to_string(ino)
+                    + "_" + newName;
+        if (newName.length() > option_.fileSystemOption.maxNameLength) {
+            newName = newName.substr(0, option_.fileSystemOption.maxNameLength);
+        }
+
+        return newName;
+    }
 
  protected:
     // mds client
@@ -297,9 +426,6 @@ class FuseClient {
 
     std::shared_ptr<LeaseExecutor> leaseExecutor_;
 
-    // dir buffer
-    std::shared_ptr<DirBuffer> dirBuf_;
-
     // filesystem info
     std::shared_ptr<FsInfo> fsInfo_;
 
@@ -309,18 +435,27 @@ class FuseClient {
     bool init_;
 
     // enable record summary info in dir inode xattr
-    bool enableSumInDir_;
+    std::atomic<bool> enableSumInDir_;
 
     std::shared_ptr<FSMetric> fsMetric_;
 
     Mountpoint mountpoint_;
 
+    // warmup manager
+    std::shared_ptr<warmup::WarmupManager> warmupManager_;
+
+    std::shared_ptr<FileSystem> fs_;
+
  private:
     MDSBaseClient* mdsBase_;
 
     Atomic<bool> isStop_;
 
     curve::common::Mutex renameMutex_;
+
+    Throttle throttle_;
+
+    bthread_timer_t throttleTimer_;
 };
 
 }  // namespace client

curvefs/src/client/fuse_common.h

@@ -34,9 +34,9 @@ extern "C" {
 #endif
 
 struct MountOption {
-    char* mountPoint;
-    char* fsName;
-    char* fsType;
+    const char* mountPoint;
+    const char* fsName;
+    const char* fsType;
     char* conf;
     char* mdsAddr;
 };

curvefs/src/client/fuse_s3_client.cpp

@@ -20,16 +20,20 @@
  * Author: xuchaojie
  */
 
-#include "curvefs/src/client/fuse_s3_client.h"
 
 #include <memory>
-#include <string>
-#include <list>
+#include <vector>
+
+#include "curvefs/src/client/fuse_s3_client.h"
+#include "curvefs/src/client/kvclient/memcache_client.h"
 
 namespace curvefs {
 namespace client {
 namespace common {
+
 DECLARE_bool(enableCto);
+DECLARE_bool(supportKVcache);
+
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs
@@ -37,14 +41,25 @@ DECLARE_bool(enableCto);
 namespace curvefs {
 namespace client {
 
+using curvefs::client::common::FLAGS_supportKVcache;
+using curvefs::client::common::FLAGS_enableCto;
+using curvefs::mds::topology::MemcacheClusterInfo;
+using curvefs::mds::topology::MemcacheServerInfo;
+
 CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
     FuseClientOption opt(option);
+
     CURVEFS_ERROR ret = FuseClient::Init(opt);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
 
-    // set fsS3Option
+    // init kvcache
+    if (FLAGS_supportKVcache && !InitKVCache(option.kvClientManagerOpt)) {
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    // set fs S3Option
     const auto& s3Info = fsInfo_->detail().s3info();
     ::curve::common::S3InfoOption fsS3Option;
     ::curvefs::client::common::S3Info2FsS3Option(s3Info, &fsS3Option);
@@ -52,10 +67,20 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
 
     auto s3Client = std::make_shared<S3ClientImpl>();
     s3Client->Init(opt.s3Opt.s3AdaptrOpt);
+
+    const uint64_t writeCacheMaxByte =
+        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
+    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
+        LOG(ERROR) << "writeCacheMaxByte is too small"
+                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
+                      ", writeCacheMaxByte = " << writeCacheMaxByte;
+        return CURVEFS_ERROR::CACHETOOSMALL;
+    }
+
     auto fsCacheManager = std::make_shared<FsCacheManager>(
         dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
-        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte);
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte, writeCacheMaxByte,
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheThreads, kvClientManager_);
     if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
         DiskCacheType::Disable) {
         auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
@@ -69,19 +94,51 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
             diskCacheManager, s3DiskCacheClient);
         ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
                                inodeManager_, mdsClient_, fsCacheManager,
-                               diskCacheManagerImpl, true);
+                               diskCacheManagerImpl, kvClientManager_, true);
     } else {
         ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
                                inodeManager_, mdsClient_, fsCacheManager,
-                               nullptr, true);
+                               nullptr, kvClientManager_, true);
     }
-
     return ret;
 }
 
+
+bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
+     // get kvcache cluster
+    MemcacheClusterInfo kvcachecluster;
+    if (!mdsClient_->AllocOrGetMemcacheCluster(fsInfo_->fsid(),
+                                               &kvcachecluster)) {
+        LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache
+                   << ", but AllocOrGetMemcacheCluster fail";
+        return false;
+    }
+
+    // init kvcache client
+    auto memcacheClient = std::make_shared<MemCachedClient>();
+    if (!memcacheClient->Init(kvcachecluster)) {
+        LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache
+                   << ", but init memcache client fail";
+        return false;
+    }
+
+    kvClientManager_ = std::make_shared<KVClientManager>();
+    if (!kvClientManager_->Init(opt, memcacheClient)) {
+        LOG(ERROR) << "FLAGS_supportKVcache = " << FLAGS_supportKVcache
+                   << ", but init kvClientManager fail";
+        return false;
+    }
+
+    if (warmupManager_ != nullptr) {
+        warmupManager_->SetKVClientManager(kvClientManager_);
+    }
+
+    return true;
+}
+
 void FuseS3Client::UnInit() {
-    s3Adaptor_->Stop();
     FuseClient::UnInit();
+    s3Adaptor_->Stop();
     curve::common::S3Adapter::Shutdown();
 }
 
@@ -98,7 +155,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
 CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                         const char *buf, size_t size, off_t off,
                                         struct fuse_file_info *fi,
-                                        size_t *wSize) {
+                                        FileOut* fileOut) {
+    size_t *wSize = &fileOut->nwritten;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -117,7 +175,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         fsMetric_->userWrite.qps.count << 1;
         uint64_t duration = butil::cpuwide_time_us() - start;
         fsMetric_->userWrite.latency << duration;
-        fsMetric_->userWriteIoSize << wRet;
+        fsMetric_->userWriteIoSize.set_value(wRet);
     }
 
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -129,21 +187,16 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
     }
 
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-    Inode *inode = inodeWrapper->GetMutableInodeUnlocked();
 
     *wSize = wRet;
     size_t changeSize = 0;
     // update file len
-    if (inode->length() < off + *wSize) {
-        changeSize = off + *wSize - inode->length();
-        inode->set_length(off + *wSize);
+    if (inodeWrapper->GetLengthLocked() < off + *wSize) {
+        changeSize = off + *wSize - inodeWrapper->GetLengthLocked();
+        inodeWrapper->SetLengthLocked(off + *wSize);
     }
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    inode->set_mtime(now.tv_sec);
-    inode->set_mtime_ns(now.tv_nsec);
-    inode->set_ctime(now.tv_sec);
-    inode->set_ctime_ns(now.tv_nsec);
+
+    inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
 
     inodeManager_->ShipToFlush(inodeWrapper);
 
@@ -152,6 +205,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
     }
 
     if (enableSumInDir_ && changeSize != 0) {
+        const Inode* inode = inodeWrapper->GetInodeLocked();
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
             std::to_string(changeSize)});
@@ -164,6 +218,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
             }
         }
     }
+
+    inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
     return ret;
 }
 
@@ -171,6 +227,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                        size_t size, off_t off,
                                        struct fuse_file_info *fi, char *buffer,
                                        size_t *rSize) {
+    (void)req;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -189,7 +246,7 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
     uint64_t fileSize = inodeWrapper->GetLength();
 
     size_t len = 0;
-    if (fileSize <= off) {
+    if (static_cast<int64_t>(fileSize) <= off) {
         *rSize = 0;
         return CURVEFS_ERROR::OK;
     } else if (fileSize < off + size) {
@@ -211,52 +268,70 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
         fsMetric_->userRead.qps.count << 1;
         uint64_t duration = butil::cpuwide_time_us() - start;
         fsMetric_->userRead.latency << duration;
-        fsMetric_->userReadIoSize << rRet;
+        fsMetric_->userReadIoSize.set_value(rRet);
     }
 
     ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-    Inode *newInode = inodeWrapper->GetMutableInodeUnlocked();
-
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    newInode->set_atime(now.tv_sec);
-    newInode->set_atime_ns(now.tv_nsec);
-
+    inodeWrapper->UpdateTimestampLocked(kAccessTime);
     inodeManager_->ShipToFlush(inodeWrapper);
 
-    VLOG(6) << "read end, read size = " << *rSize;
+    VLOG(9) << "read end, read size = " << *rSize;
     return ret;
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                          const char *name, mode_t mode,
                                          struct fuse_file_info *fi,
-                                         fuse_entry_param *e) {
+                                         EntryOut* entryOut) {
     VLOG(1) << "FuseOpCreate, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
+
+    std::shared_ptr<InodeWrapper> inode;
     CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, e);
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    return FuseOpOpen(req, e->ino, fi);
+
+    auto openFiles = fs_->BorrowMember().openFiles;
+    openFiles->Open(inode->GetInodeId(), inode);
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                        const char *name, mode_t mode,
-                                        dev_t rdev, fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        mode_t mode,
+                                        dev_t rdev,
+                                        EntryOut* entryOut) {
     VLOG(1) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, rdev, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
+                                FsFileType::TYPE_S3, rdev, false,
+                                inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    *entryOut = EntryOut(attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
-                                     fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req,
+                                       fuse_ino_t ino,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       EntryOut* entryOut) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_S3, e);
+        req, ino, newparent, newname, FsFileType::TYPE_S3, entryOut);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
@@ -268,6 +343,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
 CURVEFS_ERROR FuseS3Client::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                         int datasync,
                                         struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(1) << "FuseOpFsync, ino: " << ino << ", datasync: " << datasync;
 
     CURVEFS_ERROR ret = s3Adaptor_->Flush(ino);
@@ -290,17 +367,19 @@ CURVEFS_ERROR FuseS3Client::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
     return inodeWrapper->Sync();
 }
 
-CURVEFS_ERROR FuseS3Client::Truncate(Inode *inode, uint64_t length) {
+CURVEFS_ERROR FuseS3Client::Truncate(InodeWrapper *inode, uint64_t length) {
     return s3Adaptor_->Truncate(inode, length);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
+    (void)req;
+    (void)fi;
     VLOG(1) << "FuseOpFlush, ino: " << ino;
     CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
 
     // if enableCto, flush all write cache both in memory cache and disk cache
-    if (curvefs::client::common::FLAGS_enableCto) {
+    if (FLAGS_enableCto) {
         ret = s3Adaptor_->FlushAllCache(ino);
         if (ret != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "FuseOpFlush, flush all cache fail, ret = " << ret

curvefs/src/client/fuse_s3_client.h

@@ -25,25 +25,51 @@
 #define CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
 
 #include <memory>
+#include <string>
+#include <list>
+#include <vector>
+#include <utility>
 
 #include "curvefs/src/client/fuse_client.h"
 #include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "curvefs/src/client/warmup/warmup_manager.h"
+#include "curvefs/src/volume/common.h"
+#include "src/common/s3_adapter.h"
+
 namespace curvefs {
 namespace client {
 
+using curve::common::GetObjectAsyncContext;
+using curve::common::GetObjectAsyncCallBack;
+using curvefs::volume::kMiB;
+namespace warmup {
+class WarmupManager;
+class WarmupManagerS3Impl;
+}  // namespace warmup
+
+
 class FuseS3Client : public FuseClient {
  public:
     FuseS3Client()
-      : FuseClient(),
-        s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {}
+        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {
+        auto readFunc = [this](fuse_req_t req, fuse_ino_t ino, size_t size,
+                               off_t off, struct fuse_file_info *fi,
+                               char *buffer, size_t *rSize) {
+            return FuseOpRead(req, ino, size, off, fi, buffer, rSize);
+        };
+        warmupManager_ = std::make_shared<warmup::WarmupManagerS3Impl>(
+            metaClient_, inodeManager_, dentryManager_, fsInfo_, readFunc,
+            s3Adaptor_, nullptr);
+    }
 
     FuseS3Client(const std::shared_ptr<MdsClient> &mdsClient,
-        const std::shared_ptr<MetaServerClient> &metaClient,
-        const std::shared_ptr<InodeCacheManager> &inodeManager,
-        const std::shared_ptr<DentryCacheManager> &dentryManager,
-        const std::shared_ptr<S3ClientAdaptor> &s3Adaptor)
-        : FuseClient(mdsClient, metaClient,
-            inodeManager, dentryManager),
+                 const std::shared_ptr<MetaServerClient> &metaClient,
+                 const std::shared_ptr<InodeCacheManager> &inodeManager,
+                 const std::shared_ptr<DentryCacheManager> &dentryManager,
+                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor,
+                 const std::shared_ptr<warmup::WarmupManager> &warmupManager)
+        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
+                     warmupManager),
           s3Adaptor_(s3Adaptor) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
@@ -55,7 +81,7 @@ class FuseS3Client : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, size_t *wSize) override;
+        struct fuse_file_info *fi, FileOut* fileOut) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
         fuse_ino_t ino, size_t size, off_t off,
@@ -63,17 +89,25 @@ class FuseS3Client : public FuseClient {
         char *buffer,
         size_t *rSize) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, struct fuse_file_info *fi,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                               fuse_ino_t parent,
+                               const char* name,
+                               mode_t mode,
+                               struct fuse_file_info *fi,
+                               EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, dev_t rdev,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                              fuse_ino_t parent,
+                              const char* name,
+                              mode_t mode,
+                              dev_t rdev,
+                              EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-        fuse_ino_t newparent, const char *newname,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char *newname,
+                             EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
         const char *name) override;
@@ -84,14 +118,19 @@ class FuseS3Client : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
         struct fuse_file_info *fi) override;
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
  private:
-    CURVEFS_ERROR Truncate(Inode *inode, uint64_t length) override;
+    bool InitKVCache(const KVClientManagerOpt &opt);
 
     void FlushData() override;
 
  private:
     // s3 adaptor
     std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
+    std::shared_ptr<KVClientManager> kvClientManager_;
+
+    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };