[feat] tools-v2: add snapshot copyset --all

Signed-off-by: montaguelhz <1443171175@qq.com>

.bazelrc

@@ -1,7 +1,10 @@
+build --verbose_failures
+
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
 build --incompatible_blacklisted_protos_requires_proto_info=false
 build --copt=-fdiagnostics-color=always
+
 run --copt=-fdiagnostics-color=always

.gitignore

@@ -147,3 +147,22 @@ tools-v2/proto/curvefs/*
 tools-v2/*/*.test
 tools-v2/__debug_bin
 tools-v2/vendor/
+
+.test
+.note
+.playground
+.dumpfile
+metastore_test.dat
+GPATH
+GRTAGS
+GTAGS
+core.*
+
+test/integration/*.conf
+test/integration/client/config/client.conf*
+test/integration/snapshotcloneserver/config/*.conf
+
+.pre-commit-config.yaml
+
+*.deb
+*.whl

.obm.cfg

@@ -0,0 +1,2 @@
+container_name: curve-build-playground-master
+container_image: opencurvedocker/curve-base:build-debian9

Makefile

@@ -1,6 +1,6 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build dep install image
+.PHONY: list build dep install image playground check test
 
 stor?=""
 prefix?= "$(PWD)/projects"
@@ -70,3 +70,12 @@ install:
 
 image:
 	@bash util/image.sh $(stor) $(tag) $(os)
+
+playground:
+	@bash util/playground.sh
+
+check:
+	@bash util/check.sh $(stor)
+
+test:
+	@bash util/test.sh $(stor) $(only)

WORKSPACE

@@ -75,11 +75,14 @@ bind(
 )
 
 #import the  gtest files.
-new_git_repository(
+http_archive(
     name = "com_google_googletest",
-    build_file = "//:thirdparties/gmock.BUILD",
-    remote = "https://github.com/google/googletest",
-    tag = "release-1.8.0",
+    urls = [
+        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
+        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
+    ],
+    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
+    strip_prefix = "googletest-release-1.12.1",
 )
 
 bind(
@@ -228,6 +231,24 @@ http_archive(
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
 
+# fmt
+http_archive(
+  name = "fmt",
+  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
+  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
+  strip_prefix = "fmt-9.1.0",
+  build_file = "//:thirdparties/fmt.BUILD",
+)
+
+# spdlog
+http_archive(
+  name = "spdlog",
+  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
+  strip_prefix = "spdlog-1.11.0",
+  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
+  build_file = "//:thirdparties/spdlog.BUILD",
+)
+
 # Bazel platform rules.
 http_archive(
     name = "platforms",
@@ -248,14 +269,14 @@ new_local_repository(
 http_archive(
     name = "hedron_compile_commands",
 
-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. 
+    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
     # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
     urls = [
         "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
         "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
     ],
     strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." 
+    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()

buildfs.sh

@@ -7,12 +7,22 @@ then
     exit
 fi
 
+if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
+then
+    bazelflags=''
+else
+    bazelflags='--copt -faligned-new'
+fi
+
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi
 
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
+--define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
+-DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
+
 if [ $? -ne 0 ]
 then
     echo "build curvefs failed"
@@ -34,4 +44,5 @@ then
     echo "mds_test failed"
     exit
 fi
-fi
+fi
+echo "end compile"

conf/chunkserver.conf

@@ -9,13 +9,20 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.block_size=4096
+
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +115,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/chunkserver.conf.example

@@ -13,9 +13,12 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +111,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/client.conf

@@ -168,13 +168,6 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
 
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
-
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@@ -188,4 +181,3 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
-

conf/cs_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/mds.conf

@@ -197,6 +197,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -237,3 +239,16 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
+
+#
+## poolset rules
+#
+# for backward compatibility, rules are applied for select poolset when creating file
+#
+# for example
+#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
+#
+# when creating file reqeust doesn't have poolset, above rules are used to select poolset
+# - if filename is /dir1/file, then poolset1 is select
+# - if filename is /dir1/sub/file, then sub is select
+mds.poolset.rules=

conf/py_client.conf

@@ -153,10 +153,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/snap_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

curve-ansible/roles/generate_config/defaults/main.yml

@@ -164,7 +164,6 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
-chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -244,8 +243,6 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
-client_alignment_common: 512
-client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -14,8 +14,6 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
-# minimum alignment for io request
-global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -175,10 +175,3 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume={{ client_alignment_common }}
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume={{ client_alignment_clone }}

curvefs/conf/client.conf

@@ -77,17 +77,7 @@ rpc.healthCheckIntervalSec=0
 
 #### fuseClient
 # TODO(xuchaojie): add unit
-fuseClient.attrTimeOut=1.0
-fuseClient.entryTimeOut=1.0
 fuseClient.listDentryLimit=65536
-fuseClient.flushPeriodSec=5
-fuseClient.maxNameLength=255
-fuseClient.iCacheLruSize=65536
-fuseClient.dCacheLruSize=1000000
-fuseClient.enableICacheMetrics=true
-fuseClient.enableDCacheMetrics=true
-fuseClient.lruTimeOutSec=60
-fuseClient.cto=true
 fuseClient.downloadMaxRetryTimes=3
 
 ### kvcache opt
@@ -114,7 +104,6 @@ fuseClient.maxDataSize=1024
 fuseClient.refreshDataIntervalSec=30
 fuseClient.warmupThreadsNum=10
 
-
 # the write throttle bps of fuseClient, default no limit
 fuseClient.throttle.avgWriteBytes=0
 # the write burst bps of fuseClient, default no limit
@@ -143,6 +132,32 @@ fuseClient.throttle.burstReadIops=0
 # the times that read burst Iops can continue, default 180s
 fuseClient.throttle.burstReadIopsSecs=180
 
+#### filesystem metadata
+# {
+# fs.disableXattr:
+#   if you want to get better metadata performance,
+#   you can mount fs with |fs.disableXattr| is true
+#
+# fs.lookupCache.negativeTimeoutSec:
+#   entry which not found will be cached if |timeout| > 0
+fs.cto=true
+fs.maxNameLength=255
+fs.disableXattr=false
+fs.accessLogging=true
+fs.kernelCache.attrTimeoutSec=3600
+fs.kernelCache.dirAttrTimeoutSec=3600
+fs.kernelCache.entryTimeoutSec=3600
+fs.kernelCache.dirEntryTimeoutSec=3600
+fs.lookupCache.negativeTimeoutSec=0
+fs.lookupCache.minUses=1
+fs.lookupCache.lruSize=100000
+fs.dirCache.lruSize=5000000
+fs.openFile.lruSize=65536
+fs.attrWatcher.lruSize=5000000
+fs.rpc.listDentryLimit=65536
+fs.deferSync.delay=3
+fs.deferSync.deferDirMtime=false
+# }
 
 #### volume
 volume.bigFileSize=1048576

curvefs/conf/mds.conf

@@ -6,6 +6,9 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
 mds.loglevel=0
+# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+mds.server.idleTimeoutSec=-1
 
 #
 # space options

curvefs/conf/metaserver.conf

@@ -128,16 +128,25 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000
 
-### apply queue options for each copyset
-### apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding worker queue by certain rules
-# number of apply queue workers for each, each worker will start a indepent thread
-applyqueue.worker_count=3
+#
+# Concurrent apply queue
+### concurrent apply queue options for each copyset
+### concurrent apply queue is used to isolate raft threads, each worker has its own queue
+### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
 
-# apply queue depth for each copyset
+# worker_count: number of apply queue workers for each, each worker will start a indepent thread
+# queue_depth: apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-applyqueue.queue_depth=1
+# write apply queue workers count
+applyqueue.write_worker_count=3
+# write apply queue depth
+applyqueue.write_queue_depth=1
+# read apply queue workers count
+applyqueue.read_worker_count=2
+# read apply queue depth
+applyqueue.read_queue_depth=1
+
 
 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@@ -145,6 +154,10 @@ applyqueue.queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto
 
+# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+server.idleTimeoutSec=-1
+
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need

curvefs/src/client/BUILD

@@ -43,6 +43,8 @@ cc_library(
             "s3/*.h",
             "volume/*.cpp",
             "volume/*.h",
+            "filesystem/*.cpp",
+            "filesystem/*.h",
             "warmup/*.h",
             "warmup/*.cpp",
         ],
@@ -71,11 +73,14 @@ cc_library(
         "//curvefs/src/volume",
         "//curvefs/src/common:metric_utils",
         "//curvefs/src/common:dynamic_vlog",
+        "//curvefs/src/common:threading",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/synchronization",
+        "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",
         "@com_google_googletest//:gtest_prod",
+        "@spdlog//:spdlog",
     ],
 )

curvefs/src/client/async_request_closure.cpp

@@ -28,7 +28,7 @@
 #include <mutex>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 namespace curvefs {
@@ -63,7 +63,7 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
         cond_.wait(lk);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(GetStatusCode());
+    return ToFSError(GetStatusCode());
 }
 
 void UpdateVolumeExtentClosure::Run() {

curvefs/src/client/async_request_closure.h

@@ -28,12 +28,15 @@
 
 #include <memory>
 
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
+using ::curvefs::client::filesystem::ToFSError;
+
 class InodeWrapper;
 
 namespace internal {

curvefs/src/client/client_operator.cpp

@@ -24,13 +24,14 @@
 
 #include "src/common/uuid.h"
 #include "curvefs/src/client/client_operator.h"
-
+#include "curvefs/src/client/filesystem/error.h"
 namespace curvefs {
 namespace client {
 
 using ::curve::common::UUIDGenerator;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::mds::topology::PartitionTxId;
+using ::curvefs::client::filesystem::ToFSError;
 
 #define LOG_ERROR(action, rc) \
     LOG(ERROR) << action << " failed, retCode = " << rc \
@@ -95,7 +96,7 @@ CURVEFS_ERROR RenameOperator::GetTxId(uint32_t fsId,
     if (rc != MetaStatusCode::OK) {
         LOG_ERROR("GetTxId", rc);
     }
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::GetLatestTxIdWithLock() {
@@ -204,7 +205,7 @@ CURVEFS_ERROR RenameOperator::PrepareRenameTx(
         LOG_ERROR("PrepareRenameTx", rc);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::PrepareTx() {
@@ -418,8 +419,6 @@ CURVEFS_ERROR RenameOperator::UpdateInodeCtime() {
 }
 
 void RenameOperator::UpdateCache() {
-    dentryManager_->DeleteCache(parentId_, name_);
-    dentryManager_->InsertOrReplaceCache(newDentry_);
     SetTxId(srcPartitionId_, srcTxId_ + 1);
     SetTxId(dstPartitionId_, dstTxId_ + 1);
 }

curvefs/src/client/common/config.cpp

@@ -50,6 +50,7 @@ DEFINE_bool(enableCto, true, "acheieve cto consistency");
 DEFINE_bool(useFakeS3, false,
             "Use fake s3 to inject more metadata for testing metaserver");
 DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing");
+DEFINE_bool(access_logging, true, "enable access log");
 
 /**
  * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true
@@ -292,6 +293,55 @@ void InitKVClientManagerOpt(Configuration *conf,
                               &config->getThreadPooln);
 }
 
+void InitFileSystemOption(Configuration* c, FileSystemOption* option) {
+    c->GetValueFatalIfFail("fs.cto", &option->cto);
+    c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto);
+    c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr);
+    c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength);
+    c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging);
+    {  // kernel cache option
+        auto o = &option->kernelCacheOption;
+        c->GetValueFatalIfFail("fs.kernelCache.attrTimeoutSec",
+                               &o->attrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeoutSec",
+                               &o->dirAttrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.entryTimeoutSec",
+                               &o->entryTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeoutSec",
+                               &o->dirEntryTimeoutSec);
+    }
+    {  // lookup cache option
+        auto o = &option->lookupCacheOption;
+        c->GetValueFatalIfFail("fs.lookupCache.lruSize",
+                               &o->lruSize);
+        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeoutSec",
+                               &o->negativeTimeoutSec);
+        c->GetValueFatalIfFail("fs.lookupCache.minUses",
+                               &o->minUses);
+    }
+    {  // dir cache option
+        auto o = &option->dirCacheOption;
+        c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize);
+    }
+    {  // open file option
+        auto o = &option->openFilesOption;
+        c->GetValueFatalIfFail("fs.openFile.lruSize", &o->lruSize);
+    }
+    {  // attr watcher option
+        auto o = &option->attrWatcherOption;
+        c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize);
+    }
+    {  // rpc option
+        auto o = &option->rpcOption;
+        c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit);
+    }
+    {  // defer sync option
+        auto o = &option->deferSyncOption;
+        c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay);
+        c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime);
+    }
+}
+
 void SetBrpcOpt(Configuration *conf) {
     curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy;
     dummy.Load(conf, "defer_close_second", "rpc.defer.close.second",
@@ -312,36 +362,16 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
     InitLeaseOpt(conf, &clientOption->leaseOpt);
     InitRefreshDataOpt(conf, &clientOption->refreshDataOption);
     InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt);
+    InitFileSystemOption(conf, &clientOption->fileSystemOption);
 
-    conf->GetValueFatalIfFail("fuseClient.attrTimeOut",
-                              &clientOption->attrTimeOut);
-    conf->GetValueFatalIfFail("fuseClient.entryTimeOut",
-                              &clientOption->entryTimeOut);
     conf->GetValueFatalIfFail("fuseClient.listDentryLimit",
                               &clientOption->listDentryLimit);
     conf->GetValueFatalIfFail("fuseClient.listDentryThreads",
                               &clientOption->listDentryThreads);
-    conf->GetValueFatalIfFail("fuseClient.flushPeriodSec",
-                              &clientOption->flushPeriodSec);
-    conf->GetValueFatalIfFail("fuseClient.maxNameLength",
-                              &clientOption->maxNameLength);
-    conf->GetValueFatalIfFail("fuseClient.iCacheLruSize",
-                              &clientOption->iCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.dCacheLruSize",
-                              &clientOption->dCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.enableICacheMetrics",
-                              &clientOption->enableICacheMetrics);
-    conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
-                              &clientOption->enableDCacheMetrics);
-    conf->GetValueFatalIfFail("fuseClient.lruTimeOutSec",
-                              &clientOption->lruTimeOutSec);
     conf->GetValueFatalIfFail("client.dummyServer.startPort",
                               &clientOption->dummyServerStartPort);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                               &clientOption->enableMultiMountPointRename);
-    conf->GetValueFatalIfFail("fuseClient.disableXattr",
-                              &clientOption->disableXattr);
-    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
     conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
                               &clientOption->downloadMaxRetryTimes);
     conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
@@ -351,12 +381,6 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
         << "Not found `fuseClient.enableSplice` in conf, use default value `"
         << std::boolalpha << clientOption->enableFuseSplice << '`';
 
-    // if enableCto, attr and entry cache must invalid
-    if (FLAGS_enableCto) {
-        clientOption->attrTimeOut = 0;
-        clientOption->entryTimeOut = 0;
-    }
-
     conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteBytes",
                               &FLAGS_fuseClientAvgWriteBytes);
     conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytes",

curvefs/src/client/common/config.h

@@ -174,6 +174,59 @@ struct RefreshDataOption {
     uint64_t maxDataSize = 1024;
     uint32_t refreshDataIntervalSec = 30;
 };
+
+// { filesystem option
+struct KernelCacheOption {
+    uint32_t entryTimeoutSec;
+    uint32_t dirEntryTimeoutSec;
+    uint32_t attrTimeoutSec;
+    uint32_t dirAttrTimeoutSec;
+};
+
+struct LookupCacheOption {
+    uint64_t lruSize;
+    uint32_t negativeTimeoutSec;
+    uint32_t minUses;
+};
+
+struct DirCacheOption {
+    uint64_t lruSize;
+    uint32_t timeoutSec;
+};
+
+struct AttrWatcherOption {
+    uint64_t lruSize;
+};
+
+struct OpenFilesOption {
+    uint64_t lruSize;
+    uint32_t deferSyncSecond;
+};
+
+struct RPCOption {
+    uint32_t listDentryLimit;
+};
+
+struct DeferSyncOption {
+    uint32_t delay;
+    bool deferDirMtime;
+};
+
+struct FileSystemOption {
+    bool cto;
+    bool disableXattr;
+    uint32_t maxNameLength;
+    uint32_t blockSize = 0x10000u;
+    KernelCacheOption kernelCacheOption;
+    LookupCacheOption lookupCacheOption;
+    DirCacheOption dirCacheOption;
+    OpenFilesOption openFilesOption;
+    AttrWatcherOption attrWatcherOption;
+    RPCOption rpcOption;
+    DeferSyncOption deferSyncOption;
+};
+// }
+
 struct FuseClientOption {
     MdsOption mdsOpt;
     MetaCacheOpt metaCacheOpt;
@@ -187,22 +240,13 @@ struct FuseClientOption {
     LeaseOpt leaseOpt;
     RefreshDataOption refreshDataOption;
     KVClientManagerOpt kvClientManagerOpt;
+    FileSystemOption fileSystemOption;
 
-    double attrTimeOut;
-    double entryTimeOut;
     uint32_t listDentryLimit;
     uint32_t listDentryThreads;
-    uint32_t flushPeriodSec;
-    uint32_t maxNameLength;
-    uint64_t iCacheLruSize;
-    uint64_t dCacheLruSize;
-    bool enableICacheMetrics;
-    bool enableDCacheMetrics;
-    uint32_t lruTimeOutSec;
     uint32_t dummyServerStartPort;
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;
-    bool disableXattr = false;
     uint32_t downloadMaxRetryTimes;
     uint32_t warmupThreadsNum = 10;
 };

curvefs/src/client/curve_fuse_op.h

@@ -37,7 +37,7 @@
 extern "C" {
 #endif
 
-int InitGlog(const char *confPath, const char *argv0);
+int InitLog(const char *confPath, const char *argv0);
 
 int InitFuseClient(const struct MountOption *mountOption);
 

curvefs/src/client/dentry_cache_manager.cpp

@@ -44,31 +44,13 @@ namespace client {
 
 using curve::common::WriteLockGuard;
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
-
-void DentryCacheManagerImpl::InsertOrReplaceCache(const Dentry &dentry) {
-    std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name());
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        NameLockGuard lock(nameLock_, key);
-        dCache_->Put(key, dentry);
-    }
-}
-
-void DentryCacheManagerImpl::DeleteCache(uint64_t parentId,
-                                         const std::string &name) {
-    std::string key = GetDentryCacheKey(parentId, name);
-    NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
-}
+using ::curvefs::client::filesystem::ToFSError;
 
 CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
                                                 const std::string &name,
                                                 Dentry *out) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    bool ok = dCache_->Get(key, out);
-    if (ok) {
-        return CURVEFS_ERROR::OK;
-    }
 
     MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out);
     if (ret != MetaStatusCode::OK) {
@@ -76,10 +58,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
             << "metaClient_ GetDentry failed, MetaStatusCode = " << ret
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
             << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
-    }
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, *out);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -94,12 +73,9 @@ CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) {
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << dentry.parentinodeid()
                    << ", name = " << dentry.name();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, dentry);
-    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -108,14 +84,13 @@ CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent,
                                                    FsFileType type) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
 
     MetaStatusCode ret = metaClient_->DeleteDentry(fsId_, parent, name, type);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -149,7 +124,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", parent = " << parent << ", last = " << last
                        << ", count = " << limit << ", onlyDir = " << onlyDir;
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
 
         if (!onlyDir) {

curvefs/src/client/dentry_cache_manager.h

@@ -33,20 +33,18 @@
 #include <utility>
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
-#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
-#include "src/common/lru_cache.h"
 #include "src/common/concurrent/name_lock.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Dentry;
-using ::curve::common::TimedLRUCache;
-using ::curve::common::CacheMetrics;
 
 namespace curvefs {
 namespace client {
 
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 static const char* kDentryKeyDelimiter = ":";
 
@@ -59,13 +57,6 @@ class DentryCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-        uint32_t cacheTimeOutSec) = 0;
-
-    virtual void InsertOrReplaceCache(const Dentry& dentry) = 0;
-
-    virtual void DeleteCache(uint64_t parentId, const std::string& name) = 0;
-
     virtual CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) = 0;
 
@@ -86,30 +77,11 @@ class DentryCacheManager {
 class DentryCacheManagerImpl : public DentryCacheManager {
  public:
     DentryCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()),
-        dCache_(nullptr) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
 
     explicit DentryCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient),
-        dCache_(nullptr) {}
-
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-        uint32_t cacheTimeOutSec) override {
-        if (enableCacheMetrics) {
-            dCache_ = std::make_shared<
-                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize,
-                    std::make_shared<CacheMetrics>("dcache"));
-        } else {
-            dCache_ = std::make_shared<
-                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize);
-        }
-        return CURVEFS_ERROR::OK;
-    }
-
-    void InsertOrReplaceCache(const Dentry& dentry) override;
-
-    void DeleteCache(uint64_t parentId, const std::string& name) override;
+      : metaClient_(metaClient) {}
 
     CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) override;
@@ -130,8 +102,6 @@ class DentryCacheManagerImpl : public DentryCacheManager {
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    // key is parentId + name
-    std::shared_ptr<TimedLRUCache<std::string, Dentry>> dCache_;
     curve::common::GenericNameLock<Mutex> nameLock_;
 };
 

curvefs/src/client/error_code.cpp

@@ -1,98 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-/*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
- */
-
-#include "curvefs/src/client/error_code.h"
-
-#include <map>
-#include <string>
-
-namespace curvefs {
-namespace client {
-
-using ::curvefs::metaserver::MetaStatusCode;
-
-static const std::map<CURVEFS_ERROR, std::string> err2Msg = {
-    {CURVEFS_ERROR::OK, "OK"},
-    {CURVEFS_ERROR::INTERNAL, "internal error"},
-    {CURVEFS_ERROR::UNKNOWN, "unknown"},
-    {CURVEFS_ERROR::EXISTS, "inode or dentry already exist"},
-    {CURVEFS_ERROR::NOTEXIST, "inode or dentry not exist"},
-    {CURVEFS_ERROR::NO_SPACE, "no space to alloc"},
-    {CURVEFS_ERROR::BAD_FD, "bad fd"},
-    {CURVEFS_ERROR::INVALIDPARAM, "invalid param"},
-    {CURVEFS_ERROR::NOPERMISSION, "no permission"},
-    {CURVEFS_ERROR::NOTEMPTY, "dir not empty"},
-    {CURVEFS_ERROR::NOFLUSH, "no flush"},
-    {CURVEFS_ERROR::NOTSUPPORT, "not support"},
-    {CURVEFS_ERROR::NAMETOOLONG, "name too long"},
-    {CURVEFS_ERROR::MOUNT_POINT_EXIST, "mount point already exist"},
-    {CURVEFS_ERROR::MOUNT_FAILED, "mount failed"},
-};
-
-std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
-    os << static_cast<int>(code) << "[" << [code]() {
-        auto it = err2Msg.find(code);
-        if (it != err2Msg.end()) {
-            return it->second;
-        }
-
-        return std::string{"Unknown"};
-    }() << "]";
-
-    return os;
-}
-
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    MetaStatusCode code) {
-    CURVEFS_ERROR ret = CURVEFS_ERROR::UNKNOWN;
-    switch (code) {
-        case MetaStatusCode::OK:
-            ret = CURVEFS_ERROR::OK;
-            break;
-
-        case MetaStatusCode::NOT_FOUND:
-            ret = CURVEFS_ERROR::NOTEXIST;
-            break;
-
-        case MetaStatusCode::PARAM_ERROR:
-            ret = CURVEFS_ERROR::INVALIDPARAM;
-            break;
-
-        case MetaStatusCode::INODE_EXIST:
-        case MetaStatusCode::DENTRY_EXIST:
-            ret = CURVEFS_ERROR::EXISTS;
-            break;
-
-        case MetaStatusCode::SYM_LINK_EMPTY:
-        case MetaStatusCode::RPC_ERROR:
-            ret = CURVEFS_ERROR::INTERNAL;
-            break;
-
-        default:
-            ret = CURVEFS_ERROR::UNKNOWN;
-    }
-    return ret;
-}
-
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/README.md

@@ -0,0 +1,78 @@
+CurveFS Client Metadata Cache Design
+===
+
+```
++----------------+
++  Kernel Cache  |
++----------------+
+    |       ^
+(1) |       | (4)
+    v       |
++----------------+
++  CurveFS Fuse  |
++----------------+
+    |       ^
+(2) |       | (3)
+    v       |
++----------------+
++   MetaServer   |
++----------------+
+```
+
+(1) kernel cache timeout
+---
+
+* requests are always sent to fuse layer, except `lookup` and `getattr` request which affected by caching.
+* if entry or attribute cache timeout, vfs layer will launch request to fuse layer.
+
+(2) revalidate cache
+---
+
+* for `open` and `opendir` request, fuse layer should revalidate cache by comparing mtime, if modified, fuse layer should:
+    * `open`: return **ESTALE** to trigger vfs layer to invoke the `open` again with ignoring cache.
+    * `opendir`: drop all directory cache.
+* others, proxy request to metaserver directly.
+
+
+(3) retrive fresh metadata
+---
+
+* for `readdir` request, fuse layer should cache direcoty entries and their attributes.
+* others, no caching.
+
+(4) reply with timeout
+---
+
+* reply entry or attribute to vfs layer with corresponding cache timeout.
+* four type timeouts provided: `entryTimeout`, `dirEntryTimeout`, `attrTimeout`, `dirAtttTimeout`.
+* fuse layer should remeber the `mtime` of attribute while reply it to vfs layer.
+
+Cache Layer Level
+===
+
+```
++-------------------------------------------------------+
+|                       Kernel Cache                    |  (L1 Cache)
++-------------------------------------------------------+
+                            ^  (update length)
+                            |
++--------------------------------------------------------+
+|                       Open File Cache                  |  (L2 Cache)
+|                   (only length and chunk)              |
++--------------------------------------------------------+
+         ^                            ^
+         | (reply entry/attr)         |  (reply entry/attr)
++-----------------+                   |
+| Directory Cache |  (L3 Cache)       |
++-----------------+                   |
+         ^                            |
+         |                            |
++-----------------+ +------------------------------------+
+|     readdir     | | lookup、getattr、setattr、create... |
++-----------------+ +------------------------------------+
+         ^                            ^
+         |                            |
++--------------------------------------------------------+
+|                        MetaServer                      |
++--------------------------------------------------------+
+```

curvefs/src/client/filesystem/access_log.h

@@ -0,0 +1,89 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-17
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <unistd.h>
+#include <butil/time.h>
+#include <spdlog/spdlog.h>
+#include <spdlog/sinks/basic_file_sink.h>
+#include <spdlog/sinks/daily_file_sink.h>
+
+#include <string>
+#include <memory>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/common/config.h"
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+
+namespace curvefs {
+namespace client {
+namespace common {
+
+DECLARE_bool(access_logging);
+
+}
+namespace filesystem {
+
+using ::absl::StrFormat;
+using ::curvefs::client::common::FLAGS_access_logging;
+using MessageHandler = std::function<std::string()>;
+
+static std::shared_ptr<spdlog::logger> Logger;
+
+bool InitAccessLog(const std::string& prefix) {
+    std::string filename = StrFormat("%s/access.%d.log", prefix, getpid());
+    Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0);
+    spdlog::flush_every(std::chrono::seconds(1));
+    return true;
+}
+
+struct AccessLogGuard {
+    explicit AccessLogGuard(MessageHandler handler)
+        : enable(FLAGS_access_logging),
+          handler(handler) {
+        if (!enable) {
+            return;
+        }
+
+        timer.start();
+    }
+
+    ~AccessLogGuard() {
+        if (!enable) {
+            return;
+        }
+
+        timer.stop();
+        Logger->info("{0} <{1:.6f}>", handler(), timer.u_elapsed() / 1e6);
+    }
+
+    bool enable;
+    MessageHandler handler;
+    butil::Timer timer;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_

curvefs/src/client/filesystem/attr_watcher.cpp

@@ -0,0 +1,117 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+AttrWatcher::AttrWatcher(AttrWatcherOption option,
+                         std::shared_ptr<OpenFiles> openFiles,
+                         std::shared_ptr<DirCache> dirCache)
+    : modifiedAt_(std::make_shared<LRUType>(option.lruSize)),
+      openFiles_(openFiles),
+      dirCache_(dirCache) {}
+
+void AttrWatcher::RemeberMtime(const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    modifiedAt_->Put(attr.inodeid(), AttrMtime(attr));
+}
+
+bool AttrWatcher::GetMtime(Ino ino, TimeSpec* time) {
+    ReadLockGuard lk(rwlock_);
+    return modifiedAt_->Get(ino, time);
+}
+
+void AttrWatcher::UpdateDirEntryAttr(Ino ino, const InodeAttr& attr) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : attr.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateAttr(ino, attr);
+
+        VLOG(1) << "Write back attribute to dir entry cache: ino = " << ino
+                << ", attr = " << attr.ShortDebugString();
+    }
+}
+
+void AttrWatcher::UpdateDirEntryLength(Ino ino, const InodeAttr& open) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : open.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateLength(ino, open);
+
+        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
+                << ", attr = " << open.ShortDebugString();
+    }
+}
+
+AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                                   InodeAttr* attr,
+                                   ReplyType type,
+                                   bool writeBack)
+    : watcher(watcher), attr(attr), type(type), writeBack(writeBack) {
+    InodeAttr open;
+    Ino ino = attr->inodeid();
+    bool yes = watcher->openFiles_->GetFileAttr(ino, &open);
+    if (!yes) {
+        return;
+    }
+
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+}
+
+AttrWatcherGuard::~AttrWatcherGuard() {
+    switch (type) {
+        case ReplyType::ATTR:
+            watcher->RemeberMtime(*attr);
+            if (writeBack) {
+                watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
+            }
+            break;
+
+        case ReplyType::ONLY_LENGTH:
+            if (writeBack) {
+                watcher->UpdateDirEntryLength(attr->inodeid(), *attr);
+            }
+            break;
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/attr_watcher.h

@@ -0,0 +1,105 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::AttrWatcherOption;
+
+class AttrWatcher {
+ public:
+    using LRUType = LRUCache<Ino, struct TimeSpec>;
+
+ public:
+    AttrWatcher(AttrWatcherOption option,
+                std::shared_ptr<OpenFiles> openFiles,
+                std::shared_ptr<DirCache> dirCache);
+
+    void RemeberMtime(const InodeAttr& attr);
+
+    bool GetMtime(Ino ino, TimeSpec* time);
+
+    void UpdateDirEntryAttr(Ino ino, const InodeAttr& attr);
+
+    void UpdateDirEntryLength(Ino ino, const InodeAttr& open);
+
+ private:
+    friend class AttrWatcherGuard;
+
+ private:
+    RWLock rwlock_;
+    std::shared_ptr<LRUType> modifiedAt_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<DirCache> dirCache_;
+};
+
+
+enum class ReplyType {
+    ATTR,
+    ONLY_LENGTH
+};
+
+/*
+ * each attribute reply to kernel, the watcher will:
+ *  before reply:
+ *    1) set attibute length if the corresponding file is opened
+ *  after reply:
+ *    1) remeber attribute modified time.
+ *    2) write back attribute to dir entry cache if |writeBack| is true,
+ *       because the dir-entry attribute maybe stale.
+ */
+struct AttrWatcherGuard {
+ public:
+    AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                     InodeAttr* attr,
+                     ReplyType type,
+                     bool writeBack);
+
+    ~AttrWatcherGuard();
+
+ private:
+    std::shared_ptr<AttrWatcher> watcher;
+    InodeAttr* attr;
+    ReplyType type;
+    bool writeBack;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_

curvefs/src/client/filesystem/defer_sync.cpp

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <vector>
+#include <memory>
+
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DeferSync::DeferSync(DeferSyncOption option)
+    : option_(option),
+      mutex_(),
+      running_(false),
+      thread_(),
+      sleeper_(),
+      inodes_() {
+}
+
+void DeferSync::Start() {
+    if (!running_.exchange(true)) {
+        thread_ = std::thread(&DeferSync::SyncTask, this);
+        LOG(INFO) << "Defer sync thread start success";
+    }
+}
+
+void DeferSync::Stop() {
+    if (running_.exchange(false)) {
+        LOG(INFO) << "Stop defer sync thread...";
+        sleeper_.interrupt();
+        thread_.join();
+        LOG(INFO) << "Defer sync thread stopped";
+    }
+}
+
+void DeferSync::SyncTask() {
+    std::vector<std::shared_ptr<InodeWrapper>> inodes;
+    for ( ;; ) {
+        bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay));
+
+        {
+            LockGuard lk(mutex_);
+            inodes.swap(inodes_);
+        }
+        for (const auto& inode : inodes) {
+            UniqueLock lk(inode->GetUniqueLock());
+            inode->Async(nullptr, true);
+        }
+        inodes.clear();
+
+        if (!running) {
+            break;
+        }
+    }
+}
+
+void DeferSync::Push(const std::shared_ptr<InodeWrapper>& inode) {
+    LockGuard lk(mutex_);
+    inodes_.emplace_back(inode);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/defer_sync.h

@@ -0,0 +1,70 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+
+#include <atomic>
+#include <vector>
+#include <memory>
+
+#include "src/common/interruptible_sleeper.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::DeferSyncOption;
+
+using ::curve::common::Mutex;
+using ::curve::common::LockGuard;
+using ::curve::common::InterruptibleSleeper;
+
+class DeferSync {
+ public:
+    explicit DeferSync(DeferSyncOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Push(const std::shared_ptr<InodeWrapper>& inode);
+
+ private:
+    void SyncTask();
+
+ private:
+    DeferSyncOption option_;
+    Mutex mutex_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    InterruptibleSleeper sleeper_;
+    std::vector<std::shared_ptr<InodeWrapper>> inodes_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_

curvefs/src/client/filesystem/dir_cache.cpp

@@ -0,0 +1,189 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <utility>
+
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DirEntryList::DirEntryList()
+    : rwlock_(),
+      mtime_(),
+      entries_(),
+      attrs_() {}
+
+size_t DirEntryList::Size() {
+    ReadLockGuard lk(rwlock_);
+    return entries_.size();
+}
+
+void DirEntryList::Add(const DirEntry& dirEntry) {
+    WriteLockGuard lk(rwlock_);
+    entries_.push_back(std::move(dirEntry));
+    attrs_[dirEntry.ino] = &entries_.back();
+}
+
+bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) {
+    ReadLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+    *dirEntry = *iter->second;
+    return true;
+}
+
+bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    DirEntry* dirEntry = iter->second;
+    dirEntry->attr = std::move(attr);
+    return true;
+}
+
+bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    InodeAttr* attr = &iter->second->attr;
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+    return true;
+}
+
+void DirEntryList::Iterate(IterateHandler handler) {
+    ReadLockGuard lk(rwlock_);
+    for (auto iter = entries_.begin(); iter != entries_.end(); iter++) {
+        handler(&(*iter));
+    }
+}
+
+void DirEntryList::Clear() {
+    WriteLockGuard lk(rwlock_);
+    entries_.clear();
+    attrs_.clear();
+}
+
+void DirEntryList::SetMtime(TimeSpec mtime) {
+    WriteLockGuard lk(rwlock_);
+    mtime_ = mtime;
+}
+
+TimeSpec DirEntryList::GetMtime() {
+    ReadLockGuard lk(rwlock_);
+    return mtime_;
+}
+
+DirCache::DirCache(DirCacheOption option)
+    : rwlock_(),
+      nentries_(0),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(0);  // control size by ourself
+    mq_ = std::make_shared<MessageQueueType>("dircache", 10000);
+    mq_->Subscribe([&](const std::shared_ptr<DirEntryList>& entries){
+        entries->Clear();
+    });
+
+    LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize;
+}
+
+void DirCache::Start() {
+    mq_->Start();
+}
+
+void DirCache::Stop() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+    mq_->Stop();
+}
+
+void DirCache::Delete(Ino parent,
+                      std::shared_ptr<DirEntryList> entries,
+                      bool evit) {
+    nentries_ -= entries->Size();
+    mq_->Publish(entries);  // clear entries in background
+    lru_->Remove(parent);
+
+    VLOG(1) << "Delete directory cache (evit=" << evit << "): "
+            << "parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
+}
+
+void DirCache::Evit(size_t size) {
+    Ino parent;
+    std::shared_ptr<DirEntryList> entries;
+    while (nentries_ + size >= option_.lruSize) {
+        bool yes = lru_->GetLast(&parent, &entries);
+        if (!yes) {
+            break;
+        }
+        Delete(parent, entries, true);
+    }
+}
+
+void DirCache::Put(Ino parent, std::shared_ptr<DirEntryList> entries) {
+    WriteLockGuard lk(rwlock_);
+    Evit(entries->Size());  // it guarantee put entries success
+    lru_->Put(parent, entries);
+    nentries_ += entries->Size();
+
+    VLOG(1) << "Insert directory cache: parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
+}
+
+bool DirCache::Get(Ino parent, std::shared_ptr<DirEntryList>* entries) {
+    ReadLockGuard lk(rwlock_);
+    return lru_->Get(parent, entries);
+}
+
+void DirCache::Drop(Ino parent) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = lru_->Get(parent, &entries);
+    if (yes) {
+        Delete(parent, entries, false);
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/dir_cache.h

@@ -0,0 +1,114 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+
+#include <map>
+#include <list>
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/message_queue.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::DirCacheOption;
+
+class DirEntryList {
+ public:
+    using IterateHandler = std::function<void(DirEntry* dirEntry)>;
+
+ public:
+    DirEntryList();
+
+    size_t Size();
+
+    void Add(const DirEntry& dirEntry);
+
+    void Iterate(IterateHandler handler);
+
+    bool Get(Ino ino, DirEntry* dirEntry);
+
+    bool UpdateAttr(Ino ino, const InodeAttr& attr);
+
+    bool UpdateLength(Ino ino, const InodeAttr& open);
+
+    void Clear();
+
+    void SetMtime(TimeSpec mtime);
+
+    TimeSpec GetMtime();
+
+ private:
+    RWLock rwlock_;
+    TimeSpec mtime_;
+    std::list<DirEntry> entries_;
+    std::map<Ino, DirEntry*> attrs_;
+};
+
+class DirCache {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<DirEntryList>>;
+    using MessageType = std::shared_ptr<DirEntryList>;
+    using MessageQueueType = MessageQueue<MessageType>;
+
+ public:
+    explicit DirCache(DirCacheOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Put(Ino parent, std::shared_ptr<DirEntryList> entries);
+
+    bool Get(Ino parent, std::shared_ptr<DirEntryList>* entries);
+
+    void Drop(Ino parent);
+
+ private:
+    void Delete(Ino parent, std::shared_ptr<DirEntryList> entries, bool evit);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    size_t nentries_;
+    DirCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+    std::shared_ptr<MessageQueueType> mq_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_

curvefs/src/client/filesystem/error.cpp

@@ -0,0 +1,106 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <utility>
+#include <ostream>
+
+#include "curvefs/src/client/filesystem/error.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+static const std::map<CURVEFS_ERROR, std::pair<int, std::string>> errors = {
+    { CURVEFS_ERROR::OK, { 0, "OK"} },
+    { CURVEFS_ERROR::INTERNAL,  { EIO, "internal error" } },
+    { CURVEFS_ERROR::UNKNOWN,  { -1, "unknown" } },
+    { CURVEFS_ERROR::EXISTS,  { EEXIST, "inode or dentry already exist" } },
+    { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } },
+    { CURVEFS_ERROR::NO_SPACE,  { ENOSPC, "no space to alloc" } },
+    { CURVEFS_ERROR::BAD_FD,  { EBADF, "bad file number" } },
+    { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } },
+    { CURVEFS_ERROR::NOPERMISSION,  { EACCES, "permission denied" } },
+    { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } },
+    { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } },
+    { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } },
+    { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } },
+    { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } },
+    { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } },
+    { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } },
+    { CURVEFS_ERROR::NODATA, { ENODATA, "no data available" } },
+    { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } },
+    { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } },
+    { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } },
+};
+
+std::string StrErr(CURVEFS_ERROR code) {
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        return it->second.second;
+    }
+    return "unknown";
+}
+
+int SysErr(CURVEFS_ERROR code) {
+    int syscode = -1;
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        syscode = it->second.first;
+    }
+    return (syscode == -1) ? EIO : syscode;
+}
+
+std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
+    os << static_cast<int>(code) << "[" << [code]() {
+        auto it = errors.find(code);
+        if (it != errors.end()) {
+            return it->second.second;
+        }
+
+        return std::string{"Unknown"};
+    }() << "]";
+
+    return os;
+}
+
+CURVEFS_ERROR ToFSError(MetaStatusCode code) {
+    static std::map<MetaStatusCode, CURVEFS_ERROR> errs = {
+        { MetaStatusCode::OK, CURVEFS_ERROR::OK },
+        { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST },
+        { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM },
+        { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL },
+        { MetaStatusCode::RPC_ERROR, CURVEFS_ERROR::INTERNAL },
+    };
+
+    auto it = errs.find(code);
+    if (it != errs.end()) {
+        return it->second;
+    }
+    return CURVEFS_ERROR::UNKNOWN;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/error.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2021 NetEase Inc.
+ *  Copyright (c) 2023 NetEase Inc.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -14,22 +14,27 @@
  *  limitations under the License.
  */
 
-
 /*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
  */
 
-#ifndef CURVEFS_SRC_CLIENT_ERROR_CODE_H_
-#define CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#include <map>
+#include <string>
+#include <iostream>
 
 #include "curvefs/proto/metaserver.pb.h"
 
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+
 namespace curvefs {
 namespace client {
+namespace filesystem {
+
+using ::curvefs::metaserver::MetaStatusCode;
 
-// notice : the error code should be negative.
 enum class CURVEFS_ERROR {
     OK = 0,
     INTERNAL = -1,
@@ -50,15 +55,20 @@ enum class CURVEFS_ERROR {
     NODATA = -16,
     IO_ERROR = -17,
     CACHETOOSMALL = -18,
+    STALE = -19,
+    NOSYS = -20,
 };
 
+std::string StrErr(CURVEFS_ERROR code);
+
+int SysErr(CURVEFS_ERROR code);
 
 std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code);
 
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    ::curvefs::metaserver::MetaStatusCode code);
+CURVEFS_ERROR ToFSError(MetaStatusCode code);
 
+}  // namespace filesystem
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_

curvefs/src/client/filesystem/filesystem.cpp

@@ -0,0 +1,353 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+
+#include "curvefs/src/client/filesystem/filesystem.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
+    : option_(option), member(member) {
+    deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
+    negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
+    dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
+    openFiles_ = std::make_shared<OpenFiles>(option_.openFilesOption,
+                                             deferSync_);
+    attrWatcher_ = std::make_shared<AttrWatcher>(option_.attrWatcherOption,
+                                                 openFiles_, dirCache_);
+    handlerManager_ = std::make_shared<HandlerManager>();
+    rpc_ = std::make_shared<RPCClient>(option.rpcOption, member);
+}
+
+void FileSystem::Run() {
+    deferSync_->Start();
+    dirCache_->Start();
+}
+
+void FileSystem::Destory() {
+    openFiles_->CloseAll();
+    deferSync_->Stop();
+    dirCache_->Stop();
+}
+
+void FileSystem::Attr2Stat(InodeAttr* attr, struct stat* stat) {
+    std::memset(stat, 0, sizeof(struct stat));
+    stat->st_ino = attr->inodeid();  //  inode number
+    stat->st_mode = attr->mode();  // permission mode
+    stat->st_nlink = attr->nlink();  // number of links
+    stat->st_uid = attr->uid();  // user ID of owner
+    stat->st_gid = attr->gid();  // group ID of owner
+    stat->st_size = attr->length();  // total size, in bytes
+    stat->st_rdev = attr->rdev();  // device ID (if special file)
+    stat->st_atim.tv_sec = attr->atime();  // time of last access
+    stat->st_atim.tv_nsec = attr->atime_ns();
+    stat->st_mtim.tv_sec = attr->mtime();  // time of last modification
+    stat->st_mtim.tv_nsec = attr->mtime_ns();
+    stat->st_ctim.tv_sec = attr->ctime();  // time of last status change
+    stat->st_ctim.tv_nsec = attr->ctime_ns();
+    stat->st_blksize = option_.blockSize;  // blocksize for file system I/O
+    stat->st_blocks = 0;  // number of 512B blocks allocated
+    if (IsS3File(*attr)) {
+        stat->st_blocks = (attr->length() + 511) / 512;
+    }
+}
+
+void FileSystem::Entry2Param(EntryOut* entryOut,
+                             fuse_entry_param* e) {
+    std::memset(e, 0, sizeof(fuse_entry_param));
+    e->ino = entryOut->attr.inodeid();
+    e->generation = 0;
+    Attr2Stat(&entryOut->attr, &e->attr);
+    e->entry_timeout = entryOut->entryTimeout;
+    e->attr_timeout = entryOut->attrTimeout;
+}
+
+void FileSystem::SetEntryTimeout(EntryOut* entryOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(entryOut->attr)) {
+        entryOut->entryTimeout = option.dirEntryTimeoutSec;
+        entryOut->attrTimeout = option.dirAttrTimeoutSec;
+    } else {
+        entryOut->entryTimeout = option.entryTimeoutSec;
+        entryOut->attrTimeout = option.attrTimeoutSec;
+    }
+}
+
+void FileSystem::SetAttrTimeout(AttrOut* attrOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(attrOut->attr)) {
+        attrOut->attrTimeout = option.dirAttrTimeoutSec;
+    } else {
+        attrOut->attrTimeout = option.attrTimeoutSec;
+    }
+}
+
+// fuse reply*
+void FileSystem::ReplyError(Request req, CURVEFS_ERROR code) {
+    fuse_reply_err(req, SysErr(code));
+}
+
+void FileSystem::ReplyEntry(Request req,
+                            EntryOut* entryOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_entry(req, &e);
+}
+
+void FileSystem::ReplyAttr(Request req,
+                           AttrOut* attrOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr,
+                             ReplyType::ATTR, true);
+    struct stat stat;
+    SetAttrTimeout(attrOut);
+    Attr2Stat(&attrOut->attr, &stat);
+    fuse_reply_attr(req, &stat, attrOut->attrTimeout);
+}
+
+void FileSystem::ReplyReadlink(Request req, const std::string& link) {
+    fuse_reply_readlink(req, link.c_str());
+}
+
+void FileSystem::ReplyOpen(Request req, FileInfo* fi) {
+    fuse_reply_open(req, fi);
+}
+
+void FileSystem::ReplyOpen(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_open(req, fileOut->fi);
+}
+
+void FileSystem::ReplyData(Request req,
+                           struct fuse_bufvec *bufv,
+                           enum fuse_buf_copy_flags flags) {
+    fuse_reply_data(req, bufv, flags);
+}
+
+void FileSystem::ReplyWrite(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_write(req, fileOut->nwritten);
+}
+
+void FileSystem::ReplyBuffer(Request req, const char *buf, size_t size) {
+    fuse_reply_buf(req, buf, size);
+}
+
+void FileSystem::ReplyStatfs(Request req, const struct statvfs *stbuf) {
+    fuse_reply_statfs(req, stbuf);
+}
+
+void FileSystem::ReplyXattr(Request req, size_t size) {
+    fuse_reply_xattr(req, size);
+}
+
+void FileSystem::ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_create(req, &e, fi);
+}
+
+void FileSystem::AddDirEntry(Request req,
+                             DirBufferHead* buffer,
+                             DirEntry* dirEntry) {
+    struct stat stat;
+    std::memset(&stat, 0, sizeof(stat));
+    stat.st_ino = dirEntry->ino;
+
+    // add a directory entry to the buffer
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry(req,
+                      buffer->p + oldsize,  // char* buf
+                      buffer->size - oldsize,  // size_t bufisze
+                      name, &stat, buffer->size);
+}
+
+void FileSystem::AddDirEntryPlus(Request req,
+                                 DirBufferHead* buffer,
+                                 DirEntry* dirEntry) {
+    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr,
+                             ReplyType::ATTR, false);
+    struct fuse_entry_param e;
+    EntryOut entryOut(dirEntry->attr);
+    SetEntryTimeout(&entryOut);
+    Entry2Param(&entryOut, &e);
+
+    // add a directory entry to the buffer with the attributes
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry_plus(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry_plus(req,
+                           buffer->p + oldsize,  // char* buf
+                           buffer->size - oldsize,  // size_t bufisze
+                           name, &e, buffer->size);
+}
+
+// handler*
+std::shared_ptr<FileHandler> FileSystem::NewHandler() {
+    return handlerManager_->NewHandler();
+}
+
+std::shared_ptr<FileHandler> FileSystem::FindHandler(uint64_t fh) {
+    return handlerManager_->FindHandler(fh);
+}
+
+void FileSystem::ReleaseHandler(uint64_t fh) {
+    return handlerManager_->ReleaseHandler(fh);
+}
+
+FileSystemMember FileSystem::BorrowMember() {
+    return FileSystemMember(deferSync_, openFiles_, attrWatcher_);
+}
+
+// fuse request*
+CURVEFS_ERROR FileSystem::Lookup(Request req,
+                                 Ino parent,
+                                 const std::string& name,
+                                 EntryOut* entryOut) {
+    if (name.size() > option_.maxNameLength) {
+        return CURVEFS_ERROR::NAMETOOLONG;
+    }
+
+    bool yes = negative_->Get(parent, name);
+    if (yes) {
+        return CURVEFS_ERROR::NOTEXIST;
+    }
+
+    auto rc = rpc_->Lookup(parent, name, entryOut);
+    if (rc == CURVEFS_ERROR::OK) {
+        negative_->Delete(parent, name);
+    } else if (rc == CURVEFS_ERROR::NOTEXIST) {
+        negative_->Put(parent, name);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) {
+    InodeAttr attr;
+    auto rc = rpc_->GetAttr(ino, &attr);
+    if (rc == CURVEFS_ERROR::OK) {
+        *attrOut = AttrOut(attr);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) {
+    InodeAttr attr;
+    CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    // revalidate directory cache
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = dirCache_->Get(ino, &entries);
+    if (yes) {
+        if (entries->GetMtime() != AttrMtime(attr)) {
+            dirCache_->Drop(ino);
+        }
+    }
+
+    auto handler = NewHandler();
+    handler->mtime = AttrMtime(attr);
+    fi->fh = handler->fh;
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReadDir(Request req,
+                                  Ino ino,
+                                  FileInfo* fi,
+                                  std::shared_ptr<DirEntryList>* entries) {
+    bool yes = dirCache_->Get(ino, entries);
+    if (yes) {
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->ReadDir(ino, entries);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    (*entries)->SetMtime(FindHandler(fi->fh)->mtime);
+    dirCache_->Put(ino, *entries);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) {
+    ReleaseHandler(fi->fh);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) {
+    std::shared_ptr<InodeWrapper> inode;
+    bool yes = openFiles_->IsOpened(ino, &inode);
+    if (yes) {
+        openFiles_->Open(ino, inode);
+        // fi->keep_cache = 1;
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->Open(ino, &inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    TimeSpec mtime;
+    yes = attrWatcher_->GetMtime(ino, &mtime);
+    if (!yes) {
+        // It is rare which only arise when attribute evited for attr-watcher.
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ": attribute not found in wacther";
+        return CURVEFS_ERROR::STALE;
+    } else if (mtime != InodeMtime(inode)) {
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ", cache(" << mtime << ") vs remote("
+                     << InodeMtime(inode) << ")";
+        return CURVEFS_ERROR::STALE;
+    }
+
+    openFiles_->Open(ino, inode);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) {
+    openFiles_->Close(ino);
+    return CURVEFS_ERROR::OK;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/filesystem.h

@@ -0,0 +1,167 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+
+#include <gtest/gtest_prod.h>
+
+#include <memory>
+#include <string>
+
+#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+#include "curvefs/src/client/filesystem/rpc_client.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::FileSystemOption;
+
+struct FileSystemMember {
+    FileSystemMember(std::shared_ptr<DeferSync> deferSync,
+                     std::shared_ptr<OpenFiles> openFiles,
+                     std::shared_ptr<AttrWatcher> attrWatcher)
+        : deferSync(deferSync),
+          openFiles(openFiles),
+          attrWatcher(attrWatcher) {}
+
+    std::shared_ptr<DeferSync> deferSync;
+    std::shared_ptr<OpenFiles> openFiles;
+    std::shared_ptr<AttrWatcher> attrWatcher;
+};
+
+class FileSystem {
+ public:
+    FileSystem(struct FileSystemOption option, ExternalMember member);
+
+    void Run();
+
+    void Destory();
+
+    // fuse request
+    CURVEFS_ERROR Lookup(Request req,
+                         Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut);
+
+    CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR ReadDir(Request req,
+                          Ino ino,
+                          FileInfo* fi,
+                          std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Release(Request req, Ino ino);
+
+    // fuse reply: we control all replies to vfs layer in same entrance.
+    void ReplyError(Request req, CURVEFS_ERROR code);
+
+    void ReplyEntry(Request req, EntryOut* entryOut);
+
+    void ReplyAttr(Request req, AttrOut* attrOut);
+
+    void ReplyReadlink(Request req, const std::string& link);
+
+    void ReplyOpen(Request req, FileInfo *fi);
+
+    void ReplyOpen(Request req, FileOut* fileOut);
+
+    void ReplyData(Request req,
+                   struct fuse_bufvec *bufv,
+                   enum fuse_buf_copy_flags flags);
+
+    void ReplyWrite(Request req, FileOut* fileOut);
+
+    void ReplyBuffer(Request req, const char *buf, size_t size);
+
+    void ReplyStatfs(Request req, const struct statvfs *stbuf);
+
+    void ReplyXattr(Request req, size_t size);
+
+    void ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi);
+
+    void AddDirEntry(Request req,
+                     DirBufferHead* buffer,
+                     DirEntry* dirEntry);
+
+    void AddDirEntryPlus(Request req,
+                         DirBufferHead* buffer,
+                         DirEntry* dirEntry);
+
+    // utility: file handler
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t fh);
+
+    void ReleaseHandler(uint64_t fh);
+
+    // utility: others
+    FileSystemMember BorrowMember();
+
+ private:
+    FRIEND_TEST(FileSystemTest, Attr2Stat);
+    FRIEND_TEST(FileSystemTest, Entry2Param);
+    FRIEND_TEST(FileSystemTest, SetEntryTimeout);
+    FRIEND_TEST(FileSystemTest, SetAttrTimeout);
+
+    // utility: convert to system type.
+    void Attr2Stat(InodeAttr* attr, struct stat* stat);
+
+    void Entry2Param(EntryOut* entryOut, fuse_entry_param* e);
+
+    // utility: set entry/attribute timeout
+    void SetEntryTimeout(EntryOut* entryOut);
+
+    void SetAttrTimeout(AttrOut* attrOut);
+
+ private:
+    FileSystemOption option_;
+    ExternalMember member;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<LookupCache> negative_;
+    std::shared_ptr<DirCache> dirCache_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<AttrWatcher> attrWatcher_;
+    std::shared_ptr<HandlerManager> handlerManager_;
+    std::shared_ptr<RPCClient> rpc_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_

curvefs/src/client/filesystem/lookup_cache.cpp

@@ -0,0 +1,104 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include <ctime>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+#define RETURN_FALSE_IF_DISABLED() \
+    do {                           \
+        if (!enable_) {            \
+            return false;          \
+        }                          \
+    } while (0)
+
+
+LookupCache::LookupCache(LookupCacheOption option)
+    : enable_(option.negativeTimeoutSec > 0),
+      rwlock_(),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(option.lruSize);
+    if (enable_) {
+        LOG(INFO) << "Using lookup negative lru cache"
+                  << ", timeout = " << option.negativeTimeoutSec
+                  << ", capacity = " << option.lruSize;
+    }
+}
+
+std::string LookupCache::CacheKey(Ino parent, const std::string& name) {
+    return absl::StrFormat("%d:%s", parent, name);
+}
+
+bool LookupCache::Get(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    ReadLockGuard lk(rwlock_);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
+    if (!yes) {
+        VLOG(1) << absl::StrFormat("Lookup cache not found: key(%d,%s)",
+                                   parent, name);
+        return false;
+    } else if (entry.uses < option_.minUses) {
+        return false;
+    } else if (entry.expireTime < Now()) {
+        return false;
+    }
+    return true;
+}
+
+bool LookupCache::Put(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    WriteLockGuard lk(rwlock_);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
+    if (yes) {
+        entry.uses++;
+    } else {
+        entry.uses = 0;
+    }
+
+    entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0);
+    lru_->Put(key, entry);
+    return true;
+}
+
+bool LookupCache::Delete(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    WriteLockGuard lk(rwlock_);
+    auto key = CacheKey(parent, name);
+    lru_->Remove(key);
+    return true;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/lookup_cache.h

@@ -0,0 +1,77 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+
+#include <memory>
+#include <string>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::LookupCacheOption;
+
+// memory cache for lookup result, now we only support cache negative result,
+// and other positive entry will be cached in kernel.
+class LookupCache {
+ public:
+    struct CacheEntry {
+        uint32_t uses;
+        TimeSpec expireTime;
+    };
+
+    using LRUType = LRUCache<std::string, CacheEntry>;
+
+ public:
+    explicit LookupCache(LookupCacheOption option);
+
+    bool Get(Ino parent, const std::string& name);
+
+    bool Put(Ino parent, const std::string& name);
+
+    bool Delete(Ino parent, const std::string& name);
+
+ private:
+    std::string CacheKey(Ino parent, const std::string& name);
+
+ private:
+    bool enable_;
+    RWLock rwlock_;
+    LookupCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_

curvefs/src/client/filesystem/message_queue.h

@@ -0,0 +1,118 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+
+#include <glog/logging.h>
+
+#include <string>
+#include <atomic>
+#include <thread>
+
+#include "src/common/concurrent/task_queue.h"
+#include "curvefs/src/common/threading.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::TaskQueue;
+using ::curvefs::common::SetThreadName;
+
+template <typename MessageT>
+class MessageQueue {
+ public:
+    using MessageHandler = std::function<void(const MessageT& message)>;
+
+ public:
+    MessageQueue(const std::string& name, size_t bufferSize)
+        : name_(name),
+          running_(false),
+          thread_(),
+          handler_(),
+          queue_(bufferSize) {}
+
+    void Start() {
+        if (running_.exchange(true)) {
+            return;
+        }
+
+        thread_ = std::thread(&MessageQueue::Consumer, this);
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread start success";
+    }
+
+    void Stop() {
+        if (!running_.exchange(false)) {
+            return;
+        }
+
+        auto wakeup = []() {};
+        queue_.Push(wakeup);
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stoping...";
+
+        thread_.join();
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stopped";
+    }
+
+    void Publish(MessageT message) {
+        if (handler_ != nullptr) {
+            queue_.Push([this, message](){
+                this->handler_(message);
+            });
+        }
+    }
+
+    void Subscribe(MessageHandler handler) {
+        handler_ = handler;
+    }
+
+ private:
+    void Consumer() {
+        SetThreadName(name_.c_str());
+        while (running_.load(std::memory_order_relaxed)) {
+            queue_.Pop()();
+        }
+
+        while (queue_.Size() > 0) {
+            queue_.Pop()();
+        }
+    }
+
+ private:
+    std::string name_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    MessageHandler handler_;
+    TaskQueue queue_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_

curvefs/src/client/filesystem/meta.cpp

@@ -0,0 +1,129 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <sys/stat.h>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+HandlerManager::HandlerManager()
+    : mutex_(),
+      dirBuffer_(std::make_shared<DirBuffer>()),
+      handlers_() {}
+
+HandlerManager::~HandlerManager() {
+    dirBuffer_->DirBufferFreeAll();
+}
+
+std::shared_ptr<FileHandler> HandlerManager::NewHandler() {
+    UniqueLock lk(mutex_);
+    auto handler = std::make_shared<FileHandler>();
+    handler->fh = dirBuffer_->DirBufferNew();
+    handler->buffer = dirBuffer_->DirBufferGet(handler->fh);
+    handler->padding = false;
+    handlers_.emplace(handler->fh, handler);
+    return handler;
+}
+
+std::shared_ptr<FileHandler> HandlerManager::FindHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    auto iter = handlers_.find(fh);
+    if (iter == handlers_.end()) {
+        return nullptr;
+    }
+    return iter->second;
+}
+
+void HandlerManager::ReleaseHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    dirBuffer_->DirBufferRelease(fh);
+    handlers_.erase(fh);
+}
+
+std::string StrMode(uint16_t mode) {
+    static std::map<uint16_t, char> type2char = {
+        { S_IFSOCK, 's' },
+        { S_IFLNK, 'l' },
+        { S_IFREG, '-' },
+        { S_IFBLK, 'b' },
+        { S_IFDIR, 'd' },
+        { S_IFCHR, 'c' },
+        { S_IFIFO, 'f' },
+        { 0, '?' },
+    };
+
+    std::string s("?rwxrwxrwx");
+    s[0] = type2char[mode & (S_IFMT & 0xffff)];
+    if (mode & S_ISUID) {
+        s[3] = 's';
+    }
+    if (mode & S_ISGID) {
+        s[6] = 's';
+    }
+    if (mode & S_ISVTX) {
+        s[9] = 't';
+    }
+
+    for (auto i = 0; i < 9; i++) {
+        if ((mode & (1 << i)) == 0) {
+            if ((s[9-i] == 's') || (s[9-i] == 't')) {
+                s[9-i] &= 0xDF;
+            } else {
+                s[9-i] = '-';
+            }
+        }
+    }
+    return s;
+}
+
+namespace {
+
+std::string Attr2Str(const InodeAttr& attr) {
+    if (!attr.IsInitialized()) {
+        return "";
+    }
+
+    std::string smode;
+    return absl::StrFormat(" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])",
+        attr.inodeid(), StrMode(attr.mode()).c_str(), attr.mode(), attr.nlink(),
+        attr.uid(), attr.gid(),
+        attr.atime(), attr.mtime(), attr.ctime(),
+        attr.length());
+}
+
+}  // namespace
+
+std::string StrEntry(EntryOut entryOut) {
+    return Attr2Str(entryOut.attr);
+}
+
+std::string StrAttr(AttrOut attrOut) {
+    return Attr2Str(attrOut.attr);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/meta.h

@@ -0,0 +1,178 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <iostream>
+
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/client/fuse_common.h"
+#include "curvefs/src/client/dir_buffer.h"
+#include "curvefs/src/client/inode_wrapper.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::Mutex;
+using ::curve::common::RWLock;
+using ::curve::common::UniqueLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::metaserver::XAttr;
+using ::curvefs::metaserver::Dentry;
+using ::curvefs::metaserver::InodeAttr;
+using ::curvefs::metaserver::FsFileType;
+using ::curvefs::client::DirBufferHead;
+using ::curvefs::client::InodeWrapper;
+
+using Ino = fuse_ino_t;
+using Request = fuse_req_t;
+using FileInfo = struct fuse_file_info;
+
+struct EntryOut {
+    EntryOut() = default;
+
+    explicit EntryOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double entryTimeout;
+    double attrTimeout;
+};
+
+struct AttrOut {
+    AttrOut() = default;
+
+    explicit AttrOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double attrTimeout;
+};
+
+struct DirEntry {
+    DirEntry() = default;
+
+    DirEntry(Ino ino, const std::string& name, InodeAttr attr)
+        : ino(ino), name(name), attr(attr) {}
+
+    Ino ino;
+    std::string name;
+    InodeAttr attr;
+};
+
+struct FileOut {
+    FileOut() = default;
+
+    FileOut(FileInfo* fi, InodeAttr attr)
+        : fi(fi), attr(attr), nwritten(0) {}
+
+    FileOut(InodeAttr attr, size_t nwritten)
+        : fi(nullptr), attr(attr), nwritten(nwritten) {}
+
+    FileInfo* fi;
+    InodeAttr attr;
+    size_t nwritten;
+};
+
+struct TimeSpec {
+    TimeSpec() : seconds(0), nanoSeconds(0) {}
+
+    TimeSpec(uint64_t seconds, uint32_t nanoSeconds)
+        : seconds(seconds), nanoSeconds(nanoSeconds) {}
+
+    TimeSpec(const TimeSpec& time)
+        : seconds(time.seconds), nanoSeconds(time.nanoSeconds) {}
+
+    TimeSpec& operator=(const TimeSpec& time) = default;
+
+    TimeSpec operator+(const TimeSpec& time) const {
+        return TimeSpec(seconds + time.seconds, nanoSeconds + time.nanoSeconds);
+    }
+
+    uint64_t seconds;
+    uint32_t nanoSeconds;
+};
+
+struct FileHandler {
+    uint64_t fh;
+    DirBufferHead* buffer;
+    TimeSpec mtime;
+    bool padding;  // padding buffer
+};
+
+class HandlerManager {
+ public:
+    HandlerManager();
+
+    ~HandlerManager();
+
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t id);
+
+    void ReleaseHandler(uint64_t id);
+
+ private:
+    Mutex mutex_;
+    std::shared_ptr<DirBuffer> dirBuffer_;
+    std::map<uint64_t, std::shared_ptr<FileHandler>> handlers_;
+};
+
+inline bool operator==(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds == rhs.seconds) &&
+        (lhs.nanoSeconds == rhs.nanoSeconds);
+}
+
+inline bool operator!=(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return !(lhs == rhs);
+}
+
+inline bool operator<(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds < rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds < rhs.nanoSeconds);
+}
+
+inline bool operator>(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds > rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds > rhs.nanoSeconds);
+}
+
+inline std::ostream &operator<<(std::ostream &os, const TimeSpec& time) {
+    return os << time.seconds << "." << time.nanoSeconds;
+}
+
+std::string StrMode(uint16_t mode);
+
+std::string StrEntry(EntryOut entryOut);
+
+std::string StrAttr(AttrOut attrOut);
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_

curvefs/src/client/filesystem/openfile.cpp

@@ -0,0 +1,141 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+OpenFiles::OpenFiles(OpenFilesOption option,
+                     std::shared_ptr<DeferSync> deferSync)
+    : rwlock_(),
+      option_(option),
+      deferSync_(deferSync),
+      files_(std::make_shared<LRUType>(option.lruSize)) {
+    LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize;
+}
+
+/*
+ * Delete(...) does:
+ *   1) publish to message queue which will flush file to server with async
+ *   2) delete file from lru cache
+ */
+void OpenFiles::Delete(Ino ino,
+                       const std::shared_ptr<OpenFile>& file,
+                       bool flush) {
+    if (flush) {
+        deferSync_->Push(file->inode);
+    }
+    files_->Remove(ino);
+
+    VLOG(1) << "Delete open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+void OpenFiles::Evit(size_t size) {
+    Ino ino;
+    std::shared_ptr<OpenFile> file;
+    while (files_->Size() + size >= option_.lruSize) {
+        bool yes = files_->GetLast(&ino, &file);
+        if (!yes) {
+            break;
+        }
+        Delete(ino, file, true);
+    }
+}
+
+void OpenFiles::Open(Ino ino, std::shared_ptr<InodeWrapper> inode) {
+    WriteLockGuard lk(rwlock_);
+
+    Evit(1);
+
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (yes) {
+        file->refs++;
+        return;
+    }
+
+    file = std::make_shared<OpenFile>(inode);
+    file->refs++;
+    files_->Put(ino, file);
+
+    VLOG(1) << "Insert open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+bool OpenFiles::IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+    *inode = file->inode;
+    return true;
+}
+
+void OpenFiles::Close(Ino ino) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return;
+    }
+
+    if (file->refs > 0) {
+        file->refs--;
+    }
+
+    if (file->refs == 0) {
+        Delete(ino, file, false);  // file already flushed before close
+    }
+}
+
+/*
+ * CloseAll() does:
+ *   flush all file to server and delete from LRU cache
+ */
+void OpenFiles::CloseAll() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+}
+
+bool OpenFiles::GetFileAttr(Ino ino, InodeAttr* attr) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+
+    file->inode->GetInodeAttr(attr);
+    return true;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/openfile.h

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::InodeWrapper;
+using ::curvefs::client::common::OpenFilesOption;
+
+struct OpenFile {
+    explicit OpenFile(std::shared_ptr<InodeWrapper> inode)
+        : inode(inode), refs(0) {}
+
+    std::shared_ptr<InodeWrapper> inode;
+    uint64_t refs;
+};
+
+class OpenFiles {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<OpenFile>>;
+
+ public:
+    explicit OpenFiles(OpenFilesOption option,
+                       std::shared_ptr<DeferSync> deferSync);
+
+    void Open(Ino ino, std::shared_ptr<InodeWrapper> inode);
+
+    bool IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+    void Close(Ino ino);
+
+    void CloseAll();
+
+    bool GetFileAttr(Ino ino, InodeAttr* inode);
+
+ private:
+    void Delete(Ino ino, const std::shared_ptr<OpenFile>& file, bool flush);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    OpenFilesOption option_;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<LRUType> files_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_

curvefs/src/client/filesystem/package.h

@@ -0,0 +1,51 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+
+#include <memory>
+
+#include "curvefs/src/client/dentry_cache_manager.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/xattr_manager.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+struct ExternalMember {  // external member depended by FileSystem
+    ExternalMember() = delete;
+    ExternalMember(std::shared_ptr<DentryCacheManager> dentryManager,
+                   std::shared_ptr<InodeCacheManager> inodeManager)
+        : dentryManager(dentryManager),
+          inodeManager(inodeManager) {}
+
+    std::shared_ptr<DentryCacheManager> dentryManager;
+    std::shared_ptr<InodeCacheManager> inodeManager;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_

curvefs/src/client/filesystem/rpc_client.cpp

@@ -0,0 +1,128 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <set>
+#include <list>
+#include <string>
+#include <memory>
+#include <utility>
+
+#include "curvefs/src/client/filesystem/rpc_client.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+RPCClient::RPCClient(RPCOption option,
+                     ExternalMember member)
+    : option_(option),
+      inodeManager_(member.inodeManager),
+      dentryManager_(member.dentryManager) {}
+
+CURVEFS_ERROR RPCClient::GetAttr(Ino ino, InodeAttr* attr) {
+    CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(ino, attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(getattr::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::Lookup(Ino parent,
+                                const std::string& name,
+                                EntryOut* entryOut) {
+    Dentry dentry;
+    CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry);
+    if (rc != CURVEFS_ERROR::OK) {
+        if (rc != CURVEFS_ERROR::NOTEXIST) {
+            LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc
+                       << ", parent = " << parent << ", name = " << name;
+        }
+        return rc;
+    }
+
+    Ino ino = dentry.inodeid();
+    rc = inodeManager_->GetInodeAttr(ino, &entryOut->attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(lookup::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
+                                 std::shared_ptr<DirEntryList>* entries) {
+    uint32_t limit = option_.listDentryLimit;
+
+    std::list<Dentry> dentries;
+    CURVEFS_ERROR rc = dentryManager_->ListDentry(ino, &dentries, limit);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc
+                   << ", ino = " << ino;
+        return rc;
+    }
+
+    std::set<uint64_t> inos;
+    std::map<uint64_t, InodeAttr> attrs;
+    std::for_each(dentries.begin(), dentries.end(), [&](Dentry& dentry){
+        inos.emplace(dentry.inodeid());
+    });
+    rc = inodeManager_->BatchGetInodeAttrAsync(ino, &inos, &attrs);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::BatchGetInodeAttrAsync) failed"
+                   << ", retCode = " << rc << ", ino = " << ino;
+        return rc;
+    }
+
+    DirEntry dirEntry;
+    for (const auto& dentry : dentries) {
+        Ino ino = dentry.inodeid();
+        auto iter = attrs.find(ino);
+        if (iter == attrs.end()) {
+            LOG(WARNING) << "rpc(readdir::BatchGetInodeAttrAsync) "
+                         << "missing attribute, ino = " << ino;
+            continue;
+        }
+
+        // NOTE: we can't use std::move() for attribute for hard link
+        // which will sharing inode attribute.
+        dirEntry.ino = ino;
+        dirEntry.name = std::move(dentry.name());
+        dirEntry.attr = iter->second;
+        (*entries)->Add(dirEntry);
+    }
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR RPCClient::Open(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    CURVEFS_ERROR rc = inodeManager_->GetInode(ino, *inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(open/GetInode) failed" << ", retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/rpc_client.h

@@ -0,0 +1,65 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+
+#include <memory>
+#include <string>
+
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::RPCOption;
+
+class RPCClient {
+ public:
+    RPCClient(RPCOption option,
+              ExternalMember member);
+
+    CURVEFS_ERROR GetAttr(Ino ino, InodeAttr* attr);
+
+    CURVEFS_ERROR Lookup(Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR Open(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+ private:
+    RPCOption option_;
+    std::shared_ptr<InodeCacheManager> inodeManager_;
+    std::shared_ptr<DentryCacheManager> dentryManager_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_

curvefs/src/client/filesystem/utils.cpp

@@ -0,0 +1,69 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+bool IsDir(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_DIRECTORY;
+}
+
+bool IsS3File(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_S3;
+}
+
+bool IsVolmeFile(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_FILE;
+}
+
+bool IsSymLink(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_SYM_LINK;
+}
+
+TimeSpec AttrMtime(const InodeAttr& attr) {
+    return TimeSpec(attr.mtime(), attr.mtime_ns());
+}
+
+TimeSpec AttrCtime(const InodeAttr& attr) {
+    return TimeSpec(attr.ctime(), attr.ctime_ns());
+}
+
+TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode) {
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    return AttrMtime(attr);
+}
+
+TimeSpec Now() {
+    struct timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+    return TimeSpec(now.tv_sec, now.tv_nsec);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/utils.h

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+
+#include <memory>
+
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+// directory
+bool IsDir(const InodeAttr& attr);
+
+// file which data is stored in s3
+bool IsS3File(const InodeAttr& attr);
+
+// file which data is stored in volume
+bool IsVolmeFile(const InodeAttr& attr);
+
+// symbol link
+bool IsSymLink(const InodeAttr& attr);
+
+struct TimeSpec AttrMtime(const InodeAttr& attr);
+
+struct TimeSpec AttrCtime(const InodeAttr& attr);
+
+struct TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode);
+
+struct TimeSpec Now();
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_

curvefs/src/client/fuse_client.cpp

@@ -35,7 +35,7 @@
 
 #include "curvefs/proto/mds.pb.h"
 #include "curvefs/src/client/common/common.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/fuse_common.h"
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/inode_wrapper.h"
@@ -55,7 +55,10 @@ using ::curvefs::mds::topology::PartitionTxId;
 using ::curvefs::mds::FSStatusCode_Name;
 using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH;
 using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
-using ::curvefs::client::common::FileHandle;
+using ::curvefs::client::filesystem::ExternalMember;
+using ::curvefs::client::filesystem::DirEntry;
+using ::curvefs::client::filesystem::DirEntryList;
+using ::curvefs::client::filesystem::FileOut;
 
 #define RETURN_IF_UNSUCCESS(action)                                            \
     do {                                                                       \
@@ -154,6 +157,11 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     curve::client::ClientDummyServerInfo::GetInstance().SetPort(listenPort);
     curve::client::ClientDummyServerInfo::GetInstance().SetIP(localIp);
 
+    {
+        ExternalMember member(dentryManager_, inodeManager_);
+        fs_ = std::make_shared<FileSystem>(option_.fileSystemOption, member);
+    }
+
     MetaStatusCode ret2 =
         metaClient_->Init(option.excutorOpt, option.excutorInternalOpt,
                           metaCache, channelManager);
@@ -161,18 +169,14 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
         return CURVEFS_ERROR::INTERNAL;
     }
 
-    CURVEFS_ERROR ret3 =
-        inodeManager_->Init(option.iCacheLruSize, option.enableICacheMetrics,
-                            option.flushPeriodSec, option.refreshDataOption,
-                            option.lruTimeOutSec);
-    if (ret3 != CURVEFS_ERROR::OK) {
-        return ret3;
-    }
-    ret3 =
-        dentryManager_->Init(option.dCacheLruSize, option.enableDCacheMetrics,
-            option.lruTimeOutSec);
-    if (ret3 != CURVEFS_ERROR::OK) {
-        return ret3;
+    {  // init inode manager
+        auto member = fs_->BorrowMember();
+        CURVEFS_ERROR rc = inodeManager_->Init(option.refreshDataOption,
+                                               member.openFiles,
+                                               member.deferSync);
+        if (rc != CURVEFS_ERROR::OK) {
+            return rc;
+        }
     }
 
     if (warmupManager_ != nullptr) {
@@ -182,7 +186,7 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
 
     InitQosParam();
 
-    return ret3;
+    return CURVEFS_ERROR::OK;
 }
 
 void FuseClient::UnInit() {
@@ -200,7 +204,6 @@ void FuseClient::UnInit() {
 
 CURVEFS_ERROR FuseClient::Run() {
     if (isStop_.exchange(false)) {
-        inodeManager_->Run();
         return CURVEFS_ERROR::OK;
     }
     return CURVEFS_ERROR::INTERNAL;
@@ -208,7 +211,6 @@ CURVEFS_ERROR FuseClient::Run() {
 
 void FuseClient::Fini() {
     if (!isStop_.exchange(true)) {
-        inodeManager_->Stop();
         xattrManager_->Stop();
     }
 }
@@ -217,6 +219,7 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                      struct fuse_conn_info *conn) {
     (void)userdata;
     (void)conn;
+    fs_->Run();
     return CURVEFS_ERROR::OK;
 }
 
@@ -226,7 +229,7 @@ void FuseClient::FuseOpDestroy(void *userdata) {
     }
 
     FlushAll();
-    dirBuf_->DirBufferFreeAll();
+    fs_->Destory();
 
     // stop lease before umount fs, otherwise, lease request after umount fs
     // will add a mountpoint entry.
@@ -258,91 +261,38 @@ void FuseClient::FuseOpDestroy(void *userdata) {
               << " success!";
 }
 
-void InodeAttr2ParamAttr(const InodeAttr &inodeAttr, struct stat *attr) {
-    attr->st_ino = inodeAttr.inodeid();
-    attr->st_mode = inodeAttr.mode();
-    attr->st_nlink = inodeAttr.nlink();
-    attr->st_uid = inodeAttr.uid();
-    attr->st_gid = inodeAttr.gid();
-    attr->st_size = inodeAttr.length();
-    attr->st_rdev = inodeAttr.rdev();
-    attr->st_atim.tv_sec = inodeAttr.atime();
-    attr->st_atim.tv_nsec = inodeAttr.atime_ns();
-    attr->st_mtim.tv_sec = inodeAttr.mtime();
-    attr->st_mtim.tv_nsec = inodeAttr.mtime_ns();
-    attr->st_ctim.tv_sec = inodeAttr.ctime();
-    attr->st_ctim.tv_nsec = inodeAttr.ctime_ns();
-    attr->st_blksize = kOptimalIOBlockSize;
-
-    switch (inodeAttr.type()) {
-        case metaserver::TYPE_S3:
-            attr->st_blocks = (inodeAttr.length() + 511) / 512;
-            break;
-        default:
-            attr->st_blocks = 0;
-            break;
+CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       EntryOut* entryOut) {
+    CURVEFS_ERROR rc = fs_->Lookup(req, parent, name, entryOut);
+    if (rc != CURVEFS_ERROR::OK && rc != CURVEFS_ERROR::NOTEXIST) {
+        LOG(ERROR) << "Lookup() failed, retCode = " << rc
+                   << ", parent = " << parent << ", name = " << name;
     }
+    return rc;
 }
 
-void GetDentryParamFromInodeAttr(
-    const FuseClientOption &option,
-    const InodeAttr &inodeAttr,
-    fuse_entry_param *param) {
-    memset(param, 0, sizeof(fuse_entry_param));
-    param->ino = inodeAttr.inodeid();
-    param->generation = 0;
-    InodeAttr2ParamAttr(inodeAttr, &param->attr);
-    param->attr_timeout = option.attrTimeOut;
-    param->entry_timeout = option.entryTimeOut;
-}
-
-CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
-                                       const char *name, fuse_entry_param *e) {
-    (void)req;
-    VLOG(1) << "FuseOpLookup parent: " << parent
-            << ", name: " << name;
-    if (strlen(name) > option_.maxNameLength) {
-        return CURVEFS_ERROR::NAMETOOLONG;
-    }
-
-    Dentry dentry;
-    CURVEFS_ERROR ret = dentryManager_->GetDentry(parent, name, &dentry);
-    if (ret != CURVEFS_ERROR::OK) {
-        if (ret != CURVEFS_ERROR::NOTEXIST) {
-            LOG(WARNING) << "dentryManager_ get dentry fail, ret = " << ret
-                         << ", parent inodeid = " << parent
-                         << ", name = " << name;
-        }
-        return ret;
-    }
-
-    fuse_ino_t ino = dentry.inodeid();
-    InodeAttr attr;
-    ret = inodeManager_->GetInodeAttr(ino, &attr);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
-    }
-
-    GetDentryParamFromInodeAttr(option_, attr, e);
-    return ret;
-}
-
-CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
-                                     struct fuse_file_info *fi) {
-    (void)req;
-    VLOG(1) << "FuseOpOpen, ino: " << ino;
+CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
+                                          fuse_ino_t ino,
+                                          struct fuse_file_info* fi,
+                                          FileOut* fileOut) {
     std::shared_ptr<InodeWrapper> inodeWrapper;
+    // alredy opened
     CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
                    << ", inodeid = " << ino;
         return ret;
     }
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+
+    fileOut->fi = fi;
+    inodeWrapper->GetInodeAttr(&fileOut->attr);
+
     if (fi->flags & O_TRUNC) {
         if (fi->flags & O_WRONLY || fi->flags & O_RDWR) {
+            ::curve::common::UniqueLock lgGuard =
+                inodeWrapper->GetUniqueLock();
             uint64_t length = inodeWrapper->GetLengthLocked();
             CURVEFS_ERROR tRet = Truncate(inodeWrapper.get(), 0);
             if (tRet != CURVEFS_ERROR::OK) {
@@ -377,18 +327,29 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
                     }
                 }
             }
+            inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
         } else {
             return CURVEFS_ERROR::NOPERMISSION;
         }
     }
-    if (FLAGS_enableCto) {
-        inodeManager_->AddOpenedInode(ino);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     struct fuse_file_info* fi,
+                                     FileOut* fileOut) {
+    CURVEFS_ERROR rc = fs_->Open(req, ino, fi);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "open(" << ino << ") failed, retCode = " << rc;
+        return rc;
     }
-    return ret;
+    return HandleOpenFlags(req, ino, fi, fileOut);
 }
 
 CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
     fuse_ino_t parent, FsFileType type, NlinkChange nlink) {
+
     std::shared_ptr<InodeWrapper> parentInodeWrapper;
     auto ret = inodeManager_->GetInode(parent, parentInodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
@@ -404,16 +365,27 @@ CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
         if (FsFileType::TYPE_DIRECTORY == type) {
             parentInodeWrapper->UpdateNlinkLocked(nlink);
         }
+
+        if (option_.fileSystemOption.deferSyncOption.deferDirMtime) {
+            inodeManager_->ShipToFlush(parentInodeWrapper);
+        } else {
+            return parentInodeWrapper->SyncAttr();
+        }
     }
-    inodeManager_->ShipToFlush(parentInodeWrapper);
+
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
-                                   const char *name, mode_t mode,
-                                   FsFileType type, dev_t rdev, bool internal,
-                                   fuse_entry_param *e) {
-    if (strlen(name) > option_.maxNameLength) {
+CURVEFS_ERROR FuseClient::MakeNode(
+    fuse_req_t req,
+    fuse_ino_t parent,
+    const char* name,
+    mode_t mode,
+    FsFileType type,
+    dev_t rdev,
+    bool internal,
+    std::shared_ptr<InodeWrapper>& inodeWrapper) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -444,7 +416,6 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
     param.rdev = rdev;
     param.parent = parent;
 
-    std::shared_ptr<InodeWrapper> inodeWrapper;
     CURVEFS_ERROR ret = inodeManager_->CreateInode(param, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager CreateInode fail, ret = " << ret
@@ -515,20 +486,26 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttr(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
-                                      const char *name, mode_t mode,
-                                      fuse_entry_param *e) {
+CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char *name,
+                                      mode_t mode,
+                                      EntryOut* entryOut) {
     VLOG(1) << "FuseOpMkDir, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
     bool internal = false;
-    return MakeNode(req, parent, name, S_IFDIR | mode,
-                    FsFileType::TYPE_DIRECTORY, 0, internal, e);
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, S_IFDIR | mode,
+                                FsFileType::TYPE_DIRECTORY, 0, internal, inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
@@ -587,8 +564,6 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent,
                          << ", xattr = " << xattr.DebugString();
         }
     }
-
-    inodeManager_->ClearInodeCache(ino);
     return ret;
 }
 
@@ -601,10 +576,13 @@ std::string GetRecycleTimeDirName() {
     return now;
 }
 
-CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, uint64_t parent,
-                    const char *name, mode_t mode, ManageInodeType manageType,
-                    fuse_entry_param *e) {
-    if (strlen(name) > option_.maxNameLength) {
+CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req,
+                                           uint64_t parent,
+                                           const char* name,
+                                           mode_t mode,
+                                           ManageInodeType manageType,
+                                           EntryOut* entryOut) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -688,9 +666,7 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, uint64_t parent,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttrLocked(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
+    inodeWrapper->GetInodeAttrLocked(&entryOut->attr);
     return ret;
 }
 
@@ -704,9 +680,10 @@ CURVEFS_ERROR FuseClient::GetOrCreateRecycleDir(fuse_req_t req, Dentry *out) {
     } else if (ret == CURVEFS_ERROR::NOTEXIST) {
         LOG(INFO) << "recycle dir is not exist, create " << RECYCLENAME
                   << ", parentid = " << ROOTINODEID;
-        fuse_entry_param param;
+        EntryOut entryOut;
         ret = CreateManageNode(req, ROOTINODEID, RECYCLENAME,
-                       S_IFDIR | 0755, ManageInodeType::TYPE_RECYCLE, &param);
+                               S_IFDIR | 0755, ManageInodeType::TYPE_RECYCLE,
+                               &entryOut);
         if (ret != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "CreateManageNode failed, ret = " << ret
                        << ", inode = " << ROOTINODEID
@@ -743,17 +720,17 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
                    << ", name = " << recycleTimeDirName;
         return ret;
     } else if (ret == CURVEFS_ERROR::NOTEXIST) {
-        fuse_entry_param param;
+        std::shared_ptr<InodeWrapper> inode;
         bool internal = true;
         ret = MakeNode(req, RECYCLEINODEID, recycleTimeDirName.c_str(),
-              S_IFDIR | 0755, FsFileType::TYPE_DIRECTORY, 0, internal, &param);
+              S_IFDIR | 0755, FsFileType::TYPE_DIRECTORY, 0, internal, inode);
         if (ret != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "MakeNode failed, ret = " << ret
                        << ", inode = " << RECYCLEINODEID
                        << ", name = " << recycleTimeDirName;
             return ret;
         }
-        recycleTimeDirIno = param.ino;
+        recycleTimeDirIno = inode->GetInodeId();
     } else {
         recycleTimeDirIno = dentry.inodeid();
     }
@@ -763,7 +740,7 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
 
     // 4. move inode to recycle time dir
     ret = FuseOpRename(req, parent, name,
-                                     recycleTimeDirIno, newName.c_str());
+                       recycleTimeDirIno, newName.c_str(), 0);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "MoveToRecycle failed, ret = " << ret
                    << ", inodeid = " << ino
@@ -800,7 +777,7 @@ bool FuseClient::ShouldMoveToRecycle(fuse_ino_t parent) {
 
 CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
                                      const char *name, FsFileType type) {
-    if (strlen(name) > option_.maxNameLength) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -856,146 +833,87 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
-                                        struct fuse_file_info *fi) {
-    (void)req;
-    VLOG(1) << "FuseOpOpenDir ino = " << ino;
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
+CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        struct fuse_file_info* fi) {
+    CURVEFS_ERROR rc = fs_->OpenDir(req, ino, fi);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "opendir() failed, retCode = " << rc
+                   << ", ino = " << ino;
     }
-
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-
-    uint64_t dindex = dirBuf_->DirBufferNew();
-    fi->fh = dindex;
-    VLOG(1) << "FuseOpOpenDir, ino: " << ino << ", dindex: " << dindex;
-    return ret;
+    return rc;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
-                                           struct fuse_file_info *fi) {
-    (void)req;
-    uint64_t dindex = fi->fh;
-    VLOG(1) << "FuseOpReleaseDir, ino: " << ino << ", dindex: " << dindex;
-    dirBuf_->DirBufferRelease(dindex);
+CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        size_t size,
+                                        off_t off,
+                                        struct fuse_file_info *fi,
+                                        char** bufferOut,
+                                        size_t* rSize,
+                                        bool plus) {
+    auto handler = fs_->FindHandler(fi->fh);
+    DirBufferHead* buffer = handler->buffer;
+    if (!handler->padding) {
+        auto entries = std::make_shared<DirEntryList>();
+        CURVEFS_ERROR rc = fs_->ReadDir(req, ino, fi, &entries);
+        if (rc != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "readdir() failed, retCode = " << rc
+                       << ", ino = " << ino << ", fh = " << fi->fh;
+            return rc;
+        }
 
-    // release inodeAttr cache
-    inodeManager_->ReleaseCache(ino);
+        entries->Iterate([&](DirEntry* dirEntry){
+            if (plus) {
+                fs_->AddDirEntryPlus(req, buffer, dirEntry);
+            } else {
+                fs_->AddDirEntry(req, buffer, dirEntry);
+            }
+        });
+        handler->padding = true;
+    }
+
+    if (off < buffer->size) {
+        *bufferOut = buffer->p + off;
+        *rSize = std::min(buffer->size - off, size);
+    } else {
+        *bufferOut = nullptr;
+        *rSize = 0;
+    }
     return CURVEFS_ERROR::OK;
 }
 
-static void dirbuf_add(fuse_req_t req, struct DirBufferHead *b,
-                       const Dentry &dentry,
-                       const FuseClientOption &option,
-                       bool cacheDir,
-                       InodeAttr *attr = nullptr) {
-    struct stat stbuf;
-    struct fuse_entry_param param;
-    size_t oldsize = b->size;
-    if (!cacheDir) {
-        b->size += fuse_add_direntry(req, NULL, 0, dentry.name().c_str(),
-                                     NULL, 0);
-        b->p = static_cast<char *>(realloc(b->p, b->size));
-        memset(&stbuf, 0, sizeof(stbuf));
-        stbuf.st_ino = dentry.inodeid();
-        fuse_add_direntry(req, b->p + oldsize, b->size - oldsize,
-                          dentry.name().c_str(), &stbuf, b->size);
-    } else {
-        b->size += fuse_add_direntry_plus(req, NULL, 0, dentry.name().c_str(),
-                                          NULL, 0);
-        b->p = static_cast<char *>(realloc(b->p, b->size));
-        GetDentryParamFromInodeAttr(option, *attr, &param);
-        fuse_add_direntry_plus(req, b->p + oldsize, b->size - oldsize,
-                               dentry.name().c_str(), &param, b->size);
+CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req,
+                                           fuse_ino_t ino,
+                                           struct fuse_file_info* fi) {
+    CURVEFS_ERROR rc = fs_->ReleaseDir(req, ino, fi);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "releasedir() failed, retCode = " << rc
+                   << ", ino = " << ino;
     }
-}
-
-CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
-                                            size_t size, off_t off,
-                                            struct fuse_file_info *fi,
-                                            char **buffer, size_t *rSize,
-                                            bool cacheDir) {
-    VLOG(1) << "FuseOpReadDirPlus ino: " << ino << ", size: " << size
-            << ", off = " << off << ", cacheDir = " << cacheDir;
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
-    }
-
-    uint64_t dindex = fi->fh;
-    DirBufferHead *bufHead = dirBuf_->DirBufferGet(dindex);
-    if (!bufHead->wasRead) {
-        std::list<Dentry> dentryList;
-        std::set<uint64_t> inodeIds;
-        std::map<uint64_t, InodeAttr> inodeAttrMap;
-        auto limit = option_.listDentryLimit;
-        ret = dentryManager_->ListDentry(ino, &dentryList, limit);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
-                       << ", parent = " << ino;
-            return ret;
-        }
-
-        if (!cacheDir) {
-            for (const auto &dentry : dentryList) {
-                dirbuf_add(req, bufHead, dentry, option_, cacheDir);
-            }
-        } else {
-            for (const auto &dentry : dentryList) {
-                inodeIds.emplace(dentry.inodeid());
-            }
-            VLOG(3) << "batch get inode size = " << inodeIds.size();
-            ret = inodeManager_->BatchGetInodeAttrAsync(ino, &inodeIds,
-                                                        &inodeAttrMap);
-            if (ret != CURVEFS_ERROR::OK) {
-                LOG(ERROR) << "BatchGetInodeAttr failed when FuseOpReadDir"
-                           << ", parentId = " << ino;
-                return ret;
-            }
-
-            for (const auto &dentry : dentryList) {
-                auto iter = inodeAttrMap.find(dentry.inodeid());
-                if (iter != inodeAttrMap.end()) {
-                    dirbuf_add(req, bufHead, dentry, option_,
-                               cacheDir, &iter->second);
-                } else {
-                    LOG(WARNING) << "BatchGetInodeAttr missing some inodes,"
-                                 << " inodeId = " << dentry.inodeid();
-                }
-            }
-        }
-        bufHead->wasRead = true;
-    }
-
-    if (off < static_cast<ssize_t>(bufHead->size)) {
-        *buffer = bufHead->p + off;
-        *rSize = std::min(bufHead->size - off, size);
-    } else {
-        *buffer = nullptr;
-        *rSize = 0;
-    }
-    return ret;
+    return rc;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
                                        const char *name, fuse_ino_t newparent,
-                                       const char *newname) {
-    (void)req;
+                                       const char *newname,
+                                       unsigned int flags) {
     VLOG(1) << "FuseOpRename from (" << parent << ", " << name << ") to ("
             << newparent << ", " << newname << ")";
-    if (strlen(name) > option_.maxNameLength ||
-        strlen(newname) > option_.maxNameLength) {
+
+    // TODO(Wine93): the flag RENAME_EXCHANGE and RENAME_NOREPLACE
+    // is only used in linux interface renameat(), not required by posix,
+    // we can ignore it now
+    if (flags != 0) {
+        return CURVEFS_ERROR::INVALIDPARAM;
+    }
+
+    uint64_t maxNameLength = option_.fileSystemOption.maxNameLength;
+    if (strlen(name) > maxNameLength || strlen(newname) > maxNameLength) {
         LOG(WARNING) << "FuseOpRename name too long, name = " << name
                      << ", name len = " << strlen(name) << ", new name = "
                      << newname << ", new name len = " << strlen(newname)
-                     << ", maxNameLength = " << option_.maxNameLength;
+                     << ", maxNameLength = " << maxNameLength;
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -1034,30 +952,24 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
     return rc;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info *fi,
-                                        struct stat *attr) {
-    (void)req;
-    (void)fi;
-    VLOG(1) << "FuseOpGetAttr ino = " << ino;
-    InodeAttr inodeAttr;
-    CURVEFS_ERROR ret =
-        inodeManager_->GetInodeAttr(ino, &inodeAttr);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
+                                        AttrOut* attrOut) {
+    CURVEFS_ERROR rc = fs_->GetAttr(req, ino, attrOut);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "getattr() fail, retCode = " << rc
+                   << ", ino = " << ino;
     }
-    InodeAttr2ParamAttr(inodeAttr, attr);
-    return ret;
+    return rc;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
-                                        struct stat *attr, int to_set,
-                                        struct fuse_file_info *fi,
-                                        struct stat *attrOut) {
-    (void)req;
-    (void)fi;
+CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        struct stat* attr,
+                                        int to_set,
+                                        struct fuse_file_info* fi,
+                                        struct AttrOut* attrOut) {
     VLOG(1) << "FuseOpSetAttr to_set: " << to_set << ", ino: " << ino
             << ", attr: " << *attr;
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -1114,9 +1026,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
         if (ret != CURVEFS_ERROR::OK) {
             return ret;
         }
-        InodeAttr inodeAttr;
-        inodeWrapper->GetInodeAttrLocked(&inodeAttr);
-        InodeAttr2ParamAttr(inodeAttr, attrOut);
+        inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
 
         if (enableSumInDir_.load() && changeSize != 0) {
             // update parent summary info
@@ -1141,9 +1051,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    InodeAttr inodeAttr;
-    inodeWrapper->GetInodeAttrLocked(&inodeAttr);
-    InodeAttr2ParamAttr(inodeAttr, attrOut);
+    inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
     return ret;
 }
 
@@ -1153,8 +1061,8 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
     (void)req;
     VLOG(9) << "FuseOpGetXattr, ino: " << ino
             << ", name: " << name << ", size = " << size;
-    if (option_.disableXattr) {
-        return CURVEFS_ERROR::NOTSUPPORT;
+    if (option_.fileSystemOption.disableXattr) {
+        return CURVEFS_ERROR::NOSYS;
     }
 
     InodeAttr inodeAttr;
@@ -1191,12 +1099,6 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, const char* value,
                                          size_t size, int flags) {
-    (void)req;
-    (void)flags;
-    if (option_.disableXattr) {
-        return CURVEFS_ERROR::NOTSUPPORT;
-    }
-
     std::string strname(name);
     std::string strvalue(value, size);
     VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
@@ -1279,10 +1181,12 @@ CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
     return CURVEFS_ERROR::OUT_OF_RANGE;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
-                                        fuse_ino_t parent, const char *name,
-                                        fuse_entry_param *e) {
-    if (strlen(name) > option_.maxNameLength) {
+CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req,
+                                        const char* link,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        EntryOut* entryOut) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     const struct fuse_ctx *ctx = fuse_req_ctx(req);
@@ -1351,18 +1255,17 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttr(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
+    inodeWrapper->GetInodeAttr(&entryOut->attr);
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
+CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     fuse_ino_t newparent,
+                                     const char* newname,
                                      FsFileType type,
-                                     fuse_entry_param *e) {
-    (void)req;
-    if (strlen(newname) > option_.maxNameLength) {
+                                     EntryOut* entryOut) {
+    if (strlen(newname) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -1423,9 +1326,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttr(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
+    inodeWrapper->GetInodeAttr(&entryOut->attr);
     return ret;
 }
 
@@ -1444,24 +1345,18 @@ CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
-    (void)req;
-    (void)fi;
-    VLOG(1) << "FuseOpRelease, ino: " << ino;
-    if (FLAGS_enableCto) {
-        inodeManager_->RemoveOpenedInode(ino);
+    CURVEFS_ERROR rc = fs_->Release(req, ino);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "release() failed, ino = " << ino;
     }
-    return CURVEFS_ERROR::OK;
+    return rc;
 }
 
-void FuseClient::FlushInode() { inodeManager_->FlushInodeOnce(); }
-
-void FuseClient::FlushInodeAll() { inodeManager_->FlushAll(); }
-
 void FuseClient::FlushAll() {
     FlushData();
-    FlushInodeAll();
 }
 
 CURVEFS_ERROR

curvefs/src/client/fuse_client.h

@@ -56,6 +56,8 @@
 #include "curvefs/src/client/xattr_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
 #include "src/common/throttle.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/filesystem.h"
 
 #define DirectIOAlignment 512
 
@@ -70,6 +72,7 @@ using ::curvefs::client::metric::FSMetric;
 namespace curvefs {
 namespace client {
 
+
 namespace warmup {
 class WarmupManager;
 }
@@ -80,6 +83,10 @@ using rpcclient::MdsClient;
 using rpcclient::MdsClientImpl;
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::FileSystem;
+using ::curvefs::client::filesystem::EntryOut;
+using ::curvefs::client::filesystem::AttrOut;
+using ::curvefs::client::filesystem::FileOut;
 
 using curvefs::common::is_aligned;
 
@@ -94,7 +101,6 @@ class FuseClient {
         metaClient_(std::make_shared<MetaServerClientImpl>()),
         inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
         dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
-        dirBuf_(std::make_shared<DirBuffer>()),
         fsInfo_(nullptr),
         init_(false),
         enableSumInDir_(false),
@@ -113,7 +119,6 @@ class FuseClient {
             metaClient_(metaClient),
             inodeManager_(inodeManager),
             dentryManager_(dentryManager),
-            dirBuf_(std::make_shared<DirBuffer>()),
             fsInfo_(nullptr),
             init_(false),
             enableSumInDir_(false),
@@ -137,31 +142,42 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                       const char* buf, size_t size, off_t off,
                                       struct fuse_file_info* fi,
-                                      size_t* wSize) = 0;
+                                      FileOut* fileOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                      size_t size, off_t off,
                                      struct fuse_file_info* fi, char* buffer,
                                      size_t* rSize) = 0;
 
-    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
-                                     struct fuse_file_info* fi);
+    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     struct fuse_file_info* fi,
+                                     FileOut* fileOut);
 
-    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, mode_t mode,
+    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       mode_t mode,
                                        struct fuse_file_info* fi,
-                                       fuse_entry_param* e) = 0;
+                                       EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode, dev_t rdev,
-                                      fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      dev_t rdev,
+                                      EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode,
-                                      fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      EntryOut* entryOut);
 
     virtual CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                        const char* name) = 0;
@@ -169,30 +185,38 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
                                       const char* name);
 
-    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req,
+                                           fuse_ino_t ino,
                                            struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
-                                            size_t size, off_t off,
-                                            struct fuse_file_info* fi,
-                                            char** buffer, size_t* rSize,
-                                            bool cacheDir);
+    virtual CURVEFS_ERROR FuseOpReadDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        size_t size,
+                                        off_t off,
+                                        struct fuse_file_info* fi,
+                                        char** bufferOut,
+                                        size_t* rSize,
+                                        bool plus);
 
-    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_ino_t newparent,
-                                       const char* newname);
+    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       unsigned int flags);
 
     virtual CURVEFS_ERROR FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info* fi,
-                                        struct stat* attr);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct stat* attr, int to_set,
                                         struct fuse_file_info* fi,
-                                        struct stat* attrOut);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, std::string* value,
@@ -205,13 +229,17 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 
-    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req, const char* link,
-                                        fuse_ino_t parent, const char* name,
-                                        fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req,
+                                        const char* link,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char* newname,
-                                     fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     fuse_ino_t newparent,
+                                     const char* newname,
+                                     EntryOut* entryOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string* linkStr);
@@ -245,10 +273,12 @@ class FuseClient {
         stbuf->f_fsid = fsInfo_->fsid();
 
         stbuf->f_flag = 0;
-        stbuf->f_namemax = option_.maxNameLength;
+        stbuf->f_namemax = option_.fileSystemOption.maxNameLength;
         return CURVEFS_ERROR::OK;
     }
 
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
+
     void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
         fsInfo_ = fsInfo;
         init_ = true;
@@ -264,9 +294,9 @@ class FuseClient {
         return fsInfo_;
     }
 
-    virtual void FlushInode();
-
-    virtual void FlushInodeAll();
+    std::shared_ptr<FileSystem> GetFileSystem() {
+        return fs_;
+    }
 
     virtual void FlushAll();
 
@@ -304,17 +334,24 @@ class FuseClient {
     void InitQosParam();
 
  protected:
-    CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
-                           mode_t mode, FsFileType type, dev_t rdev,
-                           bool internal, fuse_entry_param* e);
+    CURVEFS_ERROR MakeNode(fuse_req_t req,
+                           fuse_ino_t parent,
+                           const char* name,
+                           mode_t mode,
+                           FsFileType type,
+                           dev_t rdev,
+                           bool internal,
+                           std::shared_ptr<InodeWrapper>& InodeWrapper);  // NOLINT
 
     CURVEFS_ERROR RemoveNode(fuse_req_t req, fuse_ino_t parent,
                              const char* name, FsFileType type);
 
-    CURVEFS_ERROR CreateManageNode(fuse_req_t req, uint64_t parent,
-                                   const char *name, mode_t mode,
+    CURVEFS_ERROR CreateManageNode(fuse_req_t req,
+                                   uint64_t parent,
+                                   const char* name,
+                                   mode_t mode,
                                    ManageInodeType manageType,
-                                   fuse_entry_param *e);
+                                   EntryOut* entryOut);
 
     CURVEFS_ERROR GetOrCreateRecycleDir(fuse_req_t req, Dentry *out);
 
@@ -327,10 +364,17 @@ class FuseClient {
 
     bool ShouldMoveToRecycle(fuse_ino_t parent);
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                             fuse_ino_t newparent, const char* newname,
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char* newname,
                              FsFileType type,
-                             fuse_entry_param* e);
+                             EntryOut* entryOut);
+
+    CURVEFS_ERROR HandleOpenFlags(fuse_req_t req,
+                                  fuse_ino_t ino,
+                                  struct fuse_file_info* fi,
+                                  FileOut* fileOut);
 
     int SetHostPortInMountPoint(Mountpoint* out) {
         char hostname[kMaxHostNameLength];
@@ -345,9 +389,8 @@ class FuseClient {
         return 0;
     }
 
- private:
-    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
 
+ private:
     virtual void FlushData() = 0;
 
     CURVEFS_ERROR UpdateParentMCTimeAndNlink(
@@ -358,8 +401,8 @@ class FuseClient {
         std::string newName(name);
         newName = std::to_string(parent) + "_" + std::to_string(ino)
                     + "_" + newName;
-        if (newName.length() > option_.maxNameLength) {
-            newName = newName.substr(0, option_.maxNameLength);
+        if (newName.length() > option_.fileSystemOption.maxNameLength) {
+            newName = newName.substr(0, option_.fileSystemOption.maxNameLength);
         }
 
         return newName;
@@ -383,9 +426,6 @@ class FuseClient {
 
     std::shared_ptr<LeaseExecutor> leaseExecutor_;
 
-    // dir buffer
-    std::shared_ptr<DirBuffer> dirBuf_;
-
     // filesystem info
     std::shared_ptr<FsInfo> fsInfo_;
 
@@ -404,6 +444,8 @@ class FuseClient {
     // warmup manager
     std::shared_ptr<warmup::WarmupManager> warmupManager_;
 
+    std::shared_ptr<FileSystem> fs_;
+
  private:
     MDSBaseClient* mdsBase_;
 

curvefs/src/client/fuse_s3_client.cpp

@@ -155,8 +155,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
 CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                         const char *buf, size_t size, off_t off,
                                         struct fuse_file_info *fi,
-                                        size_t *wSize) {
-    (void)req;
+                                        FileOut* fileOut) {
+    size_t *wSize = &fileOut->nwritten;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -218,6 +218,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
             }
         }
     }
+
+    inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
     return ret;
 }
 
@@ -280,33 +282,56 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                          const char *name, mode_t mode,
                                          struct fuse_file_info *fi,
-                                         fuse_entry_param *e) {
+                                         EntryOut* entryOut) {
     VLOG(1) << "FuseOpCreate, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
+
+    std::shared_ptr<InodeWrapper> inode;
     CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, e);
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    return FuseOpOpen(req, e->ino, fi);
+
+    auto openFiles = fs_->BorrowMember().openFiles;
+    openFiles->Open(inode->GetInodeId(), inode);
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                        const char *name, mode_t mode,
-                                        dev_t rdev, fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        mode_t mode,
+                                        dev_t rdev,
+                                        EntryOut* entryOut) {
     VLOG(1) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, rdev, false,
-                    e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
+                                FsFileType::TYPE_S3, rdev, false,
+                                inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    *entryOut = EntryOut(attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
-                                     fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req,
+                                       fuse_ino_t ino,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       EntryOut* entryOut) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_S3, e);
+        req, ino, newparent, newname, FsFileType::TYPE_S3, entryOut);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,

curvefs/src/client/fuse_s3_client.h

@@ -81,7 +81,7 @@ class FuseS3Client : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, size_t *wSize) override;
+        struct fuse_file_info *fi, FileOut* fileOut) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
         fuse_ino_t ino, size_t size, off_t off,
@@ -89,17 +89,25 @@ class FuseS3Client : public FuseClient {
         char *buffer,
         size_t *rSize) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, struct fuse_file_info *fi,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                               fuse_ino_t parent,
+                               const char* name,
+                               mode_t mode,
+                               struct fuse_file_info *fi,
+                               EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, dev_t rdev,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                              fuse_ino_t parent,
+                              const char* name,
+                              mode_t mode,
+                              dev_t rdev,
+                              EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-        fuse_ino_t newparent, const char *newname,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char *newname,
+                             EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
         const char *name) override;
@@ -110,11 +118,11 @@ class FuseS3Client : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
         struct fuse_file_info *fi) override;
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
  private:
     bool InitKVCache(const KVClientManagerOpt &opt);
 
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
     void FlushData() override;
 
  private:

curvefs/src/client/fuse_volume_client.cpp

@@ -32,7 +32,7 @@
 #include "absl/cleanup/cleanup.h"
 #include "absl/memory/memory.h"
 #include "curvefs/proto/mds.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/volume/default_volume_storage.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/volume/common.h"
@@ -133,11 +133,11 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
                                             size_t size,
                                             off_t off,
                                             struct fuse_file_info *fi,
-                                            size_t *wSize) {
-    (void)req;
+                                            FileOut* fileOut) {
     VLOG(9) << "write start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
+    size_t* wSize = &fileOut->nwritten;
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
               is_aligned(size, DirectIOAlignment))) {
@@ -149,7 +149,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
     butil::Timer timer;
     timer.start();
 
-    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf);
+    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf, fileOut);
     if (ret != CURVEFS_ERROR::OK) {
         if (fsMetric_) {
             fsMetric_->userWrite.eps.count << 1;
@@ -235,34 +235,57 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
 CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                              const char *name, mode_t mode,
                                              struct fuse_file_info *fi,
-                                             fuse_entry_param *e) {
+                                             EntryOut* entryOut) {
     VLOG(3) << "FuseOpCreate, parent: " << parent
               << ", name: " << name
               << ", mode: " << mode;
-    CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, false, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR ret = MakeNode(
+        req, parent, name, mode, FsFileType::TYPE_FILE, 0, false, inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    return FuseOpOpen(req, e->ino, fi);
+
+    auto openFiles = fs_->BorrowMember().openFiles;
+    openFiles->Open(inode->GetInodeId(), inode);
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                            const char *name, mode_t mode,
-                                            dev_t rdev, fuse_entry_param *e) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req,
+                                            fuse_ino_t parent,
+                                            const char* name,
+                                            mode_t mode,
+                                            dev_t rdev,
+                                            EntryOut* entryOut) {
     VLOG(3) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, rdev,
-                    false, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
+                                FsFileType::TYPE_FILE, rdev,
+                                false, inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    *entryOut = EntryOut(attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
-                                     fuse_entry_param *e) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req,
+                                           fuse_ino_t ino,
+                                           fuse_ino_t newparent,
+                                           const char* newname,
+                                           EntryOut* entryOut) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_FILE, e);
+        req, ino, newparent, newname, FsFileType::TYPE_FILE, entryOut);
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,

curvefs/src/client/fuse_volume_client.h

@@ -65,24 +65,32 @@ class FuseVolumeClient : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, size_t *wSize) override;
+        struct fuse_file_info *fi, FileOut* fileOut) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
             fuse_ino_t ino, size_t size, off_t off,
             struct fuse_file_info *fi,
             char *buffer,
             size_t *rSize) override;
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, struct fuse_file_info *fi,
-        fuse_entry_param *e) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, dev_t rdev,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                               fuse_ino_t parent,
+                               const char* name,
+                               mode_t mode,
+                               struct fuse_file_info* fi,
+                               EntryOut* entryOut) override;
+
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                              fuse_ino_t parent,
+                              const char* name,
+                              mode_t mode,
+                              dev_t rdev,
+                              EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-        fuse_ino_t newparent, const char *newname,
-        fuse_entry_param *e) override;
+                             fuse_ino_t newparent,
+                             const char *newname,
+                             EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                const char *name) override;
@@ -93,13 +101,13 @@ class FuseVolumeClient : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                               struct fuse_file_info *fi) override;
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
     void SetSpaceManagerForTesting(SpaceManager *manager);
 
     void SetVolumeStorageForTesting(VolumeStorage *storage);
 
  private:
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
     void FlushData() override;
 
  private:

curvefs/src/client/inode_cache_manager.cpp

@@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 using ::curvefs::metaserver::Inode;
@@ -46,36 +46,11 @@ DECLARE_bool(enableCto);
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::ToFSError;
+
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
 using curvefs::client::common::FLAGS_enableCto;
 
-class TrimICacheAsyncDone : public MetaServerClientDone {
- public:
-    explicit TrimICacheAsyncDone(
-        const std::shared_ptr<InodeWrapper> &inodeWrapper,
-        const std::shared_ptr<InodeCacheManagerImpl> &inodeCacheManager)
-        : inodeWrapper_(inodeWrapper), inodeCacheManager_(inodeCacheManager) {}
-
-    void Run() override {
-        std::unique_ptr<TrimICacheAsyncDone> self_guard(this);
-        MetaStatusCode ret = GetStatusCode();
-        if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
-            LOG(ERROR) << "metaClient_ UpdateInode failed, "
-                       << "MetaStatusCode: " << ret
-                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
-                       << ", inodeid: " << inodeWrapper_->GetInodeId();
-            inodeWrapper_->MarkInodeError();
-        }
-        VLOG(9) << "Trim inode " << inodeWrapper_->GetInodeId()
-                << " async success.";
-        inodeCacheManager_->RemoveICache(inodeWrapper_);
-    };
-
- private:
-    std::shared_ptr<InodeWrapper> inodeWrapper_;
-    std::shared_ptr<InodeCacheManagerImpl> inodeCacheManager_;
-};
-
 #define GET_INODE_REMOTE(FSID, INODEID, OUT, STREAMING)                        \
     MetaStatusCode ret = metaClient_->GetInode(FSID, INODEID, OUT, STREAMING); \
     if (ret != MetaStatusCode::OK) {                                           \
@@ -83,17 +58,7 @@ class TrimICacheAsyncDone : public MetaServerClientDone {
             << "metaClient_ GetInode failed, MetaStatusCode = " << ret         \
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)          \
             << ", inodeid = " << INODEID;                                      \
-        return MetaStatusCodeToCurvefsErrCode(ret);                            \
-    }
-
-#define PUT_INODE_CACHE(INODEID, INODEWRAPPER)                                 \
-    std::shared_ptr<InodeWrapper> eliminatedOne;                               \
-    bool eliminated = iCache_->Put(INODEID, INODEWRAPPER, &eliminatedOne);     \
-    if (eliminated) {                                                          \
-        VLOG(3) << "GetInode eliminate one inode, ino: "                       \
-                << eliminatedOne->GetInodeId()                                 \
-                << ", iCache does not evict inodes via put interface ";        \
-        assert(0);                                                             \
+        return ToFSError(ret);                            \
     }
 
 #define REFRESH_DATA_REMOTE(OUT, STREAMING)                                    \
@@ -106,15 +71,8 @@ CURVEFS_ERROR
 InodeCacheManagerImpl::GetInode(uint64_t inodeId,
                                 std::shared_ptr<InodeWrapper> &out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    // get inode from cache
-    bool ok = iCache_->Get(inodeId, &out);
-    if (ok && NeedUseCache(inodeId, out, false)) {
-        curve::common::UniqueLock lgGuard = out->GetUniqueLock();
-        if (out->GetType() == FsFileType::TYPE_FILE) {
-            return CURVEFS_ERROR::OK;
-        }
-
-        REFRESH_DATA_REMOTE(out, out->NeedRefreshData());
+    bool yes = openFiles_->IsOpened(inodeId, &out);
+    if (yes) {
         return CURVEFS_ERROR::OK;
     }
 
@@ -129,24 +87,12 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     // refresh data
     REFRESH_DATA_REMOTE(out, streaming);
 
-    // put to cache
-    PUT_INODE_CACHE(inodeId, out);
-
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
                                                   InodeAttr *out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    // 1. find in icache
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    bool ok = iCache_->Get(inodeId, &inodeWrapper);
-    if (ok && NeedUseCache(inodeId, inodeWrapper, true)) {
-        inodeWrapper->GetInodeAttr(out);
-        return CURVEFS_ERROR::OK;
-    }
-
-    // 2. get form metaserver
     std::set<uint64_t> inodeIds;
     std::list<InodeAttr> attrs;
     inodeIds.emplace(inodeId);
@@ -156,7 +102,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed"
                    << ", inodeId = " << inodeId << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
     if (attrs.size() != 1) {
@@ -173,21 +119,6 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
     std::set<uint64_t> *inodeIds,
     std::list<InodeAttr> *attrs) {
-    // get some inode attr in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
-            InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace_back(std::move(tmpAttr));
-            iter = inodeIds->erase(iter);
-            continue;
-        }
-        ++iter;
-    }
-
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -199,7 +130,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return MetaStatusCodeToCurvefsErrCode(ret);
+    return ToFSError(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
@@ -207,26 +138,6 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     std::set<uint64_t> *inodeIds,
     std::map<uint64_t, InodeAttr> *attrs) {
     NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    std::map<uint64_t, InodeAttr> cachedAttr;
-    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
-
-    // get some inode attr in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
-            InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace(*iter, std::move(tmpAttr));
-            iter = inodeIds->erase(iter);
-        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
-            attrs->emplace(*iter, cachedAttr[*iter]);
-            iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
-        }
-    }
 
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
@@ -238,12 +149,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
         return CURVEFS_ERROR::NOTEXIST;
     }
 
+    ::curve::common::Mutex mutex;
     std::shared_ptr<CountDownEvent> cond =
         std::make_shared<CountDownEvent>(inodeGroups.size());
     for (const auto& it : inodeGroups) {
         VLOG(3) << "BatchGetInodeAttrAsync Send " << it.size();
-        auto* done = new BatchGetInodeAttrAsyncDone(shared_from_this(),
-                                                    cond, parentId);
+        auto* done = new BatchGetInodeAttrAsyncDone(attrs, &mutex, cond);
         MetaStatusCode ret = metaClient_->BatchGetInodeAttrAsync(fsId_, it,
                                                                  done);
         if (MetaStatusCode::OK != ret) {
@@ -256,30 +167,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
 
     // wait for all sudrequest finished
     cond->Wait();
-
-    bool ok  = iAttrCache_->Get(parentId, attrs);
-    if (!ok) {
-        LOG(WARNING) << "get attrs form iAttrCache_ failed.";
-    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
     std::set<uint64_t> *inodeIds,
     std::list<XAttr> *xattr) {
-    // get some inode in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
-            xattr->emplace_back(inodeWrapper->GetXattr());
-            iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
-        }
-    }
-
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -290,7 +183,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return MetaStatusCodeToCurvefsErrCode(ret);
+    return ToFSError(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
@@ -301,23 +194,11 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ CreateInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-    uint64_t inodeid = inode.inodeid();
     out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
         s3ChunkInfoMetric_, option_.maxDataSize,
         option_.refreshDataIntervalSec);
-
-    std::shared_ptr<InodeWrapper> eliminatedOne;
-    bool eliminated = false;
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeid));
-        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
-    }
-    if (eliminated) {
-        /* iCache does not evict inodes via put interface */
-        assert(0);
-    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -330,159 +211,29 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateManageInode(
         LOG(ERROR) << "metaClient_ CreateManageInode failed, MetaStatusCode = "
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-    uint64_t inodeid = inode.inodeid();
     out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
         s3ChunkInfoMetric_, option_.maxDataSize,
         option_.refreshDataIntervalSec);
-
-    std::shared_ptr<InodeWrapper> eliminatedOne;
-    bool eliminated = false;
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeid));
-        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
-    }
-    if (eliminated) {
-        /* iCache does not evict inodes via put interface */
-        assert(0);
-    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::DeleteInode(uint64_t inodeId) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    iCache_->Remove(inodeId);
     MetaStatusCode ret = metaClient_->DeleteInode(fsId_, inodeId);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeId = " << inodeId;
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-
-    curve::common::LockGuard lg2(dirtyMapMutex_);
-    dirtyMap_.erase(inodeId);
     return CURVEFS_ERROR::OK;
 }
 
-void InodeCacheManagerImpl::AddInodeAttrs(
-    uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
-    iAttrCache_->Set(parentId, inodeAttrs);
-}
-
-void InodeCacheManagerImpl::ClearInodeCache(uint64_t inodeId) {
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeId));
-        iCache_->Remove(inodeId);
-    }
-    curve::common::LockGuard lg2(dirtyMapMutex_);
-    dirtyMap_.erase(inodeId);
-}
-
 void InodeCacheManagerImpl::ShipToFlush(
-    const std::shared_ptr<InodeWrapper> &inodeWrapper) {
-    curve::common::LockGuard lg(dirtyMapMutex_);
-    dirtyMap_.emplace(inodeWrapper->GetInodeId(), inodeWrapper);
-}
-
-void InodeCacheManagerImpl::FlushAll() {
-    while (!dirtyMap_.empty()) {
-        FlushInodeOnce();
-    }
-}
-
-void InodeCacheManagerImpl::FlushInodeOnce() {
-    std::map<uint64_t, std::shared_ptr<InodeWrapper>> temp_;
-    {
-        curve::common::LockGuard lg(dirtyMapMutex_);
-        temp_.swap(dirtyMap_);
-    }
-    for (auto it = temp_.begin(); it != temp_.end(); it++) {
-        curve::common::UniqueLock ulk = it->second->GetUniqueLock();
-        it->second->Async(nullptr, true);
-    }
-}
-
-void InodeCacheManagerImpl::ReleaseCache(uint64_t parentId) {
-    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    iAttrCache_->Release(parentId);
-}
-
-void InodeCacheManagerImpl::FlushInodeBackground() {
-    LOG(INFO) << "flush thread is start.";
-    while (!isStop_.load()) {
-        if (iCache_->Size() > maxCacheSize_) {
-            TrimIcache(iCache_->Size() - maxCacheSize_);
-        }
-        FlushInodeOnce();
-        sleeper_.wait_for(std::chrono::seconds(flushPeriodSec_));
-    }
-    LOG(INFO) << "flush thread is stop.";
-}
-
-namespace {
-// Wether an inode is dirty or not.
-// if |needLock| is true, we acquire the lock firstly and do the test,
-// otherwise, we assume the lock is already held.
-bool IsDirtyInode(InodeWrapper *ino, bool needLock) {
-    auto check = [ino]() {
-        return ino->IsDirty() || !ino->S3ChunkInfoEmptyNolock() ||
-               ino->GetMutableExtentCacheLocked()->HasDirtyExtents();
-    };
-
-    if (needLock) {
-        auto lk = ino->GetUniqueLock();
-        return check();
-    }
-
-    return check();
-}
-}  // namespace
-
-void InodeCacheManagerImpl::RemoveICache(
-    const std::shared_ptr<InodeWrapper> &inode) {
-    if (!IsDirtyInode(inode.get(), true)) {
-        uint64_t inodeId = inode->GetInodeId();
-        NameLockGuard lock(nameLock_, std::to_string(inodeId));
-        ::curve::common::UniqueLock lgGuard = inode->GetUniqueLock();
-        iCache_->Remove(inodeId);
-    }
-}
-
-void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    uint64_t inodeId;
-    VLOG(3) << "TrimIcache trimSize " << trimSize;
-    while (trimSize > 0) {
-        bool ok = iCache_->GetLast(&inodeId, &inodeWrapper);
-        if (ok) {
-            NameLockGuard lock(nameLock_, std::to_string(inodeId));
-            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-            if (IsDirtyInode(inodeWrapper.get(), false)) {
-                VLOG(9) << "TrimIcache sync dirty inode " << inodeId;
-                dirtyMapMutex_.lock();
-                dirtyMap_.erase(inodeId);
-                dirtyMapMutex_.unlock();
-                auto *done =
-                    new TrimICacheAsyncDone(inodeWrapper, shared_from_this());
-                inodeWrapper->Async(done);
-            } else {
-                VLOG(9) << "TrimIcache remove inode " << inodeId
-                        << " from iCache";
-                iCache_->Remove(inodeId);
-            }
-            trimSize--;
-            // remove the attr of the inode in iattrcache
-            auto parents = inodeWrapper->GetParentLocked();
-            for (uint64_t parent : parents) {
-                iAttrCache_->Remove(parent, inodeId);
-            }
-        } else {
-            LOG(ERROR) << "icache size " << iCache_->Size();
-            assert(0);
-        }
-    }
+    const std::shared_ptr<InodeWrapper>& inode) {
+    deferSync_->Push(inode);
 }
 
 CURVEFS_ERROR
@@ -519,57 +270,5 @@ InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
     return rc;
 }
 
-void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
-    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
-    curve::common::LockGuard lg(openInodesMutex_);
-    openedInodes_.emplace(inodeId);
-}
-
-void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
-    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
-    curve::common::LockGuard lg(openInodesMutex_);
-    auto iter = openedInodes_.find(inodeId);
-    if (iter != openedInodes_.end()) {
-        openedInodes_.erase(iter);
-    }
-}
-
-bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
-    curve::common::LockGuard lg(openInodesMutex_);
-    auto iter = openedInodes_.find(inodeId);
-    return iter != openedInodes_.end();
-}
-
-bool InodeCacheManagerImpl::NeedUseCache(uint64_t inodeId,
-    const std::shared_ptr<InodeWrapper> &inodeWrapper,
-    bool onlyAttr) {
-    auto lock = inodeWrapper->GetUniqueLock();
-    if (onlyAttr) {
-        if (inodeWrapper->IsDirty()) {
-            return true;
-        }
-    } else {
-        if (IsDirtyInode(inodeWrapper.get(), false)) {
-            return true;
-        }
-    }
-
-    if (((FLAGS_enableCto && OpenInodeCached(inodeId)) || !FLAGS_enableCto)
-        && !IsTimeOut(inodeWrapper)) {
-        return true;
-    }
-    return false;
-}
-
-bool InodeCacheManagerImpl::IsTimeOut(
-    const std::shared_ptr<InodeWrapper> &inodeWrapper) const {
-    uint32_t time = inodeWrapper->GetCachedTime();
-    if (cacheTimeOutSec_ > 0 &&
-        TimeUtility::GetTimeofDaySec() - time >= cacheTimeOutSec_) {
-        return true;
-    }
-    return false;
-}
-
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_cache_manager.h

@@ -38,12 +38,14 @@
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "src/common/concurrent/name_lock.h"
 #include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
 
 using ::curve::common::LRUCache;
 using ::curve::common::CacheMetrics;
@@ -64,57 +66,8 @@ using rpcclient::BatchGetInodeAttrDone;
 using curve::common::CountDownEvent;
 using metric::S3ChunkInfoMetric;
 using common::RefreshDataOption;
-
-class InodeAttrCache {
- public:
-    InodeAttrCache() {}
-    ~InodeAttrCache() {}
-
-    bool Get(uint64_t parentId, std::map<uint64_t, InodeAttr> *imap) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            imap->insert(iter->second.begin(), iter->second.end());
-            return true;
-        }
-        return false;
-    }
-
-    void Set(uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        VLOG(1) << "parentId = " << parentId
-                << ", iAttrCache set size = " << inodeAttrs.size();
-        auto& inner = iAttrCache_[parentId];
-        for (const auto &it : inodeAttrs) {
-            inner.emplace(it.inodeid(), it);
-        }
-    }
-
-    void Release(uint64_t parentId) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto size = iAttrCache_.size();
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            iAttrCache_.erase(iter);
-        }
-        VLOG(1) << "inodeId = " << parentId
-                << ", inode attr cache release, before = "
-                << size << ", after = " << iAttrCache_.size();
-    }
-
-    void Remove(uint64_t parentId, uint64_t inodeId) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            iter->second.erase(inodeId);
-        }
-    }
-
- private:
-    // inodeAttr cache; <parentId, <inodeId, inodeAttr>>
-    std::map<uint64_t, std::map<uint64_t, InodeAttr>> iAttrCache_;
-    curve::common::Mutex iAttrCacheMutex_;
-};
+using ::curvefs::client::filesystem::OpenFiles;
+using ::curvefs::client::filesystem::DeferSync;
 
 class InodeCacheManager {
  public:
@@ -126,14 +79,9 @@ class InodeCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-                               uint32_t flushPeriodSec,
-                               RefreshDataOption option,
-                               uint32_t cacheTimeOutSec) = 0;
-
-    virtual void Run() = 0;
-
-    virtual void Stop() = 0;
+    virtual CURVEFS_ERROR Init(RefreshDataOption option,
+                               std::shared_ptr<OpenFiles> openFiles,
+                               std::shared_ptr<DeferSync> deferSync) = 0;
 
     virtual CURVEFS_ERROR
     GetInode(uint64_t inodeId,
@@ -161,24 +109,9 @@ class InodeCacheManager {
 
     virtual CURVEFS_ERROR DeleteInode(uint64_t inodeId) = 0;
 
-    virtual void AddInodeAttrs(uint64_t parentId,
-        const RepeatedPtrField<InodeAttr>& inodeAttrs) = 0;
-
-    virtual void ClearInodeCache(uint64_t inodeId) = 0;
-
     virtual void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) = 0;
 
-    virtual void FlushAll() = 0;
-
-    virtual void FlushInodeOnce() = 0;
-
-    virtual void ReleaseCache(uint64_t parentId) = 0;
-
-    virtual void AddOpenedInode(uint64_t inodeId) = 0;
-
-    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
-
  protected:
     uint32_t fsId_;
 };
@@ -187,59 +120,22 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     public std::enable_shared_from_this<InodeCacheManagerImpl> {
  public:
     InodeCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()),
-        iCache_(nullptr),
-        iAttrCache_(nullptr),
-        isStop_(true),
-        cacheTimeOutSec_(0) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
 
     explicit InodeCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient),
-        iCache_(nullptr),
-        iAttrCache_(nullptr),
-        cacheTimeOutSec_(0) {}
+      : metaClient_(metaClient) {}
 
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-                       uint32_t flushPeriodSec,
-                       RefreshDataOption option,
-                       uint32_t cacheTimeOutSec) override {
-        if (enableCacheMetrics) {
-            iCache_ = std::make_shared<
-                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0,
-                    std::make_shared<CacheMetrics>("icache"));
-        } else {
-            iCache_ = std::make_shared<
-                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0);
-        }
-        maxCacheSize_ = cacheSize;
+    CURVEFS_ERROR Init(RefreshDataOption option,
+                       std::shared_ptr<OpenFiles> openFiles,
+                       std::shared_ptr<DeferSync> deferSync) override {
         option_ = option;
-        flushPeriodSec_ = flushPeriodSec;
-        cacheTimeOutSec_ = cacheTimeOutSec;
-        iAttrCache_ = std::make_shared<InodeAttrCache>();
         s3ChunkInfoMetric_ = std::make_shared<S3ChunkInfoMetric>();
+        openFiles_ =  openFiles;
+        deferSync_ = deferSync;
         return CURVEFS_ERROR::OK;
     }
 
-    void Run() {
-        isStop_.exchange(false);
-        flushThread_ =
-            Thread(&InodeCacheManagerImpl::FlushInodeBackground, this);
-        LOG(INFO) << "Start inodeManager flush thread ok.";
-    }
-
-    void Stop() {
-        isStop_.exchange(true);
-        LOG(INFO) << "stop inodeManager flush thread ...";
-        sleeper_.interrupt();
-        flushThread_.join();
-    }
-
-    bool IsDirtyMapExist(uint64_t inodeId) {
-        curve::common::LockGuard lg(dirtyMapMutex_);
-        return dirtyMap_.count(inodeId) > 0;
-    }
-
     CURVEFS_ERROR GetInode(uint64_t inodeId,
                            std::shared_ptr<InodeWrapper> &out) override;
 
@@ -263,78 +159,37 @@ class InodeCacheManagerImpl : public InodeCacheManager,
 
     CURVEFS_ERROR DeleteInode(uint64_t inodeId) override;
 
-    void AddInodeAttrs(uint64_t parentId,
-        const RepeatedPtrField<InodeAttr>& inodeAttrs) override;
-
-    void ClearInodeCache(uint64_t inodeId) override;
-
     void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) override;
 
-    void FlushAll() override;
-
-    void FlushInodeOnce() override;
-
-    void ReleaseCache(uint64_t parentId) override;
-
-    void RemoveICache(const std::shared_ptr<InodeWrapper> &inode);
-
-    void AddOpenedInode(uint64_t inodeId) override;
-
-    void RemoveOpenedInode(uint64_t inodeId) override;
-
-    bool NeedUseCache(uint64_t inodeId,
-        const std::shared_ptr<InodeWrapper> &inodeWrapper,
-        bool onlyAttr);
-
-    bool IsTimeOut(const std::shared_ptr<InodeWrapper> &inodeWrapper) const;
-
  private:
-    virtual void FlushInodeBackground();
-    void TrimIcache(uint64_t trimSize);
     CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
                               bool streaming = true);
-    bool OpenInodeCached(uint64_t inodeId);
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    std::shared_ptr<LRUCache<uint64_t,
-        std::shared_ptr<InodeWrapper>>> iCache_;
     std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric_;
 
-    std::shared_ptr<InodeAttrCache> iAttrCache_;
+    std::shared_ptr<OpenFiles> openFiles_;
 
-    // dirty map, key is inodeid
-    std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
-    curve::common::Mutex dirtyMapMutex_;
-
-    // record opened inode
-    std::multiset<uint64_t> openedInodes_;
-    curve::common::Mutex openInodesMutex_;
+    std::shared_ptr<DeferSync> deferSync_;
 
     curve::common::GenericNameLock<Mutex> nameLock_;
 
     curve::common::GenericNameLock<Mutex> asyncNameLock_;
 
-    uint64_t maxCacheSize_;
     RefreshDataOption option_;
-    uint32_t flushPeriodSec_;
-    Thread flushThread_;
-    InterruptibleSleeper sleeper_;
-    Atomic<bool> isStop_;
-    // cache timeout seconds, 0 means never timeout
-    uint32_t cacheTimeOutSec_;
 };
 
 class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
  public:
-    BatchGetInodeAttrAsyncDone(
-        const std::shared_ptr<InodeCacheManager> &inodeCacheManager,
-        std::shared_ptr<CountDownEvent> cond,
-        uint64_t parentId):
-        inodeCacheManager_(inodeCacheManager),
-        cond_(cond),
-        parentId_(parentId) {}
+    BatchGetInodeAttrAsyncDone(std::map<uint64_t, InodeAttr>* attrs,
+                               ::curve::common::Mutex* mutex,
+                               std::shared_ptr<CountDownEvent> cond):
+        attrs_(attrs),
+        mutex_(mutex),
+        cond_(cond) {}
+
     ~BatchGetInodeAttrAsyncDone() {}
 
     void Run() override {
@@ -342,22 +197,25 @@ class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
         MetaStatusCode ret = GetStatusCode();
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "BatchGetInodeAttrAsync failed, "
-                       << "parentId = " << parentId_
                        << ", MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret);
         } else {
             auto inodeAttrs = GetInodeAttrs();
             VLOG(3) << "BatchGetInodeAttrAsyncDone update inodeAttrCache"
                     << " size = " << inodeAttrs.size();
-            inodeCacheManager_->AddInodeAttrs(parentId_, inodeAttrs);
+
+            curve::common::LockGuard lk(*mutex_);
+            for (const auto& attr : inodeAttrs) {
+                attrs_->emplace(attr.inodeid(), attr);
+            }
         }
         cond_->Signal();
     };
 
  private:
-    std::shared_ptr<InodeCacheManager> inodeCacheManager_;
+    ::curve::common::Mutex* mutex_;
+    std::map<uint64_t, InodeAttr>* attrs_;
     std::shared_ptr<CountDownEvent> cond_;
-    uint64_t parentId_;
 };
 
 }  // namespace client

curvefs/src/client/inode_wrapper.cpp

@@ -33,7 +33,6 @@
 
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/async_request_closure.h"
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
@@ -47,6 +46,7 @@ namespace client {
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::DataIndices;
+using ::curvefs::client::filesystem::ToFSError;
 
 bvar::Adder<int64_t> g_alive_inode_count{"alive_inode_count"};
 
@@ -193,7 +193,7 @@ CURVEFS_ERROR InodeWrapper::SyncAttr(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
 
         dirty_ = false;
@@ -213,7 +213,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3ChunkInfo(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         ClearS3ChunkInfoAdd();
     }
@@ -294,7 +294,7 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
                    << "MetaStatusCode: " << ret
                    << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                    << ", inodeid: " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     auto before = s3ChunkInfoSize_;
     inode_.mutable_s3chunkinfomap()->swap(s3ChunkInfoMap);
@@ -328,7 +328,7 @@ CURVEFS_ERROR InodeWrapper::Link(uint64_t parent) {
                    <<", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
     dirty_ = false;
@@ -396,7 +396,7 @@ CURVEFS_ERROR InodeWrapper::UnLink(uint64_t parent) {
                        << ", MetaStatusCode = " << ret
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", inodeid = " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         dirty_ = false;
         dirtyAttr_.Clear();
@@ -427,7 +427,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParent(
                    << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     dirty_ = false;
     dirtyAttr_.Clear();
@@ -514,7 +514,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         ClearS3ChunkInfoAdd();
         dirty_ = false;
@@ -597,7 +597,7 @@ CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
         LOG(ERROR) << "GetVolumeExtent failed, inodeid: " << inode_.inodeid();
     }
 
-    return MetaStatusCodeToCurvefsErrCode(st);
+    return ToFSError(st);
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshNlink() {

curvefs/src/client/inode_wrapper.h

@@ -34,12 +34,12 @@
 
 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/timeutility.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Inode;
 using ::curvefs::metaserver::S3ChunkInfoList;
@@ -53,15 +53,13 @@ constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;
 
 using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 enum class InodeStatus {
     kNormal = 0,
     kError = -1,
 };
 
-// TODO(xuchaojie) : get from conf maybe?
-const uint32_t kOptimalIOBlockSize = 0x10000u;
-
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;

curvefs/src/client/main.cpp

@@ -51,13 +51,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     release : FuseOpRelease,
     fsync : FuseOpFsync,
     opendir : FuseOpOpenDir,
-    // TODO(wuhongsong): readdirplus is problematic,
-    // resulting in inconsistent metadata
-    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    // readdir : 0,
-    // #else
+    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    readdir : 0,
+    #else
     readdir : FuseOpReadDir,
-    // #endif
+    #endif
     releasedir : FuseOpReleaseDir,
     fsyncdir : 0,
     statfs : FuseOpStatFs,
@@ -81,11 +79,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     flock : 0,
     fallocate : 0,
     #endif
-    // TODO(wuhongsong): The current implementation is problematic,
-    // resulting in inconsistent metadata
-    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    readdirplus : FuseOpReadDirPlus,
+    #else
     readdirplus : 0,
-    // #endif
+    #endif
     #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 4)
     copy_file_range : 0,
     #endif
@@ -222,8 +220,8 @@ int main(int argc, char *argv[]) {
 
     fuse_daemonize(opts.foreground);
 
-    if (InitGlog(mOpts.conf, argv[0]) < 0) {
-        printf("Init glog failed, confpath = %s\n", mOpts.conf);
+    if (InitLog(mOpts.conf, argv[0]) < 0) {
+        printf("Init log failed, confpath = %s\n", mOpts.conf);
     }
 
     ret = InitFuseClient(&mOpts);

curvefs/src/client/s3/client_s3_adaptor.h

@@ -33,7 +33,7 @@
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/s3/client_s3.h"

curvefs/src/client/s3/client_s3_cache_manager.h

@@ -34,7 +34,7 @@
 #include <set>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/s3/client_s3.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/concurrent/task_thread_pool.h"

curvefs/src/client/volume/default_volume_storage.cpp

@@ -30,7 +30,7 @@
 #include <vector>
 
 #include "absl/meta/type_traits.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/volume/extent_cache.h"
@@ -74,7 +74,8 @@ std::ostream& operator<<(std::ostream& os, const std::vector<IOPart>& iov) {
 CURVEFS_ERROR DefaultVolumeStorage::Write(uint64_t ino,
                                           off_t offset,
                                           size_t len,
-                                          const char* data) {
+                                          const char* data,
+                                          FileOut* fileOut) {
     std::shared_ptr<InodeWrapper> inodeWrapper;
     LatencyUpdater updater(&metric_.writeLatency);
     auto ret = inodeCacheManager_->GetInode(ino, inodeWrapper);
@@ -116,6 +117,7 @@ CURVEFS_ERROR DefaultVolumeStorage::Write(uint64_t ino,
         }
 
         inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
+        inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
     }
 
     inodeCacheManager_->ShipToFlush(inodeWrapper);

curvefs/src/client/volume/default_volume_storage.h

@@ -31,12 +31,16 @@
 #include "curvefs/src/client/volume/volume_storage.h"
 #include "curvefs/src/volume/block_device_client.h"
 #include "curvefs/src/volume/space_manager.h"
+#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/filesystem/meta.h"
 
 namespace curvefs {
 namespace client {
 
 using ::curvefs::volume::BlockDeviceClient;
 using ::curvefs::volume::SpaceManager;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
+using ::curvefs::client::filesystem::FileOut;
 
 class InodeCacheManager;
 
@@ -66,7 +70,8 @@ class DefaultVolumeStorage final : public VolumeStorage {
     CURVEFS_ERROR Write(uint64_t ino,
                         off_t offset,
                         size_t len,
-                        const char* data) override;
+                        const char* data,
+                        FileOut* fileOut) override;
 
     CURVEFS_ERROR Flush(uint64_t ino) override;
 

curvefs/src/client/volume/volume_storage.h

@@ -28,11 +28,15 @@
 #include <cstddef>
 #include <cstdint>
 
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/filesystem/meta.h"
 
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
+using ::curvefs::client::filesystem::FileOut;
+
 class VolumeStorage {
  public:
     virtual ~VolumeStorage() = default;
@@ -45,7 +49,8 @@ class VolumeStorage {
     virtual CURVEFS_ERROR Write(uint64_t ino,
                                 off_t offset,
                                 size_t len,
-                                const char* data) = 0;
+                                const char* data,
+                                FileOut* fileOut) = 0;
 
     virtual CURVEFS_ERROR Flush(uint64_t ino) = 0;
 

curvefs/src/client/warmup/warmup_manager.cpp

@@ -473,7 +473,7 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
             }
             if (context->retCode == 0) {
                 VLOG(9) << "Get Object success: " << context->key;
-                PutObjectToCache(key, context->key, context->buf, context->len);
+                PutObjectToCache(key, context);
                 CollectMetrics(&warmupS3Metric_.warmupS3Cached, context->len,
                                start);
                 warmupS3Metric_.warmupS3CacheSize << context->len;
@@ -481,7 +481,6 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
                     VLOG(6) << "pendingReq is over";
                     cond.Signal();
                 }
-                delete[] context->buf;
                 return;
             }
             warmupS3Metric_.warmupS3Cached.eps.count << 1;
@@ -678,9 +677,8 @@ void WarmupManagerS3Impl::AddFetchS3objectsTask(fuse_ino_t key,
     }
 }
 
-void WarmupManagerS3Impl::PutObjectToCache(fuse_ino_t key,
-                                           const std::string &filename,
-                                           const char *data, uint64_t len) {
+void WarmupManagerS3Impl::PutObjectToCache(
+    fuse_ino_t key, const std::shared_ptr<GetObjectAsyncContext> &context) {
     ReadLockGuard lock(inode2ProgressMutex_);
     auto iter = FindWarmupProgressByKeyLocked(key);
     if (iter == inode2Progress_.end()) {
@@ -692,17 +690,21 @@ void WarmupManagerS3Impl::PutObjectToCache(fuse_ino_t key,
     iter->second.FinishedPlusOne();
     switch (iter->second.GetStorageType()) {
     case curvefs::client::common::WarmupStorageType::kWarmupStorageTypeDisk:
-        ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(filename, data,
-                                                                 len);
+        ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(
+            context->key, context->buf, context->len);
         if (ret < 0) {
             LOG_EVERY_SECOND(INFO)
-                << "write read directly failed, key: " << filename;
+                << "write read directly failed, key: " << context->key;
         }
+        delete[] context->buf;
         break;
     case curvefs::client::common::WarmupStorageType::kWarmupStorageTypeKvClient:
         if (kvClientManager_ != nullptr) {
-            kvClientManager_->Set(
-                std::make_shared<SetKVCacheTask>(filename, data, len));
+            kvClientManager_->Set(std::make_shared<SetKVCacheTask>(
+                context->key, context->buf, context->len,
+                [context](const std::shared_ptr<SetKVCacheTask> &) {
+                    delete[] context->buf;
+                }));
         }
         break;
     default:

curvefs/src/client/warmup/warmup_manager.h

@@ -413,8 +413,9 @@ class WarmupManagerS3Impl : public WarmupManager {
 
     void AddFetchS3objectsTask(fuse_ino_t key, std::function<void()> task);
 
-    void PutObjectToCache(fuse_ino_t key, const std::string &filename,
-                          const char *data, uint64_t len);
+    void
+    PutObjectToCache(fuse_ino_t key,
+                     const std::shared_ptr<GetObjectAsyncContext> &context);
 
  protected:
     std::deque<WarmupFilelist> warmupFilelistDeque_;

curvefs/src/client/xattr_manager.h

@@ -36,7 +36,7 @@
 
 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/client_operator.h"
 #include "src/common/interruptible_sleeper.h"
 

curvefs/src/mds/fs_storage.cpp

@@ -359,6 +359,15 @@ bool PersisKVStorage::LoadAllFs() {
 
         idToName_.emplace(fsInfo.fsid(), fsInfo.fsname());
 
+        // For compatibility when upgrading, the new field 'optional
+        // objectPrefix` in message `S3Info` needs to be set to the default
+        // value
+        if (fsInfo.fstype() == FSType::TYPE_S3) {
+            if (!fsInfo.detail().s3info().has_objectprefix()) {
+                fsInfo.mutable_detail()->mutable_s3info()->set_objectprefix(0);
+            }
+        }
+
         fs_.emplace(fsInfo.fsname(), std::move(fsInfo));
     }
 

curvefs/src/mds/mds.cpp

@@ -66,6 +66,8 @@ void MDS::InitOptions(std::shared_ptr<Configuration> conf) {
     conf_ = std::move(conf);
     conf_->GetValueFatalIfFail("mds.listen.addr", &options_.mdsListenAddr);
     conf_->GetValueFatalIfFail("mds.dummy.port", &options_.dummyPort);
+    conf_->GetValueFatalIfFail("mds.server.idleTimeoutSec",
+                               &options_.idleTimeoutSec);
 
     InitMetaServerOption(&options_.metaserverOptions);
     InitTopologyOption(&options_.topologyOptions);
@@ -275,6 +277,7 @@ void MDS::Run() {
 
     // start rpc server
     brpc::ServerOptions option;
+    option.idle_timeout_sec = options_.idleTimeoutSec;
     LOG_IF(FATAL, server.Start(options_.mdsListenAddr.c_str(), &option) != 0)
         << "start brpc server error";
     running_ = true;

curvefs/src/mds/mds.h

@@ -85,6 +85,8 @@ using ::curvefs::mds::space::MdsProxyOptions;
 struct MDSOptions {
     int dummyPort;
     std::string mdsListenAddr;
+    int idleTimeoutSec;
+
     MetaserverOptions metaserverOptions;
     // TODO(add EtcdConf): add etcd configure
 

curvefs/src/metaserver/copyset/apply_queue.cpp

@@ -1,135 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: curve
- * Date: Thu Sep  2 14:49:04 CST 2021
- * Author: wuhanqing
- */
-
-#include "curvefs/src/metaserver/copyset/apply_queue.h"
-
-#include <glog/logging.h>
-
-#include "absl/memory/memory.h"
-#include "absl/strings/str_cat.h"
-#include "curvefs/src/common/threading.h"
-#include "curvefs/src/metaserver/copyset/copyset_node.h"
-
-namespace curvefs {
-namespace metaserver {
-namespace copyset {
-
-using ::curvefs::common::SetThreadName;
-
-void ApplyQueue::StartWorkers() {
-    for (uint32_t i = 0; i < option_.workerCount; ++i) {
-        std::string name = [this, i]() -> std::string {
-            if (option_.copysetNode == nullptr) {
-                return "apply";
-            }
-
-            return absl::StrCat("apply", ":", option_.copysetNode->GetPoolId(),
-                                "_", option_.copysetNode->GetCopysetId(), ":",
-                                i);
-        }();
-        auto taskThread =
-            absl::make_unique<TaskWorker>(option_.queueDepth, std::move(name));
-        taskThread->Start();
-        workers_.emplace_back(std::move(taskThread));
-    }
-}
-
-bool ApplyQueue::Start(const ApplyQueueOption& option) {
-    if (running_) {
-        return true;
-    }
-
-    if (option.queueDepth < 1 || option.workerCount < 1) {
-        LOG(ERROR) << "ApplyQueue start failed, invalid argument, queue depth: "
-                   << option.queueDepth
-                   << ", worker count: " << option.workerCount;
-        return false;
-    }
-
-    option_ = option;
-
-    StartWorkers();
-    running_.store(true);
-    return true;
-}
-
-void ApplyQueue::Flush() {
-    if (!running_.load(std::memory_order_relaxed)) {
-        return;
-    }
-
-    curve::common::CountDownEvent event(option_.workerCount);
-
-    auto flush = [&event]() { event.Signal(); };
-
-    for (auto& worker : workers_) {
-        worker->tasks.Push(flush);
-    }
-
-    event.Wait();
-}
-
-void ApplyQueue::Stop() {
-    if (!running_.exchange(false)) {
-        return;
-    }
-
-    LOG(INFO) << "Going to stop apply queue";
-
-    for (auto& worker : workers_) {
-        worker->Stop();
-    }
-
-    workers_.clear();
-    LOG(INFO) << "Apply queue stopped";
-}
-
-void ApplyQueue::TaskWorker::Start() {
-    if (running.exchange(true)) {
-        return;
-    }
-
-    worker = std::thread(&TaskWorker::Work, this);
-}
-
-void ApplyQueue::TaskWorker::Stop() {
-    if (!running.exchange(false)) {
-        return;
-    }
-
-    auto wakeup = []() {};
-    tasks.Push(wakeup);
-
-    worker.join();
-}
-
-void ApplyQueue::TaskWorker::Work() {
-    SetThreadName(workerName_.c_str());
-
-    while (running.load(std::memory_order_relaxed)) {
-        tasks.Pop()();
-    }
-}
-
-}  // namespace copyset
-}  // namespace metaserver
-}  // namespace curvefs

curvefs/src/metaserver/copyset/apply_queue.h

@@ -1,98 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: curve
- * Date: Thu Sep  2 14:49:04 CST 2021
- * Author: wuhanqing
- */
-
-#ifndef CURVEFS_SRC_METASERVER_COPYSET_APPLY_QUEUE_H_
-#define CURVEFS_SRC_METASERVER_COPYSET_APPLY_QUEUE_H_
-
-#include <atomic>
-#include <cstdint>
-#include <memory>
-#include <string>
-#include <thread>
-#include <utility>
-#include <vector>
-
-#include "include/curve_compiler_specific.h"
-#include "src/common/concurrent/count_down_event.h"
-#include "src/common/concurrent/task_queue.h"
-
-namespace curvefs {
-namespace metaserver {
-namespace copyset {
-
-class CopysetNode;
-
-struct ApplyQueueOption {
-    uint32_t workerCount = 1;
-    uint32_t queueDepth = 1;
-    CopysetNode* copysetNode = nullptr;
-};
-
-class CURVE_CACHELINE_ALIGNMENT ApplyQueue {
- public:
-    ApplyQueue() : option_(), running_(false), workers_() {}
-
-    bool Start(const ApplyQueueOption& option);
-
-    template <typename Func, typename... Args>
-    void Push(uint64_t hash, Func&& f, Args&&... args) {
-        workers_[hash % option_.workerCount]->tasks.Push(
-            std::forward<Func>(f), std::forward<Args>(args)...);
-    }
-
-    void Flush();
-
-    void Stop();
-
- private:
-    void StartWorkers();
-
-    struct TaskWorker {
-        TaskWorker(size_t cap, std::string workerName)
-            : running(false),
-              worker(),
-              tasks(cap),
-              workerName_(std::move(workerName)) {}
-
-        void Start();
-
-        void Stop();
-
-        void Work();
-
-        std::atomic<bool> running;
-        std::thread worker;
-        curve::common::TaskQueue tasks;
-        std::string workerName_;
-    };
-
- private:
-    ApplyQueueOption option_;
-    std::atomic<bool> running_;
-    std::vector<std::unique_ptr<TaskWorker>> workers_;
-};
-
-}  // namespace copyset
-}  // namespace metaserver
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_METASERVER_COPYSET_APPLY_QUEUE_H_

curvefs/src/metaserver/copyset/concurrent_apply_queue.cpp

@@ -0,0 +1,198 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * File Created: 20230521
+ * Author: Xinlong-Chen
+ */
+
+
+#include "curvefs/src/metaserver/copyset/concurrent_apply_queue.h"
+
+#include <algorithm>
+
+namespace curvefs {
+namespace metaserver {
+namespace copyset {
+bool ApplyQueue::Init(const ApplyOption &opt) {
+    if (start_) {
+        LOG(WARNING) << "concurrent module already start!";
+        return true;
+    }
+
+    if (false == CheckOptAndInit(opt)) {
+        return false;
+    }
+
+    start_ = true;
+    cond_.Reset(opt.rconcurrentsize + opt.wconcurrentsize);
+    InitThreadPool(ThreadPoolType::READ, rconcurrentsize_, rqueuedepth_);
+    InitThreadPool(ThreadPoolType::WRITE, wconcurrentsize_, wqueuedepth_);
+
+    if (!cond_.WaitFor(5000)) {
+        LOG(ERROR) << "init concurrent module's threads fail";
+        start_ = false;
+    }
+
+    LOG(INFO) << "Init concurrent module's threads success";
+    return start_;
+}
+
+bool ApplyQueue::CheckOptAndInit(const ApplyOption &opt) {
+    if (opt.rconcurrentsize <= 0 || opt.wconcurrentsize <= 0 ||
+        opt.rqueuedepth <= 0 || opt.wqueuedepth <= 0) {
+        LOG(INFO) << "init concurrent module fail, params must >=0"
+            << ", rconcurrentsize=" << opt.rconcurrentsize
+            << ", wconcurrentsize=" << opt.wconcurrentsize
+            << ", rqueuedepth=" << opt.rqueuedepth
+            << ", wconcurrentsize=" << opt.wqueuedepth;
+        return false;
+    }
+
+    wconcurrentsize_ = opt.wconcurrentsize;
+    wqueuedepth_ = opt.wqueuedepth;
+    rconcurrentsize_ = opt.rconcurrentsize;
+    rqueuedepth_ = opt.rqueuedepth;
+
+    return true;
+}
+
+void ApplyQueue::InitThreadPool(
+    ThreadPoolType type, int concurrent, int depth) {
+    for (int i = 0; i < concurrent; i++) {
+        auto asyncth = new (std::nothrow) TaskThread(depth);
+        CHECK(asyncth != nullptr) << "allocate failed!";
+
+        switch (type) {
+        case ThreadPoolType::READ:
+            rapplyMap_.insert(std::make_pair(i, asyncth));
+            break;
+
+        case ThreadPoolType::WRITE:
+            wapplyMap_.insert(std::make_pair(i, asyncth));
+            break;
+        }
+    }
+
+    for (int i = 0; i < concurrent; i++) {
+        switch (type) {
+        case ThreadPoolType::READ:
+            rapplyMap_[i]->th =
+                    std::thread(&ApplyQueue::Run, this, type, i);
+            break;
+
+        case ThreadPoolType::WRITE:
+            wapplyMap_[i]->th =
+                    std::thread(&ApplyQueue::Run, this, type, i);
+            break;
+        }
+    }
+}
+
+void ApplyQueue::Run(ThreadPoolType type, int index) {
+    cond_.Signal();
+    while (start_) {
+        switch (type) {
+        case ThreadPoolType::READ:
+            rapplyMap_[index]->tq.Pop()();
+            break;
+
+        case ThreadPoolType::WRITE:
+            wapplyMap_[index]->tq.Pop()();
+            break;
+        }
+    }
+}
+
+void ApplyQueue::Stop() {
+    if (!start_.exchange(false)) {
+        return;
+    }
+
+    LOG(INFO) << "stop ApplyQueue...";
+    auto wakeup = []() {};
+    for (auto iter : rapplyMap_) {
+        iter.second->tq.Push(wakeup);
+        iter.second->th.join();
+        delete iter.second;
+    }
+    rapplyMap_.clear();
+
+    for (auto iter : wapplyMap_) {
+        iter.second->tq.Push(wakeup);
+        iter.second->th.join();
+        delete iter.second;
+    }
+    wapplyMap_.clear();
+
+    LOG(INFO) << "stop ApplyQueue ok.";
+}
+
+void ApplyQueue::Flush() {
+    if (!start_.load(std::memory_order_relaxed)) {
+        return;
+    }
+
+    CountDownEvent event(wconcurrentsize_);
+    auto flushtask = [&event]() {
+        event.Signal();
+    };
+
+    for (int i = 0; i < wconcurrentsize_; i++) {
+        wapplyMap_[i]->tq.Push(flushtask);
+    }
+
+    event.Wait();
+}
+
+void ApplyQueue::FlushAll() {
+    if (!start_.load(std::memory_order_relaxed)) {
+        return;
+    }
+
+    CountDownEvent event(wconcurrentsize_ + rconcurrentsize_);
+    auto flushtask = [&event]() {
+        event.Signal();
+    };
+
+    for (int i = 0; i < wconcurrentsize_; i++) {
+        wapplyMap_[i]->tq.Push(flushtask);
+    }
+
+    for (int i = 0; i < rconcurrentsize_; i++) {
+        rapplyMap_[i]->tq.Push(flushtask);
+    }
+
+    event.Wait();
+}
+
+ThreadPoolType ApplyQueue::Schedule(OperatorType optype) {
+    switch (optype) {
+    case OperatorType::GetDentry:
+    case OperatorType::ListDentry:
+    case OperatorType::GetInode:
+    case OperatorType::BatchGetInodeAttr:
+    case OperatorType::BatchGetXAttr:
+    case OperatorType::GetVolumeExtent:
+        return ThreadPoolType::READ;
+    default:
+        return ThreadPoolType::WRITE;
+    }
+}
+}   // namespace copyset
+}   // namespace metaserver
+}   // namespace curvefs

curvefs/src/metaserver/copyset/concurrent_apply_queue.h

@@ -0,0 +1,152 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * File Created: 20230521
+ * Author: Xinlong-Chen
+ */
+
+#ifndef CURVEFS_SRC_METASERVER_COPYSET_CONCURRENT_APPLY_QUEUE_H_
+#define CURVEFS_SRC_METASERVER_COPYSET_CONCURRENT_APPLY_QUEUE_H_
+
+#include <bthread/condition_variable.h>
+#include <bthread/mutex.h>
+#include <glog/logging.h>
+
+#include <atomic>
+#include <thread>
+#include <unordered_map>
+#include <utility>
+
+#include "include/curve_compiler_specific.h"
+#include "src/common/concurrent/count_down_event.h"
+#include "src/common/concurrent/task_queue.h"
+#include "curvefs/src/metaserver/copyset/operator_type.h"
+
+namespace curvefs {
+namespace metaserver {
+namespace copyset {
+
+using curve::common::CountDownEvent;
+using curve::common::GenericTaskQueue;
+
+struct ApplyOption {
+    int wconcurrentsize = 3;
+    int wqueuedepth = 1;
+    int rconcurrentsize = 1;
+    int rqueuedepth = 1;
+    ApplyOption(int wsize, int wdepth, int rsize, int rdepth) :
+        wconcurrentsize(wsize),
+        wqueuedepth(wdepth),
+        rconcurrentsize(rsize),
+        rqueuedepth(rdepth) {}
+    ApplyOption() {}
+};
+
+enum class ThreadPoolType {READ, WRITE};
+
+/*
+TODO: this moudle is same as curvebs's ConcurrentApplyModule,
+      only Schedule function is different.
+      we can make ApplyQueue to a base class,
+      and define Schedule with virtual function,
+      derive class override Schedule.
+*/ 
+
+class CURVE_CACHELINE_ALIGNMENT ApplyQueue {
+ public:
+    ApplyQueue(): start_(false),
+                  rconcurrentsize_(0),
+                  rqueuedepth_(0),
+                  wconcurrentsize_(0),
+                  wqueuedepth_(0),
+                  cond_(0) {}
+
+    /**
+     * Init: initialize ApplyQueue
+     * @param[in] wconcurrentsize: num of write threads
+     * @param[in] wqueuedepth: depth of write queue in ervery thread
+     * @param[in] rconcurrentsizee: num of read threads
+     * @param[in] wqueuedephth: depth of read queue in every thread
+     */
+    bool Init(const ApplyOption &opt);
+
+    /**
+     * Push: apply task will be push to ApplyQueue
+     * @param[in] key: used to hash task to specified queue
+     * @param[in] optype: operation type defined in proto
+     * @param[in] f: task
+     * @param[in] args: param to excute task
+     */
+    template <class F, class... Args>
+    bool Push(uint64_t key, OperatorType optype, F&& f, Args&&... args) {
+        switch (Schedule(optype)) {
+            case ThreadPoolType::READ:
+                rapplyMap_[Hash(key, rconcurrentsize_)]->tq.Push(
+                        std::forward<F>(f), std::forward<Args>(args)...);
+                break;
+            case ThreadPoolType::WRITE:
+                wapplyMap_[Hash(key, wconcurrentsize_)]->tq.Push(
+                        std::forward<F>(f), std::forward<Args>(args)...);
+                break;
+        }
+
+        return true;
+    }
+
+    /**
+     * Flush: finish all task in write threads
+     */
+    void Flush();
+    void FlushAll();
+
+    void Stop();
+
+ private:
+    bool CheckOptAndInit(const ApplyOption &option);
+
+    void Run(ThreadPoolType type, int index);
+
+    static ThreadPoolType Schedule(OperatorType optype);
+
+    void InitThreadPool(ThreadPoolType type, int concorrent, int depth);
+
+    static int Hash(uint64_t key, int concurrent) {
+        return key % concurrent;
+    }
+
+ private:
+    struct TaskThread {
+        std::thread th;
+        GenericTaskQueue<bthread::Mutex, bthread::ConditionVariable> tq;
+        explicit TaskThread(size_t capacity) : tq(capacity) {}
+    };
+
+    std::atomic<bool> start_;
+    int rconcurrentsize_;
+    int rqueuedepth_;
+    int wconcurrentsize_;
+    int wqueuedepth_;
+    CountDownEvent cond_;
+    CURVE_CACHELINE_ALIGNMENT std::unordered_map<int, TaskThread*> wapplyMap_;
+    CURVE_CACHELINE_ALIGNMENT std::unordered_map<int, TaskThread*> rapplyMap_;
+};
+}   // namespace copyset
+}   // namespace metaserver
+}   // namespace curvefs
+
+#endif  // CURVEFS_SRC_METASERVER_COPYSET_CONCURRENT_APPLY_QUEUE_H_

curvefs/src/metaserver/copyset/config.h

@@ -31,7 +31,7 @@
 #include <memory>
 #include <string>
 
-#include "curvefs/src/metaserver/copyset/apply_queue.h"
+#include "curvefs/src/metaserver/copyset/concurrent_apply_queue.h"
 #include "curvefs/src/metaserver/copyset/trash.h"
 #include "curvefs/src/metaserver/storage/config.h"
 #include "src/fs/local_filesystem.h"
@@ -74,7 +74,7 @@ struct CopysetNodeOptions {
     uint32_t checkLoadMarginIntervalMs;
 
     // apply queue options
-    ApplyQueueOption applyQueueOption;
+    ApplyOption applyQueueOption;
 
     // filesystem adaptor
     curve::fs::LocalFileSystem* localFileSystem;

curvefs/src/metaserver/copyset/copyset_node.cpp

@@ -120,9 +120,8 @@ bool CopysetNode::Init(const CopysetNodeOptions& options) {
 
     // init apply queue
     applyQueue_ = absl::make_unique<ApplyQueue>();
-    options_.applyQueueOption.copysetNode = this;
-    if (!applyQueue_->Start(options_.applyQueueOption)) {
-        LOG(ERROR) << "Start apply queue failed";
+    if (!applyQueue_->Init(options_.applyQueueOption)) {
+        LOG(ERROR) << "init concurrent apply queue failed";
         return false;
     }
 
@@ -258,6 +257,7 @@ void CopysetNode::on_apply(braft::Iterator& iter) {
                           iter.index(), doneGuard.release(),
                           TimeUtility::GetTimeofDayUs());
             applyQueue_->Push(metaClosure->GetOperator()->HashCode(),
+                              metaClosure->GetOperator()->GetOperatorType(),
                               std::move(task));
             timer.stop();
             g_concurrent_apply_wait_latency << timer.u_elapsed();
@@ -268,10 +268,11 @@ void CopysetNode::on_apply(braft::Iterator& iter) {
             butil::Timer timer;
             timer.start();
             auto hashcode = metaOperator->HashCode();
+            auto type = metaOperator->GetOperatorType();
             auto task =
                 std::bind(&MetaOperator::OnApplyFromLog, metaOperator.release(),
                           TimeUtility::GetTimeofDayUs());
-            applyQueue_->Push(hashcode, std::move(task));
+            applyQueue_->Push(hashcode, type, std::move(task));
             timer.stop();
             g_concurrent_apply_from_log_wait_latency << timer.u_elapsed();
         }
@@ -426,6 +427,9 @@ int CopysetNode::on_snapshot_load(braft::SnapshotReader* reader) {
 }
 
 void CopysetNode::on_leader_start(int64_t term) {
+    LOG(INFO) << "Copyset: " << name_ << ", peer id: " << peerId_.to_string()
+          << " going to flush apply queue, term is " << term;
+    applyQueue_->Flush();
     leaderTerm_.store(term, std::memory_order_release);
 
     LOG(INFO) << "Copyset: " << name_ << ", peer id: " << peerId_.to_string()

curvefs/src/metaserver/copyset/copyset_node.h

@@ -31,7 +31,7 @@
 #include <vector>
 
 #include "curvefs/src/metaserver/common/types.h"
-#include "curvefs/src/metaserver/copyset/apply_queue.h"
+#include "curvefs/src/metaserver/copyset/concurrent_apply_queue.h"
 #include "curvefs/src/metaserver/copyset/conf_epoch_file.h"
 #include "curvefs/src/metaserver/copyset/config.h"
 #include "curvefs/src/metaserver/copyset/copyset_conf_change.h"
@@ -123,7 +123,7 @@ class CopysetNode : public braft::StateMachine {
 #ifdef UNIT_TEST
     void SetMetaStore(MetaStore* metastore) { metaStore_.reset(metastore); }
 
-    void FlushApplyQueue() { applyQueue_->Flush(); }
+    void FlushApplyQueue() { applyQueue_->FlushAll(); }
 
     void SetRaftNode(RaftNode* raftNode) { raftNode_.reset(raftNode); }
 #endif  // UNIT_TEST

curvefs/src/metaserver/copyset/meta_operator.cpp

@@ -107,7 +107,8 @@ void MetaOperator::FastApplyTask() {
     auto task =
         std::bind(&MetaOperator::OnApply, this, node_->GetAppliedIndex(),
                   new MetaOperatorClosure(this), TimeUtility::GetTimeofDayUs());
-    node_->GetApplyQueue()->Push(HashCode(), std::move(task));
+    node_->GetApplyQueue()->Push(HashCode(),
+                                 GetOperatorType(), std::move(task));
     timer.stop();
     g_concurrent_fast_apply_wait_latency << timer.u_elapsed();
 }

curvefs/src/metaserver/metaserver.cpp

@@ -68,7 +68,6 @@ using ::curve::fs::FileSystemType;
 using ::curve::fs::LocalFsFactory;
 using ::curve::fs::LocalFileSystemOption;
 
-using ::curvefs::metaserver::copyset::ApplyQueueOption;
 using ::curvefs::client::rpcclient::MetaServerClientImpl;
 using ::curvefs::client::rpcclient::ChannelManager;
 using ::curvefs::client::rpcclient::MetaCache;
@@ -100,6 +99,9 @@ void Metaserver::InitOptions(std::shared_ptr<Configuration> conf) {
             << "Parse bthread.worker_count to int failed, string value: "
             << value;
     }
+    conf_->GetValueFatalIfFail("server.idleTimeoutSec",
+                               &options_.idleTimeoutSec);
+
 
     InitBRaftFlags(conf);
 }
@@ -456,6 +458,7 @@ void Metaserver::Run() {
     if (options_.bthreadWorkerCount != -1) {
         option.num_threads = options_.bthreadWorkerCount;
     }
+    option.idle_timeout_sec = options_.idleTimeoutSec;
     LOG_IF(FATAL, server_->Start(listenAddr, &option) != 0)
         << "start internal brpc server error";
 
@@ -653,13 +656,18 @@ void Metaserver::InitCopysetNodeOptions() {
                       "copyset.check_loadmargin_interval_ms",
                       &copysetNodeOptions_.checkLoadMarginIntervalMs));
 
-    LOG_IF(FATAL, !conf_->GetUInt32Value(
-                      "applyqueue.worker_count",
-                      &copysetNodeOptions_.applyQueueOption.workerCount));
-    LOG_IF(FATAL, !conf_->GetUInt32Value(
-                      "applyqueue.queue_depth",
-                      &copysetNodeOptions_.applyQueueOption.queueDepth));
-
+    LOG_IF(FATAL, !conf_->GetIntValue(
+                      "applyqueue.write_worker_count",
+                      &copysetNodeOptions_.applyQueueOption.wconcurrentsize));
+    LOG_IF(FATAL, !conf_->GetIntValue(
+                      "applyqueue.write_queue_depth",
+                      &copysetNodeOptions_.applyQueueOption.wqueuedepth));
+    LOG_IF(FATAL, !conf_->GetIntValue(
+                      "applyqueue.read_worker_count",
+                      &copysetNodeOptions_.applyQueueOption.rconcurrentsize));
+    LOG_IF(FATAL, !conf_->GetIntValue(
+                      "applyqueue.read_queue_depth",
+                      &copysetNodeOptions_.applyQueueOption.rqueuedepth));
     LOG_IF(FATAL,
            !conf_->GetStringValue("copyset.trash.uri",
                                   &copysetNodeOptions_.trashOptions.trashUri));

curvefs/src/metaserver/metaserver.h

@@ -28,7 +28,6 @@
 #include <memory>
 #include <string>
 
-#include "curvefs/src/metaserver/copyset/apply_queue.h"
 #include "curvefs/src/metaserver/copyset/config.h"
 #include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
 #include "curvefs/src/metaserver/copyset/raft_cli_service2.h"
@@ -52,7 +51,6 @@ namespace metaserver {
 
 using ::curve::common::Configuration;
 using ::curvefs::metaserver::storage::StorageOptions;
-using ::curvefs::metaserver::copyset::ApplyQueue;
 using ::curvefs::metaserver::copyset::CopysetNodeManager;
 using ::curvefs::metaserver::copyset::CopysetNodeOptions;
 using ::curvefs::metaserver::copyset::CopysetServiceImpl;
@@ -70,6 +68,7 @@ struct MetaserverOptions {
     std::string externalIp;
     int externalPort;
     int bthreadWorkerCount = -1;
+    int idleTimeoutSec = -1;
     bool enableExternalServer;
 };
 

curvefs/src/metaserver/trash.cpp

@@ -208,6 +208,21 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
         clientAdaptorOption.objectPrefix = s3Info.objectprefix();
         s3Adaptor_->Reinit(clientAdaptorOption, s3Info.ak(), s3Info.sk(),
             s3Info.endpoint(), s3Info.bucketname());
+        ret = inodeStorage_->PaddingInodeS3ChunkInfo(item.fsId,
+          item.inodeId, inode.mutable_s3chunkinfomap());
+        if (ret != MetaStatusCode::OK) {
+            LOG(ERROR) << "GetInode chunklist fail, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId
+                << ", retCode = " << MetaStatusCode_Name(ret);
+            return ret;
+        }
+        if (inode.s3chunkinfomap().empty()) {
+            LOG(WARNING) << "GetInode chunklist empty, fsId = " << item.fsId
+                << ", inodeId = " << item.inodeId;
+            return MetaStatusCode::NOT_FOUND;
+        }
+        VLOG(9) << "DeleteInodeAndData, inode: "
+            << inode.ShortDebugString();
         int retVal = s3Adaptor_->Delete(inode);
         if (retVal != 0) {
             LOG(ERROR) << "S3ClientAdaptor delete s3 data failed"
@@ -216,7 +231,6 @@ MetaStatusCode TrashImpl::DeleteInodeAndData(const TrashItem &item) {
             return MetaStatusCode::S3_DELETE_ERR;
         }
     }
-
     ret = inodeStorage_->Delete(Key4Inode(item.fsId, item.inodeId));
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "Delete Inode fail, fsId = " << item.fsId

curvefs/test/client/client_operator_test.cpp

@@ -34,6 +34,7 @@ namespace curvefs {
 namespace client {
 
 using ::testing::SetArgPointee;
+using ::testing::DoAll;
 using rpcclient::MockMetaServerClient;
 using rpcclient::MockMdsClient;
 

curvefs/test/client/filesystem/BUILD

@@ -0,0 +1,34 @@
+#
+#  Copyright (c) 2023 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+load("//:copts.bzl", "CURVE_TEST_COPTS")
+
+cc_test(
+    name = "curvefs_client_filesystem_test",
+    srcs = glob([
+        "*.cpp",
+        "*.h",
+        "helper/*.h",
+        "helper/*.cpp",
+    ]),
+    copts = CURVE_TEST_COPTS,
+    deps = [
+        "//curvefs/src/client:fuse_client_lib",
+        "//curvefs/test/client:mock",
+        "@com_google_googletest//:gtest",
+        "@com_google_googletest//:gtest_main",
+    ],
+)

curvefs/test/client/filesystem/attrwatcher_test.cpp

@@ -0,0 +1,78 @@
+
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-29
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+#include "curvefs/test/client/filesystem/helper/helper.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+class AttrWatcherTest : public ::testing::Test {
+ protected:
+    void SetUp() override {}
+
+    void TearDown() override {}
+};
+
+TEST_F(AttrWatcherTest, RememberMtime) {
+    auto option = AttrWatcherOption();
+    auto attrWatcher = std::make_shared<AttrWatcher>(option, nullptr, nullptr);
+
+    // remeber mtime
+    InodeAttr attr = MkAttr(100, AttrOption().mtime(123, 456));
+    attrWatcher->RemeberMtime(attr);
+
+    // get mtime
+    TimeSpec time;
+    bool yes = attrWatcher->GetMtime(100, &time);
+    ASSERT_TRUE(yes);
+    ASSERT_EQ(time, TimeSpec(123, 456));
+}
+
+TEST_F(AttrWatcherTest, EvitAttr) {
+    auto option = AttrWatcherOption{lruSize: 1};
+    auto attrWatcher = std::make_shared<AttrWatcher>(option, nullptr, nullptr);
+
+    // remeber mtime
+    for (const Ino& ino : std::vector<Ino>{100, 200}) {
+        InodeAttr attr = MkAttr(ino, AttrOption().mtime(123, 456));
+        attrWatcher->RemeberMtime(attr);
+    }
+
+    // get mtime
+    TimeSpec time;
+    bool yes = attrWatcher->GetMtime(100, &time);
+    ASSERT_FALSE(yes);
+
+    yes = attrWatcher->GetMtime(200, &time);
+    ASSERT_TRUE(yes);
+    ASSERT_EQ(time, TimeSpec(123, 456));
+}
+
+TEST_F(AttrWatcherTest, UpdateDirEntryAttr) {
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/test/client/filesystem/defer_sync_test.cpp

@@ -0,0 +1,73 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-29
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/test/client/filesystem/helper/helper.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+class DeferSyncTest : public ::testing::Test {
+ protected:
+    void SetUp() override {
+        metaClient_ = std::make_shared<MockMetaServerClient>();
+    }
+
+    void TearDown() override {
+        metaClient_ = nullptr;
+    }
+
+ protected:
+    std::shared_ptr<MockMetaServerClient> metaClient_;
+};
+
+TEST_F(DeferSyncTest, Basic) {
+    auto builder = DeferSyncBuilder();
+    auto deferSync = builder.SetOption([&](DeferSyncOption* option){
+        option->delay = 3;
+    }).Build();
+    deferSync->Start();
+
+    auto inode = MkInode(100, InodeOption().metaClient(metaClient_));
+    // EXPECT_CALL_INDOE_SYNC_TIMES(*metaClient_, 100 /* ino */, 1 /* times */);
+    inode->SetLength(100);  // make inode ditry to trigger sync
+    deferSync->Push(inode);
+    deferSync->Stop();
+}
+
+TEST_F(DeferSyncTest, Dirty) {
+    auto builder = DeferSyncBuilder();
+    auto deferSync = builder.SetOption([&](DeferSyncOption* option){
+        option->delay = 3;
+    }).Build();
+    deferSync->Start();
+
+    auto inode = MkInode(100, InodeOption().metaClient(metaClient_));
+    EXPECT_CALL_INDOE_SYNC_TIMES(*metaClient_, 100 /* ino */, 0 /* times */);
+    deferSync->Push(inode);
+    deferSync->Stop();
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/test/client/filesystem/dir_cache_test.cpp

@@ -0,0 +1,218 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-29
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <gtest/gtest.h>
+
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/test/client/filesystem/helper/helper.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+class DirEntryListTest : public ::testing::Test {
+ protected:
+    void SetUp() override {}
+
+    void TearDown() override {}
+};
+
+class DirCacheTest : public ::testing::Test {
+ protected:
+    void SetUp() override {}
+
+    void TearDown() override {}
+};
+
+TEST_F(DirEntryListTest, Size) {
+    DirEntryList entries;
+    ASSERT_EQ(entries.Size(), 0);
+
+    entries.Add(MkDirEntry(100, "f1"));
+    ASSERT_EQ(entries.Size(), 1);
+
+    entries.Clear();
+    ASSERT_EQ(entries.Size(), 0);
+}
+
+TEST_F(DirEntryListTest, Iterate) {
+    DirEntryList entries;
+
+    std::vector<std::pair<Ino, std::string>> items{
+        { 100, "f1" },
+        { 200, "f2" },
+        { 300, "f3" },
+    };
+
+    for (const auto& item : items) {
+        entries.Add(MkDirEntry(item.first, item.second));
+    }
+    ASSERT_EQ(entries.Size(), 3);
+
+    std::vector<std::pair<Ino, std::string>> out;
+    entries.Iterate([&](DirEntry* dirEntry){
+        out.push_back({dirEntry->ino, dirEntry->name});
+    });
+    ASSERT_EQ(items, out);
+}
+
+TEST_F(DirEntryListTest, Get) {
+    DirEntryList entries;
+    InodeAttr attr = MkAttr(100, AttrOption().length(1024));
+    entries.Add(MkDirEntry(100, "f1", attr));
+
+    // CASE 1: directory entry exit
+    {
+        DirEntry dirEntry;
+        bool yes = entries.Get(100, &dirEntry);
+        ASSERT_TRUE(yes);
+        ASSERT_EQ(dirEntry.attr.inodeid(), 100);
+        ASSERT_EQ(dirEntry.attr.length(), 1024);
+    }
+
+    // CASE 2: directory entry not exit
+    {
+        DirEntry dirEntry;
+        bool yes = entries.Get(200, &dirEntry);
+        ASSERT_FALSE(yes);
+    }
+}
+
+TEST_F(DirEntryListTest, UpdateAttr) {
+    DirEntryList entries;
+
+    InodeAttr attr = MkAttr(100, AttrOption().length(1024));
+    DirEntry dirEntry(100, "f1", attr);
+    entries.Add(dirEntry);
+
+    // CASE 1: update attribute success
+    {
+        InodeAttr attr = MkAttr(100, AttrOption().length(2048));
+        bool yes = entries.UpdateAttr(100, attr);
+        ASSERT_TRUE(yes);
+
+        yes = entries.Get(100, &dirEntry);
+        ASSERT_TRUE(yes);
+        ASSERT_EQ(dirEntry.attr.inodeid(), 100);
+        ASSERT_EQ(dirEntry.attr.length(), 2048);
+    }
+
+    // CASE 2: (update attribute failed) / (ino not found)
+    {
+        bool yes = entries.UpdateAttr(200, attr);
+        ASSERT_FALSE(yes);
+    }
+}
+
+TEST_F(DirEntryListTest, UpdateLength) {
+    DirEntryList entries;
+
+    DirEntry dirEntry = MkDirEntry(100, "f1", MkAttr(100, AttrOption()
+        .length(1024)
+        .mtime(100, 101)
+        .ctime(200, 201)));
+    entries.Add(dirEntry);
+
+    // CASE 1: ino not found
+    {
+        InodeAttr open = MkAttr(200);
+        bool yes = entries.UpdateLength(200, open);
+        ASSERT_FALSE(yes);
+    }
+
+    // CASE 2: update length and mtime
+    {
+        InodeAttr open = MkAttr(100, AttrOption()
+            .length(2048)
+            .mtime(100, 101)
+            .ctime(200, 202));
+        bool yes = entries.UpdateLength(100, open);
+        ASSERT_TRUE(yes);
+
+        DirEntry dirEntry;
+        yes = entries.Get(100, &dirEntry);
+        ASSERT_TRUE(yes);
+        ASSERT_EQ(dirEntry.attr.inodeid(), 100);
+        ASSERT_EQ(dirEntry.attr.length(), 2048);
+        // ASSERT_EQ(AttrMtime(dirEntry.attr), TimeSpec());
+        // ASSERT_EQ(AttrCtime(dirEntry.attr), TimeSpec());
+    }
+
+    // CASE 3: update length, mtime, ctime
+    {
+        InodeAttr open = MkAttr(100, AttrOption()
+            .length(2048)
+            .mtime(100, 101)
+            .ctime(200, 202));
+        bool yes = entries.UpdateLength(100, open);
+        ASSERT_TRUE(yes);
+
+        DirEntry dirEntry;
+        yes = entries.Get(100, &dirEntry);
+        ASSERT_TRUE(yes);
+        ASSERT_EQ(dirEntry.attr.length(), 2048);
+        // ASSERT_EQ(AttrMtime(dirEntry.attr), TimeSpec());
+        // ASSERT_EQ(AttrCtime(dirEntry.attr), TimeSpec());
+    }
+}
+
+TEST_F(DirEntryListTest, Clear) {
+    DirEntryList entries;
+    std::vector<std::pair<Ino, std::string>> items{
+        { 100, "f1" },
+        { 200, "f2" },
+        { 300, "f3" },
+    };
+
+    for (const auto& item : items) {
+        entries.Add(MkDirEntry(item.first, item.second));
+    }
+    ASSERT_EQ(entries.Size(), 3);
+
+    DirEntry dirEntry;
+    for (const auto& item : items) {
+        bool yes = entries.Get(item.first, &dirEntry);
+        ASSERT_TRUE(yes);
+        ASSERT_EQ(dirEntry.name, item.second);
+    }
+
+    entries.Clear();
+    ASSERT_EQ(entries.Size(), 0);
+    for (const auto& item : items) {
+        bool yes = entries.Get(item.first, &dirEntry);
+        ASSERT_FALSE(yes);
+    }
+}
+
+TEST_F(DirEntryListTest, Mtime) {
+    DirEntryList entries;
+    TimeSpec time = entries.GetMtime();
+    ASSERT_EQ(time, TimeSpec(0, 0));
+
+    entries.SetMtime(TimeSpec(123, 456));
+    time = entries.GetMtime();
+    ASSERT_EQ(time, TimeSpec(123, 456));
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/test/client/filesystem/error_test.cpp

@@ -0,0 +1,60 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-29
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <gtest/gtest.h>
+
+#include "curvefs/src/client/filesystem/error.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+class ErrorTest : public ::testing::Test {
+ protected:
+    void SetUp() override {}
+    void TearDown() override {}
+};
+
+TEST_F(ErrorTest, StrErr) {
+    ASSERT_EQ(StrErr(CURVEFS_ERROR::OK), "OK");
+    ASSERT_EQ(StrErr(CURVEFS_ERROR::INTERNAL), "internal error");
+    ASSERT_EQ(StrErr(CURVEFS_ERROR::INVALIDPARAM), "invalid argument");
+    ASSERT_EQ(StrErr(CURVEFS_ERROR::STALE), "stale file handler");
+    ASSERT_EQ(StrErr(CURVEFS_ERROR::UNKNOWN), "unknown");
+}
+
+TEST_F(ErrorTest, SysErr) {
+    ASSERT_EQ(SysErr(CURVEFS_ERROR::OK), 0);
+    ASSERT_EQ(SysErr(CURVEFS_ERROR::INTERNAL), EIO);
+    ASSERT_EQ(SysErr(CURVEFS_ERROR::INVALIDPARAM), EINVAL);
+    ASSERT_EQ(SysErr(CURVEFS_ERROR::STALE), ESTALE);
+    ASSERT_EQ(SysErr(CURVEFS_ERROR::UNKNOWN), EIO);
+}
+
+TEST_F(ErrorTest, ToFSError) {
+    ASSERT_EQ(ToFSError(MetaStatusCode::OK), CURVEFS_ERROR::OK);
+    ASSERT_EQ(ToFSError(MetaStatusCode::NOT_FOUND), CURVEFS_ERROR::NOTEXIST);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/test/client/filesystem/filesystem_test.cpp

@@ -0,0 +1,459 @@
+
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-04-03
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <gtest/gtest.h>
+
+#include "curvefs/test/client/filesystem/helper/helper.h"
+#include "curvefs/src/client/filesystem/filesystem.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+class FileSystemTest : public ::testing::Test {
+ protected:
+    void SetUp() override {}
+    void TearDown() override {}
+};
+
+TEST_F(FileSystemTest, Attr2Stat) {
+    InodeAttr attr = MkAttr(100, AttrOption()
+        .type(FsFileType::TYPE_S3)
+        .mode(33216)
+        .nlink(2)
+        .uid(1000)
+        .gid(1001)
+        .length(4096)
+        .rdev(2048)
+        .atime(100, 101)
+        .mtime(200, 201)
+        .ctime(300, 301));
+
+    // CASE 1: check stat field one by one
+    struct stat stat;
+    auto fs = FileSystemBuilder().Build();
+    fs->Attr2Stat(&attr, &stat);
+    ASSERT_EQ(stat.st_ino, 100);
+    ASSERT_EQ(stat.st_mode, 33216);
+    ASSERT_EQ(stat.st_nlink, 2);
+    ASSERT_EQ(stat.st_uid, 1000);
+    ASSERT_EQ(stat.st_gid, 1001);
+    ASSERT_EQ(stat.st_size, 4096);
+    ASSERT_EQ(stat.st_rdev, 2048);
+    ASSERT_EQ(stat.st_atim.tv_sec, 100);
+    ASSERT_EQ(stat.st_atim.tv_nsec, 101);
+    ASSERT_EQ(stat.st_mtim.tv_sec, 200);
+    ASSERT_EQ(stat.st_mtim.tv_nsec, 201);
+    ASSERT_EQ(stat.st_ctim.tv_sec, 300);
+    ASSERT_EQ(stat.st_ctim.tv_nsec, 301);
+    ASSERT_EQ(stat.st_blksize, 0x10000u);
+    ASSERT_EQ(stat.st_blocks, 8);
+
+    // CASE 2: convert all kind of file types
+    std::vector<FsFileType> types = {
+        FsFileType::TYPE_DIRECTORY,
+        FsFileType::TYPE_FILE,
+        FsFileType::TYPE_SYM_LINK,
+    };
+    for (const auto& type : types) {
+        attr.set_type(type);
+        fs->Attr2Stat(&attr, &stat);
+        ASSERT_EQ(stat.st_blocks, 0);
+    }
+}
+
+TEST_F(FileSystemTest, Entry2Param) {
+    EntryOut entryOut;
+    entryOut.attr = MkAttr(100, AttrOption().length(4096));
+    entryOut.entryTimeout = 1;
+    entryOut.attrTimeout = 2;
+
+    fuse_entry_param e;
+    auto fs = FileSystemBuilder().Build();
+    fs->Entry2Param(&entryOut, &e);
+    ASSERT_EQ(e.ino, 100);
+    ASSERT_EQ(e.attr.st_size, 4096);
+    ASSERT_EQ(e.generation, 0);
+    ASSERT_EQ(e.entry_timeout, 1);
+    ASSERT_EQ(e.attr_timeout, 2);
+}
+
+TEST_F(FileSystemTest, SetEntryTimeout) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.SetOption([](FileSystemOption* option){
+        option->kernelCacheOption.entryTimeoutSec = 1;
+        option->kernelCacheOption.attrTimeoutSec = 2;
+        option->kernelCacheOption.dirEntryTimeoutSec = 3;
+        option->kernelCacheOption.dirAttrTimeoutSec = 4;
+    }).Build();
+
+    // CASE 1: set kernel cache timeout for regular file or symbol link
+    std::vector<FsFileType> types = {
+        FsFileType::TYPE_S3,
+        FsFileType::TYPE_FILE,
+        FsFileType::TYPE_SYM_LINK,
+    };
+    for (const auto& type : types) {
+        auto attr = MkAttr(100, AttrOption().type(type));
+        auto entryOut = EntryOut(attr);
+        fs->SetEntryTimeout(&entryOut);
+        ASSERT_EQ(entryOut.entryTimeout, 1);
+        ASSERT_EQ(entryOut.attrTimeout, 2);
+    }
+
+    // CASE 2: set kernel cache timeout for directory
+    auto attr = MkAttr(100, AttrOption().type(FsFileType::TYPE_DIRECTORY));
+    auto entryOut = EntryOut(attr);
+    fs->SetEntryTimeout(&entryOut);
+    ASSERT_EQ(entryOut.entryTimeout, 3);
+    ASSERT_EQ(entryOut.attrTimeout, 4);
+}
+
+TEST_F(FileSystemTest, SetAttrTimeout) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.SetOption([](FileSystemOption* option){
+        option->kernelCacheOption.attrTimeoutSec = 1;
+        option->kernelCacheOption.dirAttrTimeoutSec = 2;
+    }).Build();
+
+    // CASE 1: set kernel attribute cache timeout
+    //         for regular file or symbol link
+    std::vector<FsFileType> types = {
+        FsFileType::TYPE_S3,
+        FsFileType::TYPE_FILE,
+        FsFileType::TYPE_SYM_LINK,
+    };
+    for (const auto& typ : types) {
+        auto attr = MkAttr(100, AttrOption().type(typ));
+        auto attrOut = AttrOut(attr);
+        fs->SetAttrTimeout(&attrOut);
+        ASSERT_EQ(attrOut.attrTimeout, 1);
+    }
+
+    // CASE 2: set kernel attribute cache timeout for directory
+    auto attr = MkAttr(100, AttrOption().type(FsFileType::TYPE_DIRECTORY));
+    auto attrOut = AttrOut(attr);
+    fs->SetAttrTimeout(&attrOut);
+    ASSERT_EQ(attrOut.attrTimeout, 2);
+}
+
+TEST_F(FileSystemTest, Reply) {
+    // TODO(Wine93): make it works
+}
+
+TEST_F(FileSystemTest, Lookup_Basic) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    EXPECT_CALL_RETURN_GetDentry(*builder.GetDentryManager(),
+                                 CURVEFS_ERROR::OK);
+    EXPECT_CALL_RETURN_GetInodeAttr(*builder.GetInodeManager(),
+                                    CURVEFS_ERROR::OK);
+
+    EntryOut entryOut;
+    auto rc = fs->Lookup(Request(), 1, "f1", &entryOut);
+    ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+}
+
+TEST_F(FileSystemTest, Lookup_NameTooLong) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.SetOption([](FileSystemOption* option) {
+        option->maxNameLength = 255;
+    }).Build();
+
+    EntryOut entryOut;
+    auto rc = fs->Lookup(Request(), 1, std::string(256, 'x'), &entryOut);
+    ASSERT_EQ(rc, CURVEFS_ERROR::NAMETOOLONG);
+}
+
+TEST_F(FileSystemTest, Lookup_NegativeCache) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.SetOption([](FileSystemOption* option) {
+        option->lookupCacheOption.negativeTimeoutSec = 1;
+        option->lookupCacheOption.lruSize = 100000;
+    }).Build();
+
+    EXPECT_CALL_RETURN_GetDentry(*builder.GetDentryManager(),
+                                 CURVEFS_ERROR::NOTEXIST);
+
+    EntryOut entryOut;
+    auto rc = fs->Lookup(Request(), 1, "f1", &entryOut);
+    ASSERT_EQ(rc, CURVEFS_ERROR::NOTEXIST);
+
+    rc = fs->Lookup(Request(), 1, "f1", &entryOut);
+    ASSERT_EQ(rc, CURVEFS_ERROR::NOTEXIST);
+}
+
+TEST_F(FileSystemTest, GetAttr_Basic) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    EXPECT_CALL_INVOKE_GetInodeAttr(*builder.GetInodeManager(),
+        [&](uint64_t ino, InodeAttr* attr) -> CURVEFS_ERROR {
+            attr->set_inodeid(ino);
+            attr->set_length(4096);
+            attr->set_mtime(123);
+            attr->set_mtime_ns(456);
+            return CURVEFS_ERROR::OK;
+        });
+
+    AttrOut attrOut;
+    auto rc = fs->GetAttr(Request(), 100, &attrOut);
+    ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+    ASSERT_EQ(attrOut.attr.inodeid(), 100);
+    ASSERT_EQ(attrOut.attr.length(), 4096);
+    ASSERT_EQ(attrOut.attr.mtime(), 123);
+    ASSERT_EQ(attrOut.attr.mtime_ns(), 456);
+}
+
+TEST_F(FileSystemTest, OpenDir_Basic) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    EXPECT_CALL_RETURN_GetInodeAttr(*builder.GetInodeManager(),
+                                    CURVEFS_ERROR::OK);
+
+    auto fi = FileInfo();
+    auto rc = fs->OpenDir(Request(), 1, &fi);
+    ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+}
+
+TEST_F(FileSystemTest, ReadDir_Basic) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    // mock what opendir() does:
+    auto handler = fs->NewHandler();
+    auto fi = FileInfo();
+    fi.fh = handler->fh;
+
+    // CASE 1: readdir success
+    EXPECT_CALL_RETURN_ListDentry(*builder.GetDentryManager(),
+                                  CURVEFS_ERROR::OK);
+    EXPECT_CALL_RETURN_BatchGetInodeAttrAsync(*builder.GetInodeManager(),
+                                              CURVEFS_ERROR::OK);
+
+    auto entries = std::make_shared<DirEntryList>();
+    auto rc = fs->ReadDir(Request(), 1, &fi, &entries);
+    ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+}
+
+TEST_F(FileSystemTest, ReadDir_CheckEntries) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+    Ino ino(1);
+
+    // mock what opendir() does:
+    auto handler = fs->NewHandler();
+    auto fi = FileInfo();
+    fi.fh = handler->fh;
+
+    auto CHECK_ENTRIES = [&](const std::shared_ptr<DirEntryList>& entries) {
+        std::vector<DirEntry> out;
+        entries->Iterate([&](DirEntry* dirEntry){
+            out.push_back(*dirEntry);
+        });
+        ASSERT_EQ(out.size(), 3);
+
+        int idx = 0;
+        for (auto ino = 100; ino <= 102; ino++) {
+            ASSERT_EQ(out[idx].ino, ino);
+            ASSERT_EQ(out[idx].name, StrFormat("f%d", ino));
+            ASSERT_EQ(out[idx].attr.mtime(), 123);
+            ASSERT_EQ(out[idx].attr.mtime_ns(), ino);
+            idx++;
+        }
+    };
+
+    // CASE 1: check entries
+    {
+        EXPECT_CALL_INVOKE_ListDentry(*builder.GetDentryManager(),
+            [&](uint64_t parent,
+                std::list<Dentry>* dentries,
+                uint32_t limit,
+                bool only,
+                uint32_t nlink) -> CURVEFS_ERROR {
+                for (auto ino = 100; ino <= 102; ino++) {
+                    dentries->push_back(MkDentry(ino, StrFormat("f%d", ino)));
+                }
+                return CURVEFS_ERROR::OK;
+            });
+
+        EXPECT_CALL_INVOKE_BatchGetInodeAttrAsync(*builder.GetInodeManager(),
+            [&](uint64_t parentId,
+                std::set<uint64_t>* inos,
+                std::map<uint64_t, InodeAttr>* attrs) -> CURVEFS_ERROR {
+                for (const auto& ino : *inos) {
+                    auto attr = MkAttr(ino, AttrOption().mtime(123, ino));
+                    attrs->emplace(ino, attr);
+                }
+                return CURVEFS_ERROR::OK;
+            });
+
+        auto entries = std::make_shared<DirEntryList>();
+        auto rc = fs->ReadDir(Request(), ino, &fi, &entries);
+        ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+        CHECK_ENTRIES(entries);
+    }
+
+    // CASE 2: check dir cache
+    {
+        // readdir from cache
+        auto entries = std::make_shared<DirEntryList>();
+        auto rc = fs->ReadDir(Request(), ino, &fi, &entries);
+        ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+        CHECK_ENTRIES(entries);
+    }
+}
+
+TEST_F(FileSystemTest, ReleaseDir_Basic) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    auto fi = FileInfo();
+    auto rc = fs->ReleaseDir(Request(), 1, &fi);
+    ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+}
+
+TEST_F(FileSystemTest, ReleaseDir_CheckHandler) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    // mock what opendir() does:
+    auto handler = fs->NewHandler();
+    auto fh = handler->fh;
+
+    // CASE 1: find handler success
+    ASSERT_TRUE(fs->FindHandler(fh) != nullptr);
+
+    // CASE 2: releasedir will release handler
+    auto fi = FileInfo();
+    fi.fh = fh;
+    auto rc = fs->ReleaseDir(Request(), 1, &fi);
+    ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+    ASSERT_TRUE(fs->FindHandler(fh) == nullptr);
+}
+
+TEST_F(FileSystemTest, Open_Basic) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    // mock what lookup() does:
+    Ino ino(100);
+    auto attrWatcher = fs->BorrowMember().attrWatcher;
+    attrWatcher->RemeberMtime(MkAttr(ino, AttrOption().mtime(123, 456)));
+
+    // CASE 1: open success
+    {
+        EXPECT_CALL_INVOKE_GetInode(*builder.GetInodeManager(),
+            [&](uint64_t ino,
+                std::shared_ptr<InodeWrapper>& inode) -> CURVEFS_ERROR {
+                inode = MkInode(ino, InodeOption().mtime(123, 456));
+                return CURVEFS_ERROR::OK;
+            });
+
+        auto fi = FileInfo();
+        auto rc = fs->Open(Request(), ino, &fi);
+        ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+    }
+
+    // CASE 2: file already opened
+    {
+        auto fi = FileInfo();
+        auto rc = fs->Open(Request(), ino, &fi);
+        ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+    }
+}
+
+TEST_F(FileSystemTest, Open_Stale) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    // mock lookup() does:
+    Ino ino(100);
+    auto attrWatcher = fs->BorrowMember().attrWatcher;
+    attrWatcher->RemeberMtime(MkAttr(ino, AttrOption().mtime(123, 456)));
+
+    // CASE 1: mtime(123, 456) != mtime(123, 789)
+    EXPECT_CALL_INVOKE_GetInode(*builder.GetInodeManager(),
+        [&](uint64_t ino, std::shared_ptr<InodeWrapper>& inode)
+            -> CURVEFS_ERROR {
+            inode = MkInode(ino, InodeOption().mtime(123, 789));
+            return CURVEFS_ERROR::OK;
+        });
+
+    auto fi = FileInfo();
+    auto rc = fs->Open(Request(), ino, &fi);
+    ASSERT_EQ(rc, CURVEFS_ERROR::STALE);
+}
+
+TEST_F(FileSystemTest, Open_StaleForAttrNotFound) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    EXPECT_CALL_INVOKE_GetInode(*builder.GetInodeManager(),
+       [&](uint64_t ino, std::shared_ptr<InodeWrapper>& inode)
+           -> CURVEFS_ERROR {
+           inode = MkInode(ino, InodeOption().mtime(123, 456));
+           return CURVEFS_ERROR::OK;
+       });
+
+    Ino ino(100);
+    auto fi = FileInfo();
+    auto rc = fs->Open(Request(), ino, &fi);
+    ASSERT_EQ(rc, CURVEFS_ERROR::STALE);
+}
+
+TEST_F(FileSystemTest, Release_Basic) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+    auto rc = fs->Release(Request(), 100);
+    ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+}
+
+TEST_F(FileSystemTest, Release_CheckOpenStatus) {
+    auto builder = FileSystemBuilder();
+    auto fs = builder.Build();
+
+    // mock what open() does:
+    Ino ino(100);
+    auto inode = MkInode(100);
+    auto openfiles = fs->BorrowMember().openFiles;
+    openfiles->Open(ino, inode);
+
+    // CASE 1: ino(100) is opened
+    auto out = MkInode(0);
+    bool yes = openfiles->IsOpened(ino, &out);
+    ASSERT_TRUE(yes);
+    ASSERT_EQ(inode->GetInodeId(), ino);
+
+    // CASE 2: release will close open file
+    auto rc = fs->Release(Request(), 100);
+    ASSERT_EQ(rc, CURVEFS_ERROR::OK);
+    yes = openfiles->IsOpened(ino, &out);
+    ASSERT_FALSE(yes);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/test/client/filesystem/helper/builder.h

@@ -0,0 +1,247 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-29
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_BUILDER_H_
+#define CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_BUILDER_H_
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+#include <string>
+#include <memory>
+
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/filesystem.h"
+#include "curvefs/test/client/mock_metaserver_client.h"
+#include "curvefs/test/client/mock_inode_cache_manager.h"
+#include "curvefs/test/client/mock_dentry_cache_mamager.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::KernelCacheOption;
+
+class DeferSyncBuilder {
+ public:
+    using Callback = std::function<void(DeferSyncOption* option)>;
+
+    static DeferSyncOption DefaultOption() {
+        return DeferSyncOption {
+            delay: 3,
+            deferDirMtime: false,
+        };
+    }
+
+ public:
+    DeferSyncBuilder()
+        : option_(DefaultOption()),
+          dentryManager_(std::make_shared<MockDentryCacheManager>()),
+          inodeManager_(std::make_shared<MockInodeCacheManager>()) {}
+
+    DeferSyncBuilder SetOption(Callback callback) {
+        callback(&option_);
+        return *this;
+    }
+
+    std::shared_ptr<DeferSync> Build() {
+        return std::make_shared<DeferSync>(option_);
+    }
+
+    std::shared_ptr<MockDentryCacheManager> GetDentryManager() {
+        return dentryManager_;
+    }
+
+    std::shared_ptr<MockInodeCacheManager> GetInodeManager() {
+        return inodeManager_;
+    }
+
+ private:
+    DeferSyncOption option_;
+    std::shared_ptr<MockDentryCacheManager> dentryManager_;
+    std::shared_ptr<MockInodeCacheManager> inodeManager_;
+};
+
+class DirCacheBuilder {
+ public:
+    using Callback = std::function<void(DirCacheOption* option)>;
+
+    static DirCacheOption DefaultOption() {
+        return DirCacheOption {
+            lruSize: 5000000,
+        };
+    }
+
+ public:
+    DirCacheBuilder() : option_(DefaultOption()) {}
+
+    DirCacheBuilder SetOption(Callback callback) {
+        callback(&option_);
+        return *this;
+    }
+
+    std::shared_ptr<DirCache> Build() {
+        return std::make_shared<DirCache>(option_);
+    }
+
+ private:
+    DirCacheOption option_;
+};
+
+class OpenFilesBuilder {
+ public:
+    using Callback = std::function<void(OpenFilesOption* option)>;
+
+    static OpenFilesOption DefaultOption() {
+        return OpenFilesOption {
+            lruSize: 65535,
+            deferSyncSecond: 3,
+        };
+    }
+
+ public:
+    OpenFilesBuilder()
+        : option_(DefaultOption()),
+          deferSync_(DeferSyncBuilder().Build()) {}
+
+    OpenFilesBuilder SetOption(Callback callback) {
+        callback(&option_);
+        return *this;
+    }
+
+    std::shared_ptr<OpenFiles> Build() {
+        return std::make_shared<OpenFiles>(option_, deferSync_);
+    }
+
+ private:
+    std::shared_ptr<DeferSync> deferSync_;
+    OpenFilesOption option_;
+};
+
+class RPCClientBuilder {
+ public:
+    using Callback = std::function<void(RPCOption* option)>;
+
+    static RPCOption DefaultOption() {
+        return RPCOption{ listDentryLimit: 65535 };
+    }
+
+ public:
+    RPCClientBuilder()
+        : option_(DefaultOption()),
+          dentryManager_(std::make_shared<MockDentryCacheManager>()),
+          inodeManager_(std::make_shared<MockInodeCacheManager>()) {}
+
+    RPCClientBuilder SetOption(Callback callback) {
+        callback(&option_);
+        return *this;
+    }
+
+    std::shared_ptr<RPCClient> Build() {
+        ExternalMember member(dentryManager_, inodeManager_);
+        return std::make_shared<RPCClient>(option_, member);
+    }
+
+    std::shared_ptr<MockDentryCacheManager> GetDentryManager() {
+        return dentryManager_;
+    }
+
+    std::shared_ptr<MockInodeCacheManager> GetInodeManager() {
+        return inodeManager_;
+    }
+
+ private:
+    RPCOption option_;
+    std::shared_ptr<MockDentryCacheManager> dentryManager_;
+    std::shared_ptr<MockInodeCacheManager> inodeManager_;
+};
+
+// build filesystem which you want
+class FileSystemBuilder {
+ public:
+    using Callback = std::function<void(FileSystemOption* option)>;
+
+    FileSystemOption DefaultOption() {
+        auto option = FileSystemOption();
+        auto kernelCacheOption = KernelCacheOption {
+            entryTimeoutSec: 3600,
+            dirEntryTimeoutSec: 3600,
+            attrTimeoutSec: 3600,
+            dirAttrTimeoutSec: 3600,
+        };
+        auto lookupCacheOption = LookupCacheOption {
+            lruSize: 100000,
+            negativeTimeoutSec: 0,
+        };
+        auto attrWatcherOption = AttrWatcherOption {
+            lruSize: 5000000,
+        };
+
+        option.cto = true;
+        option.disableXattr = true;
+        option.maxNameLength = 255;
+        option.blockSize = 0x10000u;
+        option.kernelCacheOption = kernelCacheOption;
+        option.lookupCacheOption = lookupCacheOption;
+        option.dirCacheOption = DirCacheBuilder::DefaultOption();
+        option.openFilesOption = OpenFilesBuilder::DefaultOption();
+        option.attrWatcherOption = attrWatcherOption;
+        option.rpcOption = RPCClientBuilder::DefaultOption();
+        option.deferSyncOption = DeferSyncBuilder::DefaultOption();
+        return option;
+    }
+
+ public:
+    FileSystemBuilder()
+        : option_(DefaultOption()),
+          dentryManager_(std::make_shared<MockDentryCacheManager>()),
+          inodeManager_(std::make_shared<MockInodeCacheManager>()) {}
+
+    FileSystemBuilder SetOption(Callback callback) {
+        callback(&option_);
+        return *this;
+    }
+
+    std::shared_ptr<FileSystem> Build() {
+        auto member = ExternalMember(dentryManager_, inodeManager_);
+        return std::make_shared<FileSystem>(option_, member);
+    }
+
+    std::shared_ptr<MockDentryCacheManager> GetDentryManager() {
+        return dentryManager_;
+    }
+
+    std::shared_ptr<MockInodeCacheManager> GetInodeManager() {
+        return inodeManager_;
+    }
+
+ private:
+    FileSystemOption option_;
+    std::shared_ptr<MockDentryCacheManager> dentryManager_;
+    std::shared_ptr<MockInodeCacheManager> inodeManager_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_BUILDER_H_

curvefs/test/client/filesystem/helper/expect.h

@@ -0,0 +1,130 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-29
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_EXPECT_H_
+#define CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_EXPECT_H_
+
+#include <gmock/gmock.h>
+
+#include "curvefs/test/client/mock_metaserver_client.h"
+#include "curvefs/test/client/mock_inode_cache_manager.h"
+#include "curvefs/test/client/mock_dentry_cache_mamager.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::testing::_;
+using ::testing::Return;
+using ::testing::Invoke;
+using ::curvefs::client::MockDentryCacheManager;
+using ::curvefs::client::MockInodeCacheManager;
+using ::curvefs::client::rpcclient::MockMetaServerClient;
+
+/*
+ * Reference:
+ *
+ * DentryCacheManager:
+ *   ListDentry(uint64_t parent,
+ *              std::list<Dentry> *dentryList,
+ *              uint32_t limit,
+ *              bool dirOnly = false,
+ *              uint32_t nlink = 0);
+ *
+ *
+ * InodeCacheManager:
+ *   GetInodeAttr(uint64_t inodeId, InodeAttr *out);
+ *
+ *   BatchGetInodeAttrAsync(uint64_t parentId,
+ *                          std::set<uint64_t>* inodeIds,
+ *                          std::map<uint64_t, InodeAttr> *attrs);
+ *
+ *   GetInode(uint64_t inodeId,
+ *            std::shared_ptr<InodeWrapper>& out);
+ */
+
+// times
+#define EXPECT_CALL_INDOE_SYNC_TIMES(CLIENT, INO, TIMES)                   \
+    EXPECT_CALL(CLIENT, UpdateInodeWithOutNlinkAsync_rvr(_, INO, _, _, _)) \
+        .Times(TIMES)
+
+// return
+#define EXPECT_CALL_RETURN_GetDentry(MANAGER, CODE) \
+do {                                                \
+    EXPECT_CALL(MANAGER, GetDentry(_, _, _))        \
+        .WillOnce(Return(CODE));                    \
+} while (0)
+
+#define EXPECT_CALL_RETURN_ListDentry(MANAGER, CODE) \
+do {                                                 \
+    EXPECT_CALL(MANAGER, ListDentry(_, _, _, _, _))  \
+        .WillOnce(Return(CODE));                     \
+} while (0)
+
+#define EXPECT_CALL_RETURN_GetInodeAttr(MANAGER, CODE) \
+do {                                                   \
+    EXPECT_CALL(MANAGER, GetInodeAttr(_, _))           \
+        .WillOnce(Return(CODE));                       \
+} while (0)
+
+#define EXPECT_CALL_RETURN_BatchGetInodeAttrAsync(MANAGER, CODE) \
+do {                                                             \
+    EXPECT_CALL(MANAGER, BatchGetInodeAttrAsync(_, _, _))        \
+        .WillOnce(Return(CODE));                                 \
+} while (0)
+
+#define EXPECT_CALL_RETURN_GetInode(MANAGER, CODE) \
+do {                                               \
+    EXPECT_CALL(MANAGER, GetInode(_, _))           \
+        .WillOnce(Return(CODE));                   \
+} while (0)
+
+// invoke
+#define EXPECT_CALL_INVOKE_ListDentry(MANAGER, CALLBACK) \
+do {                                                     \
+    EXPECT_CALL(MANAGER, ListDentry(_, _, _, _, _))      \
+        .WillOnce(Invoke(CALLBACK));                     \
+} while (0)
+
+#define EXPECT_CALL_INVOKE_GetInodeAttr(MANAGER, CALLBACK) \
+do {                                                       \
+    EXPECT_CALL(MANAGER, GetInodeAttr(_, _))               \
+        .WillOnce(Invoke(CALLBACK));                       \
+} while (0)
+
+#define EXPECT_CALL_INVOKE_BatchGetInodeAttrAsync(MANAGER, CALLBACK) \
+do {                                                                 \
+    EXPECT_CALL(MANAGER, BatchGetInodeAttrAsync(_, _, _))            \
+        .WillOnce(Invoke(CALLBACK));                                 \
+} while (0)
+
+#define EXPECT_CALL_INVOKE_GetInode(MANAGER, CALLBACK) \
+do {                                                   \
+    EXPECT_CALL(MANAGER, GetInode(_, _))               \
+        .WillOnce(Invoke(CALLBACK));                   \
+} while (0)
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_EXPECT_H_

curvefs/test/client/filesystem/helper/helper.h

@@ -0,0 +1,38 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-29
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_HELPER_H_
+#define CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_HELPER_H_
+
+#include "curvefs/test/client/filesystem/helper/builder.h"
+#include "curvefs/test/client/filesystem/helper/expect.h"
+#include "curvefs/test/client/filesystem/helper/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_HELPER_H_

curvefs/test/client/filesystem/helper/meta.cpp

@@ -0,0 +1,132 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-29
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/test/client/filesystem/helper/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+AttrOption AttrOption::type(FsFileType type) {
+    type_ = type;
+    return *this;
+}
+
+AttrOption AttrOption::mode(uint32_t mode) {
+    mode_ = mode;
+    return *this;
+}
+
+AttrOption AttrOption::nlink(uint32_t nlink) {
+    nlink_ = nlink;
+    return *this;
+}
+
+AttrOption AttrOption::uid(uint32_t uid) {
+    uid_ = uid;
+    return *this;
+}
+
+AttrOption AttrOption::gid(uint32_t gid) {
+    gid_ = gid;
+    return *this;
+}
+
+AttrOption AttrOption::length(uint64_t length) {
+    length_ = length;
+    return *this;
+}
+
+AttrOption AttrOption::rdev(uint64_t rdev) {
+    rdev_ = rdev;
+    return *this;
+}
+
+AttrOption AttrOption::atime(uint64_t seconds, uint32_t naoSeconds) {
+    atime_ = TimeSpec(seconds, naoSeconds);
+    return *this;
+}
+
+AttrOption AttrOption::mtime(uint64_t seconds, uint32_t naoSeconds) {
+    mtime_ = TimeSpec(seconds, naoSeconds);
+    return *this;
+}
+
+AttrOption AttrOption::ctime(uint64_t seconds, uint32_t naoSeconds) {
+    ctime_ = TimeSpec(seconds, naoSeconds);
+    return *this;
+}
+
+InodeOption InodeOption::mtime(uint64_t seconds, uint32_t naoSeconds) {
+    mtime_ = TimeSpec(seconds, naoSeconds);
+    return *this;
+}
+
+InodeOption InodeOption::metaClient(
+    std::shared_ptr<MetaServerClient> metaClient) {
+    metaClient_ = metaClient;
+    return *this;
+}
+
+InodeAttr MkAttr(Ino ino, AttrOption option) {
+    InodeAttr attr;
+    attr.set_inodeid(ino);
+    if (option.type_) { attr.set_type(option.type_); }
+    attr.set_mode(option.mode_);
+    attr.set_nlink(option.nlink_);
+    attr.set_uid(option.uid_);
+    attr.set_gid(option.gid_);
+    attr.set_length(option.length_);
+    attr.set_rdev(option.rdev_);
+    attr.set_atime(option.atime_.seconds);
+    attr.set_atime_ns(option.atime_.nanoSeconds);
+    attr.set_mtime(option.mtime_.seconds);
+    attr.set_mtime_ns(option.mtime_.nanoSeconds);
+    attr.set_ctime(option.ctime_.seconds);
+    attr.set_ctime_ns(option.ctime_.nanoSeconds);
+    return attr;
+}
+
+std::shared_ptr<InodeWrapper> MkInode(Ino ino, InodeOption option) {
+    Inode inode;
+    inode.set_inodeid(ino);
+    inode.set_mtime(option.mtime_.seconds);
+    inode.set_mtime_ns(option.mtime_.nanoSeconds);
+    return std::make_shared<InodeWrapper>(inode, option.metaClient_);
+}
+
+Dentry MkDentry(Ino ino, const std::string& name) {
+    Dentry dentry;
+    dentry.set_inodeid(ino);
+    dentry.set_name(name);
+    return dentry;
+}
+
+DirEntry MkDirEntry(Ino ino,
+                    const std::string& name,
+                    InodeAttr attr) {
+    return DirEntry(ino, name, attr);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/test/client/filesystem/helper/meta.h

@@ -0,0 +1,100 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-29
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_META_H_
+#define CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_META_H_
+
+#include <gmock/gmock.h>
+
+#include <string>
+#include <memory>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/filesystem.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::absl::StrFormat;
+
+struct AttrOption {
+ public:
+    AttrOption() = default;
+    AttrOption type(FsFileType type);
+    AttrOption mode(uint32_t mode);
+    AttrOption nlink(uint32_t nlink);
+    AttrOption uid(uint32_t uid);
+    AttrOption gid(uint32_t gid);
+    AttrOption length(uint64_t length);
+    AttrOption rdev(uint64_t rdev);
+    AttrOption atime(uint64_t seconds, uint32_t naoSeconds);
+    AttrOption mtime(uint64_t seconds, uint32_t naoSeconds);
+    AttrOption ctime(uint64_t seconds, uint32_t naoSeconds);
+
+ private:
+    friend InodeAttr MkAttr(Ino ino, AttrOption option);
+
+ private:
+    FsFileType type_;
+    uint32_t mode_;
+    uint32_t nlink_;
+    uint32_t uid_;
+    uint32_t gid_;
+    uint64_t length_;
+    uint64_t rdev_;
+    TimeSpec atime_;
+    TimeSpec mtime_;
+    TimeSpec ctime_;
+};
+
+class InodeOption {
+ public:
+    InodeOption() = default;
+    InodeOption mtime(uint64_t seconds, uint32_t naoSeconds);
+    InodeOption metaClient(std::shared_ptr<MetaServerClient> metaClient);
+
+ private:
+    friend std::shared_ptr<InodeWrapper> MkInode(Ino ino, InodeOption option);
+
+ private:
+    TimeSpec mtime_;
+    std::shared_ptr<MetaServerClient> metaClient_;
+};
+
+InodeAttr MkAttr(Ino ino, AttrOption option = AttrOption());
+
+std::shared_ptr<InodeWrapper> MkInode(Ino ino,
+                                      InodeOption option = InodeOption());
+
+Dentry MkDentry(Ino ino, const std::string& name);
+
+DirEntry MkDirEntry(Ino ino,
+                    const std::string& name,
+                    InodeAttr attr = MkAttr(0));
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_TEST_CLIENT_FILESYSTEM_HELPER_META_H_

curvefs/test/client/filesystem/lookup_cache_test.cpp

@@ -0,0 +1,90 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-04-03
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <gtest/gtest.h>
+
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+class LookupCacheTest : public ::testing::Test {
+ protected:
+    void SetUp() override {}
+    void TearDown() override {}
+};
+
+TEST_F(LookupCacheTest, Basic) {
+    auto option = LookupCacheOption{ lruSize: 10, negativeTimeoutSec: 1 };
+    auto cache = std::make_shared<LookupCache>(option);
+
+    ASSERT_FALSE(cache->Get(1, "f1"));
+
+    cache->Put(1, "f1");
+    ASSERT_TRUE(cache->Get(1, "f1"));
+}
+
+TEST_F(LookupCacheTest, Enable) {
+    // CASE 1: cache off, negativeTimeoutSec = 0.
+    auto option = LookupCacheOption{ lruSize: 10, negativeTimeoutSec: 0 };
+    auto cache = std::make_shared<LookupCache>(option);
+    cache->Put(1, "f1");
+    ASSERT_FALSE(cache->Get(1, "f1"));
+
+    // CASE 2: cache on, negativeTimeoutSec = 1.
+    option = LookupCacheOption{ lruSize: 10, negativeTimeoutSec: 1 };
+    cache = std::make_shared<LookupCache>(option);
+    cache->Put(1, "f1");
+    ASSERT_TRUE(cache->Get(1, "f1"));
+}
+
+TEST_F(LookupCacheTest, Timeout) {
+    auto option = LookupCacheOption{ lruSize: 10, negativeTimeoutSec: 1 };
+    auto cache = std::make_shared<LookupCache>(option);
+
+    // CASE 1: cache hit.
+    cache->Put(1, "f1");
+    ASSERT_TRUE(cache->Get(1, "f1"));
+
+    // CASE 2: cache miss due to expiration.
+    std::this_thread::sleep_for(std::chrono::seconds(1));
+    ASSERT_FALSE(cache->Get(1, "f1"));
+}
+
+TEST_F(LookupCacheTest, LRUSize) {
+    auto option = LookupCacheOption{ lruSize: 1, negativeTimeoutSec: 1 };
+    auto cache = std::make_shared<LookupCache>(option);
+
+    // CASE 1: cache hit.
+    cache->Put(1, "f1");
+    ASSERT_TRUE(cache->Get(1, "f1"));
+
+    // CASE 2: cache miss due to eviction.
+    cache->Put(1, "f2");
+    ASSERT_FALSE(cache->Get(1, "f1"));
+    ASSERT_TRUE(cache->Get(1, "f2"));
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs