fix bugs

1. fuse_volume_client just flush inode but do not flush data in Flush 2. Optimize blockgroup allocation logic: - fix the available group will always decrease in the case of insufficient allocation on the mds side - the blockgroup assigned by mds to the client is allowed to be smaller than the count required by the client - mds does not allocate blockgroups exceeding a certain water level 3. fix the issue where the cache of metaserver_client could only correspond to a single file system 4. Optimize log: add some vlog to print more details Signed-off-by: ilixiaocui <ilixiaocui@163.com>
curvefs/client: fix s3 object will not be removed
2023-07-12 18:35:19 +08:00 · 2023-07-12 18:35:19 +08:00 · 2023-06-05 09:43:23 +08:00 · 2023-05-31 16:35:59 +08:00 · 2023-05-31 16:35:59 +08:00
606 changed files with 17265 additions and 21188 deletions
--- a/.bazelrc
+++ b/.bazelrc
@ -1,10 +1,7 @@
-build --verbose_failures
-
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
 build --incompatible_blacklisted_protos_requires_proto_info=false
 build --copt=-fdiagnostics-color=always
-
 run --copt=-fdiagnostics-color=always
--- a/.gitignore
+++ b/.gitignore
@ -147,22 +147,3 @@ tools-v2/proto/curvefs/*
 tools-v2/*/*.test
 tools-v2/__debug_bin
 tools-v2/vendor/
-
-.test
-.note
-.playground
-.dumpfile
-metastore_test.dat
-GPATH
-GRTAGS
-GTAGS
-core.*
-
-test/integration/*.conf
-test/integration/client/config/client.conf*
-test/integration/snapshotcloneserver/config/*.conf
-
-.pre-commit-config.yaml
-
-*.deb
-*.whl
--- a/.obm.cfg
+++ b/.obm.cfg
@ -1,2 +0,0 @@
-container_name: curve-build-playground-master
-container_image: opencurvedocker/curve-base:build-debian9
--- a/11
+++ b/11
@ -1,6 +1,6 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.

-.PHONY: list build dep install image playground check test
+.PHONY: list build dep install image

 stor?=""
 prefix?= "$(PWD)/projects"
@ -70,12 +70,3 @@ install:

 image:
 	@bash util/image.sh $(stor) $(tag) $(os)
-
-playground:
-	@bash util/playground.sh
-
-check:
-	@bash util/check.sh $(stor)
-
-test:
-	@bash util/test.sh $(stor) $(only)
--- a/33
+++ b/33
@ -75,14 +75,11 @@ bind(
 )

 #import the  gtest files.
-http_archive(
+new_git_repository(
    name = "com_google_googletest",
-    urls = [
-        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
-        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
-    ],
-    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
-    strip_prefix = "googletest-release-1.12.1",
+    build_file = "//:thirdparties/gmock.BUILD",
+    remote = "https://github.com/google/googletest",
+    tag = "release-1.8.0",
 )

 bind(
@ -231,24 +228,6 @@ http_archive(
  sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )

-# fmt
-http_archive(
-  name = "fmt",
-  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
-  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
-  strip_prefix = "fmt-9.1.0",
-  build_file = "//:thirdparties/fmt.BUILD",
-)
-
-# spdlog
-http_archive(
-  name = "spdlog",
-  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
-  strip_prefix = "spdlog-1.11.0",
-  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
-  build_file = "//:thirdparties/spdlog.BUILD",
-)
-
 # Bazel platform rules.
 http_archive(
    name = "platforms",
@ -269,14 +248,14 @@ new_local_repository(
 http_archive(
    name = "hedron_compile_commands",

-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
+    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. 
    # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
    urls = [
        "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
        "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
    ],
    strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
+    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." 
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()
--- a/buildfs.sh
+++ b/buildfs.sh
@ -7,22 +7,12 @@ then
    exit
 fi

-if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
-then
-    bazelflags=''
-else
-    bazelflags='--copt -faligned-new'
-fi
-
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi

-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
--define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
-DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
-
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
 if [ $? -ne 0 ]
 then
    echo "build curvefs failed"
@ -44,5 +34,4 @@ then
    echo "mds_test failed"
    exit
 fi
-fi
-echo "end compile"
+fi
--- a/conf/chunkserver.conf
+++ b/conf/chunkserver.conf
@ -9,20 +9,13 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
-# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
-# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
-# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
-# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
-global.block_size=4096
-
 # clone chunk允许的最长location长度
 global.location_limit=3000
+# minimum alignment for io request
+global.min_io_alignment=512

 #
 # MDS settings
@ -115,7 +108,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=true
+copyset.enable_odsync_when_open_chunkfile=false
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB
--- a/conf/chunkserver.conf.example
+++ b/conf/chunkserver.conf.example
@ -13,12 +13,9 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
-# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
-# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
-# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
-# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
-global.block_size=4096
 global.location_limit=3000
+# minimum alignment for io request
+global.min_io_alignment=512

 #
 # MDS settings
@ -111,7 +108,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=true
+copyset.enable_odsync_when_open_chunkfile=false
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB
--- a/conf/client.conf
+++ b/conf/client.conf
@ -168,6 +168,13 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000

+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
+
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@ -181,3 +188,4 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
+
--- a/conf/cs_client.conf
+++ b/conf/cs_client.conf
@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
--- a/conf/mds.conf
+++ b/conf/mds.conf
@ -197,8 +197,6 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
-# smallest read/write unit for volume, support |512| and |4096|
-mds.curvefs.blockSize=4096

 #
 # chunkseverclient config
@ -239,16 +237,3 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
-
-#
-## poolset rules
-#
-# for backward compatibility, rules are applied for select poolset when creating file
-#
-# for example
-#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
-#
-# when creating file reqeust doesn't have poolset, above rules are used to select poolset
-# - if filename is /dir1/file, then poolset1 is select
-# - if filename is /dir1/sub/file, then sub is select
-mds.poolset.rules=
--- a/conf/py_client.conf
+++ b/conf/py_client.conf
@ -153,3 +153,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
--- a/conf/snap_client.conf
+++ b/conf/snap_client.conf
@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
--- a/curve-ansible/roles/generate_config/defaults/main.yml
+++ b/curve-ansible/roles/generate_config/defaults/main.yml
@ -164,6 +164,7 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
+chunkserver_min_io_alignment: 512

 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@ -243,6 +244,8 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
+client_alignment_common: 512
+client_alignment_clone: 4096

 # nebd默认配置
 client_config_path: /etc/curve/client.conf
--- a/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2
+++ b/curve-ansible/roles/generate_config/templates/chunkserver.conf.j2
@ -14,6 +14,8 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
+# minimum alignment for io request
+global.min_io_alignment={{ chunkserver_min_io_alignment }}

 #
 # MDS settings
--- a/curve-ansible/roles/generate_config/templates/client.conf.j2
+++ b/curve-ansible/roles/generate_config/templates/client.conf.j2
@ -175,3 +175,10 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume={{ client_alignment_common }}
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume={{ client_alignment_clone }}
--- a/curvefs/conf/client.conf
+++ b/curvefs/conf/client.conf
@ -58,10 +58,6 @@ executorOpt.maxRetryTimesBeforeConsiderSuspend=20
 # batch limit of get inode attr and xattr
 executorOpt.batchInodeAttrLimit=10000

-#### spaceserver
-spaceServer.spaceAddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
-spaceServer.rpcTimeoutMs=1000
-
 #### bdev
 # curve client's config file
 bdev.confPath=/etc/curve/client.conf
@ -77,7 +73,17 @@ rpc.healthCheckIntervalSec=0

 #### fuseClient
 # TODO(xuchaojie): add unit
+fuseClient.attrTimeOut=1.0
+fuseClient.entryTimeOut=1.0
 fuseClient.listDentryLimit=65536
+fuseClient.flushPeriodSec=5
+fuseClient.maxNameLength=255
+fuseClient.iCacheLruSize=65536
+fuseClient.dCacheLruSize=1000000
+fuseClient.enableICacheMetrics=true
+fuseClient.enableDCacheMetrics=true
+fuseClient.lruTimeOutSec=60
+fuseClient.cto=true
 fuseClient.downloadMaxRetryTimes=3

 ### kvcache opt
@ -104,6 +110,7 @@ fuseClient.maxDataSize=1024
 fuseClient.refreshDataIntervalSec=30
 fuseClient.warmupThreadsNum=10

+
 # the write throttle bps of fuseClient, default no limit
 fuseClient.throttle.avgWriteBytes=0
 # the write burst bps of fuseClient, default no limit
@ -132,32 +139,6 @@ fuseClient.throttle.burstReadIops=0
 # the times that read burst Iops can continue, default 180s
 fuseClient.throttle.burstReadIopsSecs=180

-#### filesystem metadata
-# {
-# fs.disableXattr:
-#   if you want to get better metadata performance,
-#   you can mount fs with |fs.disableXattr| is true
-#
-# fs.lookupCache.negativeTimeoutSec:
-#   entry which not found will be cached if |timeout| > 0
-fs.cto=true
-fs.maxNameLength=255
-fs.disableXattr=false
-fs.accessLogging=true
-fs.kernelCache.attrTimeoutSec=3600
-fs.kernelCache.dirAttrTimeoutSec=3600
-fs.kernelCache.entryTimeoutSec=3600
-fs.kernelCache.dirEntryTimeoutSec=3600
-fs.lookupCache.negativeTimeoutSec=0
-fs.lookupCache.minUses=1
-fs.lookupCache.lruSize=100000
-fs.dirCache.lruSize=5000000
-fs.openFile.lruSize=65536
-fs.attrWatcher.lruSize=5000000
-fs.rpc.listDentryLimit=65536
-fs.deferSync.delay=3
-fs.deferSync.deferDirMtime=false
-# }

 #### volume
 volume.bigFileSize=1048576
@ -177,6 +158,15 @@ volume.bitmapAllocator.smallAllocProportion=0.2
 # number of block groups that allocated once
 volume.blockGroup.allocateOnce=4

+## spaceserver
+# the space used by the blockgroup exceeds this percentage and can
+# be returned to mds [0.8-1]
+volume.space.useThreshold=0.95
+
+# the background thread calculates the time interval for returning
+# the blockgroup to mds
+volume.space.releaseInterSec=300
+
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false
--- a/curvefs/conf/curvebs_client.conf
+++ b/curvefs/conf/curvebs_client.conf
@ -0,0 +1,191 @@
+#
+################### mds一侧配置信息 ##################
+#
+
+# mds的地址信息，对于mds集群，地址以逗号隔开
+mds.listen.addr=127.0.0.1:6666
+
+# 初始化阶段向mds注册开关，默认为开
+mds.registerToMDS=true
+
+# 与mds通信的rpc超时时间
+mds.rpcTimeoutMS=500
+
+# 与mds通信rpc最大的超时时间, 指数退避的超时间不能超过这个值
+mds.maxRPCTimeoutMS=2000
+
+# 与mds通信重试总时间
+mds.maxRetryMS=8000
+
+# 在当前mds上连续重试次数超过该限制就切换, 这个失败次数包含超时重试次数
+mds.maxFailedTimesBeforeChangeMDS=2
+
+# 与MDS一侧保持一个lease时间内多少次续约
+mds.refreshTimesPerLease=4
+
+# mds RPC接口每次重试之前需要先睡眠一段时间
+mds.rpcRetryIntervalUS=100000
+
+# The normal retry times for trigger wait strategy
+mds.normalRetryTimesBeforeTriggerWait=3
+
+# Max retry time for IO-Path request
+mds.maxRetryMsInIOPath=86400000
+
+# Sleep interval for wait
+mds.waitSleepMs=10000
+
+#
+################# metacache配置信息 ################
+#
+
+# 获取leader的rpc超时时间
+metacache.getLeaderTimeOutMS=500
+
+# 获取leader的重试次数
+metacache.getLeaderRetry=5
+
+# 获取leader接口每次重试之前需要先睡眠一段时间
+metacache.rpcRetryIntervalUS=100000
+
+#
+############### 调度层的配置信息 #############
+#
+
+# 调度层队列大小，每个文件对应一个队列
+# 调度队列的深度会影响client端整体吞吐，这个队列存放的是异步IO任务。。
+schedule.queueCapacity=1000000
+
+# 队列的执行线程数量
+# 执行线程所要做的事情就是将IO取出，然后发到网络就返回取下一个网络任务。一个任务从
+# 队列取出到发送完rpc请求大概在(20us-100us)，20us是正常情况下不需要获取leader的时候
+# 如果在发送的时候需要获取leader，时间会在100us左右，一个线程的吞吐在10w-50w
+# 性能已经满足需求
+schedule.threadpoolSize=2
+
+# 为隔离qemu侧线程引入的任务队列，因为qemu一侧只有一个IO线程
+# 当qemu一侧调用aio接口的时候直接将调用push到任务队列就返回，
+# 这样libcurve不占用qemu的线程，不阻塞其异步调用
+isolation.taskQueueCapacity=1000000
+
+# 隔离qemu线程的任务队列线程池大小, 默认值为1个线程
+isolation.taskThreadPoolSize=1
+
+
+#
+################ 与chunkserver通信相关配置 #############
+#
+# 读写接口失败的OP之间重试睡眠
+chunkserver.opRetryIntervalUS=100000
+
+# 失败的OP重试次数
+chunkserver.opMaxRetry=2500000
+
+# 与chunkserver通信的rpc超时时间
+chunkserver.rpcTimeoutMS=1000
+
+# 开启基于appliedindex的读，用于性能优化
+chunkserver.enableAppliedIndexRead=1
+
+# 重试请求之间睡眠最长时间
+# 因为当网络拥塞的时候或者chunkserver出现过载的时候，需要增加睡眠时间
+# 这个时间最大为maxRetrySleepIntervalUs
+chunkserver.maxRetrySleepIntervalUS=8000000
+
+# 重试请求的超时rpc时间最大值，超时时间会遵循指数退避策略
+# 因为当网络拥塞的时候出现超时，需要增加RPC超时时间
+# 这个时间最大为maxTimeoutMS
+chunkserver.maxRPCTimeoutMS=8000
+
+# 同一个chunkserver连续超时上限次数
+# 如果超过这个值，就会进行健康检查，健康检查失败后，会标记为unstable
+chunkserver.maxStableTimeoutTimes=10
+# chunkserver上rpc连续超时后，健康检查请求的超时间
+chunkserver.checkHealthTimeoutMs=100
+# 同一个server上unstable的chunkserver数量超过这个值之后
+# 所有的chunkserver都会标记为unstable
+chunkserver.serverStableThreshold=3
+
+# 当底层chunkserver压力大时，可能也会触发unstable
+# 由于copyset leader may change，会导致请求超时时间设置为默认值，从而导致IO hang
+# 真正宕机的情况下，请求重试一定次数后会处理完成
+# 如果一直重试，则不是宕机情况，这时候超时时间还是要进入指数退避逻辑
+# 当一个请求重试次数超过这个值时，其超时时间一定进入指数退避
+chunkserver.minRetryTimesForceTimeoutBackoff=5
+
+# 当一个rpc重试超过次数maxRetryTimesBeforeConsiderSuspend的时候
+# 记为悬挂IO，metric会报警
+chunkserver.maxRetryTimesBeforeConsiderSuspend=20
+
+#
+################# 文件级别配置项 #############
+#
+# libcurve底层rpc调度允许最大的未返回rpc数量，每个文件的inflight RPC独立
+global.fileMaxInFlightRPCNum=128
+
+# 文件IO下发到底层chunkserver最大的分片KB
+global.fileIOSplitMaxSizeKB=64
+
+#
+################# log相关配置 ###############
+#
+# enable logging or not
+global.logging.enable=True
+#
+# log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
+global.logLevel=0
+# 设置log的路径
+global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ /curvebs/client/logs __CURVEADM_TEMPLATE__
+# 单元测试情况下
+# logpath=./runlog/
+
+#
+################# 读源卷相关配置 ###############
+#
+# 读取源卷时打开的fd超时关闭时间300s
+closefd.timeout=300
+# 读取源卷时打开的fd后台线程每600s扫描一遍fdMap，关闭超时fd
+closefd.timeInterval=600
+
+#
+############### metric 配置信息 #############
+#
+global.metricDummyServerStartPort=9000
+
+# 是否关闭健康检查: true/关闭 false/不关闭
+global.turnOffHealthCheck=true
+
+#
+### throttle config
+#
+throttle.enable=false
+
+##### discard configurations #####
+# enable/disable discard
+discard.enable=true
+# discard granularity
+discard.granularity=4096
+# discard cleanup task delay times in millisecond
+discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
+
+##### chunkserver client option #####
+# chunkserver client rpc timeout time
+csClientOpt.rpcTimeoutMs=500
+# chunkserver client rpc max try
+csClientOpt.rpcMaxTry=86400000
+# chunkserver client rpc retry interval
+csClientOpt.rpcIntervalUs=100000
+# chunkserver client rpc max timeout time
+csClientOpt.rpcMaxTimeoutMs=8000
+
+##### chunkserver broadcaster option #####
+# broad cast max machine num
+csBroadCasterOpt.broadCastMaxNum=200
+
--- a/curvefs/conf/mds.conf
+++ b/curvefs/conf/mds.conf
@ -6,9 +6,6 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
 mds.loglevel=0
-# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
-# Default value is -1 which disables the feature.
-mds.server.idleTimeoutSec=-1

 #
 # space options
@ -162,3 +159,6 @@ bs.mds.maxFailedTimesBeforeChangeMDS=2
 bs.mds.normalRetryTimesBeforeTriggerWait=3
 # sleep interval in ms for wait
 bs.mds.waitSleepMs=1000
+
+#### Options for volume space deallcatable setting
+mds.space.calIntervalSec=60
--- a/curvefs/conf/metaserver.conf
+++ b/curvefs/conf/metaserver.conf
@ -128,25 +128,16 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000

-#
-# Concurrent apply queue
-### concurrent apply queue options for each copyset
-### concurrent apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
+### apply queue options for each copyset
+### apply queue is used to isolate raft threads, each worker has its own queue
+### when a task can be applied it's been pushed into a corresponding worker queue by certain rules
+# number of apply queue workers for each, each worker will start a indepent thread
+applyqueue.worker_count=3

-# worker_count: number of apply queue workers for each, each worker will start a indepent thread
-# queue_depth: apply queue depth for each copyset
+# apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-# write apply queue workers count
-applyqueue.write_worker_count=3
-# write apply queue depth
-applyqueue.write_queue_depth=1
-# read apply queue workers count
-applyqueue.read_worker_count=2
-# read apply queue depth
-applyqueue.read_queue_depth=1
-
+applyqueue.queue_depth=1

 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@ -154,10 +145,6 @@ applyqueue.read_queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto

-# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
-# Default value is -1 which disables the feature.
-server.idleTimeoutSec=-1
-
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need
@ -320,3 +307,9 @@ metaCacheOpt.metacacheGetLeaderRetry=3
 metaCacheOpt.metacacheRPCRetryIntervalUS=100000
 # RPC timeout of get leader
 metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000
+
+#### volume deallocate
+volume.deallocate.enable=true
+volume.deallocate.workerNum=5
+volume.deallocate.batchClean=10
+volume.sdk.confPath=/etc/curvefs/curvebs_client.conf
--- a/curvefs/proto/BUILD
+++ b/curvefs/proto/BUILD
@ -113,6 +113,7 @@ proto_library(
    name = "curvefs_heartbeat_proto",
    srcs = ["heartbeat.proto"],
    deps = [":curvefs_common_proto",
+            ":metaserver_proto",
            "//proto:heartbeat_proto"],
 )

--- a/curvefs/proto/common.proto
+++ b/curvefs/proto/common.proto
@ -31,6 +31,13 @@ enum BitmapLocation {
    AtEnd = 2;
 }

+message EmptyMsg {}
+
+message BlockGroupID {
+    required uint64 fsId = 1;
+    required uint64 offset = 2;
+}
+
 // When creating fs, `volumeSize` and `extendAlignment` are fetched from the bs cluster
 message Volume {
    optional uint64 volumeSize = 1;
--- a/curvefs/proto/heartbeat.proto
+++ b/curvefs/proto/heartbeat.proto
@ -16,6 +16,7 @@

 syntax = "proto2";
 import "curvefs/proto/common.proto";
+import "curvefs/proto/metaserver.proto";
 import "proto/heartbeat.proto";
 package curvefs.mds.heartbeat;
 option cc_generic_services = true;
@ -27,6 +28,17 @@ option go_package = "curvefs/proto/heartbeat";
 //     required uint64 diskUsedByte = 2;  // the disk this copyset used
 // };

+enum BlockGroupDeallcateStatusCode {
+    BGDP_PROCESSING = 0;
+    BGDP_DONE = 1;
+}
+
+message BlockGroupStatInfo {
+    required uint32 fsId = 1;
+    repeated metaserver.DeallocatableBlockGroup deallocatableBlockGroups = 2;
+    map<uint64, BlockGroupDeallcateStatusCode> blockGroupDeallocateStatus = 3;
+}
+
 message CopySetInfo {
    required uint32 poolId = 1;
    required uint32 copysetId = 2;
@ -81,6 +93,7 @@ message MetaServerHeartbeatRequest {
    required uint32 leaderCount = 7;
    required uint32 copysetCount = 8;
    required MetaServerSpaceStatus spaceStatus = 9;
+    repeated BlockGroupStatInfo blockGroupStatInfos = 10;
 };

 message CopySetConf {
@ -107,11 +120,14 @@ enum HeartbeatStatusCode {
    hbMetaServerIpPortNotMatch = 2;
    hbMetaServerTokenNotMatch = 3;
    hbAnalyseCopysetError = 4;
+    hbMetaServerFSUnkown = 5;
 }

 message MetaServerHeartbeatResponse {
    required HeartbeatStatusCode statusCode = 1;
    repeated CopySetConf needUpdateCopysets = 2;
+    // key is fsid, value is blockgroup offset
+    map<uint64, uint64> issuedBlockGroups = 3;
 };

 service HeartbeatService {
--- a/curvefs/proto/mds.proto
+++ b/curvefs/proto/mds.proto
@ -60,6 +60,7 @@ enum FSStatusCode {
    INSERT_MANAGE_INODE_FAIL = 35;
    DELETE_DENTRY_FAIL = 36;
    UPDATE_FS_FAIL = 37;
+    SPACE_RELEASE_FAIL = 38;
 }

 // fs interface
--- a/curvefs/proto/metaserver.proto
+++ b/curvefs/proto/metaserver.proto
@ -167,6 +167,46 @@ enum FsFileType {
    TYPE_S3 = 4;
 };

+message DeallocatableBlockGroup {
+    required uint64 blockGroupOffset = 1;
+    optional uint64 deallocatableSize = 2;
+    repeated uint64 inodeIdlist = 3;
+    repeated uint64 inodeIdUnderDeallocate = 4;
+
+    oneof type {
+        IncreaseDeallocatableBlockGroup increase = 5;
+        DecreaseDeallocatableBlockGroup decrease = 6;
+        MarkDeallocatableBlockGroup mark = 7;
+    }
+}
+
+message IncreaseDeallocatableBlockGroup {
+    required uint64 increaseDeallocatableSize = 1;
+    repeated uint64 inodeIdlistAdd = 2;
+}
+
+message DecreaseDeallocatableBlockGroup {
+    required uint64 decreaseDeallocatableSize = 1;
+    repeated uint64 inodedDeallocated = 2;
+}
+
+message MarkDeallocatableBlockGroup {
+    repeated uint64 inodeIdUnderDeallocate = 2;
+}
+
+message UpdateDeallocatableBlockGroupRequest {
+    required uint32 poolId = 1;
+    required uint32 copysetId = 2;
+    required uint32 partitionId = 3;
+    required uint64 fsId = 4;
+    repeated DeallocatableBlockGroup update = 5;
+}
+
+message UpdateDeallocatableBlockGroupResponse {
+    required MetaStatusCode statusCode = 1; 
+    optional uint64 appliedIndex = 2;
+}
+
 message VolumeExtent {
    required uint64 fsOffset = 1;
    required uint64 volumeOffset = 2;
@ -179,7 +219,7 @@ message VolumeExtentSlice {
    repeated VolumeExtent extents = 2;
 }

-message VolumeExtentList {
+message VolumeExtentSliceList {
    repeated VolumeExtentSlice slices = 1;
 }

@ -317,7 +357,7 @@ message UpdateInodeRequest {
    map<string, bytes> xattr = 20;
    repeated uint64 parent = 21;
    map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
-    optional VolumeExtentList volumeExtents = 23;
+    optional VolumeExtentSliceList volumeExtents = 23;
 }

 message UpdateInodeResponse {
@ -464,7 +504,7 @@ message GetVolumeExtentRequest {
 message GetVolumeExtentResponse {
    required MetaStatusCode statusCode = 1;
    optional uint64 appliedIndex = 2;
-    optional VolumeExtentList slices = 3;
+    optional VolumeExtentSliceList slices = 3;
 }

 message UpdateVolumeExtentRequest {
@ -473,7 +513,7 @@ message UpdateVolumeExtentRequest {
    required uint32 partitionId = 3;
    required uint32 fsId = 4;
    required uint64 inodeId = 5;
-    required VolumeExtentList extents = 6;
+    required VolumeExtentSliceList extents = 6;
 }

 message UpdateVolumeExtentResponse {
@ -508,4 +548,7 @@ service MetaServerService {
    // volume extent interface
    rpc GetVolumeExtent(GetVolumeExtentRequest) returns (GetVolumeExtentResponse);
    rpc UpdateVolumeExtent(UpdateVolumeExtentRequest) returns (UpdateVolumeExtentResponse);
+
+    // block group with deallocatable inode list interface
+    rpc UpdateDeallocatableBlockGroup(UpdateDeallocatableBlockGroupRequest) returns (UpdateDeallocatableBlockGroupResponse);
 }
--- a/curvefs/proto/space.proto
+++ b/curvefs/proto/space.proto
@ -34,6 +34,7 @@ enum SpaceErrCode {
    SpaceErrParam = 10;
    SpaceErrNotSupport = 11;
    SpaceErrExtendVolumeError = 12;
+    SpaceErrRelease = 13;
 }

 message BlockGroup {
@ -48,6 +49,10 @@ message BlockGroup {
    required common.BitmapLocation bitmaplocation = 4;
    // owner, who owns this block group
    optional string owner = 5;
+    // deallocating, metaserver who is deallocate this block group
+    repeated uint32 deallocating = 6;
+    // deallocated, metaserver who finish deallocate this block group
+    repeated uint32 deallocated = 7;
 }

 message AllocateBlockGroupRequest {
--- a/curvefs/src/client/BUILD
+++ b/curvefs/src/client/BUILD
@ -43,10 +43,13 @@ cc_library(
            "s3/*.h",
            "volume/*.cpp",
            "volume/*.h",
-            "filesystem/*.cpp",
-            "filesystem/*.h",
            "warmup/*.h",
            "warmup/*.cpp",
+            "cache/*.h",
+            "cache/*.cpp",
+            "cache/diskcache/*.h",
+            "cache/diskcache/*.cpp",
+
        ],
        exclude = ["main.cpp"],
    ),
@ -73,14 +76,11 @@ cc_library(
        "//curvefs/src/volume",
        "//curvefs/src/common:metric_utils",
        "//curvefs/src/common:dynamic_vlog",
-        "//curvefs/src/common:threading",
        "@com_google_absl//absl/memory",
        "@com_google_absl//absl/strings",
        "@com_google_absl//absl/synchronization",
-        "@com_google_absl//absl/strings:str_format",
        "@com_google_absl//absl/meta:type_traits",
        "@com_google_absl//absl/types:optional",
        "@com_google_googletest//:gtest_prod",
-        "@spdlog//:spdlog",
    ],
 )
--- a/curvefs/src/client/async_request_closure.cpp
+++ b/curvefs/src/client/async_request_closure.cpp
@ -28,7 +28,7 @@
 #include <mutex>

 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/inode_wrapper.h"

 namespace curvefs {
@ -63,19 +63,26 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
        cond_.wait(lk);
    }

-    return ToFSError(GetStatusCode());
+    return MetaStatusCodeToCurvefsErrCode(GetStatusCode());
 }

 void UpdateVolumeExtentClosure::Run() {
    auto st = GetStatusCode();
    if (!IsOK(st)) {
-        LOG(ERROR) << "UpdateVolumeExtent failed, error: "
-                   << MetaStatusCode_Name(st)
-                   << ", inodeid: " << inode_->GetInodeId();
-        inode_->MarkInodeError();
+        if (inode_ != nullptr) {
+            inode_->MarkInodeError();
+            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
+                       << MetaStatusCode_Name(st)
+                       << ", inodeid: " << inode_->GetInodeId();
+        } else {
+            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
+                       << MetaStatusCode_Name(st);
+        }
    }

-    inode_->syncingVolumeExtentsMtx_.unlock();
+    if (inode_ != nullptr) {
+        inode_->syncingVolumeExtentsMtx_.unlock();
+    }

    if (sync_) {
        std::lock_guard<bthread::Mutex> lk(mtx_);
--- a/curvefs/src/client/async_request_closure.h
+++ b/curvefs/src/client/async_request_closure.h
@ -28,15 +28,12 @@

 #include <memory>

+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
-#include "curvefs/src/client/filesystem/error.h"

 namespace curvefs {
 namespace client {

-using ::curvefs::client::filesystem::CURVEFS_ERROR;
-using ::curvefs::client::filesystem::ToFSError;
-
 class InodeWrapper;

 namespace internal {
--- a/curvefs/src/client/cache/diskcache/disk_cache_base.cpp
+++ b/curvefs/src/client/cache/diskcache/disk_cache_base.cpp
@ -28,7 +28,7 @@
 #include <dirent.h>
 #include <functional>

-#include "curvefs/src/client/s3/disk_cache_base.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_base.h"

 namespace curvefs {

--- a/curvefs/src/client/cache/diskcache/disk_cache_base.h
+++ b/curvefs/src/client/cache/diskcache/disk_cache_base.h
@ -20,8 +20,8 @@
 * Author: wuhongsong
 */

-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_BASE_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_BASE_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_BASE_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_BASE_H_

 #include <glog/logging.h>

@ -71,4 +71,4 @@ class DiskCacheBase {
 }  // namespace client
 }  // namespace curvefs

-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_BASE_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_BASE_H_
--- a/curvefs/src/client/cache/diskcache/disk_cache_manager.cpp
+++ b/curvefs/src/client/cache/diskcache/disk_cache_manager.cpp
@ -19,6 +19,9 @@
 * Created Date: 21-08-13
 * Author: hzwuhongsong
 */
+
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager.h"
+
 #include <sys/vfs.h>
 #include <errno.h>
 #include <string>
@ -27,7 +30,6 @@
 #include <list>

 #include "curvefs/src/client/s3/client_s3_adaptor.h"
-#include "curvefs/src/client/s3/disk_cache_manager.h"
 #include "curvefs/src/common/s3util.h"

 namespace curvefs {
@ -230,7 +232,12 @@ int DiskCacheManager::WriteDiskFile(const std::string fileName, const char *buf,
                                    uint64_t length, bool force) {
    // write throttle
    diskCacheThrottle_.Add(false, length);
+    uint64_t start = butil::cpuwide_time_us();
    int ret = cacheWrite_->WriteDiskFile(fileName, buf, length, force);
+    if ((ret >= 0) &&
+      metric_.get() != nullptr) {
+        CollectMetrics(&metric_->adaptorWriteDiskCache, length, start);
+    }
    if (ret > 0)
        AddDiskUsedBytes(ret);
    return ret;
@ -244,14 +251,26 @@ int DiskCacheManager::ReadDiskFile(const std::string name, char *buf,
                                   uint64_t offset, uint64_t length) {
    // read throttle
    diskCacheThrottle_.Add(true, length);
-    return cacheRead_->ReadDiskFile(name, buf, offset, length);
+    uint64_t start = butil::cpuwide_time_us();
+    int ret = cacheRead_->ReadDiskFile(
+      name, buf, offset, length);
+    if ((ret >= 0) &&
+      metric_.get() != nullptr) {
+        CollectMetrics(&metric_->adaptorWriteDiskCache, length, start);
+    }
+    return ret;
 }

 int DiskCacheManager::WriteReadDirect(const std::string fileName,
                                      const char *buf, uint64_t length) {
    // write hrottle
    diskCacheThrottle_.Add(false, length);
+    uint64_t start = butil::cpuwide_time_us();
    int ret = cacheRead_->WriteDiskFile(fileName, buf, length);
+    if ((ret >= 0) &&
+      metric_.get() != nullptr) {
+        CollectMetrics(&metric_->adaptorWriteDiskCache, length, start);
+    }
    if (ret > 0)
        AddDiskUsedBytes(ret);
    return ret;
--- a/curvefs/src/client/cache/diskcache/disk_cache_manager.h
+++ b/curvefs/src/client/cache/diskcache/disk_cache_manager.h
@ -19,8 +19,9 @@
 * Created Date: 21-08-13
 * Author: wuhongsong
 */
-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_
+
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_H_

 #include <bthread/mutex.h>

@ -38,8 +39,8 @@
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/common/utils.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/s3/disk_cache_write.h"
-#include "curvefs/src/client/s3/disk_cache_read.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_write.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_read.h"
 #include "curvefs/src/client/common/config.h"
 namespace curvefs {
 namespace client {
@ -59,7 +60,7 @@ class DiskCacheManager {
                     std::shared_ptr<DiskCacheWrite> cacheWrite,
                     std::shared_ptr<DiskCacheRead> cacheRead);
    DiskCacheManager() {}
-    virtual ~DiskCacheManager() { TrimStop(); }
+    virtual ~DiskCacheManager() { TrimStop();}

    virtual int Init(std::shared_ptr<S3Client> client,
        const S3ClientAdaptorOption option);
@ -76,33 +77,48 @@ class DiskCacheManager {
    void AddCache(const std::string &name);

    int CreateDir();
+
    std::string GetCacheReadFullDir();
+
    std::string GetCacheWriteFullDir();

    int WriteDiskFile(const std::string fileName, const char *buf,
                      uint64_t length, bool force = true);
-    void AsyncUploadEnqueue(const std::string objName);
+
    virtual int WriteReadDirect(const std::string fileName, const char *buf,
                                uint64_t length);
+
    int ReadDiskFile(const std::string name, char *buf, uint64_t offset,
                     uint64_t length);
+
    int LinkWriteToRead(const std::string fileName,
                        const std::string fullWriteDir,
                        const std::string fullReadDir);
+
+    void AsyncUploadEnqueue(const std::string objName);
+
    int UploadAllCacheWriteFile();
+
    int UploadWriteCacheByInode(const std::string &inode);
+
    int ClearReadCache(const std::list<std::string> &files);
+
    /**
     * @brief get use ratio of cache disk
     * @return the use ratio
     */
    int64_t SetDiskFsUsedRatio();
+
    virtual bool IsDiskCacheFull();
+
    bool IsDiskCacheSafe();
+
    /**
     * @brief: start trim thread.
     */
+
    int TrimRun();
+
    /**
     * @brief: stop trim thread.
     */
@ -129,6 +145,7 @@ class DiskCacheManager {
                << ", now is: " << usedBytes_.load();
        return;
    }
+
    /**
     * @brief dec the used bytes of disk cache.
     * can not dec disk used bytes after file have been loaded,
@ -143,12 +160,15 @@ class DiskCacheManager {
                << ", now is: " << usedBytes_.load();
        return;
    }
+
    void SetDiskInitUsedBytes();
+
    uint64_t GetDiskUsedbytes() {
        return usedBytes_.load();
    }

    void InitQosParam();
+
    /**
     * @brief trim cache func.
     */
@ -164,6 +184,14 @@ class DiskCacheManager {
     */
    bool IsCacheClean();

+    void CollectMetrics(InterfaceMetric *interface,
+      int count, uint64_t start) {
+        interface->bps.count << count;
+        interface->qps.count << 1;
+        interface->latency << (butil::cpuwide_time_us() - start);
+    }
+
+ private:
    curve::common::Thread backEndThread_;
    curve::common::Atomic<bool> isRunning_;
    curve::common::InterruptibleSleeper sleeper_;
@ -202,4 +230,4 @@ class DiskCacheManager {
 }  // namespace client
 }  // namespace curvefs

-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_H_
--- a/curvefs/src/client/cache/diskcache/disk_cache_manager_impl.cpp
+++ b/curvefs/src/client/cache/diskcache/disk_cache_manager_impl.cpp
@ -27,7 +27,7 @@
 #include <memory>

 #include "curvefs/src/client/s3/client_s3_adaptor.h"
-#include "curvefs/src/client/s3/disk_cache_manager_impl.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h"

 namespace curvefs {

@ -70,8 +70,6 @@ void DiskCacheManagerImpl::Enqueue(
    taskPool_.Enqueue(task);
 }

-
-
 int DiskCacheManagerImpl::WriteReadDirectClosure(
    std::shared_ptr<PutObjectAsyncContext> context) {
        VLOG(9) << "WriteReadClosure start, name: " << context->key;
--- a/curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h
+++ b/curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h
@ -19,8 +19,8 @@
 * Created Date: 21-08-13
 * Author: wuhongsong
 */
-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_IMPL_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_IMPL_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_IMPL_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_IMPL_H_

 #include <bthread/mutex.h>

@ -35,9 +35,9 @@
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/s3/disk_cache_write.h"
-#include "curvefs/src/client/s3/disk_cache_read.h"
-#include "curvefs/src/client/s3/disk_cache_manager.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_write.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_read.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager.h"

 namespace curvefs {
 namespace client {
@ -46,6 +46,7 @@ using curvefs::common::PosixWrapper;
 using curvefs::client::common::DiskCacheType;
 using curve::common::TaskThreadPool;

+// disk cache opt
 struct DiskCacheOption {
    DiskCacheType diskCacheType;
    uint64_t trimCheckIntervalSec;
@ -72,13 +73,15 @@ class DiskCacheManagerImpl {
        std::shared_ptr<S3Client> client);
    DiskCacheManagerImpl() {}
    virtual ~DiskCacheManagerImpl() {}
+
    /**
     * @brief init DiskCacheManagerImpl
     * @param[in] client S3Client
     * @param[in] option config option
     * @return success: 0, fail : < 0
-     */
-    int Init(const S3ClientAdaptorOption option);
+    */
+    virtual int Init(const S3ClientAdaptorOption option);
+
    /**
     * @brief Write obj
     * @param[in] name obj name
@ -87,12 +90,14 @@ class DiskCacheManagerImpl {
     * @return success: write length, fail : < 0
     */
    int Write(const std::string name, const char *buf, uint64_t length);
+
    /**
     * @brief whether obj is cached in cached disk
     * @param[in] name obj name
     * @return cached: true, not cached : < 0
     */
    bool IsCached(const std::string name);
+
    /**
     * @brief read obj
     * @param[in] name obj name
@ -109,9 +114,18 @@ class DiskCacheManagerImpl {
     */
    int UmountDiskCache();

+    /// @brief disk cache is full or not
+    /// @return true: full, false: not full
    bool IsDiskCacheFull();
+
+    /// @brief write to cache read disk
+    /// @param fileName file name
+    /// @param buf data need to write
+    /// @param length data length need to write
+    /// @return data length write to cache read
    int WriteReadDirect(const std::string fileName, const char *buf,
                        uint64_t length);
+
    void InitMetrics(std::string fsName);

    virtual int UploadWriteCacheByInode(const std::string &inode);
@ -124,14 +138,17 @@ class DiskCacheManagerImpl {
 private:
    int WriteDiskFile(const std::string name, const char *buf, uint64_t length);

-    std::shared_ptr<DiskCacheManager> diskCacheManager_;
-
-    bool forceFlush_;
-    std::shared_ptr<S3Client> client_;
-
    int WriteClosure(std::shared_ptr<PutObjectAsyncContext> context);

    int WriteReadDirectClosure(std::shared_ptr<PutObjectAsyncContext> context);
+
+ private:
+    // disk cache manager
+    std::shared_ptr<DiskCacheManager> diskCacheManager_;
+    // use fdatasync or not
+    bool forceFlush_;
+    // s3 client interface(put/get object from s3)
+    std::shared_ptr<S3Client> client_;
    // threads for disk cache
    uint32_t threads_;
    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
@ -141,4 +158,4 @@ class DiskCacheManagerImpl {
 }  // namespace client
 }  // namespace curvefs

-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_IMPL_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_IMPL_H_
--- a/curvefs/src/client/cache/diskcache/disk_cache_read.cpp
+++ b/curvefs/src/client/cache/diskcache/disk_cache_read.cpp
@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>

-#include "curvefs/src/client/s3/disk_cache_read.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_read.h"
 #include "curvefs/src/common/s3util.h"

 namespace curvefs {
--- a/curvefs/src/client/cache/diskcache/disk_cache_read.h
+++ b/curvefs/src/client/cache/diskcache/disk_cache_read.h
@ -19,8 +19,8 @@
 * Created Date: 21-08-13
 * Author: wuhongsong
 */
-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_READ_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_READ_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_READ_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_READ_H_

 #include <list>
 #include <string>
@ -32,7 +32,7 @@
 #include "src/common/interruptible_sleeper.h"
 #include "src/common/lru_cache.h"
 #include "curvefs/src/common/wrap_posix.h"
-#include "curvefs/src/client/s3/disk_cache_base.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_base.h"

 namespace curvefs {
 namespace client {
@ -73,4 +73,4 @@ class DiskCacheRead : public DiskCacheBase {

 }  // namespace client
 }  // namespace curvefs
-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_READ_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_READ_H_
--- a/curvefs/src/client/cache/diskcache/disk_cache_write.cpp
+++ b/curvefs/src/client/cache/diskcache/disk_cache_write.cpp
@ -27,7 +27,8 @@
 #include <dirent.h>

 #include <vector>
-#include "curvefs/src/client/s3/disk_cache_write.h"
+
+#include "curvefs/src/client/cache/diskcache/disk_cache_write.h"
 #include "curvefs/src/common/s3util.h"

 namespace curvefs {
--- a/curvefs/src/client/cache/diskcache/disk_cache_write.h
+++ b/curvefs/src/client/cache/diskcache/disk_cache_write.h
@ -19,8 +19,8 @@
 * Created Date: 21-08-13
 * Author: wuhongsong
 */
-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_WRITE_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_WRITE_H_

 #include <sys/stat.h>
 #include <bthread/condition_variable.h>
@ -39,9 +39,8 @@
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/common/utils.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/s3/disk_cache_read.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/s3/disk_cache_base.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_base.h"

 namespace curvefs {
 namespace client {
@ -167,4 +166,4 @@ class DiskCacheWrite : public DiskCacheBase {
 }  // namespace client
 }  // namespace curvefs

-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_WRITE_H_
--- a/curvefs/src/client/cache/fuse_client_cache_manager.cpp
+++ b/curvefs/src/client/cache/fuse_client_cache_manager.cpp
--- a/curvefs/src/client/cache/fuse_client_cache_manager.h
+++ b/curvefs/src/client/cache/fuse_client_cache_manager.h
@ -19,8 +19,8 @@
 * Created Date: 21-8-18
 * Author: huyao
 */
-#ifndef CURVEFS_SRC_CLIENT_S3_CLIENT_S3_CACHE_MANAGER_H_
-#define CURVEFS_SRC_CLIENT_S3_CLIENT_S3_CACHE_MANAGER_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_FUSE_CLIENT_CACHE_MANAGER_H_
+#define CURVEFS_SRC_CLIENT_CACHE_FUSE_CLIENT_CACHE_MANAGER_H_

 #include <algorithm>
 #include <cstring>
@ -34,7 +34,7 @@
 #include <set>

 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/s3/client_s3.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/concurrent/task_thread_pool.h"
@ -48,19 +48,16 @@ using curve::common::WriteLockGuard;
 namespace curvefs {
 namespace client {

-class S3ClientAdaptorImpl;
+class StorageAdaptor;
 class ChunkCacheManager;
 class FileCacheManager;
 class FsCacheManager;
 class DataCache;
-class S3ReadRequest;
 using FileCacheManagerPtr = std::shared_ptr<FileCacheManager>;
 using ChunkCacheManagerPtr = std::shared_ptr<ChunkCacheManager>;
 using DataCachePtr = std::shared_ptr<DataCache>;
 using WeakDataCachePtr = std::weak_ptr<DataCache>;
-using curve::common::GetObjectAsyncCallBack;
-using curve::common::PutObjectAsyncCallBack;
-using curve::common::S3Adapter;
+using curve::common::TaskThreadPool;
 using curvefs::metaserver::Inode;
 using curvefs::metaserver::S3ChunkInfo;
 using curvefs::metaserver::S3ChunkInfoList;
@ -87,34 +84,22 @@ struct ReadRequest {
    }
 };

-struct S3ReadRequest {
-    uint64_t chunkId;
-    uint64_t offset;  // file offset
-    uint64_t len;
-    uint64_t objectOffset;  // s3 object's begin in the block
-    uint64_t readOffset;    // read buf offset
-    uint64_t fsId;
+struct UperFlushRequest {
    uint64_t inodeId;
-    uint64_t compaction;
-
-    std::string DebugString() const {
-        std::ostringstream os;
-        os << "S3ReadRequest ( chunkId = " << chunkId << ", offset = " << offset
-           << ", len = " << len << ", objectOffset = " << objectOffset
-           << ", readOffset = " << readOffset << ", fsId = " << fsId
-           << ", inodeId = " << inodeId << ", compaction = " << compaction
-           << " )";
-        return os.str();
-    }
+    const char *buf;
+    uint64_t length;
+    uint64_t offset;   // offset at inode
+    uint64_t chunkId;
+    uint64_t chunkPos;
+    bool sync;
 };

-inline std::string
-S3ReadRequestVecDebugString(const std::vector<S3ReadRequest> &reqs) {
-    std::ostringstream os;
-    for_each(reqs.begin(), reqs.end(),
-             [&](const S3ReadRequest &req) { os << req.DebugString() << " "; });
-    return os.str();
-}
+struct  UperReadRequest {
+    std::vector<ReadRequest> requests;
+    uint64_t inodeId;
+    std::shared_ptr<InodeWrapper> inodeWapper;
+    char *buf;
+};

 struct ObjectChunkInfo {
    S3ChunkInfo s3ChunkInfo;
@ -134,7 +119,7 @@ enum DataCacheStatus {

 class DataCache : public std::enable_shared_from_this<DataCache> {
 public:
-    DataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
+    DataCache(StorageAdaptor *s3ClientAdaptor,
              ChunkCacheManagerPtr chunkCacheManager, uint64_t chunkPos,
              uint64_t len, const char *data,
              std::shared_ptr<KVClientManager> kvClientManager);
@ -209,21 +194,10 @@ class DataCache : public std::enable_shared_from_this<DataCache> {
                             const char *data);
    void AddDataBefore(uint64_t len, const char *data);

-    CURVEFS_ERROR PrepareFlushTasks(
-        uint64_t inodeId, char *data,
-        std::vector<std::shared_ptr<PutObjectAsyncContext>> *s3Tasks,
-        std::vector<std::shared_ptr<SetKVCacheTask>> *kvCacheTasks,
-        uint64_t *chunkId, uint64_t *writeOffset);
-
-    void FlushTaskExecute(
-        CachePolicy cachePolicy,
-        const std::vector<std::shared_ptr<PutObjectAsyncContext>> &s3Tasks,
-        const std::vector<std::shared_ptr<SetKVCacheTask>> &kvCacheTasks);
-
    CachePolicy GetCachePolicy(bool toS3);

 private:
-    S3ClientAdaptorImpl *s3ClientAdaptor_;
+    StorageAdaptor *storageAdaptor_;
    ChunkCacheManagerPtr chunkCacheManager_;
    uint64_t chunkPos_;  // useful chunkPos
    uint64_t len_;  // useful len
@ -238,33 +212,19 @@ class DataCache : public std::enable_shared_from_this<DataCache> {
    std::shared_ptr<KVClientManager> kvClientManager_;
 };

-class S3ReadResponse {
- public:
-    explicit S3ReadResponse(char *data, uint64_t length)
-        : data_(data), len_(length) {}
-
-    char *GetDataBuf() { return data_; }
-
-    uint64_t GetBufLen() { return len_; }
-
- private:
-    char *data_;
-    uint64_t len_;
-};
-
 class ChunkCacheManager
    : public std::enable_shared_from_this<ChunkCacheManager> {
 public:
-    ChunkCacheManager(uint64_t index, S3ClientAdaptorImpl *s3ClientAdaptor,
+    ChunkCacheManager(uint64_t index, StorageAdaptor *s3ClientAdaptor,
                      std::shared_ptr<KVClientManager> kvClientManager)
-        : index_(index), s3ClientAdaptor_(s3ClientAdaptor),
+        : index_(index), storageAdaptor_(s3ClientAdaptor),
          flushingDataCache_(nullptr),
          kvClientManager_(std::move(kvClientManager)) {}
    virtual ~ChunkCacheManager() = default;
-    void ReadChunk(uint64_t index, uint64_t chunkPos, uint64_t readLen,
+    virtual void ReadChunk(uint64_t index, uint64_t chunkPos, uint64_t readLen,
                   char *dataBuf, uint64_t dataBufOffset,
                   std::vector<ReadRequest> *requests);
-    virtual void WriteNewDataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
+    virtual void WriteNewDataCache(StorageAdaptor *s3ClientAdaptor,
                                   uint32_t chunkPos, uint32_t len,
                                   const char *data);
    virtual void AddReadDataCache(DataCachePtr dataCache);
@ -320,7 +280,7 @@ class ChunkCacheManager
        dataRCacheMap_;  // first is pos in chunk

    RWLock rwLockRead_;  //  for read cache
-    S3ClientAdaptorImpl *s3ClientAdaptor_;
+    StorageAdaptor *storageAdaptor_;
    curve::common::Mutex flushMtx_;
    DataCachePtr flushingDataCache_;
    curve::common::Mutex flushingDataCacheMtx_;
@ -331,10 +291,10 @@ class ChunkCacheManager
 class FileCacheManager {
 public:
    FileCacheManager(uint32_t fsid, uint64_t inode,
-                     S3ClientAdaptorImpl *s3ClientAdaptor,
-                     std::shared_ptr<KVClientManager> kvClientManager,
-                     std::shared_ptr<TaskThreadPool<>> threadPool)
-        : fsId_(fsid), inode_(inode), s3ClientAdaptor_(s3ClientAdaptor),
+      StorageAdaptor *s3ClientAdaptor,
+      std::shared_ptr<KVClientManager> kvClientManager,
+      std::shared_ptr<TaskThreadPool<>> threadPool)
+        : fsId_(fsid), inode_(inode), storageAdaptor_(s3ClientAdaptor),
          kvClientManager_(std::move(kvClientManager)),
          readTaskPool_(threadPool) {}
    FileCacheManager() = default;
@ -353,6 +313,8 @@ class FileCacheManager {
    virtual int Read(uint64_t inodeId, uint64_t offset, uint64_t length,
                     char *dataBuf);

+    // for test
+
    bool IsEmpty() { return chunkCacheMap_.empty(); }

    uint64_t GetInodeId() const { return inode_; }
@ -369,24 +331,6 @@ class FileCacheManager {
 private:
    void WriteChunk(uint64_t index, uint64_t chunkPos, uint64_t writeLen,
                    const char *dataBuf);
-    void GenerateS3Request(ReadRequest request,
-                           const S3ChunkInfoList &s3ChunkInfoList,
-                           char *dataBuf, std::vector<S3ReadRequest> *requests,
-                           uint64_t fsId, uint64_t inodeId);
-
-    void PrefetchS3Objs(
-        const std::vector<std::pair<std::string, uint64_t>> &prefetchObjs);
-
-    void HandleReadRequest(const ReadRequest &request,
-                           const S3ChunkInfo &s3ChunkInfo,
-                           std::vector<ReadRequest> *addReadRequests,
-                           std::vector<uint64_t> *deletingReq,
-                           std::vector<S3ReadRequest> *requests, char *dataBuf,
-                           uint64_t fsId, uint64_t inodeId);
-
-    int HandleReadRequest(const std::vector<S3ReadRequest> &requests,
-                          std::vector<S3ReadResponse> *responses,
-                          uint64_t fileLen);

    // GetChunkLoc: get chunk info according to offset
    void GetChunkLoc(uint64_t offset, uint64_t *index, uint64_t *chunkPos,
@ -401,12 +345,7 @@ class FileCacheManager {
                          uint64_t *actualReadLen,
                          std::vector<ReadRequest> *memCacheMissRequest);

-    // miss read from memory read/write cache, need read from
-    // kv(localdisk/remote cache/s3)
-    int GenerateKVRequest(const std::shared_ptr<InodeWrapper> &inodeWrapper,
-                          const std::vector<ReadRequest> &readRequest,
-                          char *dataBuf, std::vector<S3ReadRequest> *kvRequest);
-
+/*  whs
    enum class ReadStatus {
        OK = 0,
        S3_READ_FAIL = -1,
@ -421,39 +360,7 @@ class FileCacheManager {
        }
        return st;
    }
-
-    // read kv request, need
-    ReadStatus ReadKVRequest(const std::vector<S3ReadRequest> &kvRequests,
-                             char *dataBuf, uint64_t fileLen);
-
-    // thread function for ReadKVRequest
-    void ProcessKVRequest(const S3ReadRequest &req, char *dataBuf,
-                          uint64_t fileLen,
-                          std::once_flag &cancelFlag,     // NOLINT
-                          std::atomic<bool> &isCanceled,  // NOLINT
-                          std::atomic<int> &retCode);     // NOLINT
-
-    // read kv request from local disk cache
-    bool ReadKVRequestFromLocalCache(const std::string &name, char *databuf,
-                                     uint64_t offset, uint64_t len);
-
-    // read kv request from remote cache like memcached
-    bool ReadKVRequestFromRemoteCache(const std::string &name, char *databuf,
-                                      uint64_t offset, uint64_t length);
-
-    // read kv request from s3
-    bool ReadKVRequestFromS3(const std::string &name, char *databuf,
-                             uint64_t offset, uint64_t length, int *ret);
-
-    // read retry policy when read from s3 occur not exist error
-    int HandleReadS3NotExist(uint32_t retry,
-                             const std::shared_ptr<InodeWrapper> &inodeWrapper);
-
-    // prefetch for block
-    void PrefetchForBlock(const S3ReadRequest &req, uint64_t fileLen,
-                         uint64_t blockSize, uint64_t chunkSize,
-                         uint64_t startBlockIndex);
-
+*/
 private:
    friend class AsyncPrefetchCallback;

@ -462,9 +369,7 @@ class FileCacheManager {
    std::map<uint64_t, ChunkCacheManagerPtr> chunkCacheMap_;  // first is index
    RWLock rwLock_;
    curve::common::Mutex mtx_;
-    S3ClientAdaptorImpl *s3ClientAdaptor_;
-    curve::common::Mutex downloadMtx_;
-    std::set<std::string> downloadingObj_;
+    StorageAdaptor *storageAdaptor_;

    std::shared_ptr<KVClientManager> kvClientManager_;
    std::shared_ptr<TaskThreadPool<>> readTaskPool_;
@ -472,19 +377,27 @@ class FileCacheManager {

 class FsCacheManager {
 public:
-    FsCacheManager(S3ClientAdaptorImpl *s3ClientAdaptor,
+    FsCacheManager(StorageAdaptor *s3ClientAdaptor,
                   uint64_t readCacheMaxByte, uint64_t writeCacheMaxByte,
                   uint32_t readCacheThreads,
                   std::shared_ptr<KVClientManager> kvClientManager)
        : lruByte_(0), wDataCacheNum_(0), wDataCacheByte_(0),
          readCacheMaxByte_(readCacheMaxByte),
          writeCacheMaxByte_(writeCacheMaxByte),
-          s3ClientAdaptor_(s3ClientAdaptor), isWaiting_(false),
+          storageAdaptor_(s3ClientAdaptor), isWaiting_(false),
          kvClientManager_(std::move(kvClientManager)) {
-        readTaskPool_->Start(readCacheThreads);
+            LOG(INFO) << "whs read task pool start. " << readCacheThreads;
+            readTaskPool_->Start(readCacheThreads);
+            LOG(INFO) << "whs read task pool start end";
    }
+
    FsCacheManager() = default;
-    virtual ~FsCacheManager() { readTaskPool_->Stop(); }
+    virtual ~FsCacheManager() {
+        LOG(INFO) << "whs read task pool stop";
+        readTaskPool_->Stop();
+        LOG(INFO) << "whs read task pool stop end";
+    }
+
    virtual FileCacheManagerPtr FindFileCacheManager(uint64_t inodeId);
    virtual FileCacheManagerPtr FindOrCreateFileCacheManager(uint64_t fsId,
                                                     uint64_t inodeId);
@ -547,6 +460,11 @@ class FsCacheManager {
        assert(ret.second);
        (void)ret;
    }
+
+    std::shared_ptr<TaskThreadPool<>> GetReadTaskPool() {
+        return readTaskPool_;
+    }
+
    void DataCacheNumInc();
    void DataCacheNumFetchSub(uint64_t v);
    void DataCacheByteInc(uint64_t v);
@ -585,7 +503,7 @@ class FsCacheManager {
    std::atomic<uint64_t> wDataCacheByte_;
    uint64_t readCacheMaxByte_;
    uint64_t writeCacheMaxByte_;
-    S3ClientAdaptorImpl *s3ClientAdaptor_;
+    StorageAdaptor *storageAdaptor_;
    bool isWaiting_;
    std::mutex mutex_;
    std::condition_variable cond_;
@ -601,4 +519,4 @@ class FsCacheManager {
 }  // namespace client
 }  // namespace curvefs

-#endif  // CURVEFS_SRC_CLIENT_S3_CLIENT_S3_CACHE_MANAGER_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_FUSE_CLIENT_CACHE_MANAGER_H_
--- a/curvefs/src/client/client_operator.cpp
+++ b/curvefs/src/client/client_operator.cpp
@ -24,14 +24,13 @@

 #include "src/common/uuid.h"
 #include "curvefs/src/client/client_operator.h"
-#include "curvefs/src/client/filesystem/error.h"
+
 namespace curvefs {
 namespace client {

 using ::curve::common::UUIDGenerator;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::mds::topology::PartitionTxId;
-using ::curvefs::client::filesystem::ToFSError;

 #define LOG_ERROR(action, rc) \
    LOG(ERROR) << action << " failed, retCode = " << rc \
@ -96,7 +95,7 @@ CURVEFS_ERROR RenameOperator::GetTxId(uint32_t fsId,
    if (rc != MetaStatusCode::OK) {
        LOG_ERROR("GetTxId", rc);
    }
-    return ToFSError(rc);
+    return MetaStatusCodeToCurvefsErrCode(rc);
 }

 CURVEFS_ERROR RenameOperator::GetLatestTxIdWithLock() {
@ -205,7 +204,7 @@ CURVEFS_ERROR RenameOperator::PrepareRenameTx(
        LOG_ERROR("PrepareRenameTx", rc);
    }

-    return ToFSError(rc);
+    return MetaStatusCodeToCurvefsErrCode(rc);
 }

 CURVEFS_ERROR RenameOperator::PrepareTx() {
@ -419,6 +418,8 @@ CURVEFS_ERROR RenameOperator::UpdateInodeCtime() {
 }

 void RenameOperator::UpdateCache() {
+    dentryManager_->DeleteCache(parentId_, name_);
+    dentryManager_->InsertOrReplaceCache(newDentry_);
    SetTxId(srcPartitionId_, srcTxId_ + 1);
    SetTxId(dstPartitionId_, dstTxId_ + 1);
 }
--- a/curvefs/src/client/client_storage_adaptor.cpp
+++ b/curvefs/src/client/client_storage_adaptor.cpp
@ -0,0 +1,301 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*
+ * Project: curve
+ * Created Date: Thur March 14 2023
+ * Author: wuhongsong
+ */
+
+#include "curvefs/src/client/client_storage_adaptor.h"
+
+#include <brpc/channel.h>
+#include <brpc/controller.h>
+#include <algorithm>
+#include <list>
+
+#include "absl/memory/memory.h"
+
+
+#include "curvefs/src/common/s3util.h"
+
+namespace curvefs {
+namespace client {
+
+CURVEFS_ERROR
+StorageAdaptor::Init(const FuseClientOption &fuseOption,
+    std::shared_ptr<InodeCacheManager> inodeManager,
+    std::shared_ptr<MdsClient> mdsClient,
+    std::shared_ptr<FsCacheManager> fsCacheManager,
+    std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+    std::shared_ptr<KVClientManager> kvClientManager,
+    std::shared_ptr<FsInfo> fsInfo) {
+        const S3ClientAdaptorOption option =
+          fuseOption.s3Opt.s3ClientAdaptorOpt;
+        LOG(INFO) << "StorageAdaptor Init. block size:" << blockSize_
+            << ", chunk size: " << chunkSize_
+            << ", intervalSec: " << option.intervalSec
+            << ", flushIntervalSec: " << option.flushIntervalSec
+            << ", writeCacheMaxByte: " << option.writeCacheMaxByte
+            << ", readCacheMaxByte: " << option.readCacheMaxByte
+            << ", nearfullRatio: " << option.nearfullRatio
+            << ", baseSleepUs: " << option.baseSleepUs;
+        pageSize_ = option.pageSize;
+        if (chunkSize_ % blockSize_ != 0) {
+            LOG(ERROR) << "chunkSize:" << chunkSize_
+                    << " is not integral multiple for the blockSize:"
+                    << blockSize_;
+            return CURVEFS_ERROR::INVALIDPARAM;
+        }
+        diskCacheType_ = option.diskCacheOpt.diskCacheType;
+        memCacheNearfullRatio_ = option.nearfullRatio;
+        throttleBaseSleepUs_ = option.baseSleepUs;
+        flushIntervalSec_ = option.flushIntervalSec;
+        chunkFlushThreads_ = option.chunkFlushThreads;
+        inodeManager_ = inodeManager;
+        mdsClient_ = mdsClient;
+        fsCacheManager_ = fsCacheManager;
+        waitInterval_.Init(option.intervalSec * 1000);
+        fsInfo_ = fsInfo;
+
+        if (nullptr != diskCacheManagerImpl) {
+            diskCacheManagerImpl_ = diskCacheManagerImpl;
+            if (diskCacheManagerImpl_->Init(option) < 0) {
+                LOG(ERROR) << "Init disk cache failed";
+                return CURVEFS_ERROR::INTERNAL;
+            }
+        }
+
+        if (enableBgFlush_) {        // start background flush thread
+            toStop_.store(false, std::memory_order_release);
+            bgFlushThread_ = Thread(&StorageAdaptor::BackGroundFlush, this);
+        }
+
+        // start chunk flush threads
+        taskPool_.Start(chunkFlushThreads_);
+        LOG(INFO) << "storage Adaptor init success.";
+        return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR StorageAdaptor::FuseOpInit(void *userdata,
+  struct fuse_conn_info *conn) {
+    LOG(INFO) << " storage adaptor fuse init start.";
+    SetFsId(fsInfo_->fsid());
+    InitMetrics(fsInfo_->fsname());
+    return CURVEFS_ERROR::OK;
+}
+
+int StorageAdaptor::Stop() {
+    LOG(INFO) << " stop storage adaptor start.";
+    waitInterval_.StopWait();
+    toStop_.store(true, std::memory_order_release);
+    FsSyncSignal();
+    if (bgFlushThread_.joinable()) {
+        bgFlushThread_.join();
+    }
+    if (HasDiskCache()) {
+        diskCacheManagerImpl_->UmountDiskCache();
+    }
+    taskPool_.Stop();
+    LOG(INFO) << " stop storage adaptor success.";
+    return 0;
+}
+
+int StorageAdaptor::Write(uint64_t inodeId, uint64_t offset,
+  uint64_t length, const char *buf) {
+    VLOG(6) << "write start offset:" << offset << ", len:" << length
+            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
+    uint64_t start = butil::cpuwide_time_us();
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindOrCreateFileCacheManager(fsId_, inodeId);
+    {
+        std::lock_guard<std::mutex> lockguard(ioMtx_);
+        fsCacheManager_->DataCacheByteInc(length);
+        uint64_t size = fsCacheManager_->GetDataCacheSize();
+        uint64_t maxSize = fsCacheManager_->GetDataCacheMaxSize();
+        if (size >= maxSize) {
+            VLOG(6) << "write cache is full, wait flush. size: " << size
+                    << ", maxSize:" << maxSize;
+            // offer to do flush
+            waitInterval_.StopWait();
+            fsCacheManager_->WaitFlush();
+        }
+    }
+    uint64_t memCacheRatio = fsCacheManager_->MemCacheRatio();
+    int64_t exceedRatio = memCacheRatio - memCacheNearfullRatio_;
+    if (exceedRatio > 0) {
+        // offer to do flush
+        waitInterval_.StopWait();
+        // upload to s3 derectly or cache disk full
+        bool needSleep =
+            (DisableDiskCache() || IsReadCache()) ||
+            (IsReadWriteCache() && diskCacheManagerImpl_->IsDiskCacheFull());
+        if (needSleep) {
+            uint32_t exponent = pow(2, (exceedRatio) / 10);
+            bthread_usleep(throttleBaseSleepUs_ * exceedRatio * exponent);
+            VLOG(6) << "write cache nearfull and use ratio is: "
+                    << memCacheRatio << ", exponent is: " << exponent;
+        }
+    }
+    int ret = fileCacheManager->Write(offset, length, buf);
+    fsCacheManager_->DataCacheByteDec(length);
+    if (nullptr != GetMetric()) {
+        CollectMetrics(&ioMetric_->adaptorWrite, ret, start);
+    }
+    VLOG(6) << "write end inodeId:" << inodeId << ",ret:" << ret;
+    return ret;
+}
+
+int StorageAdaptor::Read(uint64_t inodeId, uint64_t offset,
+  uint64_t length, char *buf) {
+    VLOG(6) << "read start offset:" << offset << ", len:" << length
+            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
+    uint64_t start = butil::cpuwide_time_us();
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindOrCreateFileCacheManager(fsId_, inodeId);
+
+    int ret = fileCacheManager->Read(inodeId, offset, length, buf);
+    VLOG(6) << "read end inodeId:" << inodeId << ",ret:" << ret;
+    if (ret < 0) {
+        return ret;
+    }
+    if (nullptr != GetMetric()) {
+        CollectMetrics(&ioMetric_->adaptorRead, ret, start);
+    }
+    VLOG(6) << "read end offset:" << offset << ", len:" << length
+            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
+    return ret;
+}
+
+void StorageAdaptor::BackGroundFlush() {
+    while (!toStop_.load(std::memory_order_acquire)) {
+        {
+            std::unique_lock<std::mutex> lck(mtx_);
+            if (fsCacheManager_->GetDataCacheNum() == 0) {
+                VLOG(3) << "BackGroundFlush has no write cache, so wait";
+                cond_.wait(lck);
+            }
+        }
+        uint64_t start = butil::cpuwide_time_us();
+        if (fsCacheManager_->MemCacheRatio() > memCacheNearfullRatio_) {
+            VLOG(3) << "BackGroundFlush radically, write cache num is: "
+                    << fsCacheManager_->GetDataCacheNum()
+                    << "cache ratio is: " << fsCacheManager_->MemCacheRatio();
+            fsCacheManager_->FsSync(true);
+
+        } else {
+            waitInterval_.WaitForNextExcution();
+            VLOG(6) << "BackGroundFlush, write cache num is:"
+                    << fsCacheManager_->GetDataCacheNum()
+                    << "cache ratio is: " << fsCacheManager_->MemCacheRatio();
+            fsCacheManager_->FsSync(false);
+            VLOG(6) << "background fssync end";
+        }
+        if (nullptr != GetMetric()) {
+            CollectMetrics(&ioMetric_->adaptorBgFlush, 1, start);
+        }
+    }
+    return;
+}
+
+
+CURVEFS_ERROR StorageAdaptor::Flush(uint64_t inodeId) {
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindFileCacheManager(inodeId);
+    if (nullptr == fileCacheManager) {
+        return CURVEFS_ERROR::OK;
+    }
+    VLOG(6) << "Flush data of inodeId:" << inodeId;
+    return fileCacheManager->Flush(true, false);
+}
+
+CURVEFS_ERROR StorageAdaptor::FlushAllCache(uint64_t inodeId) {
+    VLOG(6) << "FlushAllCache, inodeId:" << inodeId;
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindFileCacheManager(inodeId);
+    if (!fileCacheManager) {
+        return CURVEFS_ERROR::OK;
+    }
+
+    // force flush data in memory to s3
+    VLOG(6) << "FlushAllCache, flush memory data of inodeId:" << inodeId;
+    CURVEFS_ERROR ret = fileCacheManager->Flush(true, false);
+    if (ret != CURVEFS_ERROR::OK) {
+        return ret;
+    }
+
+    // force flush data in diskcache to s3
+    if (HasDiskCache()) {
+        VLOG(6) << "FlushAllCache, wait inodeId:" << inodeId
+                << "related chunk upload to s3";
+        if (ClearDiskCache(inodeId) < 0) {
+            return CURVEFS_ERROR::INTERNAL;
+        }
+    }
+
+    return ret;
+}
+
+CURVEFS_ERROR StorageAdaptor::FsSync() {
+    return fsCacheManager_->FsSync(true);
+}
+
+int StorageAdaptor::ClearDiskCache(int64_t inodeId) {
+    // flush disk cache. read cache do not need clean
+    int ret =
+        diskCacheManagerImpl_->UploadWriteCacheByInode(std::to_string(inodeId));
+    LOG_IF(ERROR, ret < 0) << "FlushAllCache, inode:" << inodeId
+                           << ", upload write cache fail";
+    return ret;
+}
+
+void StorageAdaptor::ReleaseCache(uint64_t inodeId) {
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindFileCacheManager(inodeId);
+    if (!fileCacheManager) {
+        return;
+    }
+    VLOG(9) << "ReleaseCache inode:" << inodeId;
+    fileCacheManager->ReleaseCache();
+    fsCacheManager_->ReleaseFileCacheManager(inodeId);
+    return;
+}
+
+void StorageAdaptor::Enqueue(
+  std::shared_ptr<FlushChunkCacheContext> context) {
+    auto task = [this, context]() {
+        this->FlushChunkClosure(context);
+    };
+    taskPool_.Enqueue(task);
+}
+
+int StorageAdaptor::FlushChunkClosure(
+  std::shared_ptr<FlushChunkCacheContext> context) {
+    VLOG(9) << "FlushChunkCacheClosure start: " << context->inode;
+    CURVEFS_ERROR ret = context->chunkCacheManptr->Flush(
+      context->inode, context->force);
+    // set the returned value
+    // it is need in FlushChunkCacheCallBack
+    context->retCode = ret;
+    context->cb(context);
+    VLOG(9) << "FlushChunkCacheClosure end: " << context->inode;
+    return 0;
+}
+
+}  // namespace client
+}  // namespace curvefs
+
--- a/curvefs/src/client/client_storage_adaptor.h
+++ b/curvefs/src/client/client_storage_adaptor.h
@ -0,0 +1,332 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*
+ * Project: curve
+ * Created Date: Thur March 14 2023
+ * Author: wuhongsong
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_CLIENT_STORAGE_ADAPTOR_H_
+#define CURVEFS_SRC_CLIENT_CLIENT_STORAGE_ADAPTOR_H_
+
+#include <memory>
+#include <string>
+
+#include "curvefs/proto/common.pb.h"
+#include "curvefs/proto/mds.pb.h"
+#include "curvefs/proto/metaserver.pb.h"
+
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h"
+#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
+#include "curvefs/src/client/common/common.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/common/define.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/metric/client_metric.h"
+#include "curvefs/src/client/rpcclient/mds_client.h"
+#include "curvefs/src/client/s3/client_s3.h"
+
+using ::curve::common::Thread;
+using ::curve::common::TaskThreadPool;
+using curvefs::client::common::FuseClientOption;
+using curvefs::client::common::DiskCacheType;
+using curvefs::client::metric::IoMetric;
+using curvefs::client::rpcclient::MdsClient;
+using curvefs::metaserver::Inode;
+
+namespace curvefs {
+namespace client {
+
+class DiskCacheManagerImpl;
+class FlushChunkCacheContext;
+class ChunkCacheManager;
+
+// callback function for FlushChunkCache
+using FlushChunkCacheCallBack = std::function<
+  void(const std::shared_ptr<FlushChunkCacheContext>&)>;
+
+
+/// @brief flush chunk cache context
+/// @param inode inode id
+/// @param chunkCacheManptr chunk cache manager
+/// @param force force flush memory cache
+/// @param cb callback function
+/// @param retCode error code
+struct FlushChunkCacheContext {
+    uint64_t inode;
+    ChunkCacheManagerPtr chunkCacheManptr;
+    bool force;
+    FlushChunkCacheCallBack cb;
+    CURVEFS_ERROR retCode;
+};
+
+// the base class of the underlying storage adaptation layer
+class StorageAdaptor {
+ public:
+    StorageAdaptor() {}
+    virtual ~StorageAdaptor() {
+        Stop();
+    }
+
+    /// @brief init storage adaptor
+    /// @param option fuse client option
+    /// @param inodeManager inode cache manager
+    /// @param mdsClient mds client
+    /// @param fsCacheManager fscache manager
+    /// @param diskCacheManagerImpl disk cache manager
+    /// @param kvClientManager kv client manager
+    /// @param fsInfo file system information
+    /// @return error code
+    virtual CURVEFS_ERROR
+    Init(const FuseClientOption &option,
+        std::shared_ptr<InodeCacheManager> inodeManager,
+        std::shared_ptr<MdsClient> mdsClient,
+        std::shared_ptr<FsCacheManager> fsCacheManager,
+        std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+        std::shared_ptr<KVClientManager> kvClientManager,
+        std::shared_ptr<FsInfo> fsInfo);
+
+    virtual CURVEFS_ERROR FuseOpInit(void *userdata,
+        struct fuse_conn_info *conn);
+
+    virtual int Stop();
+
+    /// @brief flush data cache to backend storage(s3 or volume)
+    /// @param req flush request
+    /// @param writeOffset length that has been written to backend storage
+    /// @return error code
+    virtual CURVEFS_ERROR FlushDataCache(const UperFlushRequest& req,
+      uint64_t* writeOffset) = 0;
+
+    /// @brief read data from backend storage(s3 or volume)
+    /// @param request read request
+    /// @return error code
+    virtual CURVEFS_ERROR ReadFromLowlevel(UperReadRequest request) = 0;
+
+    /// @brief wrire data to memory cache
+    /// @param inodeId file inode id
+    /// @param offset offset of data in file
+    /// @param length length of data
+    /// @param buf data need to write
+    /// @return length of data that has been written to memory cache
+    virtual int Write(uint64_t inodeId, uint64_t offset,
+      uint64_t length, const char *buf);
+
+    /// @brief read data from memory cache
+    /// @param inodeId file inode id
+    /// @param offset offset of data in file
+    /// @param length length need to read
+    /// @param buf data buffer
+    /// @return length of data that has been read from memory cache
+    virtual int Read(uint64_t inodeId, uint64_t offset,
+      uint64_t length, char *buf);
+
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inodeWrapper,
+      uint64_t size) = 0;
+
+    /// @brief allocate chunk id from mds
+    /// @param fsId file system id
+    /// @param idNum chunks number need to allocate
+    /// @param chunkId chunk id that has been allocated
+    /// @return error code
+    FSStatusCode AllocChunkId(uint32_t fsId,
+      uint32_t idNum, uint64_t *chunkId) {
+        return mdsClient_->AllocS3ChunkId(fsId, idNum, chunkId);
+    }
+
+    /// @brief release all cache of file
+    /// @param inodeId file inode id
+    void ReleaseCache(uint64_t inodeId);
+
+    virtual CURVEFS_ERROR Flush(uint64_t inodeId);
+    virtual CURVEFS_ERROR FlushAllCache(uint64_t inodeId);
+    virtual CURVEFS_ERROR FsSync();
+
+    /// @brief enqueue flush chunk cache context
+    void Enqueue(std::shared_ptr<FlushChunkCacheContext> context);
+
+    void FsSyncSignal() {
+        std::lock_guard<std::mutex> lk(mtx_);
+        VLOG(3) << "fs sync signal";
+        cond_.notify_one();
+    }
+
+    void FsSyncSignalAndDataCacheInc() {
+        std::lock_guard<std::mutex> lk(mtx_);
+        fsCacheManager_->DataCacheNumInc();
+        VLOG(3) << "fs sync signal";
+        cond_.notify_one();
+    }
+
+/*** get and set element ***/
+
+    void SetFsId(uint32_t fsId) {
+        fsId_ = fsId;
+    }
+
+    uint64_t GetBlockSize() {
+        return blockSize_;
+    }
+
+    uint64_t GetChunkSize() {
+        return chunkSize_;
+    }
+
+    void SetBlockSize(const uint64_t& blockSize) {
+        blockSize_ = blockSize;
+    }
+
+    void SetChunkSize(const uint64_t& chunkSize) {
+        chunkSize_ = chunkSize;
+    }
+
+    std::shared_ptr<FsCacheManager> GetFsCacheManager() {
+        return fsCacheManager_;
+    }
+
+    uint32_t GetFlushInterval() { return flushIntervalSec_; }
+
+    uint32_t GetDiskCacheType() {
+        return diskCacheType_;
+    }
+
+    bool DisableDiskCache() {
+        return diskCacheType_ == DiskCacheType::Disable;
+    }
+
+    bool HasDiskCache() {
+        return diskCacheType_ != DiskCacheType::Disable;
+    }
+
+    bool IsReadCache() {
+        return diskCacheType_ == DiskCacheType::OnlyRead;
+    }
+
+    bool IsReadWriteCache() {
+        return diskCacheType_ == DiskCacheType::ReadWrite;
+    }
+    virtual std::shared_ptr<InodeCacheManager> GetInodeCacheManager() {
+        return inodeManager_;
+    }
+
+    std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() {
+        return diskCacheManagerImpl_;
+    }
+
+    uint32_t GetFsId() {
+        return fsId_;
+    }
+
+    uint32_t GetPageSize() {
+        return pageSize_;
+    }
+
+    void DisableBgFlush() {
+        enableBgFlush_ = false;
+    }
+
+    std::shared_ptr<IoMetric> GetMetric() {
+      return ioMetric_;
+    }
+
+    void CollectMetrics(InterfaceMetric *interface,
+      int count, uint64_t start) {
+        interface->bps.count << count;
+        interface->qps.count << 1;
+        interface->latency << (butil::cpuwide_time_us() - start);
+    }
+
+    void SetDiskCache(DiskCacheType type) {
+       diskCacheType_ = type;
+    }
+
+    CachePolicy GetCachePolicy(bool sync) {
+      const bool mayCache =
+        HasDiskCache() && !GetDiskCacheManager()->IsDiskCacheFull() && !sync;
+        if (IsReadCache() && mayCache) {
+            return CachePolicy::RCache;
+        } else if (IsReadWriteCache() && mayCache) {
+            return CachePolicy::WRCache;
+        } else {
+            return CachePolicy::NCache;
+        }
+    }
+
+    std::shared_ptr<MdsClient> GetMdsClient() {
+        return mdsClient_;
+    }
+
+    std::string GetMountOwner() {
+        return mountOwner_;
+    }
+
+    void SetMountOwner(const std::string& mountOwner) {
+        mountOwner_ = mountOwner;
+    }
+
+ private:
+    int FlushChunkClosure(std::shared_ptr<FlushChunkCacheContext> context);
+    int ClearDiskCache(int64_t inodeId);
+    void BackGroundFlush();
+
+    void InitMetrics(const std::string &fsName) {
+        fsName_ = fsName;
+        ioMetric_ = std::make_shared<IoMetric>(fsName);
+        // init disk cache metrics(needed)
+        if (HasDiskCache()) {
+        diskCacheManagerImpl_->InitMetrics(fsName);
+        }
+    }
+
+ protected:
+  std::shared_ptr<FsCacheManager> fsCacheManager_;
+  std::shared_ptr<MdsClient> mdsClient_;
+  std::shared_ptr<FsInfo> fsInfo_;
+
+ private:
+    uint64_t blockSize_;
+    uint64_t chunkSize_;
+    uint32_t pageSize_;
+    uint32_t flushIntervalSec_;
+    uint32_t chunkFlushThreads_;
+    uint32_t memCacheNearfullRatio_;
+    uint32_t throttleBaseSleepUs_;
+    Thread bgFlushThread_;
+    std::atomic<bool> toStop_;
+    std::mutex mtx_;
+    std::mutex ioMtx_;
+    std::condition_variable cond_;
+    curve::common::WaitInterval waitInterval_;
+    std::shared_ptr<InodeCacheManager> inodeManager_;
+    std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl_;
+    DiskCacheType diskCacheType_;
+    uint32_t fsId_;
+    std::string fsName_;
+    std::string mountOwner_;
+    std::shared_ptr<IoMetric> ioMetric_;
+    bool enableBgFlush_ = true;
+
+    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
+        taskPool_;
+};
+
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_CLIENT_STORAGE_ADAPTOR_H_
--- a/curvefs/src/client/common/common.h
+++ b/curvefs/src/client/common/common.h
@ -60,6 +60,7 @@ enum class MetaServerOpType {
    GetVolumeExtent,
    UpdateVolumeExtent,
    CreateManageInode,
+    UpdateDeallocatableBlockGroup,
 };

 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);
--- a/curvefs/src/client/common/config.cpp
+++ b/curvefs/src/client/common/config.cpp
@ -50,7 +50,6 @@ DEFINE_bool(enableCto, true, "acheieve cto consistency");
 DEFINE_bool(useFakeS3, false,
            "Use fake s3 to inject more metadata for testing metaserver");
 DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing");
-DEFINE_bool(access_logging, true, "enable access log");

 /**
 * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true
@ -205,6 +204,7 @@ void InitDiskCacheOption(Configuration *conf,
                              &diskCacheOption->avgReadFileIops);
 }

+// TODO(@hzwuhongsong)
 void InitS3Option(Configuration *conf, S3Option *s3Opt) {
    conf->GetValueFatalIfFail("s3.fakeS3", &FLAGS_useFakeS3);
    conf->GetValueFatalIfFail("s3.pageSize",
@ -245,6 +245,10 @@ void InitVolumeOption(Configuration *conf, VolumeOption *volumeOpt) {
    conf->GetValueFatalIfFail("volume.fsBlockSize", &volumeOpt->fsBlockSize);
    conf->GetValueFatalIfFail("volume.allocator.type",
                              &volumeOpt->allocatorOption.type);
+    conf->GetValueFatalIfFail("volume.space.useThreshold",
+                              &volumeOpt->threshold);
+    conf->GetValueFatalIfFail("volume.space.releaseInterSec",
+                              &volumeOpt->releaseInterSec);

    conf->GetValueFatalIfFail(
        "volume.blockGroup.allocateOnce",
@ -293,55 +297,6 @@ void InitKVClientManagerOpt(Configuration *conf,
                              &config->getThreadPooln);
 }

-void InitFileSystemOption(Configuration* c, FileSystemOption* option) {
-    c->GetValueFatalIfFail("fs.cto", &option->cto);
-    c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto);
-    c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr);
-    c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength);
-    c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging);
-    {  // kernel cache option
-        auto o = &option->kernelCacheOption;
-        c->GetValueFatalIfFail("fs.kernelCache.attrTimeoutSec",
-                               &o->attrTimeoutSec);
-        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeoutSec",
-                               &o->dirAttrTimeoutSec);
-        c->GetValueFatalIfFail("fs.kernelCache.entryTimeoutSec",
-                               &o->entryTimeoutSec);
-        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeoutSec",
-                               &o->dirEntryTimeoutSec);
-    }
-    {  // lookup cache option
-        auto o = &option->lookupCacheOption;
-        c->GetValueFatalIfFail("fs.lookupCache.lruSize",
-                               &o->lruSize);
-        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeoutSec",
-                               &o->negativeTimeoutSec);
-        c->GetValueFatalIfFail("fs.lookupCache.minUses",
-                               &o->minUses);
-    }
-    {  // dir cache option
-        auto o = &option->dirCacheOption;
-        c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize);
-    }
-    {  // open file option
-        auto o = &option->openFilesOption;
-        c->GetValueFatalIfFail("fs.openFile.lruSize", &o->lruSize);
-    }
-    {  // attr watcher option
-        auto o = &option->attrWatcherOption;
-        c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize);
-    }
-    {  // rpc option
-        auto o = &option->rpcOption;
-        c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit);
-    }
-    {  // defer sync option
-        auto o = &option->deferSyncOption;
-        c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay);
-        c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime);
-    }
-}
-
 void SetBrpcOpt(Configuration *conf) {
    curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy;
    dummy.Load(conf, "defer_close_second", "rpc.defer.close.second",
@ -362,16 +317,36 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
    InitLeaseOpt(conf, &clientOption->leaseOpt);
    InitRefreshDataOpt(conf, &clientOption->refreshDataOption);
    InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt);
-    InitFileSystemOption(conf, &clientOption->fileSystemOption);

+    conf->GetValueFatalIfFail("fuseClient.attrTimeOut",
+                              &clientOption->attrTimeOut);
+    conf->GetValueFatalIfFail("fuseClient.entryTimeOut",
+                              &clientOption->entryTimeOut);
    conf->GetValueFatalIfFail("fuseClient.listDentryLimit",
                              &clientOption->listDentryLimit);
    conf->GetValueFatalIfFail("fuseClient.listDentryThreads",
                              &clientOption->listDentryThreads);
+    conf->GetValueFatalIfFail("fuseClient.flushPeriodSec",
+                              &clientOption->flushPeriodSec);
+    conf->GetValueFatalIfFail("fuseClient.maxNameLength",
+                              &clientOption->maxNameLength);
+    conf->GetValueFatalIfFail("fuseClient.iCacheLruSize",
+                              &clientOption->iCacheLruSize);
+    conf->GetValueFatalIfFail("fuseClient.dCacheLruSize",
+                              &clientOption->dCacheLruSize);
+    conf->GetValueFatalIfFail("fuseClient.enableICacheMetrics",
+                              &clientOption->enableICacheMetrics);
+    conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
+                              &clientOption->enableDCacheMetrics);
+    conf->GetValueFatalIfFail("fuseClient.lruTimeOutSec",
+                              &clientOption->lruTimeOutSec);
    conf->GetValueFatalIfFail("client.dummyServer.startPort",
                              &clientOption->dummyServerStartPort);
    conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                              &clientOption->enableMultiMountPointRename);
+    conf->GetValueFatalIfFail("fuseClient.disableXattr",
+                              &clientOption->disableXattr);
+    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
    conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
                              &clientOption->downloadMaxRetryTimes);
    conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
@ -381,6 +356,12 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
        << "Not found `fuseClient.enableSplice` in conf, use default value `"
        << std::boolalpha << clientOption->enableFuseSplice << '`';

+    // if enableCto, attr and entry cache must invalid
+    if (FLAGS_enableCto) {
+        clientOption->attrTimeOut = 0;
+        clientOption->entryTimeOut = 0;
+    }
+
    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteBytes",
                              &FLAGS_fuseClientAvgWriteBytes);
    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytes",
@ -420,6 +401,9 @@ void SetFuseClientS3Option(FuseClientOption *clientOption,
    clientOption->s3Opt.s3AdaptrOpt.ak = fsS3Opt.ak;
    clientOption->s3Opt.s3AdaptrOpt.sk = fsS3Opt.sk;
    clientOption->s3Opt.s3AdaptrOpt.bucketName = fsS3Opt.bucketName;
+    VLOG(1) << "fuse s3 option, chunk size is: " << fsS3Opt.chunkSize
+            << "block size is: " << fsS3Opt.blockSize
+            << "s3 adress is: " << fsS3Opt.s3Address;
 }

 void S3Info2FsS3Option(const curvefs::common::S3Info& s3,
--- a/curvefs/src/client/common/config.h
+++ b/curvefs/src/client/common/config.h
@ -73,11 +73,6 @@ struct LeaseOpt {
    uint32_t leaseTimeUs = 20000000;
 };

-struct SpaceAllocServerOption {
-    std::string spaceaddr;
-    uint64_t rpcTimeoutMs;
-};
-
 struct KVClientManagerOpt {
    int setThreadPooln = 4;
    int getThreadPooln = 4;
@ -164,6 +159,9 @@ struct VolumeOption {
    uint64_t volBlockSize;
    uint64_t fsBlockSize;
    VolumeAllocatorOption allocatorOption;
+
+    double threshold{1.0};
+    uint64_t releaseInterSec{300};
 };

 struct ExtentManagerOption {
@ -174,65 +172,11 @@ struct RefreshDataOption {
    uint64_t maxDataSize = 1024;
    uint32_t refreshDataIntervalSec = 30;
 };
-
-// { filesystem option
-struct KernelCacheOption {
-    uint32_t entryTimeoutSec;
-    uint32_t dirEntryTimeoutSec;
-    uint32_t attrTimeoutSec;
-    uint32_t dirAttrTimeoutSec;
-};
-
-struct LookupCacheOption {
-    uint64_t lruSize;
-    uint32_t negativeTimeoutSec;
-    uint32_t minUses;
-};
-
-struct DirCacheOption {
-    uint64_t lruSize;
-    uint32_t timeoutSec;
-};
-
-struct AttrWatcherOption {
-    uint64_t lruSize;
-};
-
-struct OpenFilesOption {
-    uint64_t lruSize;
-    uint32_t deferSyncSecond;
-};
-
-struct RPCOption {
-    uint32_t listDentryLimit;
-};
-
-struct DeferSyncOption {
-    uint32_t delay;
-    bool deferDirMtime;
-};
-
-struct FileSystemOption {
-    bool cto;
-    bool disableXattr;
-    uint32_t maxNameLength;
-    uint32_t blockSize = 0x10000u;
-    KernelCacheOption kernelCacheOption;
-    LookupCacheOption lookupCacheOption;
-    DirCacheOption dirCacheOption;
-    OpenFilesOption openFilesOption;
-    AttrWatcherOption attrWatcherOption;
-    RPCOption rpcOption;
-    DeferSyncOption deferSyncOption;
-};
-// }
-
 struct FuseClientOption {
    MdsOption mdsOpt;
    MetaCacheOpt metaCacheOpt;
    ExcutorOpt excutorOpt;
    ExcutorOpt excutorInternalOpt;
-    SpaceAllocServerOption spaceOpt;
    BlockDeviceClientOptions bdevOpt;
    S3Option s3Opt;
    ExtentManagerOption extentManagerOpt;
@ -240,13 +184,22 @@ struct FuseClientOption {
    LeaseOpt leaseOpt;
    RefreshDataOption refreshDataOption;
    KVClientManagerOpt kvClientManagerOpt;
-    FileSystemOption fileSystemOption;

+    double attrTimeOut;
+    double entryTimeOut;
    uint32_t listDentryLimit;
    uint32_t listDentryThreads;
+    uint32_t flushPeriodSec;
+    uint32_t maxNameLength;
+    uint64_t iCacheLruSize;
+    uint64_t dCacheLruSize;
+    bool enableICacheMetrics;
+    bool enableDCacheMetrics;
+    uint32_t lruTimeOutSec;
    uint32_t dummyServerStartPort;
    bool enableMultiMountPointRename = false;
    bool enableFuseSplice = false;
+    bool disableXattr = false;
    uint32_t downloadMaxRetryTimes;
    uint32_t warmupThreadsNum = 10;
 };
--- a/curvefs/src/client/curve_fuse_op.cpp
+++ b/curvefs/src/client/curve_fuse_op.cpp
--- a/curvefs/src/client/curve_fuse_op.h
+++ b/curvefs/src/client/curve_fuse_op.h
@ -37,7 +37,7 @@
 extern "C" {
 #endif

-int InitLog(const char *confPath, const char *argv0);
+int InitGlog(const char *confPath, const char *argv0);

 int InitFuseClient(const struct MountOption *mountOption);

--- a/curvefs/src/client/dentry_cache_manager.cpp
+++ b/curvefs/src/client/dentry_cache_manager.cpp
@ -44,13 +44,31 @@ namespace client {

 using curve::common::WriteLockGuard;
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
-using ::curvefs::client::filesystem::ToFSError;
+
+void DentryCacheManagerImpl::InsertOrReplaceCache(const Dentry &dentry) {
+    std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name());
+    if (!curvefs::client::common::FLAGS_enableCto) {
+        NameLockGuard lock(nameLock_, key);
+        dCache_->Put(key, dentry);
+    }
+}
+
+void DentryCacheManagerImpl::DeleteCache(uint64_t parentId,
+                                         const std::string &name) {
+    std::string key = GetDentryCacheKey(parentId, name);
+    NameLockGuard lock(nameLock_, key);
+    dCache_->Remove(key);
+}

 CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
                                                const std::string &name,
                                                Dentry *out) {
    std::string key = GetDentryCacheKey(parent, name);
    NameLockGuard lock(nameLock_, key);
+    bool ok = dCache_->Get(key, out);
+    if (ok) {
+        return CURVEFS_ERROR::OK;
+    }

    MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out);
    if (ret != MetaStatusCode::OK) {
@ -58,7 +76,10 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
            << "metaClient_ GetDentry failed, MetaStatusCode = " << ret
            << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
            << ", parent = " << parent << ", name = " << name;
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
+    }
+    if (!curvefs::client::common::FLAGS_enableCto) {
+        dCache_->Put(key, *out);
    }
    return CURVEFS_ERROR::OK;
 }
@ -73,9 +94,12 @@ CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) {
                   << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                   << ", parent = " << dentry.parentinodeid()
                   << ", name = " << dentry.name();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
    }

+    if (!curvefs::client::common::FLAGS_enableCto) {
+        dCache_->Put(key, dentry);
+    }
    return CURVEFS_ERROR::OK;
 }

@ -84,13 +108,14 @@ CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent,
                                                   FsFileType type) {
    std::string key = GetDentryCacheKey(parent, name);
    NameLockGuard lock(nameLock_, key);
+    dCache_->Remove(key);

    MetaStatusCode ret = metaClient_->DeleteDentry(fsId_, parent, name, type);
    if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
        LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                   << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                   << ", parent = " << parent << ", name = " << name;
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
    }
    return CURVEFS_ERROR::OK;
 }
@ -124,7 +149,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                       << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                       << ", parent = " << parent << ", last = " << last
                       << ", count = " << limit << ", onlyDir = " << onlyDir;
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
        }

        if (!onlyDir) {
--- a/curvefs/src/client/dentry_cache_manager.h
+++ b/curvefs/src/client/dentry_cache_manager.h
@ -33,18 +33,20 @@
 #include <utility>

 #include "curvefs/src/client/rpcclient/metaserver_client.h"
+#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
+#include "src/common/lru_cache.h"
 #include "src/common/concurrent/name_lock.h"
-#include "curvefs/src/client/filesystem/error.h"

 using ::curvefs::metaserver::Dentry;
+using ::curve::common::TimedLRUCache;
+using ::curve::common::CacheMetrics;

 namespace curvefs {
 namespace client {

 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
-using ::curvefs::client::filesystem::CURVEFS_ERROR;

 static const char* kDentryKeyDelimiter = ":";

@ -57,6 +59,13 @@ class DentryCacheManager {
        fsId_ = fsId;
    }

+    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+        uint32_t cacheTimeOutSec) = 0;
+
+    virtual void InsertOrReplaceCache(const Dentry& dentry) = 0;
+
+    virtual void DeleteCache(uint64_t parentId, const std::string& name) = 0;
+
    virtual CURVEFS_ERROR GetDentry(uint64_t parent,
        const std::string &name, Dentry *out) = 0;

@ -77,11 +86,30 @@ class DentryCacheManager {
 class DentryCacheManagerImpl : public DentryCacheManager {
 public:
    DentryCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()),
+        dCache_(nullptr) {}

    explicit DentryCacheManagerImpl(
        const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient) {}
+      : metaClient_(metaClient),
+        dCache_(nullptr) {}
+
+    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+        uint32_t cacheTimeOutSec) override {
+        if (enableCacheMetrics) {
+            dCache_ = std::make_shared<
+                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize,
+                    std::make_shared<CacheMetrics>("dcache"));
+        } else {
+            dCache_ = std::make_shared<
+                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize);
+        }
+        return CURVEFS_ERROR::OK;
+    }
+
+    void InsertOrReplaceCache(const Dentry& dentry) override;
+
+    void DeleteCache(uint64_t parentId, const std::string& name) override;

    CURVEFS_ERROR GetDentry(uint64_t parent,
        const std::string &name, Dentry *out) override;
@ -102,6 +130,8 @@ class DentryCacheManagerImpl : public DentryCacheManager {

 private:
    std::shared_ptr<MetaServerClient> metaClient_;
+    // key is parentId + name
+    std::shared_ptr<TimedLRUCache<std::string, Dentry>> dCache_;
    curve::common::GenericNameLock<Mutex> nameLock_;
 };

--- a/curvefs/src/client/error_code.cpp
+++ b/curvefs/src/client/error_code.cpp
@ -0,0 +1,98 @@
+/*
+ *  Copyright (c) 2021 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*
+ * Project: curve
+ * Created Date: Thur May 27 2021
+ * Author: xuchaojie
+ */
+
+#include "curvefs/src/client/error_code.h"
+
+#include <map>
+#include <string>
+
+namespace curvefs {
+namespace client {
+
+using ::curvefs::metaserver::MetaStatusCode;
+
+static const std::map<CURVEFS_ERROR, std::string> err2Msg = {
+    {CURVEFS_ERROR::OK, "OK"},
+    {CURVEFS_ERROR::INTERNAL, "internal error"},
+    {CURVEFS_ERROR::UNKNOWN, "unknown"},
+    {CURVEFS_ERROR::EXISTS, "inode or dentry already exist"},
+    {CURVEFS_ERROR::NOTEXIST, "inode or dentry not exist"},
+    {CURVEFS_ERROR::NO_SPACE, "no space to alloc"},
+    {CURVEFS_ERROR::BAD_FD, "bad fd"},
+    {CURVEFS_ERROR::INVALIDPARAM, "invalid param"},
+    {CURVEFS_ERROR::NOPERMISSION, "no permission"},
+    {CURVEFS_ERROR::NOTEMPTY, "dir not empty"},
+    {CURVEFS_ERROR::NOFLUSH, "no flush"},
+    {CURVEFS_ERROR::NOTSUPPORT, "not support"},
+    {CURVEFS_ERROR::NAMETOOLONG, "name too long"},
+    {CURVEFS_ERROR::MOUNT_POINT_EXIST, "mount point already exist"},
+    {CURVEFS_ERROR::MOUNT_FAILED, "mount failed"},
+};
+
+std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
+    os << static_cast<int>(code) << "[" << [code]() {
+        auto it = err2Msg.find(code);
+        if (it != err2Msg.end()) {
+            return it->second;
+        }
+
+        return std::string{"Unknown"};
+    }() << "]";
+
+    return os;
+}
+
+CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
+    MetaStatusCode code) {
+    CURVEFS_ERROR ret = CURVEFS_ERROR::UNKNOWN;
+    switch (code) {
+        case MetaStatusCode::OK:
+            ret = CURVEFS_ERROR::OK;
+            break;
+
+        case MetaStatusCode::NOT_FOUND:
+            ret = CURVEFS_ERROR::NOTEXIST;
+            break;
+
+        case MetaStatusCode::PARAM_ERROR:
+            ret = CURVEFS_ERROR::INVALIDPARAM;
+            break;
+
+        case MetaStatusCode::INODE_EXIST:
+        case MetaStatusCode::DENTRY_EXIST:
+            ret = CURVEFS_ERROR::EXISTS;
+            break;
+
+        case MetaStatusCode::SYM_LINK_EMPTY:
+        case MetaStatusCode::RPC_ERROR:
+            ret = CURVEFS_ERROR::INTERNAL;
+            break;
+
+        default:
+            ret = CURVEFS_ERROR::UNKNOWN;
+    }
+    return ret;
+}
+
+}  // namespace client
+}  // namespace curvefs
--- a/curvefs/src/client/filesystem/error.h
+++ b/curvefs/src/client/filesystem/error.h
@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2023 NetEase Inc.
+ *  Copyright (c) 2021 NetEase Inc.
 *
 *  Licensed under the Apache License, Version 2.0 (the "License");
 *  you may not use this file except in compliance with the License.
@ -14,27 +14,22 @@
 *  limitations under the License.
 */

+
 /*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
+ * Project: curve
+ * Created Date: Thur May 27 2021
+ * Author: xuchaojie
 */

-#include <map>
-#include <string>
-#include <iostream>
+#ifndef CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#define CURVEFS_SRC_CLIENT_ERROR_CODE_H_

 #include "curvefs/proto/metaserver.pb.h"

-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
-
 namespace curvefs {
 namespace client {
-namespace filesystem {
-
-using ::curvefs::metaserver::MetaStatusCode;

+// notice : the error code should be negative.
 enum class CURVEFS_ERROR {
    OK = 0,
    INTERNAL = -1,
@ -55,20 +50,15 @@ enum class CURVEFS_ERROR {
    NODATA = -16,
    IO_ERROR = -17,
    CACHETOOSMALL = -18,
-    STALE = -19,
-    NOSYS = -20,
 };

-std::string StrErr(CURVEFS_ERROR code);
-
-int SysErr(CURVEFS_ERROR code);

 std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code);

-CURVEFS_ERROR ToFSError(MetaStatusCode code);
+CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
+    ::curvefs::metaserver::MetaStatusCode code);

-}  // namespace filesystem
 }  // namespace client
 }  // namespace curvefs

-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+#endif  // CURVEFS_SRC_CLIENT_ERROR_CODE_H_
--- a/curvefs/src/client/filesystem/README.md
+++ b/curvefs/src/client/filesystem/README.md
@ -1,78 +0,0 @@
-CurveFS Client Metadata Cache Design
-===
-
-```
-+----------------+
-+  Kernel Cache  |
-+----------------+
-    |       ^
-(1) |       | (4)
-    v       |
-+----------------+
-+  CurveFS Fuse  |
-+----------------+
-    |       ^
-(2) |       | (3)
-    v       |
-+----------------+
-+   MetaServer   |
-+----------------+
-```
-
-(1) kernel cache timeout
---
-
-* requests are always sent to fuse layer, except `lookup` and `getattr` request which affected by caching.
-* if entry or attribute cache timeout, vfs layer will launch request to fuse layer.
-
-(2) revalidate cache
---
-
-* for `open` and `opendir` request, fuse layer should revalidate cache by comparing mtime, if modified, fuse layer should:
-    * `open`: return **ESTALE** to trigger vfs layer to invoke the `open` again with ignoring cache.
-    * `opendir`: drop all directory cache.
-* others, proxy request to metaserver directly.
-
-
-(3) retrive fresh metadata
---
-
-* for `readdir` request, fuse layer should cache direcoty entries and their attributes.
-* others, no caching.
-
-(4) reply with timeout
---
-
-* reply entry or attribute to vfs layer with corresponding cache timeout.
-* four type timeouts provided: `entryTimeout`, `dirEntryTimeout`, `attrTimeout`, `dirAtttTimeout`.
-* fuse layer should remeber the `mtime` of attribute while reply it to vfs layer.
-
-Cache Layer Level
-===
-
-```
-+-------------------------------------------------------+
-|                       Kernel Cache                    |  (L1 Cache)
-+-------------------------------------------------------+
-                            ^  (update length)
-                            |
-+--------------------------------------------------------+
-|                       Open File Cache                  |  (L2 Cache)
-|                   (only length and chunk)              |
-+--------------------------------------------------------+
-         ^                            ^
-         | (reply entry/attr)         |  (reply entry/attr)
-+-----------------+                   |
-| Directory Cache |  (L3 Cache)       |
-+-----------------+                   |
-         ^                            |
-         |                            |
-+-----------------+ +------------------------------------+
-|     readdir     | | lookup、getattr、setattr、create... |
-+-----------------+ +------------------------------------+
-         ^                            ^
-         |                            |
-+--------------------------------------------------------+
-|                        MetaServer                      |
-+--------------------------------------------------------+
-```
--- a/curvefs/src/client/filesystem/access_log.h
+++ b/curvefs/src/client/filesystem/access_log.h
@ -1,89 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-17
- * Author: Jingli Chen (Wine93)
- */
-
-#include <unistd.h>
-#include <butil/time.h>
-#include <spdlog/spdlog.h>
-#include <spdlog/sinks/basic_file_sink.h>
-#include <spdlog/sinks/daily_file_sink.h>
-
-#include <string>
-#include <memory>
-
-#include "absl/strings/str_format.h"
-#include "curvefs/src/client/common/config.h"
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
-
-namespace curvefs {
-namespace client {
-namespace common {
-
-DECLARE_bool(access_logging);
-
-}
-namespace filesystem {
-
-using ::absl::StrFormat;
-using ::curvefs::client::common::FLAGS_access_logging;
-using MessageHandler = std::function<std::string()>;
-
-static std::shared_ptr<spdlog::logger> Logger;
-
-bool InitAccessLog(const std::string& prefix) {
-    std::string filename = StrFormat("%s/access.%d.log", prefix, getpid());
-    Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0);
-    spdlog::flush_every(std::chrono::seconds(1));
-    return true;
-}
-
-struct AccessLogGuard {
-    explicit AccessLogGuard(MessageHandler handler)
-        : enable(FLAGS_access_logging),
-          handler(handler) {
-        if (!enable) {
-            return;
-        }
-
-        timer.start();
-    }
-
-    ~AccessLogGuard() {
-        if (!enable) {
-            return;
-        }
-
-        timer.stop();
-        Logger->info("{0} <{1:.6f}>", handler(), timer.u_elapsed() / 1e6);
-    }
-
-    bool enable;
-    MessageHandler handler;
-    butil::Timer timer;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
--- a/curvefs/src/client/filesystem/attr_watcher.cpp
+++ b/curvefs/src/client/filesystem/attr_watcher.cpp
@ -1,117 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#include "curvefs/src/client/filesystem/utils.h"
-#include "curvefs/src/client/filesystem/attr_watcher.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-AttrWatcher::AttrWatcher(AttrWatcherOption option,
-                         std::shared_ptr<OpenFiles> openFiles,
-                         std::shared_ptr<DirCache> dirCache)
-    : modifiedAt_(std::make_shared<LRUType>(option.lruSize)),
-      openFiles_(openFiles),
-      dirCache_(dirCache) {}
-
-void AttrWatcher::RemeberMtime(const InodeAttr& attr) {
-    WriteLockGuard lk(rwlock_);
-    modifiedAt_->Put(attr.inodeid(), AttrMtime(attr));
-}
-
-bool AttrWatcher::GetMtime(Ino ino, TimeSpec* time) {
-    ReadLockGuard lk(rwlock_);
-    return modifiedAt_->Get(ino, time);
-}
-
-void AttrWatcher::UpdateDirEntryAttr(Ino ino, const InodeAttr& attr) {
-    std::shared_ptr<DirEntryList> entries;
-    for (const auto parent : attr.parent()) {
-        bool yes = dirCache_->Get(parent, &entries);
-        if (!yes) {
-            continue;
-        }
-
-        entries->UpdateAttr(ino, attr);
-
-        VLOG(1) << "Write back attribute to dir entry cache: ino = " << ino
-                << ", attr = " << attr.ShortDebugString();
-    }
-}
-
-void AttrWatcher::UpdateDirEntryLength(Ino ino, const InodeAttr& open) {
-    std::shared_ptr<DirEntryList> entries;
-    for (const auto parent : open.parent()) {
-        bool yes = dirCache_->Get(parent, &entries);
-        if (!yes) {
-            continue;
-        }
-
-        entries->UpdateLength(ino, open);
-
-        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
-                << ", attr = " << open.ShortDebugString();
-    }
-}
-
-AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
-                                   InodeAttr* attr,
-                                   ReplyType type,
-                                   bool writeBack)
-    : watcher(watcher), attr(attr), type(type), writeBack(writeBack) {
-    InodeAttr open;
-    Ino ino = attr->inodeid();
-    bool yes = watcher->openFiles_->GetFileAttr(ino, &open);
-    if (!yes) {
-        return;
-    }
-
-    attr->set_length(open.length());
-    attr->set_mtime(open.mtime());
-    attr->set_mtime_ns(open.mtime_ns());
-    if (AttrCtime(open) > AttrCtime(*attr)) {
-        attr->set_ctime(open.ctime());
-        attr->set_ctime_ns(open.ctime_ns());
-    }
-}
-
-AttrWatcherGuard::~AttrWatcherGuard() {
-    switch (type) {
-        case ReplyType::ATTR:
-            watcher->RemeberMtime(*attr);
-            if (writeBack) {
-                watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
-            }
-            break;
-
-        case ReplyType::ONLY_LENGTH:
-            if (writeBack) {
-                watcher->UpdateDirEntryLength(attr->inodeid(), *attr);
-            }
-            break;
-    }
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/attr_watcher.h
+++ b/curvefs/src/client/filesystem/attr_watcher.h
@ -1,105 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
-
-#include <memory>
-
-#include "src/common/lru_cache.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/openfile.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::RWLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::common::AttrWatcherOption;
-
-class AttrWatcher {
- public:
-    using LRUType = LRUCache<Ino, struct TimeSpec>;
-
- public:
-    AttrWatcher(AttrWatcherOption option,
-                std::shared_ptr<OpenFiles> openFiles,
-                std::shared_ptr<DirCache> dirCache);
-
-    void RemeberMtime(const InodeAttr& attr);
-
-    bool GetMtime(Ino ino, TimeSpec* time);
-
-    void UpdateDirEntryAttr(Ino ino, const InodeAttr& attr);
-
-    void UpdateDirEntryLength(Ino ino, const InodeAttr& open);
-
- private:
-    friend class AttrWatcherGuard;
-
- private:
-    RWLock rwlock_;
-    std::shared_ptr<LRUType> modifiedAt_;
-    std::shared_ptr<OpenFiles> openFiles_;
-    std::shared_ptr<DirCache> dirCache_;
-};
-
-
-enum class ReplyType {
-    ATTR,
-    ONLY_LENGTH
-};
-
-/*
- * each attribute reply to kernel, the watcher will:
- *  before reply:
- *    1) set attibute length if the corresponding file is opened
- *  after reply:
- *    1) remeber attribute modified time.
- *    2) write back attribute to dir entry cache if |writeBack| is true,
- *       because the dir-entry attribute maybe stale.
- */
-struct AttrWatcherGuard {
- public:
-    AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
-                     InodeAttr* attr,
-                     ReplyType type,
-                     bool writeBack);
-
-    ~AttrWatcherGuard();
-
- private:
-    std::shared_ptr<AttrWatcher> watcher;
-    InodeAttr* attr;
-    ReplyType type;
-    bool writeBack;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
--- a/curvefs/src/client/filesystem/defer_sync.cpp
+++ b/curvefs/src/client/filesystem/defer_sync.cpp
@ -1,86 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <vector>
-#include <memory>
-
-#include "curvefs/src/client/filesystem/defer_sync.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-DeferSync::DeferSync(DeferSyncOption option)
-    : option_(option),
-      mutex_(),
-      running_(false),
-      thread_(),
-      sleeper_(),
-      inodes_() {
-}
-
-void DeferSync::Start() {
-    if (!running_.exchange(true)) {
-        thread_ = std::thread(&DeferSync::SyncTask, this);
-        LOG(INFO) << "Defer sync thread start success";
-    }
-}
-
-void DeferSync::Stop() {
-    if (running_.exchange(false)) {
-        LOG(INFO) << "Stop defer sync thread...";
-        sleeper_.interrupt();
-        thread_.join();
-        LOG(INFO) << "Defer sync thread stopped";
-    }
-}
-
-void DeferSync::SyncTask() {
-    std::vector<std::shared_ptr<InodeWrapper>> inodes;
-    for ( ;; ) {
-        bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay));
-
-        {
-            LockGuard lk(mutex_);
-            inodes.swap(inodes_);
-        }
-        for (const auto& inode : inodes) {
-            UniqueLock lk(inode->GetUniqueLock());
-            inode->Async(nullptr, true);
-        }
-        inodes.clear();
-
-        if (!running) {
-            break;
-        }
-    }
-}
-
-void DeferSync::Push(const std::shared_ptr<InodeWrapper>& inode) {
-    LockGuard lk(mutex_);
-    inodes_.emplace_back(inode);
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/defer_sync.h
+++ b/curvefs/src/client/filesystem/defer_sync.h
@ -1,70 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
-
-#include <atomic>
-#include <vector>
-#include <memory>
-
-#include "src/common/interruptible_sleeper.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curvefs::client::common::DeferSyncOption;
-
-using ::curve::common::Mutex;
-using ::curve::common::LockGuard;
-using ::curve::common::InterruptibleSleeper;
-
-class DeferSync {
- public:
-    explicit DeferSync(DeferSyncOption option);
-
-    void Start();
-
-    void Stop();
-
-    void Push(const std::shared_ptr<InodeWrapper>& inode);
-
- private:
-    void SyncTask();
-
- private:
-    DeferSyncOption option_;
-    Mutex mutex_;
-    std::atomic<bool> running_;
-    std::thread thread_;
-    InterruptibleSleeper sleeper_;
-    std::vector<std::shared_ptr<InodeWrapper>> inodes_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
--- a/curvefs/src/client/filesystem/dir_cache.cpp
+++ b/curvefs/src/client/filesystem/dir_cache.cpp
@ -1,189 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#include <utility>
-
-#include "curvefs/src/client/filesystem/dir_cache.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-DirEntryList::DirEntryList()
-    : rwlock_(),
-      mtime_(),
-      entries_(),
-      attrs_() {}
-
-size_t DirEntryList::Size() {
-    ReadLockGuard lk(rwlock_);
-    return entries_.size();
-}
-
-void DirEntryList::Add(const DirEntry& dirEntry) {
-    WriteLockGuard lk(rwlock_);
-    entries_.push_back(std::move(dirEntry));
-    attrs_[dirEntry.ino] = &entries_.back();
-}
-
-bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) {
-    ReadLockGuard lk(rwlock_);
-    auto iter = attrs_.find(ino);
-    if (iter == attrs_.end()) {
-        return false;
-    }
-    *dirEntry = *iter->second;
-    return true;
-}
-
-bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) {
-    WriteLockGuard lk(rwlock_);
-    auto iter = attrs_.find(ino);
-    if (iter == attrs_.end()) {
-        return false;
-    }
-
-    DirEntry* dirEntry = iter->second;
-    dirEntry->attr = std::move(attr);
-    return true;
-}
-
-bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) {
-    WriteLockGuard lk(rwlock_);
-    auto iter = attrs_.find(ino);
-    if (iter == attrs_.end()) {
-        return false;
-    }
-
-    InodeAttr* attr = &iter->second->attr;
-    attr->set_length(open.length());
-    attr->set_mtime(open.mtime());
-    attr->set_mtime_ns(open.mtime_ns());
-    if (AttrCtime(open) > AttrCtime(*attr)) {
-        attr->set_ctime(open.ctime());
-        attr->set_ctime_ns(open.ctime_ns());
-    }
-    return true;
-}
-
-void DirEntryList::Iterate(IterateHandler handler) {
-    ReadLockGuard lk(rwlock_);
-    for (auto iter = entries_.begin(); iter != entries_.end(); iter++) {
-        handler(&(*iter));
-    }
-}
-
-void DirEntryList::Clear() {
-    WriteLockGuard lk(rwlock_);
-    entries_.clear();
-    attrs_.clear();
-}
-
-void DirEntryList::SetMtime(TimeSpec mtime) {
-    WriteLockGuard lk(rwlock_);
-    mtime_ = mtime;
-}
-
-TimeSpec DirEntryList::GetMtime() {
-    ReadLockGuard lk(rwlock_);
-    return mtime_;
-}
-
-DirCache::DirCache(DirCacheOption option)
-    : rwlock_(),
-      nentries_(0),
-      option_(option) {
-    lru_ = std::make_shared<LRUType>(0);  // control size by ourself
-    mq_ = std::make_shared<MessageQueueType>("dircache", 10000);
-    mq_->Subscribe([&](const std::shared_ptr<DirEntryList>& entries){
-        entries->Clear();
-    });
-
-    LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize;
-}
-
-void DirCache::Start() {
-    mq_->Start();
-}
-
-void DirCache::Stop() {
-    WriteLockGuard lk(rwlock_);
-    Evit(option_.lruSize);
-    mq_->Stop();
-}
-
-void DirCache::Delete(Ino parent,
-                      std::shared_ptr<DirEntryList> entries,
-                      bool evit) {
-    nentries_ -= entries->Size();
-    mq_->Publish(entries);  // clear entries in background
-    lru_->Remove(parent);
-
-    VLOG(1) << "Delete directory cache (evit=" << evit << "): "
-            << "parent = " << parent
-            << ", mtime = " << entries->GetMtime()
-            << ", size = " << entries->Size()
-            << ", nentries = " << nentries_;
-}
-
-void DirCache::Evit(size_t size) {
-    Ino parent;
-    std::shared_ptr<DirEntryList> entries;
-    while (nentries_ + size >= option_.lruSize) {
-        bool yes = lru_->GetLast(&parent, &entries);
-        if (!yes) {
-            break;
-        }
-        Delete(parent, entries, true);
-    }
-}
-
-void DirCache::Put(Ino parent, std::shared_ptr<DirEntryList> entries) {
-    WriteLockGuard lk(rwlock_);
-    Evit(entries->Size());  // it guarantee put entries success
-    lru_->Put(parent, entries);
-    nentries_ += entries->Size();
-
-    VLOG(1) << "Insert directory cache: parent = " << parent
-            << ", mtime = " << entries->GetMtime()
-            << ", size = " << entries->Size()
-            << ", nentries = " << nentries_;
-}
-
-bool DirCache::Get(Ino parent, std::shared_ptr<DirEntryList>* entries) {
-    ReadLockGuard lk(rwlock_);
-    return lru_->Get(parent, entries);
-}
-
-void DirCache::Drop(Ino parent) {
-    WriteLockGuard lk(rwlock_);
-    std::shared_ptr<DirEntryList> entries;
-    bool yes = lru_->Get(parent, &entries);
-    if (yes) {
-        Delete(parent, entries, false);
-    }
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/dir_cache.h
+++ b/curvefs/src/client/filesystem/dir_cache.h
@ -1,114 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
-
-#include <map>
-#include <list>
-#include <memory>
-
-#include "src/common/lru_cache.h"
-#include "src/common/concurrent/concurrent.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/message_queue.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::RWLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::common::DirCacheOption;
-
-class DirEntryList {
- public:
-    using IterateHandler = std::function<void(DirEntry* dirEntry)>;
-
- public:
-    DirEntryList();
-
-    size_t Size();
-
-    void Add(const DirEntry& dirEntry);
-
-    void Iterate(IterateHandler handler);
-
-    bool Get(Ino ino, DirEntry* dirEntry);
-
-    bool UpdateAttr(Ino ino, const InodeAttr& attr);
-
-    bool UpdateLength(Ino ino, const InodeAttr& open);
-
-    void Clear();
-
-    void SetMtime(TimeSpec mtime);
-
-    TimeSpec GetMtime();
-
- private:
-    RWLock rwlock_;
-    TimeSpec mtime_;
-    std::list<DirEntry> entries_;
-    std::map<Ino, DirEntry*> attrs_;
-};
-
-class DirCache {
- public:
-    using LRUType = LRUCache<Ino, std::shared_ptr<DirEntryList>>;
-    using MessageType = std::shared_ptr<DirEntryList>;
-    using MessageQueueType = MessageQueue<MessageType>;
-
- public:
-    explicit DirCache(DirCacheOption option);
-
-    void Start();
-
-    void Stop();
-
-    void Put(Ino parent, std::shared_ptr<DirEntryList> entries);
-
-    bool Get(Ino parent, std::shared_ptr<DirEntryList>* entries);
-
-    void Drop(Ino parent);
-
- private:
-    void Delete(Ino parent, std::shared_ptr<DirEntryList> entries, bool evit);
-
-    void Evit(size_t size);
-
- private:
-    RWLock rwlock_;
-    size_t nentries_;
-    DirCacheOption option_;
-    std::shared_ptr<LRUType> lru_;
-    std::shared_ptr<MessageQueueType> mq_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
--- a/curvefs/src/client/filesystem/error.cpp
+++ b/curvefs/src/client/filesystem/error.cpp
@ -1,106 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <map>
-#include <utility>
-#include <ostream>
-
-#include "curvefs/src/client/filesystem/error.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-static const std::map<CURVEFS_ERROR, std::pair<int, std::string>> errors = {
-    { CURVEFS_ERROR::OK, { 0, "OK"} },
-    { CURVEFS_ERROR::INTERNAL,  { EIO, "internal error" } },
-    { CURVEFS_ERROR::UNKNOWN,  { -1, "unknown" } },
-    { CURVEFS_ERROR::EXISTS,  { EEXIST, "inode or dentry already exist" } },
-    { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } },
-    { CURVEFS_ERROR::NO_SPACE,  { ENOSPC, "no space to alloc" } },
-    { CURVEFS_ERROR::BAD_FD,  { EBADF, "bad file number" } },
-    { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } },
-    { CURVEFS_ERROR::NOPERMISSION,  { EACCES, "permission denied" } },
-    { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } },
-    { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } },
-    { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } },
-    { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } },
-    { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } },
-    { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } },
-    { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } },
-    { CURVEFS_ERROR::NODATA, { ENODATA, "no data available" } },
-    { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } },
-    { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } },
-    { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } },
-};
-
-std::string StrErr(CURVEFS_ERROR code) {
-    auto it = errors.find(code);
-    if (it != errors.end()) {
-        return it->second.second;
-    }
-    return "unknown";
-}
-
-int SysErr(CURVEFS_ERROR code) {
-    int syscode = -1;
-    auto it = errors.find(code);
-    if (it != errors.end()) {
-        syscode = it->second.first;
-    }
-    return (syscode == -1) ? EIO : syscode;
-}
-
-std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
-    os << static_cast<int>(code) << "[" << [code]() {
-        auto it = errors.find(code);
-        if (it != errors.end()) {
-            return it->second.second;
-        }
-
-        return std::string{"Unknown"};
-    }() << "]";
-
-    return os;
-}
-
-CURVEFS_ERROR ToFSError(MetaStatusCode code) {
-    static std::map<MetaStatusCode, CURVEFS_ERROR> errs = {
-        { MetaStatusCode::OK, CURVEFS_ERROR::OK },
-        { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST },
-        { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM },
-        { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS },
-        { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS },
-        { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL },
-        { MetaStatusCode::RPC_ERROR, CURVEFS_ERROR::INTERNAL },
-    };
-
-    auto it = errs.find(code);
-    if (it != errs.end()) {
-        return it->second;
-    }
-    return CURVEFS_ERROR::UNKNOWN;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/filesystem.cpp
+++ b/curvefs/src/client/filesystem/filesystem.cpp
@ -1,353 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-08
- * Author: Jingli Chen (Wine93)
- */
-
-#include <map>
-
-#include "curvefs/src/client/filesystem/filesystem.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
-    : option_(option), member(member) {
-    deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
-    negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
-    dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
-    openFiles_ = std::make_shared<OpenFiles>(option_.openFilesOption,
-                                             deferSync_);
-    attrWatcher_ = std::make_shared<AttrWatcher>(option_.attrWatcherOption,
-                                                 openFiles_, dirCache_);
-    handlerManager_ = std::make_shared<HandlerManager>();
-    rpc_ = std::make_shared<RPCClient>(option.rpcOption, member);
-}
-
-void FileSystem::Run() {
-    deferSync_->Start();
-    dirCache_->Start();
-}
-
-void FileSystem::Destory() {
-    openFiles_->CloseAll();
-    deferSync_->Stop();
-    dirCache_->Stop();
-}
-
-void FileSystem::Attr2Stat(InodeAttr* attr, struct stat* stat) {
-    std::memset(stat, 0, sizeof(struct stat));
-    stat->st_ino = attr->inodeid();  //  inode number
-    stat->st_mode = attr->mode();  // permission mode
-    stat->st_nlink = attr->nlink();  // number of links
-    stat->st_uid = attr->uid();  // user ID of owner
-    stat->st_gid = attr->gid();  // group ID of owner
-    stat->st_size = attr->length();  // total size, in bytes
-    stat->st_rdev = attr->rdev();  // device ID (if special file)
-    stat->st_atim.tv_sec = attr->atime();  // time of last access
-    stat->st_atim.tv_nsec = attr->atime_ns();
-    stat->st_mtim.tv_sec = attr->mtime();  // time of last modification
-    stat->st_mtim.tv_nsec = attr->mtime_ns();
-    stat->st_ctim.tv_sec = attr->ctime();  // time of last status change
-    stat->st_ctim.tv_nsec = attr->ctime_ns();
-    stat->st_blksize = option_.blockSize;  // blocksize for file system I/O
-    stat->st_blocks = 0;  // number of 512B blocks allocated
-    if (IsS3File(*attr)) {
-        stat->st_blocks = (attr->length() + 511) / 512;
-    }
-}
-
-void FileSystem::Entry2Param(EntryOut* entryOut,
-                             fuse_entry_param* e) {
-    std::memset(e, 0, sizeof(fuse_entry_param));
-    e->ino = entryOut->attr.inodeid();
-    e->generation = 0;
-    Attr2Stat(&entryOut->attr, &e->attr);
-    e->entry_timeout = entryOut->entryTimeout;
-    e->attr_timeout = entryOut->attrTimeout;
-}
-
-void FileSystem::SetEntryTimeout(EntryOut* entryOut) {
-    auto option = option_.kernelCacheOption;
-    if (IsDir(entryOut->attr)) {
-        entryOut->entryTimeout = option.dirEntryTimeoutSec;
-        entryOut->attrTimeout = option.dirAttrTimeoutSec;
-    } else {
-        entryOut->entryTimeout = option.entryTimeoutSec;
-        entryOut->attrTimeout = option.attrTimeoutSec;
-    }
-}
-
-void FileSystem::SetAttrTimeout(AttrOut* attrOut) {
-    auto option = option_.kernelCacheOption;
-    if (IsDir(attrOut->attr)) {
-        attrOut->attrTimeout = option.dirAttrTimeoutSec;
-    } else {
-        attrOut->attrTimeout = option.attrTimeoutSec;
-    }
-}
-
-// fuse reply*
-void FileSystem::ReplyError(Request req, CURVEFS_ERROR code) {
-    fuse_reply_err(req, SysErr(code));
-}
-
-void FileSystem::ReplyEntry(Request req,
-                            EntryOut* entryOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
-                             ReplyType::ATTR, true);
-    fuse_entry_param e;
-    SetEntryTimeout(entryOut);
-    Entry2Param(entryOut, &e);
-    fuse_reply_entry(req, &e);
-}
-
-void FileSystem::ReplyAttr(Request req,
-                           AttrOut* attrOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr,
-                             ReplyType::ATTR, true);
-    struct stat stat;
-    SetAttrTimeout(attrOut);
-    Attr2Stat(&attrOut->attr, &stat);
-    fuse_reply_attr(req, &stat, attrOut->attrTimeout);
-}
-
-void FileSystem::ReplyReadlink(Request req, const std::string& link) {
-    fuse_reply_readlink(req, link.c_str());
-}
-
-void FileSystem::ReplyOpen(Request req, FileInfo* fi) {
-    fuse_reply_open(req, fi);
-}
-
-void FileSystem::ReplyOpen(Request req, FileOut* fileOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
-                             ReplyType::ONLY_LENGTH, true);
-    fuse_reply_open(req, fileOut->fi);
-}
-
-void FileSystem::ReplyData(Request req,
-                           struct fuse_bufvec *bufv,
-                           enum fuse_buf_copy_flags flags) {
-    fuse_reply_data(req, bufv, flags);
-}
-
-void FileSystem::ReplyWrite(Request req, FileOut* fileOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
-                             ReplyType::ONLY_LENGTH, true);
-    fuse_reply_write(req, fileOut->nwritten);
-}
-
-void FileSystem::ReplyBuffer(Request req, const char *buf, size_t size) {
-    fuse_reply_buf(req, buf, size);
-}
-
-void FileSystem::ReplyStatfs(Request req, const struct statvfs *stbuf) {
-    fuse_reply_statfs(req, stbuf);
-}
-
-void FileSystem::ReplyXattr(Request req, size_t size) {
-    fuse_reply_xattr(req, size);
-}
-
-void FileSystem::ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi) {
-    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
-                             ReplyType::ATTR, true);
-    fuse_entry_param e;
-    SetEntryTimeout(entryOut);
-    Entry2Param(entryOut, &e);
-    fuse_reply_create(req, &e, fi);
-}
-
-void FileSystem::AddDirEntry(Request req,
-                             DirBufferHead* buffer,
-                             DirEntry* dirEntry) {
-    struct stat stat;
-    std::memset(&stat, 0, sizeof(stat));
-    stat.st_ino = dirEntry->ino;
-
-    // add a directory entry to the buffer
-    size_t oldsize = buffer->size;
-    const char* name = dirEntry->name.c_str();
-    buffer->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);
-    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
-    fuse_add_direntry(req,
-                      buffer->p + oldsize,  // char* buf
-                      buffer->size - oldsize,  // size_t bufisze
-                      name, &stat, buffer->size);
-}
-
-void FileSystem::AddDirEntryPlus(Request req,
-                                 DirBufferHead* buffer,
-                                 DirEntry* dirEntry) {
-    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr,
-                             ReplyType::ATTR, false);
-    struct fuse_entry_param e;
-    EntryOut entryOut(dirEntry->attr);
-    SetEntryTimeout(&entryOut);
-    Entry2Param(&entryOut, &e);
-
-    // add a directory entry to the buffer with the attributes
-    size_t oldsize = buffer->size;
-    const char* name = dirEntry->name.c_str();
-    buffer->size += fuse_add_direntry_plus(req, NULL, 0, name, NULL, 0);
-    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
-    fuse_add_direntry_plus(req,
-                           buffer->p + oldsize,  // char* buf
-                           buffer->size - oldsize,  // size_t bufisze
-                           name, &e, buffer->size);
-}
-
-// handler*
-std::shared_ptr<FileHandler> FileSystem::NewHandler() {
-    return handlerManager_->NewHandler();
-}
-
-std::shared_ptr<FileHandler> FileSystem::FindHandler(uint64_t fh) {
-    return handlerManager_->FindHandler(fh);
-}
-
-void FileSystem::ReleaseHandler(uint64_t fh) {
-    return handlerManager_->ReleaseHandler(fh);
-}
-
-FileSystemMember FileSystem::BorrowMember() {
-    return FileSystemMember(deferSync_, openFiles_, attrWatcher_);
-}
-
-// fuse request*
-CURVEFS_ERROR FileSystem::Lookup(Request req,
-                                 Ino parent,
-                                 const std::string& name,
-                                 EntryOut* entryOut) {
-    if (name.size() > option_.maxNameLength) {
-        return CURVEFS_ERROR::NAMETOOLONG;
-    }
-
-    bool yes = negative_->Get(parent, name);
-    if (yes) {
-        return CURVEFS_ERROR::NOTEXIST;
-    }
-
-    auto rc = rpc_->Lookup(parent, name, entryOut);
-    if (rc == CURVEFS_ERROR::OK) {
-        negative_->Delete(parent, name);
-    } else if (rc == CURVEFS_ERROR::NOTEXIST) {
-        negative_->Put(parent, name);
-    }
-    return rc;
-}
-
-CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) {
-    InodeAttr attr;
-    auto rc = rpc_->GetAttr(ino, &attr);
-    if (rc == CURVEFS_ERROR::OK) {
-        *attrOut = AttrOut(attr);
-    }
-    return rc;
-}
-
-CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) {
-    InodeAttr attr;
-    CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    // revalidate directory cache
-    std::shared_ptr<DirEntryList> entries;
-    bool yes = dirCache_->Get(ino, &entries);
-    if (yes) {
-        if (entries->GetMtime() != AttrMtime(attr)) {
-            dirCache_->Drop(ino);
-        }
-    }
-
-    auto handler = NewHandler();
-    handler->mtime = AttrMtime(attr);
-    fi->fh = handler->fh;
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::ReadDir(Request req,
-                                  Ino ino,
-                                  FileInfo* fi,
-                                  std::shared_ptr<DirEntryList>* entries) {
-    bool yes = dirCache_->Get(ino, entries);
-    if (yes) {
-        return CURVEFS_ERROR::OK;
-    }
-
-    CURVEFS_ERROR rc = rpc_->ReadDir(ino, entries);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    (*entries)->SetMtime(FindHandler(fi->fh)->mtime);
-    dirCache_->Put(ino, *entries);
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) {
-    ReleaseHandler(fi->fh);
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) {
-    std::shared_ptr<InodeWrapper> inode;
-    bool yes = openFiles_->IsOpened(ino, &inode);
-    if (yes) {
-        openFiles_->Open(ino, inode);
-        // fi->keep_cache = 1;
-        return CURVEFS_ERROR::OK;
-    }
-
-    CURVEFS_ERROR rc = rpc_->Open(ino, &inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    TimeSpec mtime;
-    yes = attrWatcher_->GetMtime(ino, &mtime);
-    if (!yes) {
-        // It is rare which only arise when attribute evited for attr-watcher.
-        LOG(WARNING) << "open(" << ino << "): stale file handler"
-                     << ": attribute not found in wacther";
-        return CURVEFS_ERROR::STALE;
-    } else if (mtime != InodeMtime(inode)) {
-        LOG(WARNING) << "open(" << ino << "): stale file handler"
-                     << ", cache(" << mtime << ") vs remote("
-                     << InodeMtime(inode) << ")";
-        return CURVEFS_ERROR::STALE;
-    }
-
-    openFiles_->Open(ino, inode);
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) {
-    openFiles_->Close(ino);
-    return CURVEFS_ERROR::OK;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/filesystem.h
+++ b/curvefs/src/client/filesystem/filesystem.h
@ -1,167 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-08
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
-
-#include <gtest/gtest_prod.h>
-
-#include <memory>
-#include <string>
-
-#include "curvefs/src/client/filesystem/error.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/package.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/lookup_cache.h"
-#include "curvefs/src/client/filesystem/dir_cache.h"
-#include "curvefs/src/client/filesystem/openfile.h"
-#include "curvefs/src/client/filesystem/attr_watcher.h"
-#include "curvefs/src/client/filesystem/rpc_client.h"
-#include "curvefs/src/client/filesystem/defer_sync.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curvefs::client::common::FileSystemOption;
-
-struct FileSystemMember {
-    FileSystemMember(std::shared_ptr<DeferSync> deferSync,
-                     std::shared_ptr<OpenFiles> openFiles,
-                     std::shared_ptr<AttrWatcher> attrWatcher)
-        : deferSync(deferSync),
-          openFiles(openFiles),
-          attrWatcher(attrWatcher) {}
-
-    std::shared_ptr<DeferSync> deferSync;
-    std::shared_ptr<OpenFiles> openFiles;
-    std::shared_ptr<AttrWatcher> attrWatcher;
-};
-
-class FileSystem {
- public:
-    FileSystem(struct FileSystemOption option, ExternalMember member);
-
-    void Run();
-
-    void Destory();
-
-    // fuse request
-    CURVEFS_ERROR Lookup(Request req,
-                         Ino parent,
-                         const std::string& name,
-                         EntryOut* entryOut);
-
-    CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut);
-
-    CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi);
-
-    CURVEFS_ERROR ReadDir(Request req,
-                          Ino ino,
-                          FileInfo* fi,
-                          std::shared_ptr<DirEntryList>* entries);
-
-    CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi);
-
-    CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi);
-
-    CURVEFS_ERROR Release(Request req, Ino ino);
-
-    // fuse reply: we control all replies to vfs layer in same entrance.
-    void ReplyError(Request req, CURVEFS_ERROR code);
-
-    void ReplyEntry(Request req, EntryOut* entryOut);
-
-    void ReplyAttr(Request req, AttrOut* attrOut);
-
-    void ReplyReadlink(Request req, const std::string& link);
-
-    void ReplyOpen(Request req, FileInfo *fi);
-
-    void ReplyOpen(Request req, FileOut* fileOut);
-
-    void ReplyData(Request req,
-                   struct fuse_bufvec *bufv,
-                   enum fuse_buf_copy_flags flags);
-
-    void ReplyWrite(Request req, FileOut* fileOut);
-
-    void ReplyBuffer(Request req, const char *buf, size_t size);
-
-    void ReplyStatfs(Request req, const struct statvfs *stbuf);
-
-    void ReplyXattr(Request req, size_t size);
-
-    void ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi);
-
-    void AddDirEntry(Request req,
-                     DirBufferHead* buffer,
-                     DirEntry* dirEntry);
-
-    void AddDirEntryPlus(Request req,
-                         DirBufferHead* buffer,
-                         DirEntry* dirEntry);
-
-    // utility: file handler
-    std::shared_ptr<FileHandler> NewHandler();
-
-    std::shared_ptr<FileHandler> FindHandler(uint64_t fh);
-
-    void ReleaseHandler(uint64_t fh);
-
-    // utility: others
-    FileSystemMember BorrowMember();
-
- private:
-    FRIEND_TEST(FileSystemTest, Attr2Stat);
-    FRIEND_TEST(FileSystemTest, Entry2Param);
-    FRIEND_TEST(FileSystemTest, SetEntryTimeout);
-    FRIEND_TEST(FileSystemTest, SetAttrTimeout);
-
-    // utility: convert to system type.
-    void Attr2Stat(InodeAttr* attr, struct stat* stat);
-
-    void Entry2Param(EntryOut* entryOut, fuse_entry_param* e);
-
-    // utility: set entry/attribute timeout
-    void SetEntryTimeout(EntryOut* entryOut);
-
-    void SetAttrTimeout(AttrOut* attrOut);
-
- private:
-    FileSystemOption option_;
-    ExternalMember member;
-    std::shared_ptr<DeferSync> deferSync_;
-    std::shared_ptr<LookupCache> negative_;
-    std::shared_ptr<DirCache> dirCache_;
-    std::shared_ptr<OpenFiles> openFiles_;
-    std::shared_ptr<AttrWatcher> attrWatcher_;
-    std::shared_ptr<HandlerManager> handlerManager_;
-    std::shared_ptr<RPCClient> rpc_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
--- a/curvefs/src/client/filesystem/lookup_cache.cpp
+++ b/curvefs/src/client/filesystem/lookup_cache.cpp
@ -1,104 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-31
- * Author: Jingli Chen (Wine93)
- */
-
-#include <glog/logging.h>
-
-#include <ctime>
-
-#include "absl/strings/str_format.h"
-#include "curvefs/src/client/filesystem/utils.h"
-#include "curvefs/src/client/filesystem/lookup_cache.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-#define RETURN_FALSE_IF_DISABLED() \
-    do {                           \
-        if (!enable_) {            \
-            return false;          \
-        }                          \
-    } while (0)
-
-
-LookupCache::LookupCache(LookupCacheOption option)
-    : enable_(option.negativeTimeoutSec > 0),
-      rwlock_(),
-      option_(option) {
-    lru_ = std::make_shared<LRUType>(option.lruSize);
-    if (enable_) {
-        LOG(INFO) << "Using lookup negative lru cache"
-                  << ", timeout = " << option.negativeTimeoutSec
-                  << ", capacity = " << option.lruSize;
-    }
-}
-
-std::string LookupCache::CacheKey(Ino parent, const std::string& name) {
-    return absl::StrFormat("%d:%s", parent, name);
-}
-
-bool LookupCache::Get(Ino parent, const std::string& name) {
-    RETURN_FALSE_IF_DISABLED();
-    ReadLockGuard lk(rwlock_);
-    CacheEntry entry;
-    auto key = CacheKey(parent, name);
-    bool yes = lru_->Get(key, &entry);
-    if (!yes) {
-        VLOG(1) << absl::StrFormat("Lookup cache not found: key(%d,%s)",
-                                   parent, name);
-        return false;
-    } else if (entry.uses < option_.minUses) {
-        return false;
-    } else if (entry.expireTime < Now()) {
-        return false;
-    }
-    return true;
-}
-
-bool LookupCache::Put(Ino parent, const std::string& name) {
-    RETURN_FALSE_IF_DISABLED();
-    WriteLockGuard lk(rwlock_);
-    CacheEntry entry;
-    auto key = CacheKey(parent, name);
-    bool yes = lru_->Get(key, &entry);
-    if (yes) {
-        entry.uses++;
-    } else {
-        entry.uses = 0;
-    }
-
-    entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0);
-    lru_->Put(key, entry);
-    return true;
-}
-
-bool LookupCache::Delete(Ino parent, const std::string& name) {
-    RETURN_FALSE_IF_DISABLED();
-    WriteLockGuard lk(rwlock_);
-    auto key = CacheKey(parent, name);
-    lru_->Remove(key);
-    return true;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/lookup_cache.h
+++ b/curvefs/src/client/filesystem/lookup_cache.h
@ -1,77 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-31
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
-
-#include <memory>
-#include <string>
-
-#include "src/common/lru_cache.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::RWLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::common::LookupCacheOption;
-
-// memory cache for lookup result, now we only support cache negative result,
-// and other positive entry will be cached in kernel.
-class LookupCache {
- public:
-    struct CacheEntry {
-        uint32_t uses;
-        TimeSpec expireTime;
-    };
-
-    using LRUType = LRUCache<std::string, CacheEntry>;
-
- public:
-    explicit LookupCache(LookupCacheOption option);
-
-    bool Get(Ino parent, const std::string& name);
-
-    bool Put(Ino parent, const std::string& name);
-
-    bool Delete(Ino parent, const std::string& name);
-
- private:
-    std::string CacheKey(Ino parent, const std::string& name);
-
- private:
-    bool enable_;
-    RWLock rwlock_;
-    LookupCacheOption option_;
-    std::shared_ptr<LRUType> lru_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
--- a/curvefs/src/client/filesystem/message_queue.h
+++ b/curvefs/src/client/filesystem/message_queue.h
@ -1,118 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
-
-#include <glog/logging.h>
-
-#include <string>
-#include <atomic>
-#include <thread>
-
-#include "src/common/concurrent/task_queue.h"
-#include "curvefs/src/common/threading.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::TaskQueue;
-using ::curvefs::common::SetThreadName;
-
-template <typename MessageT>
-class MessageQueue {
- public:
-    using MessageHandler = std::function<void(const MessageT& message)>;
-
- public:
-    MessageQueue(const std::string& name, size_t bufferSize)
-        : name_(name),
-          running_(false),
-          thread_(),
-          handler_(),
-          queue_(bufferSize) {}
-
-    void Start() {
-        if (running_.exchange(true)) {
-            return;
-        }
-
-        thread_ = std::thread(&MessageQueue::Consumer, this);
-        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
-                  << "consumer thread start success";
-    }
-
-    void Stop() {
-        if (!running_.exchange(false)) {
-            return;
-        }
-
-        auto wakeup = []() {};
-        queue_.Push(wakeup);
-
-        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
-                  << "consumer thread stoping...";
-
-        thread_.join();
-
-        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
-                  << "consumer thread stopped";
-    }
-
-    void Publish(MessageT message) {
-        if (handler_ != nullptr) {
-            queue_.Push([this, message](){
-                this->handler_(message);
-            });
-        }
-    }
-
-    void Subscribe(MessageHandler handler) {
-        handler_ = handler;
-    }
-
- private:
-    void Consumer() {
-        SetThreadName(name_.c_str());
-        while (running_.load(std::memory_order_relaxed)) {
-            queue_.Pop()();
-        }
-
-        while (queue_.Size() > 0) {
-            queue_.Pop()();
-        }
-    }
-
- private:
-    std::string name_;
-    std::atomic<bool> running_;
-    std::thread thread_;
-    MessageHandler handler_;
-    TaskQueue queue_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
--- a/curvefs/src/client/filesystem/meta.cpp
+++ b/curvefs/src/client/filesystem/meta.cpp
@ -1,129 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <sys/stat.h>
-
-#include "absl/strings/str_format.h"
-#include "curvefs/src/client/filesystem/meta.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-HandlerManager::HandlerManager()
-    : mutex_(),
-      dirBuffer_(std::make_shared<DirBuffer>()),
-      handlers_() {}
-
-HandlerManager::~HandlerManager() {
-    dirBuffer_->DirBufferFreeAll();
-}
-
-std::shared_ptr<FileHandler> HandlerManager::NewHandler() {
-    UniqueLock lk(mutex_);
-    auto handler = std::make_shared<FileHandler>();
-    handler->fh = dirBuffer_->DirBufferNew();
-    handler->buffer = dirBuffer_->DirBufferGet(handler->fh);
-    handler->padding = false;
-    handlers_.emplace(handler->fh, handler);
-    return handler;
-}
-
-std::shared_ptr<FileHandler> HandlerManager::FindHandler(uint64_t fh) {
-    UniqueLock lk(mutex_);
-    auto iter = handlers_.find(fh);
-    if (iter == handlers_.end()) {
-        return nullptr;
-    }
-    return iter->second;
-}
-
-void HandlerManager::ReleaseHandler(uint64_t fh) {
-    UniqueLock lk(mutex_);
-    dirBuffer_->DirBufferRelease(fh);
-    handlers_.erase(fh);
-}
-
-std::string StrMode(uint16_t mode) {
-    static std::map<uint16_t, char> type2char = {
-        { S_IFSOCK, 's' },
-        { S_IFLNK, 'l' },
-        { S_IFREG, '-' },
-        { S_IFBLK, 'b' },
-        { S_IFDIR, 'd' },
-        { S_IFCHR, 'c' },
-        { S_IFIFO, 'f' },
-        { 0, '?' },
-    };
-
-    std::string s("?rwxrwxrwx");
-    s[0] = type2char[mode & (S_IFMT & 0xffff)];
-    if (mode & S_ISUID) {
-        s[3] = 's';
-    }
-    if (mode & S_ISGID) {
-        s[6] = 's';
-    }
-    if (mode & S_ISVTX) {
-        s[9] = 't';
-    }
-
-    for (auto i = 0; i < 9; i++) {
-        if ((mode & (1 << i)) == 0) {
-            if ((s[9-i] == 's') || (s[9-i] == 't')) {
-                s[9-i] &= 0xDF;
-            } else {
-                s[9-i] = '-';
-            }
-        }
-    }
-    return s;
-}
-
-namespace {
-
-std::string Attr2Str(const InodeAttr& attr) {
-    if (!attr.IsInitialized()) {
-        return "";
-    }
-
-    std::string smode;
-    return absl::StrFormat(" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])",
-        attr.inodeid(), StrMode(attr.mode()).c_str(), attr.mode(), attr.nlink(),
-        attr.uid(), attr.gid(),
-        attr.atime(), attr.mtime(), attr.ctime(),
-        attr.length());
-}
-
-}  // namespace
-
-std::string StrEntry(EntryOut entryOut) {
-    return Attr2Str(entryOut.attr);
-}
-
-std::string StrAttr(AttrOut attrOut) {
-    return Attr2Str(attrOut.attr);
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/meta.h
+++ b/curvefs/src/client/filesystem/meta.h
@ -1,178 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
-
-#include <map>
-#include <memory>
-#include <string>
-#include <iostream>
-
-#include "src/common/concurrent/concurrent.h"
-#include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/fuse_common.h"
-#include "curvefs/src/client/dir_buffer.h"
-#include "curvefs/src/client/inode_wrapper.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::Mutex;
-using ::curve::common::RWLock;
-using ::curve::common::UniqueLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::metaserver::XAttr;
-using ::curvefs::metaserver::Dentry;
-using ::curvefs::metaserver::InodeAttr;
-using ::curvefs::metaserver::FsFileType;
-using ::curvefs::client::DirBufferHead;
-using ::curvefs::client::InodeWrapper;
-
-using Ino = fuse_ino_t;
-using Request = fuse_req_t;
-using FileInfo = struct fuse_file_info;
-
-struct EntryOut {
-    EntryOut() = default;
-
-    explicit EntryOut(InodeAttr attr) : attr(attr) {}
-
-    InodeAttr attr;
-    double entryTimeout;
-    double attrTimeout;
-};
-
-struct AttrOut {
-    AttrOut() = default;
-
-    explicit AttrOut(InodeAttr attr) : attr(attr) {}
-
-    InodeAttr attr;
-    double attrTimeout;
-};
-
-struct DirEntry {
-    DirEntry() = default;
-
-    DirEntry(Ino ino, const std::string& name, InodeAttr attr)
-        : ino(ino), name(name), attr(attr) {}
-
-    Ino ino;
-    std::string name;
-    InodeAttr attr;
-};
-
-struct FileOut {
-    FileOut() = default;
-
-    FileOut(FileInfo* fi, InodeAttr attr)
-        : fi(fi), attr(attr), nwritten(0) {}
-
-    FileOut(InodeAttr attr, size_t nwritten)
-        : fi(nullptr), attr(attr), nwritten(nwritten) {}
-
-    FileInfo* fi;
-    InodeAttr attr;
-    size_t nwritten;
-};
-
-struct TimeSpec {
-    TimeSpec() : seconds(0), nanoSeconds(0) {}
-
-    TimeSpec(uint64_t seconds, uint32_t nanoSeconds)
-        : seconds(seconds), nanoSeconds(nanoSeconds) {}
-
-    TimeSpec(const TimeSpec& time)
-        : seconds(time.seconds), nanoSeconds(time.nanoSeconds) {}
-
-    TimeSpec& operator=(const TimeSpec& time) = default;
-
-    TimeSpec operator+(const TimeSpec& time) const {
-        return TimeSpec(seconds + time.seconds, nanoSeconds + time.nanoSeconds);
-    }
-
-    uint64_t seconds;
-    uint32_t nanoSeconds;
-};
-
-struct FileHandler {
-    uint64_t fh;
-    DirBufferHead* buffer;
-    TimeSpec mtime;
-    bool padding;  // padding buffer
-};
-
-class HandlerManager {
- public:
-    HandlerManager();
-
-    ~HandlerManager();
-
-    std::shared_ptr<FileHandler> NewHandler();
-
-    std::shared_ptr<FileHandler> FindHandler(uint64_t id);
-
-    void ReleaseHandler(uint64_t id);
-
- private:
-    Mutex mutex_;
-    std::shared_ptr<DirBuffer> dirBuffer_;
-    std::map<uint64_t, std::shared_ptr<FileHandler>> handlers_;
-};
-
-inline bool operator==(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return (lhs.seconds == rhs.seconds) &&
-        (lhs.nanoSeconds == rhs.nanoSeconds);
-}
-
-inline bool operator!=(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return !(lhs == rhs);
-}
-
-inline bool operator<(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return (lhs.seconds < rhs.seconds) ||
-        (lhs.seconds == rhs.seconds && lhs.nanoSeconds < rhs.nanoSeconds);
-}
-
-inline bool operator>(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return (lhs.seconds > rhs.seconds) ||
-        (lhs.seconds == rhs.seconds && lhs.nanoSeconds > rhs.nanoSeconds);
-}
-
-inline std::ostream &operator<<(std::ostream &os, const TimeSpec& time) {
-    return os << time.seconds << "." << time.nanoSeconds;
-}
-
-std::string StrMode(uint16_t mode);
-
-std::string StrEntry(EntryOut entryOut);
-
-std::string StrAttr(AttrOut attrOut);
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
--- a/curvefs/src/client/filesystem/openfile.cpp
+++ b/curvefs/src/client/filesystem/openfile.cpp
@ -1,141 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#include "curvefs/src/client/filesystem/openfile.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-OpenFiles::OpenFiles(OpenFilesOption option,
-                     std::shared_ptr<DeferSync> deferSync)
-    : rwlock_(),
-      option_(option),
-      deferSync_(deferSync),
-      files_(std::make_shared<LRUType>(option.lruSize)) {
-    LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize;
-}
-
-/*
- * Delete(...) does:
- *   1) publish to message queue which will flush file to server with async
- *   2) delete file from lru cache
- */
-void OpenFiles::Delete(Ino ino,
-                       const std::shared_ptr<OpenFile>& file,
-                       bool flush) {
-    if (flush) {
-        deferSync_->Push(file->inode);
-    }
-    files_->Remove(ino);
-
-    VLOG(1) << "Delete open file cache: ino = " << ino
-            << ", refs = " << file->refs
-            << ", mtime = " << InodeMtime(file->inode);
-}
-
-void OpenFiles::Evit(size_t size) {
-    Ino ino;
-    std::shared_ptr<OpenFile> file;
-    while (files_->Size() + size >= option_.lruSize) {
-        bool yes = files_->GetLast(&ino, &file);
-        if (!yes) {
-            break;
-        }
-        Delete(ino, file, true);
-    }
-}
-
-void OpenFiles::Open(Ino ino, std::shared_ptr<InodeWrapper> inode) {
-    WriteLockGuard lk(rwlock_);
-
-    Evit(1);
-
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (yes) {
-        file->refs++;
-        return;
-    }
-
-    file = std::make_shared<OpenFile>(inode);
-    file->refs++;
-    files_->Put(ino, file);
-
-    VLOG(1) << "Insert open file cache: ino = " << ino
-            << ", refs = " << file->refs
-            << ", mtime = " << InodeMtime(file->inode);
-}
-
-bool OpenFiles::IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
-    ReadLockGuard lk(rwlock_);
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (!yes) {
-        return false;
-    }
-    *inode = file->inode;
-    return true;
-}
-
-void OpenFiles::Close(Ino ino) {
-    WriteLockGuard lk(rwlock_);
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (!yes) {
-        return;
-    }
-
-    if (file->refs > 0) {
-        file->refs--;
-    }
-
-    if (file->refs == 0) {
-        Delete(ino, file, false);  // file already flushed before close
-    }
-}
-
-/*
- * CloseAll() does:
- *   flush all file to server and delete from LRU cache
- */
-void OpenFiles::CloseAll() {
-    WriteLockGuard lk(rwlock_);
-    Evit(option_.lruSize);
-}
-
-bool OpenFiles::GetFileAttr(Ino ino, InodeAttr* attr) {
-    ReadLockGuard lk(rwlock_);
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (!yes) {
-        return false;
-    }
-
-    file->inode->GetInodeAttr(attr);
-    return true;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/openfile.h
+++ b/curvefs/src/client/filesystem/openfile.h
@ -1,86 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
-
-#include <memory>
-
-#include "src/common/lru_cache.h"
-#include "curvefs/src/client/inode_wrapper.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/defer_sync.h"
-#include "curvefs/src/client/filesystem/dir_cache.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::InodeWrapper;
-using ::curvefs::client::common::OpenFilesOption;
-
-struct OpenFile {
-    explicit OpenFile(std::shared_ptr<InodeWrapper> inode)
-        : inode(inode), refs(0) {}
-
-    std::shared_ptr<InodeWrapper> inode;
-    uint64_t refs;
-};
-
-class OpenFiles {
- public:
-    using LRUType = LRUCache<Ino, std::shared_ptr<OpenFile>>;
-
- public:
-    explicit OpenFiles(OpenFilesOption option,
-                       std::shared_ptr<DeferSync> deferSync);
-
-    void Open(Ino ino, std::shared_ptr<InodeWrapper> inode);
-
-    bool IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode);
-
-    void Close(Ino ino);
-
-    void CloseAll();
-
-    bool GetFileAttr(Ino ino, InodeAttr* inode);
-
- private:
-    void Delete(Ino ino, const std::shared_ptr<OpenFile>& file, bool flush);
-
-    void Evit(size_t size);
-
- private:
-    RWLock rwlock_;
-    OpenFilesOption option_;
-    std::shared_ptr<DeferSync> deferSync_;
-    std::shared_ptr<LRUType> files_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
--- a/curvefs/src/client/filesystem/package.h
+++ b/curvefs/src/client/filesystem/package.h
@ -1,51 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
-
-#include <memory>
-
-#include "curvefs/src/client/dentry_cache_manager.h"
-#include "curvefs/src/client/inode_cache_manager.h"
-#include "curvefs/src/client/xattr_manager.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-struct ExternalMember {  // external member depended by FileSystem
-    ExternalMember() = delete;
-    ExternalMember(std::shared_ptr<DentryCacheManager> dentryManager,
-                   std::shared_ptr<InodeCacheManager> inodeManager)
-        : dentryManager(dentryManager),
-          inodeManager(inodeManager) {}
-
-    std::shared_ptr<DentryCacheManager> dentryManager;
-    std::shared_ptr<InodeCacheManager> inodeManager;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
--- a/curvefs/src/client/filesystem/rpc_client.cpp
+++ b/curvefs/src/client/filesystem/rpc_client.cpp
@ -1,128 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#include <map>
-#include <set>
-#include <list>
-#include <string>
-#include <memory>
-#include <utility>
-
-#include "curvefs/src/client/filesystem/rpc_client.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-RPCClient::RPCClient(RPCOption option,
-                     ExternalMember member)
-    : option_(option),
-      inodeManager_(member.inodeManager),
-      dentryManager_(member.dentryManager) {}
-
-CURVEFS_ERROR RPCClient::GetAttr(Ino ino, InodeAttr* attr) {
-    CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(ino, attr);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(getattr::GetInodeAttr) failed, retCode = " << rc
-                   << ", ino = " << ino;
-    }
-    return rc;
-}
-
-CURVEFS_ERROR RPCClient::Lookup(Ino parent,
-                                const std::string& name,
-                                EntryOut* entryOut) {
-    Dentry dentry;
-    CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry);
-    if (rc != CURVEFS_ERROR::OK) {
-        if (rc != CURVEFS_ERROR::NOTEXIST) {
-            LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc
-                       << ", parent = " << parent << ", name = " << name;
-        }
-        return rc;
-    }
-
-    Ino ino = dentry.inodeid();
-    rc = inodeManager_->GetInodeAttr(ino, &entryOut->attr);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(lookup::GetInodeAttr) failed, retCode = " << rc
-                   << ", ino = " << ino;
-    }
-    return rc;
-}
-
-CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
-                                 std::shared_ptr<DirEntryList>* entries) {
-    uint32_t limit = option_.listDentryLimit;
-
-    std::list<Dentry> dentries;
-    CURVEFS_ERROR rc = dentryManager_->ListDentry(ino, &dentries, limit);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc
-                   << ", ino = " << ino;
-        return rc;
-    }
-
-    std::set<uint64_t> inos;
-    std::map<uint64_t, InodeAttr> attrs;
-    std::for_each(dentries.begin(), dentries.end(), [&](Dentry& dentry){
-        inos.emplace(dentry.inodeid());
-    });
-    rc = inodeManager_->BatchGetInodeAttrAsync(ino, &inos, &attrs);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(readdir::BatchGetInodeAttrAsync) failed"
-                   << ", retCode = " << rc << ", ino = " << ino;
-        return rc;
-    }
-
-    DirEntry dirEntry;
-    for (const auto& dentry : dentries) {
-        Ino ino = dentry.inodeid();
-        auto iter = attrs.find(ino);
-        if (iter == attrs.end()) {
-            LOG(WARNING) << "rpc(readdir::BatchGetInodeAttrAsync) "
-                         << "missing attribute, ino = " << ino;
-            continue;
-        }
-
-        // NOTE: we can't use std::move() for attribute for hard link
-        // which will sharing inode attribute.
-        dirEntry.ino = ino;
-        dirEntry.name = std::move(dentry.name());
-        dirEntry.attr = iter->second;
-        (*entries)->Add(dirEntry);
-    }
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR RPCClient::Open(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
-    CURVEFS_ERROR rc = inodeManager_->GetInode(ino, *inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(open/GetInode) failed" << ", retCode = " << rc
-                   << ", ino = " << ino;
-    }
-    return rc;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/rpc_client.h
+++ b/curvefs/src/client/filesystem/rpc_client.h
@ -1,65 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
-
-#include <memory>
-#include <string>
-
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/package.h"
-#include "curvefs/src/client/filesystem/dir_cache.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curvefs::client::common::RPCOption;
-
-class RPCClient {
- public:
-    RPCClient(RPCOption option,
-              ExternalMember member);
-
-    CURVEFS_ERROR GetAttr(Ino ino, InodeAttr* attr);
-
-    CURVEFS_ERROR Lookup(Ino parent,
-                         const std::string& name,
-                         EntryOut* entryOut);
-
-    CURVEFS_ERROR ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries);
-
-    CURVEFS_ERROR Open(Ino ino, std::shared_ptr<InodeWrapper>* inode);
-
- private:
-    RPCOption option_;
-    std::shared_ptr<InodeCacheManager> inodeManager_;
-    std::shared_ptr<DentryCacheManager> dentryManager_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
--- a/curvefs/src/client/filesystem/utils.cpp
+++ b/curvefs/src/client/filesystem/utils.cpp
@ -1,69 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <glog/logging.h>
-
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-bool IsDir(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_DIRECTORY;
-}
-
-bool IsS3File(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_S3;
-}
-
-bool IsVolmeFile(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_FILE;
-}
-
-bool IsSymLink(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_SYM_LINK;
-}
-
-TimeSpec AttrMtime(const InodeAttr& attr) {
-    return TimeSpec(attr.mtime(), attr.mtime_ns());
-}
-
-TimeSpec AttrCtime(const InodeAttr& attr) {
-    return TimeSpec(attr.ctime(), attr.ctime_ns());
-}
-
-TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode) {
-    InodeAttr attr;
-    inode->GetInodeAttr(&attr);
-    return AttrMtime(attr);
-}
-
-TimeSpec Now() {
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    return TimeSpec(now.tv_sec, now.tv_nsec);
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
--- a/curvefs/src/client/filesystem/utils.h
+++ b/curvefs/src/client/filesystem/utils.h
@ -1,59 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
-
-#include <memory>
-
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/package.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-// directory
-bool IsDir(const InodeAttr& attr);
-
-// file which data is stored in s3
-bool IsS3File(const InodeAttr& attr);
-
-// file which data is stored in volume
-bool IsVolmeFile(const InodeAttr& attr);
-
-// symbol link
-bool IsSymLink(const InodeAttr& attr);
-
-struct TimeSpec AttrMtime(const InodeAttr& attr);
-
-struct TimeSpec AttrCtime(const InodeAttr& attr);
-
-struct TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode);
-
-struct TimeSpec Now();
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
--- a/curvefs/src/client/fuse_client.cpp
+++ b/curvefs/src/client/fuse_client.cpp
@ -35,10 +35,11 @@

 #include "curvefs/proto/mds.pb.h"
 #include "curvefs/src/client/common/common.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/fuse_common.h"
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/kvclient/memcache_client.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
 #include "curvefs/src/client/xattr_manager.h"
 #include "curvefs/src/common/define.h"
@ -49,16 +50,9 @@

 #define PORT_LIMIT 65535

-using ::curvefs::common::S3Info;
-using ::curvefs::common::Volume;
-using ::curvefs::mds::topology::PartitionTxId;
 using ::curvefs::mds::FSStatusCode_Name;
 using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH;
 using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
-using ::curvefs::client::filesystem::ExternalMember;
-using ::curvefs::client::filesystem::DirEntry;
-using ::curvefs::client::filesystem::DirEntryList;
-using ::curvefs::client::filesystem::FileOut;

 #define RETURN_IF_UNSUCCESS(action)                                            \
    do {                                                                       \
@ -117,16 +111,25 @@ using common::FLAGS_fuseClientAvgReadBytes;
 using common::FLAGS_fuseClientBurstReadBytes;
 using common::FLAGS_fuseClientBurstReadBytesSecs;

-static void on_throttle_timer(void *arg) {
-    FuseClient *fuseClient = reinterpret_cast<FuseClient *>(arg);
-    fuseClient->InitQosParam();
-}
+// static void on_throttle_timer(void *arg) {
+//     FuseClient *fuseClient = reinterpret_cast<FuseClient *>(arg);
+//     fuseClient->InitQosParam();
+// }

 CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
+    LOG(INFO) << "fuse client init start.";
    option_ = option;
+    const uint64_t writeCacheMaxByte =
+      option_.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
+    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
+        LOG(ERROR) << "writeCacheMaxByte is too small"
+                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
+                      ", writeCacheMaxByte = " << writeCacheMaxByte;
+        return CURVEFS_ERROR::CACHETOOSMALL;
+    }

    mdsBase_ = new MDSBaseClient();
-    FSStatusCode ret = mdsClient_->Init(option.mdsOpt, mdsBase_);
+    auto ret = mdsClient_->Init(option.mdsOpt, mdsBase_);
    if (ret != FSStatusCode::OK) {
        return CURVEFS_ERROR::INTERNAL;
    }
@ -157,26 +160,25 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
    curve::client::ClientDummyServerInfo::GetInstance().SetPort(listenPort);
    curve::client::ClientDummyServerInfo::GetInstance().SetIP(localIp);

-    {
-        ExternalMember member(dentryManager_, inodeManager_);
-        fs_ = std::make_shared<FileSystem>(option_.fileSystemOption, member);
-    }
-
-    MetaStatusCode ret2 =
+    auto ret2 =
        metaClient_->Init(option.excutorOpt, option.excutorInternalOpt,
                          metaCache, channelManager);
    if (ret2 != MetaStatusCode::OK) {
        return CURVEFS_ERROR::INTERNAL;
    }

-    {  // init inode manager
-        auto member = fs_->BorrowMember();
-        CURVEFS_ERROR rc = inodeManager_->Init(option.refreshDataOption,
-                                               member.openFiles,
-                                               member.deferSync);
-        if (rc != CURVEFS_ERROR::OK) {
-            return rc;
-        }
+    auto ret3 =
+        inodeManager_->Init(option.iCacheLruSize, option.enableICacheMetrics,
+                            option.flushPeriodSec, option.refreshDataOption,
+                            option.lruTimeOutSec);
+    if (ret3 != CURVEFS_ERROR::OK) {
+        return ret3;
+    }
+    ret3 =
+        dentryManager_->Init(option.dCacheLruSize, option.enableDCacheMetrics,
+            option.lruTimeOutSec);
+    if (ret3 != CURVEFS_ERROR::OK) {
+        return ret3;
    }

    if (warmupManager_ != nullptr) {
@ -185,8 +187,8 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
    }

    InitQosParam();
-
-    return CURVEFS_ERROR::OK;
+    LOG(INFO) << "fuse client init success.";
+    return ret3;
 }

 void FuseClient::UnInit() {
@ -197,13 +199,15 @@ void FuseClient::UnInit() {
    delete mdsBase_;
    mdsBase_ = nullptr;

-    while (bthread_timer_del(throttleTimer_) == 1) {
-        bthread_usleep(1000);
-    }
+    // while (bthread_timer_del(throttleTimer_) == 1) {
+    //     bthread_usleep(1000);
+    // }
+    // s3Adaptor_->Stop();
 }

 CURVEFS_ERROR FuseClient::Run() {
    if (isStop_.exchange(false)) {
+        inodeManager_->Run();
        return CURVEFS_ERROR::OK;
    }
    return CURVEFS_ERROR::INTERNAL;
@ -211,6 +215,7 @@ CURVEFS_ERROR FuseClient::Run() {

 void FuseClient::Fini() {
    if (!isStop_.exchange(true)) {
+        inodeManager_->Stop();
        xattrManager_->Stop();
    }
 }
@ -219,7 +224,6 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                     struct fuse_conn_info *conn) {
    (void)userdata;
    (void)conn;
-    fs_->Run();
    return CURVEFS_ERROR::OK;
 }

@ -229,7 +233,7 @@ void FuseClient::FuseOpDestroy(void *userdata) {
    }

    FlushAll();
-    fs_->Destory();
+    dirBuf_->DirBufferFreeAll();

    // stop lease before umount fs, otherwise, lease request after umount fs
    // will add a mountpoint entry.
@ -261,38 +265,91 @@ void FuseClient::FuseOpDestroy(void *userdata) {
              << " success!";
 }

-CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       EntryOut* entryOut) {
-    CURVEFS_ERROR rc = fs_->Lookup(req, parent, name, entryOut);
-    if (rc != CURVEFS_ERROR::OK && rc != CURVEFS_ERROR::NOTEXIST) {
-        LOG(ERROR) << "Lookup() failed, retCode = " << rc
-                   << ", parent = " << parent << ", name = " << name;
+void InodeAttr2ParamAttr(const InodeAttr &inodeAttr, struct stat *attr) {
+    attr->st_ino = inodeAttr.inodeid();
+    attr->st_mode = inodeAttr.mode();
+    attr->st_nlink = inodeAttr.nlink();
+    attr->st_uid = inodeAttr.uid();
+    attr->st_gid = inodeAttr.gid();
+    attr->st_size = inodeAttr.length();
+    attr->st_rdev = inodeAttr.rdev();
+    attr->st_atim.tv_sec = inodeAttr.atime();
+    attr->st_atim.tv_nsec = inodeAttr.atime_ns();
+    attr->st_mtim.tv_sec = inodeAttr.mtime();
+    attr->st_mtim.tv_nsec = inodeAttr.mtime_ns();
+    attr->st_ctim.tv_sec = inodeAttr.ctime();
+    attr->st_ctim.tv_nsec = inodeAttr.ctime_ns();
+    attr->st_blksize = kOptimalIOBlockSize;
+    switch (inodeAttr.type()) {
+        // TODO(@hzwuhongsong): type_volume?
+        case metaserver::TYPE_S3:
+            attr->st_blocks = (inodeAttr.length() + 511) / 512;
+            break;
+        default:
+            attr->st_blocks = 0;
+            break;
    }
-    return rc;
 }

-CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
-                                          fuse_ino_t ino,
-                                          struct fuse_file_info* fi,
-                                          FileOut* fileOut) {
+void GetDentryParamFromInodeAttr(
+    const FuseClientOption &option,
+    const InodeAttr &inodeAttr,
+    fuse_entry_param *param) {
+    memset(param, 0, sizeof(fuse_entry_param));
+    param->ino = inodeAttr.inodeid();
+    param->generation = 0;
+    InodeAttr2ParamAttr(inodeAttr, &param->attr);
+    param->attr_timeout = option.attrTimeOut;
+    param->entry_timeout = option.entryTimeOut;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
+                                       const char *name, fuse_entry_param *e) {
+    (void)req;
+    VLOG(1) << "FuseOpLookup parent: " << parent
+            << ", name: " << name;
+    if (strlen(name) > option_.maxNameLength) {
+        return CURVEFS_ERROR::NAMETOOLONG;
+    }
+
+    Dentry dentry;
+    CURVEFS_ERROR ret = dentryManager_->GetDentry(parent, name, &dentry);
+    if (ret != CURVEFS_ERROR::OK) {
+        if (ret != CURVEFS_ERROR::NOTEXIST) {
+            LOG(WARNING) << "dentryManager_ get dentry fail, ret = " << ret
+                         << ", parent inodeid = " << parent
+                         << ", name = " << name;
+        }
+        return ret;
+    }
+
+    fuse_ino_t ino = dentry.inodeid();
+    InodeAttr attr;
+    ret = inodeManager_->GetInodeAttr(ino, &attr);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
+    }
+
+    GetDentryParamFromInodeAttr(option_, attr, e);
+    return ret;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
+                                     struct fuse_file_info *fi) {
+    (void)req;
+    VLOG(1) << "FuseOpOpen, ino: " << ino;
    std::shared_ptr<InodeWrapper> inodeWrapper;
-    // alredy opened
    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
    if (ret != CURVEFS_ERROR::OK) {
        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
                   << ", inodeid = " << ino;
        return ret;
    }
-
-    fileOut->fi = fi;
-    inodeWrapper->GetInodeAttr(&fileOut->attr);
-
+    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
    if (fi->flags & O_TRUNC) {
        if (fi->flags & O_WRONLY || fi->flags & O_RDWR) {
-            ::curve::common::UniqueLock lgGuard =
-                inodeWrapper->GetUniqueLock();
            uint64_t length = inodeWrapper->GetLengthLocked();
            CURVEFS_ERROR tRet = Truncate(inodeWrapper.get(), 0);
            if (tRet != CURVEFS_ERROR::OK) {
@ -327,29 +384,18 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
                    }
                }
            }
-            inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
        } else {
            return CURVEFS_ERROR::NOPERMISSION;
        }
    }
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     struct fuse_file_info* fi,
-                                     FileOut* fileOut) {
-    CURVEFS_ERROR rc = fs_->Open(req, ino, fi);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "open(" << ino << ") failed, retCode = " << rc;
-        return rc;
+    if (FLAGS_enableCto) {
+        inodeManager_->AddOpenedInode(ino);
    }
-    return HandleOpenFlags(req, ino, fi, fileOut);
+    return ret;
 }

 CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
    fuse_ino_t parent, FsFileType type, NlinkChange nlink) {
-
    std::shared_ptr<InodeWrapper> parentInodeWrapper;
    auto ret = inodeManager_->GetInode(parent, parentInodeWrapper);
    if (ret != CURVEFS_ERROR::OK) {
@ -365,27 +411,16 @@ CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
        if (FsFileType::TYPE_DIRECTORY == type) {
            parentInodeWrapper->UpdateNlinkLocked(nlink);
        }
-
-        if (option_.fileSystemOption.deferSyncOption.deferDirMtime) {
-            inodeManager_->ShipToFlush(parentInodeWrapper);
-        } else {
-            return parentInodeWrapper->SyncAttr();
-        }
    }
-
+    inodeManager_->ShipToFlush(parentInodeWrapper);
    return CURVEFS_ERROR::OK;
 }

-CURVEFS_ERROR FuseClient::MakeNode(
-    fuse_req_t req,
-    fuse_ino_t parent,
-    const char* name,
-    mode_t mode,
-    FsFileType type,
-    dev_t rdev,
-    bool internal,
-    std::shared_ptr<InodeWrapper>& inodeWrapper) {
-    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
+                                   const char *name, mode_t mode,
+                                   FsFileType type, dev_t rdev, bool internal,
+                                   fuse_entry_param *e) {
+    if (strlen(name) > option_.maxNameLength) {
        return CURVEFS_ERROR::NAMETOOLONG;
    }

@ -416,6 +451,7 @@ CURVEFS_ERROR FuseClient::MakeNode(
    param.rdev = rdev;
    param.parent = parent;

+    std::shared_ptr<InodeWrapper> inodeWrapper;
    CURVEFS_ERROR ret = inodeManager_->CreateInode(param, inodeWrapper);
    if (ret != CURVEFS_ERROR::OK) {
        LOG(ERROR) << "inodeManager CreateInode fail, ret = " << ret
@ -486,26 +522,20 @@ CURVEFS_ERROR FuseClient::MakeNode(
        }
    }

+    InodeAttr attr;
+    inodeWrapper->GetInodeAttr(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
    return ret;
 }

-CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req,
-                                      fuse_ino_t parent,
-                                      const char *name,
-                                      mode_t mode,
-                                      EntryOut* entryOut) {
+CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
+                                      const char *name, mode_t mode,
+                                      fuse_entry_param *e) {
    VLOG(1) << "FuseOpMkDir, parent: " << parent << ", name: " << name
            << ", mode: " << mode;
    bool internal = false;
-    std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR rc = MakeNode(req, parent, name, S_IFDIR | mode,
-                                FsFileType::TYPE_DIRECTORY, 0, internal, inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    inode->GetInodeAttr(&entryOut->attr);
-    return CURVEFS_ERROR::OK;
+    return MakeNode(req, parent, name, S_IFDIR | mode,
+                    FsFileType::TYPE_DIRECTORY, 0, internal, e);
 }

 CURVEFS_ERROR FuseClient::FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
@ -564,6 +594,8 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent,
                         << ", xattr = " << xattr.DebugString();
        }
    }
+
+    inodeManager_->ClearInodeCache(ino);
    return ret;
 }

@ -576,13 +608,10 @@ std::string GetRecycleTimeDirName() {
    return now;
 }

-CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req,
-                                           uint64_t parent,
-                                           const char* name,
-                                           mode_t mode,
-                                           ManageInodeType manageType,
-                                           EntryOut* entryOut) {
-    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, uint64_t parent,
+                    const char *name, mode_t mode, ManageInodeType manageType,
+                    fuse_entry_param *e) {
+    if (strlen(name) > option_.maxNameLength) {
        return CURVEFS_ERROR::NAMETOOLONG;
    }

@ -666,7 +695,9 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req,
        }
    }

-    inodeWrapper->GetInodeAttrLocked(&entryOut->attr);
+    InodeAttr attr;
+    inodeWrapper->GetInodeAttrLocked(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
    return ret;
 }

@ -680,10 +711,9 @@ CURVEFS_ERROR FuseClient::GetOrCreateRecycleDir(fuse_req_t req, Dentry *out) {
    } else if (ret == CURVEFS_ERROR::NOTEXIST) {
        LOG(INFO) << "recycle dir is not exist, create " << RECYCLENAME
                  << ", parentid = " << ROOTINODEID;
-        EntryOut entryOut;
+        fuse_entry_param param;
        ret = CreateManageNode(req, ROOTINODEID, RECYCLENAME,
-                               S_IFDIR | 0755, ManageInodeType::TYPE_RECYCLE,
-                               &entryOut);
+                       S_IFDIR | 0755, ManageInodeType::TYPE_RECYCLE, &param);
        if (ret != CURVEFS_ERROR::OK) {
            LOG(ERROR) << "CreateManageNode failed, ret = " << ret
                       << ", inode = " << ROOTINODEID
@ -720,17 +750,17 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
                   << ", name = " << recycleTimeDirName;
        return ret;
    } else if (ret == CURVEFS_ERROR::NOTEXIST) {
-        std::shared_ptr<InodeWrapper> inode;
+        fuse_entry_param param;
        bool internal = true;
        ret = MakeNode(req, RECYCLEINODEID, recycleTimeDirName.c_str(),
-              S_IFDIR | 0755, FsFileType::TYPE_DIRECTORY, 0, internal, inode);
+              S_IFDIR | 0755, FsFileType::TYPE_DIRECTORY, 0, internal, &param);
        if (ret != CURVEFS_ERROR::OK) {
            LOG(ERROR) << "MakeNode failed, ret = " << ret
                       << ", inode = " << RECYCLEINODEID
                       << ", name = " << recycleTimeDirName;
            return ret;
        }
-        recycleTimeDirIno = inode->GetInodeId();
+        recycleTimeDirIno = param.ino;
    } else {
        recycleTimeDirIno = dentry.inodeid();
    }
@ -740,7 +770,7 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino,

    // 4. move inode to recycle time dir
    ret = FuseOpRename(req, parent, name,
-                       recycleTimeDirIno, newName.c_str(), 0);
+                                     recycleTimeDirIno, newName.c_str());
    if (ret != CURVEFS_ERROR::OK) {
        LOG(ERROR) << "MoveToRecycle failed, ret = " << ret
                   << ", inodeid = " << ino
@ -777,7 +807,7 @@ bool FuseClient::ShouldMoveToRecycle(fuse_ino_t parent) {

 CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
                                     const char *name, FsFileType type) {
-    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+    if (strlen(name) > option_.maxNameLength) {
        return CURVEFS_ERROR::NAMETOOLONG;
    }

@ -833,87 +863,146 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
    return CURVEFS_ERROR::OK;
 }

-CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req,
-                                        fuse_ino_t ino,
-                                        struct fuse_file_info* fi) {
-    CURVEFS_ERROR rc = fs_->OpenDir(req, ino, fi);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "opendir() failed, retCode = " << rc
-                   << ", ino = " << ino;
+CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
+                                        struct fuse_file_info *fi) {
+    (void)req;
+    VLOG(1) << "FuseOpOpenDir ino = " << ino;
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
    }
-    return rc;
+
+    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+
+    uint64_t dindex = dirBuf_->DirBufferNew();
+    fi->fh = dindex;
+    VLOG(1) << "FuseOpOpenDir, ino: " << ino << ", dindex: " << dindex;
+    return ret;
 }

-CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req,
-                                        fuse_ino_t ino,
-                                        size_t size,
-                                        off_t off,
-                                        struct fuse_file_info *fi,
-                                        char** bufferOut,
-                                        size_t* rSize,
-                                        bool plus) {
-    auto handler = fs_->FindHandler(fi->fh);
-    DirBufferHead* buffer = handler->buffer;
-    if (!handler->padding) {
-        auto entries = std::make_shared<DirEntryList>();
-        CURVEFS_ERROR rc = fs_->ReadDir(req, ino, fi, &entries);
-        if (rc != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "readdir() failed, retCode = " << rc
-                       << ", ino = " << ino << ", fh = " << fi->fh;
-            return rc;
-        }
+CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
+                                           struct fuse_file_info *fi) {
+    (void)req;
+    uint64_t dindex = fi->fh;
+    VLOG(1) << "FuseOpReleaseDir, ino: " << ino << ", dindex: " << dindex;
+    dirBuf_->DirBufferRelease(dindex);

-        entries->Iterate([&](DirEntry* dirEntry){
-            if (plus) {
-                fs_->AddDirEntryPlus(req, buffer, dirEntry);
-            } else {
-                fs_->AddDirEntry(req, buffer, dirEntry);
-            }
-        });
-        handler->padding = true;
-    }
-
-    if (off < buffer->size) {
-        *bufferOut = buffer->p + off;
-        *rSize = std::min(buffer->size - off, size);
-    } else {
-        *bufferOut = nullptr;
-        *rSize = 0;
-    }
+    // release inodeAttr cache
+    inodeManager_->ReleaseCache(ino);
    return CURVEFS_ERROR::OK;
 }

-CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req,
-                                           fuse_ino_t ino,
-                                           struct fuse_file_info* fi) {
-    CURVEFS_ERROR rc = fs_->ReleaseDir(req, ino, fi);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "releasedir() failed, retCode = " << rc
-                   << ", ino = " << ino;
+static void dirbuf_add(fuse_req_t req, struct DirBufferHead *b,
+                       const Dentry &dentry,
+                       const FuseClientOption &option,
+                       bool cacheDir,
+                       InodeAttr *attr = nullptr) {
+    struct stat stbuf;
+    struct fuse_entry_param param;
+    size_t oldsize = b->size;
+    if (!cacheDir) {
+        b->size += fuse_add_direntry(req, NULL, 0, dentry.name().c_str(),
+                                     NULL, 0);
+        b->p = static_cast<char *>(realloc(b->p, b->size));
+        memset(&stbuf, 0, sizeof(stbuf));
+        stbuf.st_ino = dentry.inodeid();
+        fuse_add_direntry(req, b->p + oldsize, b->size - oldsize,
+                          dentry.name().c_str(), &stbuf, b->size);
+    } else {
+        b->size += fuse_add_direntry_plus(req, NULL, 0, dentry.name().c_str(),
+                                          NULL, 0);
+        b->p = static_cast<char *>(realloc(b->p, b->size));
+        GetDentryParamFromInodeAttr(option, *attr, &param);
+        fuse_add_direntry_plus(req, b->p + oldsize, b->size - oldsize,
+                               dentry.name().c_str(), &param, b->size);
    }
-    return rc;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
+                                            size_t size, off_t off,
+                                            struct fuse_file_info *fi,
+                                            char **buffer, size_t *rSize,
+                                            bool cacheDir) {
+    VLOG(1) << "FuseOpReadDirPlus ino: " << ino << ", size: " << size
+            << ", off = " << off << ", cacheDir = " << cacheDir;
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
+    }
+
+    uint64_t dindex = fi->fh;
+    DirBufferHead *bufHead = dirBuf_->DirBufferGet(dindex);
+    if (!bufHead->wasRead) {
+        std::list<Dentry> dentryList;
+        std::set<uint64_t> inodeIds;
+        std::map<uint64_t, InodeAttr> inodeAttrMap;
+        auto limit = option_.listDentryLimit;
+        ret = dentryManager_->ListDentry(ino, &dentryList, limit);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
+                       << ", parent = " << ino;
+            return ret;
+        }
+
+        if (!cacheDir) {
+            for (const auto &dentry : dentryList) {
+                dirbuf_add(req, bufHead, dentry, option_, cacheDir);
+            }
+        } else {
+            for (const auto &dentry : dentryList) {
+                inodeIds.emplace(dentry.inodeid());
+            }
+            VLOG(3) << "batch get inode size = " << inodeIds.size();
+            ret = inodeManager_->BatchGetInodeAttrAsync(ino, &inodeIds,
+                                                        &inodeAttrMap);
+            if (ret != CURVEFS_ERROR::OK) {
+                LOG(ERROR) << "BatchGetInodeAttr failed when FuseOpReadDir"
+                           << ", parentId = " << ino;
+                return ret;
+            }
+
+            for (const auto &dentry : dentryList) {
+                auto iter = inodeAttrMap.find(dentry.inodeid());
+                if (iter != inodeAttrMap.end()) {
+                    dirbuf_add(req, bufHead, dentry, option_,
+                               cacheDir, &iter->second);
+                } else {
+                    LOG(WARNING) << "BatchGetInodeAttr missing some inodes,"
+                                 << " inodeId = " << dentry.inodeid();
+                }
+            }
+        }
+        bufHead->wasRead = true;
+    }
+
+    if (off < static_cast<ssize_t>(bufHead->size)) {
+        *buffer = bufHead->p + off;
+        *rSize = std::min(bufHead->size - off, size);
+    } else {
+        *buffer = nullptr;
+        *rSize = 0;
+    }
+    return ret;
 }

 CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
                                       const char *name, fuse_ino_t newparent,
-                                       const char *newname,
-                                       unsigned int flags) {
+                                       const char *newname) {
+    (void)req;
    VLOG(1) << "FuseOpRename from (" << parent << ", " << name << ") to ("
            << newparent << ", " << newname << ")";
-
-    // TODO(Wine93): the flag RENAME_EXCHANGE and RENAME_NOREPLACE
-    // is only used in linux interface renameat(), not required by posix,
-    // we can ignore it now
-    if (flags != 0) {
-        return CURVEFS_ERROR::INVALIDPARAM;
-    }
-
-    uint64_t maxNameLength = option_.fileSystemOption.maxNameLength;
-    if (strlen(name) > maxNameLength || strlen(newname) > maxNameLength) {
+    if (strlen(name) > option_.maxNameLength ||
+        strlen(newname) > option_.maxNameLength) {
        LOG(WARNING) << "FuseOpRename name too long, name = " << name
                     << ", name len = " << strlen(name) << ", new name = "
                     << newname << ", new name len = " << strlen(newname)
-                     << ", maxNameLength = " << maxNameLength;
+                     << ", maxNameLength = " << option_.maxNameLength;
        return CURVEFS_ERROR::NAMETOOLONG;
    }

@ -952,24 +1041,30 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
    return rc;
 }

-CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req,
-                                        fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                        struct fuse_file_info *fi,
-                                        AttrOut* attrOut) {
-    CURVEFS_ERROR rc = fs_->GetAttr(req, ino, attrOut);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "getattr() fail, retCode = " << rc
-                   << ", ino = " << ino;
+                                        struct stat *attr) {
+    (void)req;
+    (void)fi;
+    VLOG(1) << "FuseOpGetAttr ino = " << ino;
+    InodeAttr inodeAttr;
+    CURVEFS_ERROR ret =
+        inodeManager_->GetInodeAttr(ino, &inodeAttr);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
    }
-    return rc;
+    InodeAttr2ParamAttr(inodeAttr, attr);
+    return ret;
 }

-CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
-                                        fuse_ino_t ino,
-                                        struct stat* attr,
-                                        int to_set,
-                                        struct fuse_file_info* fi,
-                                        struct AttrOut* attrOut) {
+CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
+                                        struct stat *attr, int to_set,
+                                        struct fuse_file_info *fi,
+                                        struct stat *attrOut) {
+    (void)req;
+    (void)fi;
    VLOG(1) << "FuseOpSetAttr to_set: " << to_set << ", ino: " << ino
            << ", attr: " << *attr;
    std::shared_ptr<InodeWrapper> inodeWrapper;
@ -1026,7 +1121,9 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
        if (ret != CURVEFS_ERROR::OK) {
            return ret;
        }
-        inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
+        InodeAttr inodeAttr;
+        inodeWrapper->GetInodeAttrLocked(&inodeAttr);
+        InodeAttr2ParamAttr(inodeAttr, attrOut);

        if (enableSumInDir_.load() && changeSize != 0) {
            // update parent summary info
@ -1051,7 +1148,9 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
    if (ret != CURVEFS_ERROR::OK) {
        return ret;
    }
-    inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
+    InodeAttr inodeAttr;
+    inodeWrapper->GetInodeAttrLocked(&inodeAttr);
+    InodeAttr2ParamAttr(inodeAttr, attrOut);
    return ret;
 }

@ -1061,8 +1160,8 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
    (void)req;
    VLOG(9) << "FuseOpGetXattr, ino: " << ino
            << ", name: " << name << ", size = " << size;
-    if (option_.fileSystemOption.disableXattr) {
-        return CURVEFS_ERROR::NOSYS;
+    if (option_.disableXattr) {
+        return CURVEFS_ERROR::NOTSUPPORT;
    }

    InodeAttr inodeAttr;
@ -1099,6 +1198,12 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
                                         const char* name, const char* value,
                                         size_t size, int flags) {
+    (void)req;
+    (void)flags;
+    if (option_.disableXattr) {
+        return CURVEFS_ERROR::NOTSUPPORT;
+    }
+
    std::string strname(name);
    std::string strvalue(value, size);
    VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
@ -1181,12 +1286,10 @@ CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
    return CURVEFS_ERROR::OUT_OF_RANGE;
 }

-CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req,
-                                        const char* link,
-                                        fuse_ino_t parent,
-                                        const char* name,
-                                        EntryOut* entryOut) {
-    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
+                                        fuse_ino_t parent, const char *name,
+                                        fuse_entry_param *e) {
+    if (strlen(name) > option_.maxNameLength) {
        return CURVEFS_ERROR::NAMETOOLONG;
    }
    const struct fuse_ctx *ctx = fuse_req_ctx(req);
@ -1255,17 +1358,18 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req,
        }
    }

-    inodeWrapper->GetInodeAttr(&entryOut->attr);
+    InodeAttr attr;
+    inodeWrapper->GetInodeAttr(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
    return ret;
 }

-CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     fuse_ino_t newparent,
-                                     const char* newname,
+CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char *newname,
                                     FsFileType type,
-                                     EntryOut* entryOut) {
-    if (strlen(newname) > option_.fileSystemOption.maxNameLength) {
+                                     fuse_entry_param *e) {
+    (void)req;
+    if (strlen(newname) > option_.maxNameLength) {
        return CURVEFS_ERROR::NAMETOOLONG;
    }
    std::shared_ptr<InodeWrapper> inodeWrapper;
@ -1326,7 +1430,9 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
        }
    }

-    inodeWrapper->GetInodeAttr(&entryOut->attr);
+    InodeAttr attr;
+    inodeWrapper->GetInodeAttr(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
    return ret;
 }

@ -1345,18 +1451,24 @@ CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
    return CURVEFS_ERROR::OK;
 }

-CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req,
-                                        fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino,
                                        struct fuse_file_info *fi) {
-    CURVEFS_ERROR rc = fs_->Release(req, ino);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "release() failed, ino = " << ino;
+    (void)req;
+    (void)fi;
+    VLOG(1) << "FuseOpRelease, ino: " << ino;
+    if (FLAGS_enableCto) {
+        inodeManager_->RemoveOpenedInode(ino);
    }
-    return rc;
+    return CURVEFS_ERROR::OK;
 }

+void FuseClient::FlushInode() { inodeManager_->FlushInodeOnce(); }
+
+void FuseClient::FlushInodeAll() { inodeManager_->FlushAll(); }
+
 void FuseClient::FlushAll() {
    FlushData();
+    FlushInodeAll();
 }

 CURVEFS_ERROR
@ -1431,11 +1543,11 @@ void FuseClient::InitQosParam() {

    throttle_.UpdateThrottleParams(params);

-    int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1),
-                                on_throttle_timer, this);
-    if (ret != 0) {
-        LOG(ERROR) << "Create fuse client throttle timer failed!";
-    }
+    // int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1),
+    //                             on_throttle_timer, this);
+    // if (ret != 0) {
+    //     LOG(ERROR) << "Create fuse client throttle timer failed!";
+    // }
 }

 }  // namespace client
--- a/curvefs/src/client/fuse_client.h
+++ b/curvefs/src/client/fuse_client.h
@ -45,6 +45,7 @@
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
+#include "curvefs/src/client/volume/client_volume_adaptor.h"
 #include "curvefs/src/common/fast_align.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/concurrent/concurrent.h"
@ -56,8 +57,6 @@
 #include "curvefs/src/client/xattr_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
 #include "src/common/throttle.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/filesystem.h"

 #define DirectIOAlignment 512

@ -68,11 +67,11 @@ using ::curvefs::common::FSType;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::metaserver::ManageInodeType;
 using ::curvefs::client::metric::FSMetric;
+using curvefs::volume::kMiB;

 namespace curvefs {
 namespace client {

-
 namespace warmup {
 class WarmupManager;
 }
@ -83,10 +82,6 @@ using rpcclient::MdsClient;
 using rpcclient::MdsClientImpl;
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
-using ::curvefs::client::filesystem::FileSystem;
-using ::curvefs::client::filesystem::EntryOut;
-using ::curvefs::client::filesystem::AttrOut;
-using ::curvefs::client::filesystem::FileOut;

 using curvefs::common::is_aligned;

@ -97,16 +92,17 @@ using mds::Mountpoint;
 class FuseClient {
 public:
    FuseClient()
-      : mdsClient_(std::make_shared<MdsClientImpl>()),
-        metaClient_(std::make_shared<MetaServerClientImpl>()),
-        inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
-        dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
-        fsInfo_(nullptr),
-        init_(false),
-        enableSumInDir_(false),
-        warmupManager_(nullptr),
-        mdsBase_(nullptr),
-        isStop_(true) {}
+    : mdsClient_(std::make_shared<MdsClientImpl>()),
+      metaClient_(std::make_shared<MetaServerClientImpl>()),
+      inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
+      dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
+      dirBuf_(std::make_shared<DirBuffer>()),
+      fsInfo_(nullptr),
+      init_(false),
+      enableSumInDir_(false),
+      warmupManager_(nullptr),
+      mdsBase_(nullptr),
+      isStop_(true) {}

    virtual ~FuseClient() {}

@ -119,6 +115,7 @@ class FuseClient {
            metaClient_(metaClient),
            inodeManager_(inodeManager),
            dentryManager_(dentryManager),
+            dirBuf_(std::make_shared<DirBuffer>()),
            fsInfo_(nullptr),
            init_(false),
            enableSumInDir_(false),
@ -134,6 +131,8 @@ class FuseClient {

    virtual void Fini();

+/*** fuse op***/
+
    virtual CURVEFS_ERROR FuseOpInit(
        void* userdata, struct fuse_conn_info* conn);

@ -142,42 +141,31 @@ class FuseClient {
    virtual CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                      const char* buf, size_t size, off_t off,
                                      struct fuse_file_info* fi,
-                                      FileOut* fileOut) = 0;
+                                      size_t* wSize) = 0;

    virtual CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                     size_t size, off_t off,
                                     struct fuse_file_info* fi, char* buffer,
                                     size_t* rSize) = 0;

-    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       EntryOut* entryOut);
+    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
+                                       const char* name, fuse_entry_param* e);

-    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     struct fuse_file_info* fi,
-                                     FileOut* fileOut);
+    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
+                                     struct fuse_file_info* fi);

-    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       mode_t mode,
+    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
+                                       const char* name, mode_t mode,
                                       struct fuse_file_info* fi,
-                                       EntryOut* entryOut) = 0;
+                                       fuse_entry_param* e) = 0;

-    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
-                                      fuse_ino_t parent,
-                                      const char* name,
-                                      mode_t mode,
-                                      dev_t rdev,
-                                      EntryOut* entryOut) = 0;
+    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+                                      const char* name, mode_t mode, dev_t rdev,
+                                      fuse_entry_param* e) = 0;

-    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req,
-                                      fuse_ino_t parent,
-                                      const char* name,
-                                      mode_t mode,
-                                      EntryOut* entryOut);
+    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
+                                      const char* name, mode_t mode,
+                                      fuse_entry_param* e);

    virtual CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                       const char* name) = 0;
@ -185,38 +173,30 @@ class FuseClient {
    virtual CURVEFS_ERROR FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
                                      const char* name);

-    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req,
-                                        fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
                                        struct fuse_file_info* fi);

-    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req,
-                                           fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
                                           struct fuse_file_info* fi);

-    virtual CURVEFS_ERROR FuseOpReadDir(fuse_req_t req,
-                                        fuse_ino_t ino,
-                                        size_t size,
-                                        off_t off,
-                                        struct fuse_file_info* fi,
-                                        char** bufferOut,
-                                        size_t* rSize,
-                                        bool plus);
+    virtual CURVEFS_ERROR FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
+                                            size_t size, off_t off,
+                                            struct fuse_file_info* fi,
+                                            char** buffer, size_t* rSize,
+                                            bool cacheDir);

-    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       fuse_ino_t newparent,
-                                       const char* newname,
-                                       unsigned int flags);
+    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req, fuse_ino_t parent,
+                                       const char* name, fuse_ino_t newparent,
+                                       const char* newname);

    virtual CURVEFS_ERROR FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                        struct fuse_file_info* fi,
-                                        struct AttrOut* out);
+                                        struct stat* attr);

    virtual CURVEFS_ERROR FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                                        struct stat* attr, int to_set,
                                        struct fuse_file_info* fi,
-                                        struct AttrOut* out);
+                                        struct stat* attrOut);

    virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
                                         const char* name, std::string* value,
@ -229,17 +209,13 @@ class FuseClient {
    virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                            char *value, size_t size, size_t *realSize);

-    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req,
-                                        const char* link,
-                                        fuse_ino_t parent,
-                                        const char* name,
-                                        EntryOut* entryOut);
+    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req, const char* link,
+                                        fuse_ino_t parent, const char* name,
+                                        fuse_entry_param* e);

-    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     fuse_ino_t newparent,
-                                     const char* newname,
-                                     EntryOut* entryOut) = 0;
+    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char* newname,
+                                     fuse_entry_param* e) = 0;

    virtual CURVEFS_ERROR FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                         std::string* linkStr);
@ -250,13 +226,9 @@ class FuseClient {
    virtual CURVEFS_ERROR FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                      int datasync,
                                      struct fuse_file_info* fi) = 0;
+
    virtual CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
-                                      struct fuse_file_info *fi) {
-        (void)req;
-        (void)ino;
-        (void)fi;
-        return CURVEFS_ERROR::OK;
-    }
+                                      struct fuse_file_info *fi) = 0;

    virtual CURVEFS_ERROR FuseOpStatFs(fuse_req_t req, fuse_ino_t ino,
                                       struct statvfs* stbuf) {
@ -273,11 +245,19 @@ class FuseClient {
        stbuf->f_fsid = fsInfo_->fsid();

        stbuf->f_flag = 0;
-        stbuf->f_namemax = option_.fileSystemOption.maxNameLength;
+        stbuf->f_namemax = option_.maxNameLength;
        return CURVEFS_ERROR::OK;
    }

-    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
+/*** flush inode ***/
+
+    virtual void FlushInode();
+
+    virtual void FlushInodeAll();
+
+    virtual void FlushAll();
+
+/*** get or set some elements***/

    void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
        fsInfo_ = fsInfo;
@ -294,12 +274,6 @@ class FuseClient {
        return fsInfo_;
    }

-    std::shared_ptr<FileSystem> GetFileSystem() {
-        return fs_;
-    }
-
-    virtual void FlushAll();
-
    // for unit test
    void SetEnableSumInDir(bool enable) {
        enableSumInDir_ = enable;
@ -332,26 +306,22 @@ class FuseClient {
    void Add(bool isRead, size_t size) { throttle_.Add(isRead, size); }

    void InitQosParam();
+    Mountpoint& GetMountPoint() {
+        return mountpoint_;
+    }

 protected:
-    CURVEFS_ERROR MakeNode(fuse_req_t req,
-                           fuse_ino_t parent,
-                           const char* name,
-                           mode_t mode,
-                           FsFileType type,
-                           dev_t rdev,
-                           bool internal,
-                           std::shared_ptr<InodeWrapper>& InodeWrapper);  // NOLINT
+    CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
+                           mode_t mode, FsFileType type, dev_t rdev,
+                           bool internal, fuse_entry_param* e);

    CURVEFS_ERROR RemoveNode(fuse_req_t req, fuse_ino_t parent,
                             const char* name, FsFileType type);

-    CURVEFS_ERROR CreateManageNode(fuse_req_t req,
-                                   uint64_t parent,
-                                   const char* name,
-                                   mode_t mode,
+    CURVEFS_ERROR CreateManageNode(fuse_req_t req, uint64_t parent,
+                                   const char *name, mode_t mode,
                                   ManageInodeType manageType,
-                                   EntryOut* entryOut);
+                                   fuse_entry_param *e);

    CURVEFS_ERROR GetOrCreateRecycleDir(fuse_req_t req, Dentry *out);

@ -364,17 +334,10 @@ class FuseClient {

    bool ShouldMoveToRecycle(fuse_ino_t parent);

-    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
-                             fuse_ino_t ino,
-                             fuse_ino_t newparent,
-                             const char* newname,
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                             fuse_ino_t newparent, const char* newname,
                             FsFileType type,
-                             EntryOut* entryOut);
-
-    CURVEFS_ERROR HandleOpenFlags(fuse_req_t req,
-                                  fuse_ino_t ino,
-                                  struct fuse_file_info* fi,
-                                  FileOut* fileOut);
+                             fuse_entry_param* e);

    int SetHostPortInMountPoint(Mountpoint* out) {
        char hostname[kMaxHostNameLength];
@ -389,8 +352,9 @@ class FuseClient {
        return 0;
    }

-
 private:
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
+
    virtual void FlushData() = 0;

    CURVEFS_ERROR UpdateParentMCTimeAndNlink(
@ -401,8 +365,8 @@ class FuseClient {
        std::string newName(name);
        newName = std::to_string(parent) + "_" + std::to_string(ino)
                    + "_" + newName;
-        if (newName.length() > option_.fileSystemOption.maxNameLength) {
-            newName = newName.substr(0, option_.fileSystemOption.maxNameLength);
+        if (newName.length() > option_.maxNameLength) {
+            newName = newName.substr(0, option_.maxNameLength);
        }

        return newName;
@ -426,6 +390,9 @@ class FuseClient {

    std::shared_ptr<LeaseExecutor> leaseExecutor_;

+    // dir buffer
+    std::shared_ptr<DirBuffer> dirBuf_;
+
    // filesystem info
    std::shared_ptr<FsInfo> fsInfo_;

@ -444,8 +411,6 @@ class FuseClient {
    // warmup manager
    std::shared_ptr<warmup::WarmupManager> warmupManager_;

-    std::shared_ptr<FileSystem> fs_;
-
 private:
    MDSBaseClient* mdsBase_;

@ -456,6 +421,8 @@ class FuseClient {
    Throttle throttle_;

    bthread_timer_t throttleTimer_;
+
+    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };

 }  // namespace client
--- a/curvefs/src/client/inode_cache_manager.cpp
+++ b/curvefs/src/client/inode_cache_manager.cpp
@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/inode_wrapper.h"

 using ::curvefs::metaserver::Inode;
@ -46,11 +46,36 @@ DECLARE_bool(enableCto);
 namespace curvefs {
 namespace client {

-using ::curvefs::client::filesystem::ToFSError;
-
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
 using curvefs::client::common::FLAGS_enableCto;

+class TrimICacheAsyncDone : public MetaServerClientDone {
+ public:
+    explicit TrimICacheAsyncDone(
+        const std::shared_ptr<InodeWrapper> &inodeWrapper,
+        const std::shared_ptr<InodeCacheManagerImpl> &inodeCacheManager)
+        : inodeWrapper_(inodeWrapper), inodeCacheManager_(inodeCacheManager) {}
+
+    void Run() override {
+        std::unique_ptr<TrimICacheAsyncDone> self_guard(this);
+        MetaStatusCode ret = GetStatusCode();
+        if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
+            LOG(ERROR) << "metaClient_ UpdateInode failed, "
+                       << "MetaStatusCode: " << ret
+                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
+                       << ", inodeid: " << inodeWrapper_->GetInodeId();
+            inodeWrapper_->MarkInodeError();
+        }
+        VLOG(9) << "Trim inode " << inodeWrapper_->GetInodeId()
+                << " async success.";
+        inodeCacheManager_->RemoveICache(inodeWrapper_);
+    };
+
+ private:
+    std::shared_ptr<InodeWrapper> inodeWrapper_;
+    std::shared_ptr<InodeCacheManagerImpl> inodeCacheManager_;
+};
+
 #define GET_INODE_REMOTE(FSID, INODEID, OUT, STREAMING)                        \
    MetaStatusCode ret = metaClient_->GetInode(FSID, INODEID, OUT, STREAMING); \
    if (ret != MetaStatusCode::OK) {                                           \
@ -58,7 +83,17 @@ using curvefs::client::common::FLAGS_enableCto;
            << "metaClient_ GetInode failed, MetaStatusCode = " << ret         \
            << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)          \
            << ", inodeid = " << INODEID;                                      \
-        return ToFSError(ret);                            \
+        return MetaStatusCodeToCurvefsErrCode(ret);                            \
+    }
+
+#define PUT_INODE_CACHE(INODEID, INODEWRAPPER)                                 \
+    std::shared_ptr<InodeWrapper> eliminatedOne;                               \
+    bool eliminated = iCache_->Put(INODEID, INODEWRAPPER, &eliminatedOne);     \
+    if (eliminated) {                                                          \
+        VLOG(3) << "GetInode eliminate one inode, ino: "                       \
+                << eliminatedOne->GetInodeId()                                 \
+                << ", iCache does not evict inodes via put interface ";        \
+        assert(0);                                                             \
    }

 #define REFRESH_DATA_REMOTE(OUT, STREAMING)                                    \
@ -71,8 +106,16 @@ CURVEFS_ERROR
 InodeCacheManagerImpl::GetInode(uint64_t inodeId,
                                std::shared_ptr<InodeWrapper> &out) {
    NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    bool yes = openFiles_->IsOpened(inodeId, &out);
-    if (yes) {
+    // get inode from cache
+    bool ok = iCache_->Get(inodeId, &out);
+    if (ok && NeedUseCache(inodeId, out, false)) {
+        curve::common::UniqueLock lgGuard = out->GetUniqueLock();
+        if (out->GetType() == FsFileType::TYPE_FILE) {
+            return CURVEFS_ERROR::OK;
+        }
+        VLOG(9) << "get inode: " << inodeId
+                << " from icache ok, need refresh data";
+        REFRESH_DATA_REMOTE(out, out->NeedRefreshData());
        return CURVEFS_ERROR::OK;
    }

@ -85,14 +128,27 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
        option_.refreshDataIntervalSec);

    // refresh data
+    VLOG(9) << "get inode: " << inodeId << " from icache fail, get from remote";
    REFRESH_DATA_REMOTE(out, streaming);

+    // put to cache
+    PUT_INODE_CACHE(inodeId, out);
+
    return CURVEFS_ERROR::OK;
 }

 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
                                                  InodeAttr *out) {
    NameLockGuard lock(nameLock_, std::to_string(inodeId));
+    // 1. find in icache
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    bool ok = iCache_->Get(inodeId, &inodeWrapper);
+    if (ok && NeedUseCache(inodeId, inodeWrapper, true)) {
+        inodeWrapper->GetInodeAttr(out);
+        return CURVEFS_ERROR::OK;
+    }
+
+    // 2. get form metaserver
    std::set<uint64_t> inodeIds;
    std::list<InodeAttr> attrs;
    inodeIds.emplace(inodeId);
@ -102,7 +158,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
        LOG(ERROR) << "metaClient BatchGetInodeAttr failed"
                   << ", inodeId = " << inodeId << ", MetaStatusCode = " << ret
                   << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
    }

    if (attrs.size() != 1) {
@ -119,6 +175,21 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
    std::set<uint64_t> *inodeIds,
    std::list<InodeAttr> *attrs) {
+    // get some inode attr in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
+            InodeAttr tmpAttr;
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace_back(std::move(tmpAttr));
+            iter = inodeIds->erase(iter);
+            continue;
+        }
+        ++iter;
+    }
+
    if (inodeIds->empty()) {
        return CURVEFS_ERROR::OK;
    }
@ -130,7 +201,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
                   << ret << ", MetaStatusCode_Name = "
                   << MetaStatusCode_Name(ret);
    }
-    return ToFSError(ret);
+    return MetaStatusCodeToCurvefsErrCode(ret);
 }

 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
@ -138,6 +209,26 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
    std::set<uint64_t> *inodeIds,
    std::map<uint64_t, InodeAttr> *attrs) {
    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
+    std::map<uint64_t, InodeAttr> cachedAttr;
+    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
+
+    // get some inode attr in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
+            InodeAttr tmpAttr;
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace(*iter, std::move(tmpAttr));
+            iter = inodeIds->erase(iter);
+        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
+            attrs->emplace(*iter, cachedAttr[*iter]);
+            iter = inodeIds->erase(iter);
+        } else {
+            ++iter;
+        }
+    }

    if (inodeIds->empty()) {
        return CURVEFS_ERROR::OK;
@ -149,12 +240,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
        return CURVEFS_ERROR::NOTEXIST;
    }

-    ::curve::common::Mutex mutex;
    std::shared_ptr<CountDownEvent> cond =
        std::make_shared<CountDownEvent>(inodeGroups.size());
    for (const auto& it : inodeGroups) {
        VLOG(3) << "BatchGetInodeAttrAsync Send " << it.size();
-        auto* done = new BatchGetInodeAttrAsyncDone(attrs, &mutex, cond);
+        auto* done = new BatchGetInodeAttrAsyncDone(shared_from_this(),
+                                                    cond, parentId);
        MetaStatusCode ret = metaClient_->BatchGetInodeAttrAsync(fsId_, it,
                                                                 done);
        if (MetaStatusCode::OK != ret) {
@ -167,12 +258,30 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(

    // wait for all sudrequest finished
    cond->Wait();
+
+    bool ok  = iAttrCache_->Get(parentId, attrs);
+    if (!ok) {
+        LOG(WARNING) << "get attrs form iAttrCache_ failed.";
+    }
    return CURVEFS_ERROR::OK;
 }

 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
    std::set<uint64_t> *inodeIds,
    std::list<XAttr> *xattr) {
+    // get some inode in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
+            xattr->emplace_back(inodeWrapper->GetXattr());
+            iter = inodeIds->erase(iter);
+        } else {
+            ++iter;
+        }
+    }
+
    if (inodeIds->empty()) {
        return CURVEFS_ERROR::OK;
    }
@ -183,7 +292,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
                   << ret << ", MetaStatusCode_Name = "
                   << MetaStatusCode_Name(ret);
    }
-    return ToFSError(ret);
+    return MetaStatusCodeToCurvefsErrCode(ret);
 }

 CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
@ -194,11 +303,23 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
    if (ret != MetaStatusCode::OK) {
        LOG(ERROR) << "metaClient_ CreateInode failed, MetaStatusCode = " << ret
                   << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
    }
+    uint64_t inodeid = inode.inodeid();
    out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
        s3ChunkInfoMetric_, option_.maxDataSize,
        option_.refreshDataIntervalSec);
+
+    std::shared_ptr<InodeWrapper> eliminatedOne;
+    bool eliminated = false;
+    {
+        NameLockGuard lock(nameLock_, std::to_string(inodeid));
+        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
+    }
+    if (eliminated) {
+        /* iCache does not evict inodes via put interface */
+        assert(0);
+    }
    return CURVEFS_ERROR::OK;
 }

@ -211,29 +332,159 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateManageInode(
        LOG(ERROR) << "metaClient_ CreateManageInode failed, MetaStatusCode = "
                   << ret << ", MetaStatusCode_Name = "
                   << MetaStatusCode_Name(ret);
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
    }
+    uint64_t inodeid = inode.inodeid();
    out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
        s3ChunkInfoMetric_, option_.maxDataSize,
        option_.refreshDataIntervalSec);
+
+    std::shared_ptr<InodeWrapper> eliminatedOne;
+    bool eliminated = false;
+    {
+        NameLockGuard lock(nameLock_, std::to_string(inodeid));
+        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
+    }
+    if (eliminated) {
+        /* iCache does not evict inodes via put interface */
+        assert(0);
+    }
    return CURVEFS_ERROR::OK;
 }

 CURVEFS_ERROR InodeCacheManagerImpl::DeleteInode(uint64_t inodeId) {
    NameLockGuard lock(nameLock_, std::to_string(inodeId));
+    iCache_->Remove(inodeId);
    MetaStatusCode ret = metaClient_->DeleteInode(fsId_, inodeId);
    if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
        LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                   << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                   << ", inodeId = " << inodeId;
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
    }
+
+    curve::common::LockGuard lg2(dirtyMapMutex_);
+    dirtyMap_.erase(inodeId);
    return CURVEFS_ERROR::OK;
 }

+void InodeCacheManagerImpl::AddInodeAttrs(
+    uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
+    iAttrCache_->Set(parentId, inodeAttrs);
+}
+
+void InodeCacheManagerImpl::ClearInodeCache(uint64_t inodeId) {
+    {
+        NameLockGuard lock(nameLock_, std::to_string(inodeId));
+        iCache_->Remove(inodeId);
+    }
+    curve::common::LockGuard lg2(dirtyMapMutex_);
+    dirtyMap_.erase(inodeId);
+}
+
 void InodeCacheManagerImpl::ShipToFlush(
-    const std::shared_ptr<InodeWrapper>& inode) {
-    deferSync_->Push(inode);
+    const std::shared_ptr<InodeWrapper> &inodeWrapper) {
+    curve::common::LockGuard lg(dirtyMapMutex_);
+    dirtyMap_.emplace(inodeWrapper->GetInodeId(), inodeWrapper);
+}
+
+void InodeCacheManagerImpl::FlushAll() {
+    while (!dirtyMap_.empty()) {
+        FlushInodeOnce();
+    }
+}
+
+void InodeCacheManagerImpl::FlushInodeOnce() {
+    std::map<uint64_t, std::shared_ptr<InodeWrapper>> temp_;
+    {
+        curve::common::LockGuard lg(dirtyMapMutex_);
+        temp_.swap(dirtyMap_);
+    }
+    for (auto it = temp_.begin(); it != temp_.end(); it++) {
+        curve::common::UniqueLock ulk = it->second->GetUniqueLock();
+        it->second->Async(nullptr, true);
+    }
+}
+
+void InodeCacheManagerImpl::ReleaseCache(uint64_t parentId) {
+    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
+    iAttrCache_->Release(parentId);
+}
+
+void InodeCacheManagerImpl::FlushInodeBackground() {
+    LOG(INFO) << "flush thread is start.";
+    while (!isStop_.load()) {
+        if (iCache_->Size() > maxCacheSize_) {
+            TrimIcache(iCache_->Size() - maxCacheSize_);
+        }
+        FlushInodeOnce();
+        sleeper_.wait_for(std::chrono::seconds(flushPeriodSec_));
+    }
+    LOG(INFO) << "flush thread is stop.";
+}
+
+namespace {
+// Wether an inode is dirty or not.
+// if |needLock| is true, we acquire the lock firstly and do the test,
+// otherwise, we assume the lock is already held.
+bool IsDirtyInode(InodeWrapper *ino, bool needLock) {
+    auto check = [ino]() {
+        return ino->IsDirty() || !ino->S3ChunkInfoEmptyNolock() ||
+               ino->GetMutableExtentCacheLocked()->HasDirtyExtents();
+    };
+
+    if (needLock) {
+        auto lk = ino->GetUniqueLock();
+        return check();
+    }
+
+    return check();
+}
+}  // namespace
+
+void InodeCacheManagerImpl::RemoveICache(
+    const std::shared_ptr<InodeWrapper> &inode) {
+    if (!IsDirtyInode(inode.get(), true)) {
+        uint64_t inodeId = inode->GetInodeId();
+        NameLockGuard lock(nameLock_, std::to_string(inodeId));
+        ::curve::common::UniqueLock lgGuard = inode->GetUniqueLock();
+        iCache_->Remove(inodeId);
+    }
+}
+
+void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    uint64_t inodeId;
+    VLOG(3) << "TrimIcache trimSize " << trimSize;
+    while (trimSize > 0) {
+        bool ok = iCache_->GetLast(&inodeId, &inodeWrapper);
+        if (ok) {
+            NameLockGuard lock(nameLock_, std::to_string(inodeId));
+            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+            if (IsDirtyInode(inodeWrapper.get(), false)) {
+                VLOG(9) << "TrimIcache sync dirty inode " << inodeId;
+                dirtyMapMutex_.lock();
+                dirtyMap_.erase(inodeId);
+                dirtyMapMutex_.unlock();
+                auto *done =
+                    new TrimICacheAsyncDone(inodeWrapper, shared_from_this());
+                inodeWrapper->Async(done);
+            } else {
+                VLOG(9) << "TrimIcache remove inode " << inodeId
+                        << " from iCache";
+                iCache_->Remove(inodeId);
+            }
+            trimSize--;
+            // remove the attr of the inode in iattrcache
+            auto parents = inodeWrapper->GetParentLocked();
+            for (uint64_t parent : parents) {
+                iAttrCache_->Remove(parent, inodeId);
+            }
+        } else {
+            LOG(ERROR) << "icache size " << iCache_->Size();
+            assert(0);
+        }
+    }
 }

 CURVEFS_ERROR
@ -270,5 +521,60 @@ InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
    return rc;
 }

+void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    openedInodes_.emplace(inodeId);
+}
+
+void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    if (iter != openedInodes_.end()) {
+        openedInodes_.erase(iter);
+    }
+}
+
+bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    return iter != openedInodes_.end();
+}
+
+bool InodeCacheManagerImpl::NeedUseCache(uint64_t inodeId,
+    const std::shared_ptr<InodeWrapper> &inodeWrapper,
+    bool onlyAttr) {
+    auto lock = inodeWrapper->GetUniqueLock();
+
+    if (onlyAttr) {
+        if (inodeWrapper->IsDirty()) {
+            return true;
+        }
+    } else {
+        if (IsDirtyInode(inodeWrapper.get(), false)) {
+            return true;
+        }
+    }
+
+    if (((FLAGS_enableCto && OpenInodeCached(inodeId)) || !FLAGS_enableCto)
+        && !IsTimeOut(inodeWrapper)) {
+        return true;
+    }
+
+    VLOG(9) << "inodeId " << inodeId << " is not need use cache";
+    return false;
+}
+
+bool InodeCacheManagerImpl::IsTimeOut(
+    const std::shared_ptr<InodeWrapper> &inodeWrapper) const {
+    uint32_t time = inodeWrapper->GetCachedTime();
+    if (cacheTimeOutSec_ > 0 &&
+        TimeUtility::GetTimeofDaySec() - time >= cacheTimeOutSec_) {
+        return true;
+    }
+    return false;
+}
+
 }  // namespace client
 }  // namespace curvefs
--- a/curvefs/src/client/inode_cache_manager.h
+++ b/curvefs/src/client/inode_cache_manager.h
@ -38,14 +38,12 @@

 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "src/common/concurrent/name_lock.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/openfile.h"
-#include "curvefs/src/client/filesystem/defer_sync.h"

 using ::curve::common::LRUCache;
 using ::curve::common::CacheMetrics;
@ -66,8 +64,57 @@ using rpcclient::BatchGetInodeAttrDone;
 using curve::common::CountDownEvent;
 using metric::S3ChunkInfoMetric;
 using common::RefreshDataOption;
-using ::curvefs::client::filesystem::OpenFiles;
-using ::curvefs::client::filesystem::DeferSync;
+
+class InodeAttrCache {
+ public:
+    InodeAttrCache() {}
+    ~InodeAttrCache() {}
+
+    bool Get(uint64_t parentId, std::map<uint64_t, InodeAttr> *imap) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            imap->insert(iter->second.begin(), iter->second.end());
+            return true;
+        }
+        return false;
+    }
+
+    void Set(uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        VLOG(1) << "parentId = " << parentId
+                << ", iAttrCache set size = " << inodeAttrs.size();
+        auto& inner = iAttrCache_[parentId];
+        for (const auto &it : inodeAttrs) {
+            inner.emplace(it.inodeid(), it);
+        }
+    }
+
+    void Release(uint64_t parentId) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto size = iAttrCache_.size();
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            iAttrCache_.erase(iter);
+        }
+        VLOG(1) << "inodeId = " << parentId
+                << ", inode attr cache release, before = "
+                << size << ", after = " << iAttrCache_.size();
+    }
+
+    void Remove(uint64_t parentId, uint64_t inodeId) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            iter->second.erase(inodeId);
+        }
+    }
+
+ private:
+    // inodeAttr cache; <parentId, <inodeId, inodeAttr>>
+    std::map<uint64_t, std::map<uint64_t, InodeAttr>> iAttrCache_;
+    curve::common::Mutex iAttrCacheMutex_;
+};

 class InodeCacheManager {
 public:
@ -79,9 +126,14 @@ class InodeCacheManager {
        fsId_ = fsId;
    }

-    virtual CURVEFS_ERROR Init(RefreshDataOption option,
-                               std::shared_ptr<OpenFiles> openFiles,
-                               std::shared_ptr<DeferSync> deferSync) = 0;
+    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+                               uint32_t flushPeriodSec,
+                               RefreshDataOption option,
+                               uint32_t cacheTimeOutSec) = 0;
+
+    virtual void Run() = 0;
+
+    virtual void Stop() = 0;

    virtual CURVEFS_ERROR
    GetInode(uint64_t inodeId,
@ -109,9 +161,24 @@ class InodeCacheManager {

    virtual CURVEFS_ERROR DeleteInode(uint64_t inodeId) = 0;

+    virtual void AddInodeAttrs(uint64_t parentId,
+        const RepeatedPtrField<InodeAttr>& inodeAttrs) = 0;
+
+    virtual void ClearInodeCache(uint64_t inodeId) = 0;
+
    virtual void ShipToFlush(
        const std::shared_ptr<InodeWrapper> &inodeWrapper) = 0;

+    virtual void FlushAll() = 0;
+
+    virtual void FlushInodeOnce() = 0;
+
+    virtual void ReleaseCache(uint64_t parentId) = 0;
+
+    virtual void AddOpenedInode(uint64_t inodeId) = 0;
+
+    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
+
 protected:
    uint32_t fsId_;
 };
@ -120,22 +187,59 @@ class InodeCacheManagerImpl : public InodeCacheManager,
    public std::enable_shared_from_this<InodeCacheManagerImpl> {
 public:
    InodeCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()),
+        iCache_(nullptr),
+        iAttrCache_(nullptr),
+        isStop_(true),
+        cacheTimeOutSec_(0) {}

    explicit InodeCacheManagerImpl(
        const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient) {}
+      : metaClient_(metaClient),
+        iCache_(nullptr),
+        iAttrCache_(nullptr),
+        cacheTimeOutSec_(0) {}

-    CURVEFS_ERROR Init(RefreshDataOption option,
-                       std::shared_ptr<OpenFiles> openFiles,
-                       std::shared_ptr<DeferSync> deferSync) override {
+    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+                       uint32_t flushPeriodSec,
+                       RefreshDataOption option,
+                       uint32_t cacheTimeOutSec) override {
+        if (enableCacheMetrics) {
+            iCache_ = std::make_shared<
+                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0,
+                    std::make_shared<CacheMetrics>("icache"));
+        } else {
+            iCache_ = std::make_shared<
+                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0);
+        }
+        maxCacheSize_ = cacheSize;
        option_ = option;
+        flushPeriodSec_ = flushPeriodSec;
+        cacheTimeOutSec_ = cacheTimeOutSec;
+        iAttrCache_ = std::make_shared<InodeAttrCache>();
        s3ChunkInfoMetric_ = std::make_shared<S3ChunkInfoMetric>();
-        openFiles_ =  openFiles;
-        deferSync_ = deferSync;
        return CURVEFS_ERROR::OK;
    }

+    void Run() {
+        isStop_.exchange(false);
+        flushThread_ =
+            Thread(&InodeCacheManagerImpl::FlushInodeBackground, this);
+        LOG(INFO) << "Start inodeManager flush thread ok.";
+    }
+
+    void Stop() {
+        isStop_.exchange(true);
+        LOG(INFO) << "stop inodeManager flush thread ...";
+        sleeper_.interrupt();
+        flushThread_.join();
+    }
+
+    bool IsDirtyMapExist(uint64_t inodeId) {
+        curve::common::LockGuard lg(dirtyMapMutex_);
+        return dirtyMap_.count(inodeId) > 0;
+    }
+
    CURVEFS_ERROR GetInode(uint64_t inodeId,
                           std::shared_ptr<InodeWrapper> &out) override;

@ -159,37 +263,78 @@ class InodeCacheManagerImpl : public InodeCacheManager,

    CURVEFS_ERROR DeleteInode(uint64_t inodeId) override;

+    void AddInodeAttrs(uint64_t parentId,
+        const RepeatedPtrField<InodeAttr>& inodeAttrs) override;
+
+    void ClearInodeCache(uint64_t inodeId) override;
+
    void ShipToFlush(
        const std::shared_ptr<InodeWrapper> &inodeWrapper) override;

+    void FlushAll() override;
+
+    void FlushInodeOnce() override;
+
+    void ReleaseCache(uint64_t parentId) override;
+
+    void RemoveICache(const std::shared_ptr<InodeWrapper> &inode);
+
+    void AddOpenedInode(uint64_t inodeId) override;
+
+    void RemoveOpenedInode(uint64_t inodeId) override;
+
+    bool NeedUseCache(uint64_t inodeId,
+        const std::shared_ptr<InodeWrapper> &inodeWrapper,
+        bool onlyAttr);
+
+    bool IsTimeOut(const std::shared_ptr<InodeWrapper> &inodeWrapper) const;
+
 private:
+    virtual void FlushInodeBackground();
+    void TrimIcache(uint64_t trimSize);
    CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
                              bool streaming = true);
+    bool OpenInodeCached(uint64_t inodeId);

 private:
    std::shared_ptr<MetaServerClient> metaClient_;
+    std::shared_ptr<LRUCache<uint64_t,
+        std::shared_ptr<InodeWrapper>>> iCache_;
    std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric_;

-    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<InodeAttrCache> iAttrCache_;

-    std::shared_ptr<DeferSync> deferSync_;
+    // dirty map, key is inodeid
+    std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
+    curve::common::Mutex dirtyMapMutex_;
+
+    // record opened inode
+    std::multiset<uint64_t> openedInodes_;
+    curve::common::Mutex openInodesMutex_;

    curve::common::GenericNameLock<Mutex> nameLock_;

    curve::common::GenericNameLock<Mutex> asyncNameLock_;

+    uint64_t maxCacheSize_;
    RefreshDataOption option_;
+    uint32_t flushPeriodSec_;
+    Thread flushThread_;
+    InterruptibleSleeper sleeper_;
+    Atomic<bool> isStop_;
+    // cache timeout seconds, 0 means never timeout
+    uint32_t cacheTimeOutSec_;
 };

 class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
 public:
-    BatchGetInodeAttrAsyncDone(std::map<uint64_t, InodeAttr>* attrs,
-                               ::curve::common::Mutex* mutex,
-                               std::shared_ptr<CountDownEvent> cond):
-        attrs_(attrs),
-        mutex_(mutex),
-        cond_(cond) {}
-
+    BatchGetInodeAttrAsyncDone(
+        const std::shared_ptr<InodeCacheManager> &inodeCacheManager,
+        std::shared_ptr<CountDownEvent> cond,
+        uint64_t parentId):
+        inodeCacheManager_(inodeCacheManager),
+        cond_(cond),
+        parentId_(parentId) {}
    ~BatchGetInodeAttrAsyncDone() {}

    void Run() override {
@ -197,25 +342,22 @@ class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
        MetaStatusCode ret = GetStatusCode();
        if (ret != MetaStatusCode::OK) {
            LOG(ERROR) << "BatchGetInodeAttrAsync failed, "
+                       << "parentId = " << parentId_
                       << ", MetaStatusCode: " << ret
                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret);
        } else {
            auto inodeAttrs = GetInodeAttrs();
            VLOG(3) << "BatchGetInodeAttrAsyncDone update inodeAttrCache"
                    << " size = " << inodeAttrs.size();
-
-            curve::common::LockGuard lk(*mutex_);
-            for (const auto& attr : inodeAttrs) {
-                attrs_->emplace(attr.inodeid(), attr);
-            }
+            inodeCacheManager_->AddInodeAttrs(parentId_, inodeAttrs);
        }
        cond_->Signal();
    };

 private:
-    ::curve::common::Mutex* mutex_;
-    std::map<uint64_t, InodeAttr>* attrs_;
+    std::shared_ptr<InodeCacheManager> inodeCacheManager_;
    std::shared_ptr<CountDownEvent> cond_;
+    uint64_t parentId_;
 };

 }  // namespace client
--- a/curvefs/src/client/inode_wrapper.cpp
+++ b/curvefs/src/client/inode_wrapper.cpp
@ -33,6 +33,7 @@

 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/async_request_closure.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
@ -46,7 +47,6 @@ namespace client {
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::DataIndices;
-using ::curvefs::client::filesystem::ToFSError;

 bvar::Adder<int64_t> g_alive_inode_count{"alive_inode_count"};

@ -193,7 +193,7 @@ CURVEFS_ERROR InodeWrapper::SyncAttr(bool internal) {
                       << "MetaStatusCode: " << ret
                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                       << ", inodeid: " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
        }

        dirty_ = false;
@ -213,7 +213,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3ChunkInfo(bool internal) {
                       << "MetaStatusCode: " << ret
                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                       << ", inodeid: " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
        }
        ClearS3ChunkInfoAdd();
    }
@ -251,6 +251,8 @@ void InodeWrapper::FlushS3ChunkInfoAsync() {
 CURVEFS_ERROR InodeWrapper::FlushVolumeExtent() {
    std::lock_guard<::curve::common::Mutex> guard(syncingVolumeExtentsMtx_);
    if (!extentCache_.HasDirtyExtents()) {
+        VLOG(9) << "FlushVolumeExtent, ino: " << inode_.inodeid()
+                << ", no dirty extents";
        return CURVEFS_ERROR::OK;
    }

@ -294,7 +296,7 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
                   << "MetaStatusCode: " << ret
                   << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                   << ", inodeid: " << inode_.inodeid();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
    }
    auto before = s3ChunkInfoSize_;
    inode_.mutable_s3chunkinfomap()->swap(s3ChunkInfoMap);
@ -328,7 +330,7 @@ CURVEFS_ERROR InodeWrapper::Link(uint64_t parent) {
                   <<", MetaStatusCode = " << ret
                   << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                   << ", inodeid = " << inode_.inodeid();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
    }

    dirty_ = false;
@ -396,7 +398,7 @@ CURVEFS_ERROR InodeWrapper::UnLink(uint64_t parent) {
                       << ", MetaStatusCode = " << ret
                       << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                       << ", inodeid = " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
        }
        dirty_ = false;
        dirtyAttr_.Clear();
@ -427,7 +429,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParent(
                   << ", MetaStatusCode = " << ret
                   << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                   << ", inodeid = " << inode_.inodeid();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
    }
    dirty_ = false;
    dirtyAttr_.Clear();
@ -476,12 +478,18 @@ void InodeWrapper::Async(MetaServerClientDone *done, bool internal) {

 void InodeWrapper::AsyncFlushAttrAndExtents(MetaServerClientDone *done,
                                            bool /*internal*/) {
+    VLOG(9) << "async inode: " << inode_.ShortDebugString()
+            << ", is dirty: " << dirty_
+            << ", has dirty extents: " << extentCache_.HasDirtyExtents();
    if (dirty_ || extentCache_.HasDirtyExtents()) {
        LockSyncingInode();
        syncingVolumeExtentsMtx_.lock();
        DataIndices indices;
        if (extentCache_.HasDirtyExtents()) {
            indices.volumeExtents = extentCache_.GetDirtyExtents();
+            VLOG(9) << "aync inode: " << inode_.ShortDebugString()
+                    << ", volume extents: "
+                    << indices.volumeExtents->ShortDebugString();
        }

        metaClient_->UpdateInodeWithOutNlinkAsync(
@ -514,7 +522,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3(bool internal) {
                       << "MetaStatusCode: " << ret
                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                       << ", inodeid: " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
        }
        ClearS3ChunkInfoAdd();
        dirty_ = false;
@ -583,7 +591,7 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
 }

 CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
-    VolumeExtentList extents;
+    VolumeExtentSliceList extents;
    auto st = metaClient_->GetVolumeExtent(inode_.fsid(), inode_.inodeid(),
                                           true, &extents);
    VLOG(9) << "RefreshVolumeExtent, ino: " << inode_.inodeid()
@ -597,7 +605,7 @@ CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
        LOG(ERROR) << "GetVolumeExtent failed, inodeid: " << inode_.inodeid();
    }

-    return ToFSError(st);
+    return MetaStatusCodeToCurvefsErrCode(st);
 }

 CURVEFS_ERROR InodeWrapper::RefreshNlink() {
--- a/curvefs/src/client/inode_wrapper.h
+++ b/curvefs/src/client/inode_wrapper.h
@ -34,12 +34,12 @@

 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/timeutility.h"
-#include "curvefs/src/client/filesystem/error.h"

 using ::curvefs::metaserver::Inode;
 using ::curvefs::metaserver::S3ChunkInfoList;
@ -52,14 +52,16 @@ constexpr int kAccessTime = 1 << 0;
 constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;

-using ::curvefs::metaserver::VolumeExtentList;
-using ::curvefs::client::filesystem::CURVEFS_ERROR;
+using ::curvefs::metaserver::VolumeExtentSliceList;

 enum class InodeStatus {
    kNormal = 0,
    kError = -1,
 };

+// TODO(xuchaojie) : get from conf maybe?
+const uint32_t kOptimalIOBlockSize = 0x10000u;
+
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;
--- a/curvefs/src/client/kvclient/kvclient_manager.cpp
+++ b/curvefs/src/client/kvclient/kvclient_manager.cpp
@ -41,6 +41,7 @@ namespace client {

 bool KVClientManager::Init(const KVClientManagerOpt &config,
                           const std::shared_ptr<KVClient> &kvclient) {
+    LOG(INFO) << "KVClientManager init.";
    client_ = kvclient;
    return threadPool_.Start(config.setThreadPooln) == 0;
 }
--- a/curvefs/src/client/kvclient/kvclient_manager.h
+++ b/curvefs/src/client/kvclient/kvclient_manager.h
@ -57,6 +57,7 @@ struct SetKVCacheTask {
    std::string key;
    const char *value;
    uint64_t length;
+    uint64_t startTime;
    SetKVCacheDone done;
    SetKVCacheTask() = default;
    SetKVCacheTask(
@ -71,6 +72,7 @@ struct GetKVCacheTask {
    uint64_t offset;
    uint64_t length;
    bool res;
+    uint64_t startTime;
    GetKVCacheDone done;
    GetKVCacheTask(const std::string &k, char *v, uint64_t off, uint64_t len)
        : key(k), value(v), offset(off), length(len), res(false) {
--- a/curvefs/src/client/main.cpp
+++ b/curvefs/src/client/main.cpp
@ -51,11 +51,13 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
    release : FuseOpRelease,
    fsync : FuseOpFsync,
    opendir : FuseOpOpenDir,
-    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    readdir : 0,
-    #else
+    // TODO(wuhongsong): readdirplus is problematic,
+    // resulting in inconsistent metadata
+    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    // readdir : 0,
+    // #else
    readdir : FuseOpReadDir,
-    #endif
+    // #endif
    releasedir : FuseOpReleaseDir,
    fsyncdir : 0,
    statfs : FuseOpStatFs,
@ -79,11 +81,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
    flock : 0,
    fallocate : 0,
    #endif
-    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    readdirplus : FuseOpReadDirPlus,
-    #else
+    // TODO(wuhongsong): The current implementation is problematic,
+    // resulting in inconsistent metadata
+    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
    readdirplus : 0,
-    #endif
+    // #endif
    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 4)
    copy_file_range : 0,
    #endif
@ -220,8 +222,8 @@ int main(int argc, char *argv[]) {

    fuse_daemonize(opts.foreground);

-    if (InitLog(mOpts.conf, argv[0]) < 0) {
-        printf("Init log failed, confpath = %s\n", mOpts.conf);
+    if (InitGlog(mOpts.conf, argv[0]) < 0) {
+        printf("Init glog failed, confpath = %s\n", mOpts.conf);
    }

    ret = InitFuseClient(&mOpts);
--- a/curvefs/src/client/metric/client_metric.cpp
+++ b/curvefs/src/client/metric/client_metric.cpp
@ -31,7 +31,7 @@ const std::string MetaServerClientMetric::prefix = "curvefs_metaserver_client";
 const std::string ClientOpMetric::prefix = "curvefs_client";  // NOLINT
 const std::string S3MultiManagerMetric::prefix = "curvefs_client_manager";  // NOLINT
 const std::string FSMetric::prefix = "curvefs_client";  // NOLINT
-const std::string S3Metric::prefix = "curvefs_s3";  // NOLINT
+const std::string IoMetric::prefix = "curvefs_s3";  // NOLINT
 const std::string DiskCacheMetric::prefix = "curvefs_disk_cache";  // NOLINT
 const std::string KVClientMetric::prefix = "curvefs_kvclient";  // NOLINT
 const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info";  // NOLINT
--- a/curvefs/src/client/metric/client_metric.h
+++ b/curvefs/src/client/metric/client_metric.h
@ -92,13 +92,12 @@ struct MetaServerClientMetric {
    // volume extent
    InterfaceMetric updateVolumeExtent;
    InterfaceMetric getVolumeExtent;
+    InterfaceMetric updateDeallocatableBlockGroup;

    MetaServerClientMetric()
-        : getDentry(prefix, "getDentry"),
-          listDentry(prefix, "listDentry"),
+        : getDentry(prefix, "getDentry"), listDentry(prefix, "listDentry"),
          createDentry(prefix, "createDentry"),
-          deleteDentry(prefix, "deleteDentry"),
-          getInode(prefix, "getInode"),
+          deleteDentry(prefix, "deleteDentry"), getInode(prefix, "getInode"),
          batchGetInodeAttr(prefix, "batchGetInodeAttr"),
          batchGetXattr(prefix, "batchGetXattr"),
          createInode(prefix, "createInode"),
@ -107,7 +106,9 @@ struct MetaServerClientMetric {
          appendS3ChunkInfo(prefix, "appendS3ChunkInfo"),
          prepareRenameTx(prefix, "prepareRenameTx"),
          updateVolumeExtent(prefix, "updateVolumeExtent"),
-          getVolumeExtent(prefix, "getVolumeExtent") {}
+          getVolumeExtent(prefix, "getVolumeExtent"),
+          updateDeallocatableBlockGroup(prefix,
+                                        "updateDeallocatableBlockGroup") {}
 };

 struct InflightGuard {
@ -227,27 +228,35 @@ struct FSMetric {
          userReadIoSize(prefix, fsName + "_userReadIoSize", 0) {}
 };

-struct S3Metric {
-    static const std::string prefix;

+struct IoMetric {
+    static const std::string prefix;
    std::string fsName;
    InterfaceMetric adaptorWrite;
    InterfaceMetric adaptorRead;
+    InterfaceMetric adaptorBgFlush;
+    InterfaceMetric adaptorFlushBackend;
    InterfaceMetric adaptorWriteS3;
-    InterfaceMetric adaptorWriteDiskCache;
    InterfaceMetric adaptorReadS3;
+    InterfaceMetric adaptorWriteKvCache;
+    InterfaceMetric adaptorReadKvCache;
+    InterfaceMetric adaptorReadBackend;
    InterfaceMetric adaptorReadDiskCache;
    bvar::Status<uint32_t> readSize;
    bvar::Status<uint32_t> writeSize;

-    explicit S3Metric(const std::string &name = "")
+    explicit IoMetric(const std::string &name = "")
        : fsName(!name.empty() ? name
                               : prefix + curve::common::ToHexString(this)),
          adaptorWrite(prefix, fsName + "_adaptor_write"),
          adaptorRead(prefix, fsName + "_adaptor_read"),
+          adaptorBgFlush(prefix, fsName + "_adaptor_bg_flush"),
+          adaptorFlushBackend(prefix, fsName + "_adaptor_flush_backend"),
+          adaptorReadBackend(prefix, fsName + "_adaptor_read_backend"),
          adaptorWriteS3(prefix, fsName + "_adaptor_write_s3"),
-          adaptorWriteDiskCache(prefix, fsName + "_adaptor_write_disk_cache"),
          adaptorReadS3(prefix, fsName + "_adaptor_read_s3"),
+          adaptorWriteKvCache(prefix, fsName + "_adaptor_write_kv_cache"),
+          adaptorReadKvCache(prefix, fsName + "_adaptor_read_kv_cache"),
          adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache"),
          readSize(prefix, fsName + "_adaptor_read_size", 0),
          writeSize(prefix, fsName + "_adaptor_write_size", 0) {}
@ -257,12 +266,16 @@ struct DiskCacheMetric {
    static const std::string prefix;

    std::string fsName;
+    InterfaceMetric adaptorWriteDiskCache;
+    InterfaceMetric adaptorReadDiskCache;
    InterfaceMetric writeS3;
    bvar::Status<uint64_t> diskUsedBytes;

    explicit DiskCacheMetric(const std::string &name = "")
        : fsName(!name.empty() ? name
                               : prefix + curve::common::ToHexString(this)),
+          adaptorWriteDiskCache(prefix, fsName + "_adaptor_write_disk_cache"),
+          adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache"),
          writeS3(prefix, fsName + "_write_s3"),
          diskUsedBytes(prefix, fsName + "_diskcache_usedbytes", 0) {}
 };
--- a/curvefs/src/client/rpcclient/base_client.cpp
+++ b/curvefs/src/client/rpcclient/base_client.cpp
@ -180,7 +180,6 @@ void MDSBaseClient::AllocateVolumeBlockGroup(
    request.set_fsid(fsId);
    request.set_count(count);
    request.set_owner(owner);
-
    SpaceService_Stub stub(channel);
    stub.AllocateBlockGroup(cntl, &request, response, nullptr);
 }
--- a/curvefs/src/client/rpcclient/mds_client.cpp
+++ b/curvefs/src/client/rpcclient/mds_client.cpp
@ -710,14 +710,17 @@ SpaceErrCode MdsClientImpl::AllocateVolumeBlockGroup(
        AllocateBlockGroupResponse response;
        mdsbasecli_->AllocateVolumeBlockGroup(fsId, count, owner, &response,
                                              cntl, channel);
+        VLOG(9) << "AllocateVolumeBlockGroup, response: "
+                    << response.DebugString();

        CHECK_RPC_AND_RETRY_IF_ERROR("AllocateVolumeBlockGroup");

        auto status = response.status();
-        if (status != SpaceErrCode::SpaceOk) {
-            LOG(WARNING) << "Allocate volume block group failed, err: "
-                         << SpaceErrCode_Name(status);
-        } else if (response.blockgroups_size() == 0) {
+        LOG_IF(WARNING, status != SpaceErrCode::SpaceOk)
+            << "Allocate volume block group failed, err: "
+            << SpaceErrCode_Name(status);
+
+        if (response.blockgroups_size() == 0) {
            LOG(WARNING) << "Allocate volume block group failed, no block "
                            "group allcoated";
            return SpaceErrCode::SpaceErrNoSpace;
--- a/curvefs/src/client/rpcclient/metaserver_client.cpp
+++ b/curvefs/src/client/rpcclient/metaserver_client.cpp
@ -65,6 +65,7 @@ using BatchGetXAttrExcutor = TaskExecutor;
 using GetOrModifyS3ChunkInfoExcutor = TaskExecutor;
 using UpdateVolumeExtentExecutor = TaskExecutor;
 using GetVolumeExtentExecutor = TaskExecutor;
+using UpdateDeallocatableBlockGroupExcutor = TaskExecutor;

 using ::curvefs::common::LatencyUpdater;
 using ::curvefs::common::StreamConnection;
@ -970,6 +971,7 @@ void MetaServerClientImpl::UpdateInodeAsync(const UpdateInodeRequest &request,
        req.set_poolid(poolID);
        req.set_copysetid(copysetID);
        req.set_partitionid(partitionID);
+        VLOG(9) << "update inode async req: " << req.ShortDebugString();

        auto *rpcDone = new UpdateInodeRpcDone(taskExecutorDone, &metric_);
        curvefs::metaserver::MetaServerService_Stub stub(channel);
@ -1355,8 +1357,8 @@ MetaStatusCode MetaServerClientImpl::CreateManageInode(const InodeParam &param,

    auto taskCtx = std::make_shared<TaskContext>(
        MetaServerOpType::CreateManageInode, task, param.fsId, 0);
-    CreateInodeExcutor excutor(opt_, metaCache_, channelManager_,
-                               std::move(taskCtx));
+    CreateManagerInodeExcutor excutor(opt_, metaCache_, channelManager_,
+                                      std::move(taskCtx));
    return ConvertToMetaStatusCode(excutor.DoRPCTask());
 }

@ -1465,7 +1467,7 @@ void UpdateVolumeExtentRpcDone::Run() {
    } while (0)

 void MetaServerClientImpl::AsyncUpdateVolumeExtent(
-    uint32_t fsId, uint64_t inodeId, const VolumeExtentList &extents,
+    uint32_t fsId, uint64_t inodeId, const VolumeExtentSliceList &extents,
    MetaServerClientDone *done) {
    auto task = AsyncRPCTask {
        (void)txId;
@ -1473,7 +1475,7 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
        metric_.updateVolumeExtent.qps.count << 1;
        metaserver::UpdateVolumeExtentRequest request;
        SET_COMMON_FIELDS;
-        request.set_allocated_extents(new VolumeExtentList{extents});
+        request.set_allocated_extents(new VolumeExtentSliceList{extents});

        auto *rpcDone =
            new UpdateVolumeExtentRpcDone(taskExecutorDone, &metric_);
@ -1493,7 +1495,8 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
 namespace {

 struct ParseVolumeExtentCallBack {
-    explicit ParseVolumeExtentCallBack(VolumeExtentList *ext) : extents(ext) {}
+    explicit ParseVolumeExtentCallBack(VolumeExtentSliceList *ext)
+        : extents(ext) {}

    bool operator()(butil::IOBuf *data) const {
        metaserver::VolumeExtentSlice slice;
@ -1506,7 +1509,7 @@ struct ParseVolumeExtentCallBack {
        return true;
    }

-    VolumeExtentList *extents;
+    VolumeExtentSliceList *extents;
 };

 }  // namespace
@ -1514,7 +1517,7 @@ struct ParseVolumeExtentCallBack {
 MetaStatusCode
 MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
                                      bool streaming,
-                                      VolumeExtentList *extents) {
+                                      VolumeExtentSliceList *extents) {
    auto task = RPCTask {
        (void)txId;
        (void)applyIndex;
@ -1596,9 +1599,66 @@ MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
    return ConvertToMetaStatusCode(executor.DoRPCTask());
 }

-MetaStatusCode MetaServerClientImpl::GetInodeAttr(uint32_t fsId,
-                                                  uint64_t inodeid,
-                                                  InodeAttr *attr) {
+MetaStatusCode MetaServerClientImpl::UpdateDeallocatableBlockGroup(
+    uint32_t fsId, uint64_t inodeId, DeallocatableBlockGroupMap *statistic) {
+    auto task = RPCTask {
+        metric_.updateDeallocatableBlockGroup.qps.count << 1;
+        LatencyUpdater updater(&metric_.updateDeallocatableBlockGroup.latency);
+
+        metaserver::UpdateDeallocatableBlockGroupRequest request;
+        metaserver::UpdateDeallocatableBlockGroupResponse response;
+        request.set_poolid(poolID);
+        request.set_copysetid(copysetID);
+        request.set_partitionid(partitionID);
+        request.set_fsid(fsId);
+        auto *upadte = request.mutable_update();
+        for (auto &it : *statistic) {
+            upadte->Add()->CopyFrom(std::move(it.second));
+        }
+
+        curvefs::metaserver::MetaServerService_Stub stub(channel);
+        stub.UpdateDeallocatableBlockGroup(cntl, &request, &response, nullptr);
+
+        if (cntl->Failed()) {
+            metric_.updateDeallocatableBlockGroup.eps.count << 1;
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup failed"
+                         << ", errorCode = " << cntl->ErrorCode()
+                         << ", errorText = " << cntl->ErrorText()
+                         << ", logId = " << cntl->log_id();
+            return -cntl->ErrorCode();
+        }
+
+        auto rc = response.statuscode();
+        if (rc != MetaStatusCode::OK) {
+            metric_.updateDeallocatableBlockGroup.eps.count << 1;
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup: retCode = " << rc
+                         << ", message = " << MetaStatusCode_Name(rc);
+        } else if (response.has_appliedindex()) {
+            metaCache_->UpdateApplyIndex(CopysetGroupID(poolID, copysetID),
+                                         response.appliedindex());
+        } else {
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup OK"
+                         << ", but applyIndex not set in response:"
+                         << response.DebugString();
+            return -1;
+        }
+
+        VLOG(6) << "UpdateDeallocatableBlockGroup done, request: "
+                << request.DebugString()
+                << "response: " << response.DebugString();
+        return rc;
+    };
+
+    auto taskCtx = std::make_shared<TaskContext>(
+        MetaServerOpType::UpdateDeallocatableBlockGroup, task, fsId, inodeId);
+
+    UpdateDeallocatableBlockGroupExcutor excutor(
+        opt_, metaCache_, channelManager_, std::move(taskCtx));
+    return ConvertToMetaStatusCode(excutor.DoRPCTask());
+}
+
+MetaStatusCode MetaServerClientImpl::GetInodeAttr(
+    uint32_t fsId, uint64_t inodeid, InodeAttr *attr) {
    std::set<uint64_t> inodeIds;
    inodeIds.insert(inodeid);
    std::list<InodeAttr> attrs;
--- a/curvefs/src/client/rpcclient/metaserver_client.h
+++ b/curvefs/src/client/rpcclient/metaserver_client.h
@ -30,6 +30,7 @@
 #include <set>
 #include <unordered_map>
 #include <utility>
+#include <map>

 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/metaserver.pb.h"
@ -49,21 +50,24 @@ using ::curvefs::metaserver::InodeAttr;
 using ::curvefs::metaserver::XAttr;
 using ::curvefs::metaserver::MetaStatusCode;
 using ::curvefs::metaserver::S3ChunkInfoList;
+using ::curvefs::metaserver::DeallocatableBlockGroup;
 using ::curvefs::common::StreamStatus;
 using ::curvefs::common::StreamClient;
-using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 using ::curvefs::metaserver::Time;

+using DeallocatableBlockGroupMap = std::map<uint64_t, DeallocatableBlockGroup>;
+using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+
 namespace curvefs {
 namespace client {
 namespace rpcclient {

 using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtentSliceList;

 struct DataIndices {
    absl::optional<S3ChunkInfoMap> s3ChunkInfoMap;
-    absl::optional<VolumeExtentList> volumeExtents;
+    absl::optional<VolumeExtentSliceList> volumeExtents;
 };

 class MetaServerClient {
@ -161,13 +165,17 @@ class MetaServerClient {

    virtual void AsyncUpdateVolumeExtent(uint32_t fsId,
                                         uint64_t inodeId,
-                                         const VolumeExtentList &extents,
+                                         const VolumeExtentSliceList &extents,
                                         MetaServerClientDone *done) = 0;

    virtual MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                           uint64_t inodeId,
                                           bool streaming,
-                                           VolumeExtentList *extents) = 0;
+                                           VolumeExtentSliceList *extents) = 0;
+
+    virtual MetaStatusCode
+    UpdateDeallocatableBlockGroup(uint32_t fsId, uint64_t inodeId,
+                                  DeallocatableBlockGroupMap *statistic) = 0;
 };

 class MetaServerClientImpl : public MetaServerClient {
@ -265,13 +273,17 @@ class MetaServerClientImpl : public MetaServerClient {

    void AsyncUpdateVolumeExtent(uint32_t fsId,
                                 uint64_t inodeId,
-                                 const VolumeExtentList &extents,
+                                 const VolumeExtentSliceList &extents,
                                 MetaServerClientDone *done) override;

    MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                   uint64_t inodeId,
                                   bool streaming,
-                                   VolumeExtentList *extents) override;
+                                   VolumeExtentSliceList *extents) override;
+
+    MetaStatusCode UpdateDeallocatableBlockGroup(
+        uint32_t fsId, uint64_t inodeId,
+        DeallocatableBlockGroupMap *statistic) override;

 private:
    MetaStatusCode UpdateInode(const UpdateInodeRequest &request,
--- a/curvefs/src/client/rpcclient/task_excutor.cpp
+++ b/curvefs/src/client/rpcclient/task_excutor.cpp
@ -27,8 +27,10 @@

 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/common/define.h"

 using ::curvefs::metaserver::MetaStatusCode;
+using ::curvefs::RECYCLEINODEID;

 namespace curvefs {
 namespace client {
@ -339,6 +341,17 @@ bool CreateInodeExcutor::GetTarget() {
    return true;
 }

+bool CreateManagerInodeExcutor::GetTarget() {
+    if (!metaCache_->GetTarget(task_->fsID, RECYCLEINODEID, &task_->target,
+                               &task_->applyIndex)) {
+        LOG(ERROR) << "CreateManagerInodeExcutor select target for task fail, "
+                   << task_->TaskContextStr()
+                   << ", recycleInodeId = " << RECYCLEINODEID;
+        return false;
+    }
+    return true;
+}
+
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs
--- a/curvefs/src/client/rpcclient/task_excutor.h
+++ b/curvefs/src/client/rpcclient/task_excutor.h
@ -275,6 +275,18 @@ class CreateInodeExcutor : public TaskExecutor {
    bool GetTarget() override;
 };

+class CreateManagerInodeExcutor : public TaskExecutor {
+ public:
+    explicit CreateManagerInodeExcutor(
+        const ExcutorOpt &opt, const std::shared_ptr<MetaCache> &metaCache,
+        const std::shared_ptr<ChannelManager<MetaserverID>> &channelManager,
+        const std::shared_ptr<TaskContext> &task)
+        : TaskExecutor(opt, metaCache, channelManager, task) {}
+
+ protected:
+    bool GetTarget() override;
+};
+
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs
--- a/curvefs/src/client/s3/client_s3_adaptor.cpp
+++ b/curvefs/src/client/s3/client_s3_adaptor.cpp
--- a/curvefs/src/client/s3/client_s3_adaptor.h
+++ b/curvefs/src/client/s3/client_s3_adaptor.h
@ -19,12 +19,15 @@
 * Created Date: 21-5-31
 * Author: huyao
 */
+
 #ifndef CURVEFS_SRC_CLIENT_S3_CLIENT_S3_ADAPTOR_H_
 #define CURVEFS_SRC_CLIENT_S3_CLIENT_S3_ADAPTOR_H_

 #include <bthread/execution_queue.h>

 #include <memory>
+#include <utility>
+#include <set>
 #include <string>
 #include <vector>

@ -33,204 +36,216 @@
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/client_storage_adaptor.h"
+#include "curvefs/src/client/kvclient/kvclient_manager.h"
+#include "curvefs/src/client/kvclient/kvclient.h"
+#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/s3/client_s3_cache_manager.h"
-#include "curvefs/src/client/s3/disk_cache_manager_impl.h"
+#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h"
 #include "src/common/wait_interval.h"
+
 namespace curvefs {
 namespace client {

-using ::curve::common::Thread;
-using ::curve::common::TaskThreadPool;
+using curve::common::GetObjectAsyncCallBack;
+using curve::common::PutObjectAsyncCallBack;
+using curve::common::S3Adapter;
 using curvefs::client::common::S3ClientAdaptorOption;
-using curvefs::client::common::DiskCacheType;
-using curvefs::metaserver::Inode;
-using curvefs::metaserver::S3ChunkInfo;
-using curvefs::metaserver::S3ChunkInfoList;
-using rpcclient::MdsClient;
-using curvefs::client::metric::S3Metric;
+using curvefs::client::metric::IoMetric;

-class DiskCacheManagerImpl;
-class FlushChunkCacheContext;
-class ChunkCacheManager;
+/// @brief s3 read request
+/// @param chunkId chunk id
+/// @param offset file offset
+/// @param len read length
+/// @param objectOffset first offset in the block
+/// @param readOffset read buf offset
+/// @param fsId file system id
+/// @param inodeId inode id
+/// @param compaction compaction flag
+struct S3ReadRequest {
+    uint64_t chunkId;
+    uint64_t offset;
+    uint64_t len;
+    uint64_t objectOffset;
+    uint64_t readOffset;
+    uint64_t fsId;
+    uint64_t inodeId;
+    uint64_t compaction;

-class S3ClientAdaptor {
- public:
-    S3ClientAdaptor() {}
-    virtual ~S3ClientAdaptor() {}
-    /**
-     * @brief Initailize s3 client
-     * @param[in] options the options for s3 client
-     */
-    virtual CURVEFS_ERROR
-    Init(const S3ClientAdaptorOption &option, std::shared_ptr<S3Client> client,
-         std::shared_ptr<InodeCacheManager> inodeManager,
-         std::shared_ptr<MdsClient> mdsClient,
-         std::shared_ptr<FsCacheManager> fsCacheManager,
-         std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
-         std::shared_ptr<KVClientManager> kvClientManager,
-         bool startBackGround = false) = 0;
-    /**
-     * @brief write data to s3
-     * @param[in] options the options for s3 client
-     */
-    virtual int Write(uint64_t inodeId, uint64_t offset, uint64_t length,
-                      const char *buf) = 0;
-    virtual int Read(uint64_t inodeId, uint64_t offset, uint64_t length,
-                     char *buf) = 0;
-    virtual CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper,
-                                   uint64_t size) = 0;
-    virtual void ReleaseCache(uint64_t inodeId) = 0;
-    virtual CURVEFS_ERROR Flush(uint64_t inodeId) = 0;
-    virtual CURVEFS_ERROR FlushAllCache(uint64_t inodeId) = 0;
-    virtual CURVEFS_ERROR FsSync() = 0;
-    virtual int Stop() = 0;
-    virtual FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum,
-                                        uint64_t *chunkId) = 0;
-    virtual void SetFsId(uint32_t fsId) = 0;
-    virtual void InitMetrics(const std::string &fsName) = 0;
-    virtual void CollectMetrics(InterfaceMetric *interface, int count,
-                                uint64_t start) = 0;
-    virtual std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() = 0;
-    virtual std::shared_ptr<S3Client> GetS3Client() = 0;
-    virtual uint64_t GetBlockSize() = 0;
-    virtual uint64_t GetChunkSize() = 0;
-    virtual uint32_t GetObjectPrefix() = 0;
-    virtual bool HasDiskCache() = 0;
+    std::string DebugString() const {
+        std::ostringstream os;
+        os << "S3ReadRequest ( chunkId = " << chunkId << ", offset = " << offset
+           << ", len = " << len << ", objectOffset = " << objectOffset
+           << ", readOffset = " << readOffset << ", fsId = " << fsId
+           << ", inodeId = " << inodeId << ", compaction = " << compaction
+           << " )";
+        return os.str();
+    }
 };

-using FlushChunkCacheCallBack = std::function<
-  void(const std::shared_ptr<FlushChunkCacheContext>&)>;
-
-struct FlushChunkCacheContext {
-    uint64_t inode;
-    ChunkCacheManagerPtr chunkCacheManptr;
-    bool force;
-    FlushChunkCacheCallBack cb;
-    CURVEFS_ERROR retCode;
-};
+inline std::string
+S3ReadRequestVecDebugString(const std::vector<S3ReadRequest> &reqs) {
+    std::ostringstream os;
+    for_each(reqs.begin(), reqs.end(),
+             [&](const S3ReadRequest &req) { os << req.DebugString() << " "; });
+    return os.str();
+}

 // client use s3 internal interface
-class S3ClientAdaptorImpl : public S3ClientAdaptor {
+class S3ClientAdaptorImpl : public StorageAdaptor {
 public:
-    S3ClientAdaptorImpl() {}
+    S3ClientAdaptorImpl() : StorageAdaptor() {}
+
+    // for unittest
+    explicit S3ClientAdaptorImpl(std::shared_ptr<
+      S3Client> client) : StorageAdaptor() {
+        client_ = client;
+    }
+
    virtual ~S3ClientAdaptorImpl() {
        LOG(INFO) << "delete S3ClientAdaptorImpl";
    }
-    /**
-     * @brief Initailize s3 client
-     * @param[in] options the options for s3 client
-     */
-    CURVEFS_ERROR
-    Init(const S3ClientAdaptorOption &option, std::shared_ptr<S3Client> client,
+
+    /// @brief init s3 storage adaptor
+    /// @param option fuse client option
+    /// @param inodeManager inode cache manager
+    /// @param mdsClient mds client
+    /// @param fsCacheManager fscache manager
+    /// @param diskCacheManagerImpl disk cache manager
+    /// @param kvClientManager kv client manager
+    /// @param fsInfo file system information
+    /// @return error code
+    CURVEFS_ERROR Init(const FuseClientOption &option,
         std::shared_ptr<InodeCacheManager> inodeManager,
         std::shared_ptr<MdsClient> mdsClient,
         std::shared_ptr<FsCacheManager> fsCacheManager,
         std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
         std::shared_ptr<KVClientManager> kvClientManager,
-         bool startBackGround = false);
-    /**
-     * @brief write data to s3
-     * @param[in] options the options for s3 client
-     */
-    int Write(uint64_t inodeId, uint64_t offset, uint64_t length,
-              const char *buf);
-    int Read(uint64_t inodeId, uint64_t offset, uint64_t length, char *buf);
+         std::shared_ptr<FsInfo> fsInfo) override;
+
+    int Stop() override;
+
+    /// @brief read data from s3 storage
+    /// @param request read request
+    /// @return error code
+    CURVEFS_ERROR FlushDataCache(const UperFlushRequest& req,
+      uint64_t* writeOffset) override;
+
+    /// @brief read data from s3 storage
+    /// @param request read request
+    /// @return error code
+    CURVEFS_ERROR ReadFromLowlevel(UperReadRequest request) override;
+
    CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper, uint64_t size);
-    void ReleaseCache(uint64_t inodeId);
-    CURVEFS_ERROR Flush(uint64_t inodeId);
-    CURVEFS_ERROR FlushAllCache(uint64_t inodeId);
-    CURVEFS_ERROR FsSync();
-    int Stop();
-    uint64_t GetBlockSize() {
-        return blockSize_;
-    }
-    uint64_t GetChunkSize() {
-        return chunkSize_;
-    }
+
    uint32_t GetObjectPrefix() {
        return objectPrefix_;
    }

-    std::shared_ptr<FsCacheManager> GetFsCacheManager() {
-        return fsCacheManager_;
-    }
-    uint32_t GetFlushInterval() { return flushIntervalSec_; }
    std::shared_ptr<S3Client> GetS3Client() { return client_; }
-    uint32_t GetPrefetchBlocks() {
-        return prefetchBlocks_;
-    }
-    uint32_t GetDiskCacheType() {
-        return diskCacheType_;
-    }
-    bool DisableDiskCache() {
-        return diskCacheType_ == DiskCacheType::Disable;
-    }
-    bool HasDiskCache() {
-        return diskCacheType_ != DiskCacheType::Disable;
-    }
-    bool IsReadCache() {
-        return diskCacheType_ == DiskCacheType::OnlyRead;
-    }
-    bool IsReadWriteCache() {
-        return diskCacheType_ == DiskCacheType::ReadWrite;
-    }
-    std::shared_ptr<InodeCacheManager> GetInodeCacheManager() {
-        return inodeManager_;
-    }
-    std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() {
-        return diskCacheManagerImpl_;
-    }
-    FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum,
-                                uint64_t *chunkId);
-    void FsSyncSignal() {
-        std::lock_guard<std::mutex> lk(mtx_);
-        VLOG(3) << "fs sync signal";
-        cond_.notify_one();
-    }
-    void FsSyncSignalAndDataCacheInc() {
-        std::lock_guard<std::mutex> lk(mtx_);
-        fsCacheManager_->DataCacheNumInc();
-        VLOG(3) << "fs sync signal";
-        cond_.notify_one();
-    }
-    void SetFsId(uint32_t fsId) {
-        fsId_ = fsId;
-    }
-    uint32_t GetFsId() {
-        return fsId_;
-    }
-    uint32_t GetPageSize() {
-        return pageSize_;
-    }
-    void InitMetrics(const std::string &fsName);
-    void CollectMetrics(InterfaceMetric *interface, int count, uint64_t start);
-    void SetDiskCache(DiskCacheType type) {
-       diskCacheType_ = type;
+
+    CURVEFS_ERROR FuseOpInit(void *userdata,
+      struct fuse_conn_info *conn) override {
+        StorageAdaptor::FuseOpInit(userdata, conn);
+        return CURVEFS_ERROR::OK;
    }

-    uint32_t GetMaxReadRetryIntervalMs() const {
-        return maxReadRetryIntervalMs_;
+ private:
+     enum class ReadStatus {
+        OK = 0,
+        S3_READ_FAIL = -1,
+        S3_NOT_EXIST = -2,
+    };
+
+    ReadStatus toReadStatus(const int retCode) {
+        ReadStatus st = ReadStatus::OK;
+        if (retCode < 0) {
+            st = (retCode == -2) ? ReadStatus::S3_NOT_EXIST
+                                 : ReadStatus::S3_READ_FAIL;
+        }
+        return st;
+    }
+
+    S3ClientAdaptorImpl::ReadStatus ReadKVRequest(
+      const std::vector<S3ReadRequest> &kvRequests,
+      char *dataBuf, uint64_t fileLen);
+
+    CURVEFS_ERROR PrepareFlushTasks(const UperFlushRequest& req,
+      std::vector<std::shared_ptr<PutObjectAsyncContext>> *s3Tasks,
+      std::vector<std::shared_ptr<SetKVCacheTask>> *kvCacheTasks,
+      uint64_t* writeOffset);
+
+    void FlushTaskExecute(CachePolicy cachePoily,
+      const std::vector<std::shared_ptr<PutObjectAsyncContext>> &s3Tasks,
+      const std::vector<std::shared_ptr<SetKVCacheTask>> &kvCacheTasks);
+
+    void PrefetchS3Objs(uint64_t inodeId,
+      const std::vector<std::pair<std::string, uint64_t>> &prefetchObjs);
+
+    void HandleReadRequest(
+      const ReadRequest &request, const S3ChunkInfo &s3ChunkInfo,
+      std::vector<ReadRequest> *addReadRequests,
+      std::vector<uint64_t> *deletingReq, std::vector<S3ReadRequest> *requests,
+      char *dataBuf, uint64_t fsId, uint64_t inodeId);
+
+    void GenerateS3Request(ReadRequest request,
+      const S3ChunkInfoList &s3ChunkInfoList,
+      char *dataBuf,
+      std::vector<S3ReadRequest> *requests,
+      uint64_t fsId,
+      uint64_t inodeId);
+
+    // miss read from memory read/write cache, need read from
+    // kv(localdisk/remote cache/s3)
+    int GenerateKVReuqest(const std::shared_ptr<InodeWrapper> &inodeWrapper,
+                          const std::vector<ReadRequest> &readRequest,
+                          char *dataBuf, std::vector<S3ReadRequest> *kvRequest);
+
+    int HandleReadS3NotExist(uint32_t retry,
+      const std::shared_ptr<InodeWrapper> &inodeWrapper);
+
+    bool ReadKVRequestFromS3(const std::string &name,
+      char *databuf, uint64_t offset, uint64_t length, int *ret);
+
+    bool ReadKVRequestFromRemoteCache(const std::string &name,
+      char *databuf, uint64_t offset, uint64_t length);
+
+    bool ReadKVRequestFromLocalCache(const std::string &name, char *databuf,
+      uint64_t offset, uint64_t len);
+
+    // thread function for ReadKVRequest
+    void ProcessKVRequest(const S3ReadRequest &req, char *dataBuf,
+        uint64_t fileLen,
+        std::once_flag &cancelFlag,     // NOLINT
+        std::atomic<bool> &isCanceled,  // NOLINT
+        std::atomic<int> &retCode);     // NOLINT
+
+    void PrefetchForBlock(const S3ReadRequest &req, uint64_t fileLen,
+      uint64_t blockSize, uint64_t chunkSize, uint64_t startBlockIndex);
+
+    void GetChunkLoc(uint64_t offset, uint64_t *index,
+      uint64_t *chunkPos, uint64_t *chunkSize);
+
+    void GetBlockLoc(uint64_t offset, uint64_t *chunkIndex, uint64_t *chunkPos,
+      uint64_t *blockIndex, uint64_t *blockPos);
+
+    uint32_t GetPrefetchBlocks() {
+        return prefetchBlocks_;
    }

    uint32_t GetReadRetryIntervalMs() const {
        return readRetryIntervalMs_;
    }

- private:
-    void BackGroundFlush();
-
    using AsyncDownloadTask = std::function<void()>;
-
    static int ExecAsyncDownloadTask(void* meta, bthread::TaskIterator<AsyncDownloadTask>& iter);  // NOLINT

-    int ClearDiskCache(int64_t inodeId);
-
 public:
-    void PushAsyncTask(const AsyncDownloadTask& task) {
+  void PushAsyncTask(const AsyncDownloadTask& task) {
        static thread_local unsigned int seed = time(nullptr);

        int idx = rand_r(&seed) % downloadTaskQueues_.size();
@ -241,47 +256,62 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
            task();
        }
    }
-    std::shared_ptr<S3Metric> s3Metric_;
-
-    void Enqueue(std::shared_ptr<FlushChunkCacheContext> context);

 private:
-    std::shared_ptr<S3Client> client_;
-    uint64_t blockSize_;
-    uint64_t chunkSize_;
-    uint32_t prefetchBlocks_;
-    uint32_t prefetchExecQueueNum_;
-    std::string allocateServerEps_;
-    uint32_t flushIntervalSec_;
-    uint32_t chunkFlushThreads_;
-    uint32_t memCacheNearfullRatio_;
-    uint32_t throttleBaseSleepUs_;
-    uint32_t maxReadRetryIntervalMs_;
-    uint32_t readRetryIntervalMs_;
-    uint32_t objectPrefix_;
-    Thread bgFlushThread_;
-    std::atomic<bool> toStop_;
-    std::mutex mtx_;
-    std::mutex ioMtx_;
-    std::condition_variable cond_;
-    curve::common::WaitInterval waitInterval_;
-    std::shared_ptr<FsCacheManager> fsCacheManager_;
-    std::shared_ptr<InodeCacheManager> inodeManager_;
-    std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl_;
-    DiskCacheType diskCacheType_;
-    std::shared_ptr<MdsClient> mdsClient_;
-    uint32_t fsId_;
-    std::string fsName_;
+  class AsyncPrefetchCallback {
+   public:
+      AsyncPrefetchCallback(uint64_t inode, S3ClientAdaptorImpl *s3Client)
+          : inode_(inode), s3Client_(s3Client) {}
+
+      void operator()(const S3Adapter *,
+                      const std::shared_ptr<GetObjectAsyncContext> &context) {
+          std::unique_ptr<char[]> guard(context->buf);
+
+          if (context->retCode != 0) {
+              LOG(WARNING) << "prefetch failed, key: " << context->key;
+              return;
+          }
+
+          int ret = s3Client_->GetDiskCacheManager()->WriteReadDirect(
+              context->key, context->buf, context->actualLen);
+          if (ret < 0) {
+              LOG_EVERY_SECOND(INFO) <<
+                "prefetch failed, write read directly failed, key: "
+                << context->key;
+          }
+          {
+            curve::common::LockGuard lg(s3Client_->downloadMtx_);
+            s3Client_->downloadingObj_.erase(context->key);
+          }
+          VLOG(9) << "prefetch end, objectname is: " << context->key
+                  << ", len is: " << context->len
+                  << ", actual len is: " <<  context->actualLen;
+      }
+
+   private:
+      const uint64_t inode_;
+      S3ClientAdaptorImpl *s3Client_;
+  };
+
+ protected:
+  curve::common::Mutex downloadMtx_;
+  std::set<std::string> downloadingObj_;
+
+ private:
    std::vector<bthread::ExecutionQueueId<AsyncDownloadTask>>
      downloadTaskQueues_;
-    uint32_t pageSize_;
-
-    int FlushChunkClosure(std::shared_ptr<FlushChunkCacheContext> context);
-
-    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
-        taskPool_;
-
-    std::shared_ptr<KVClientManager> kvClientManager_ = nullptr;
+    // prefetch blocks nums
+    uint32_t prefetchBlocks_;
+    // prefetch thread nums
+    uint32_t prefetchExecQueueNum_;
+    // read and max retry times when read s3 failed
+    uint32_t readRetryIntervalMs_;
+    uint32_t objectPrefix_;
+    uint32_t maxReadRetryIntervalMs_;
+    // s3 client manager(put or get object form s3)
+    std::shared_ptr<S3Client> client_;
+    // kv client manager
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };

 }  // namespace client
--- a/curvefs/src/client/s3/fuse_s3_client.cpp
+++ b/curvefs/src/client/s3/fuse_s3_client.cpp
@ -24,7 +24,7 @@
 #include <memory>
 #include <vector>

-#include "curvefs/src/client/fuse_s3_client.h"
+#include "curvefs/src/client/s3/fuse_s3_client.h"
 #include "curvefs/src/client/kvclient/memcache_client.h"

 namespace curvefs {
@ -47,63 +47,30 @@ using curvefs::mds::topology::MemcacheClusterInfo;
 using curvefs::mds::topology::MemcacheServerInfo;

 CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
-    FuseClientOption opt(option);
-
-    CURVEFS_ERROR ret = FuseClient::Init(opt);
+    LOG(INFO) << "fuse init start.";
+    auto ret = FuseClient::Init(option);
    if (ret != CURVEFS_ERROR::OK) {
+        LOG(INFO) << "fuse init failed, " << ret;
        return ret;
    }

    // init kvcache
    if (FLAGS_supportKVcache && !InitKVCache(option.kvClientManagerOpt)) {
+        LOG(INFO) << "init kv cache failed.";
        return CURVEFS_ERROR::INTERNAL;
    }

-    // set fs S3Option
-    const auto& s3Info = fsInfo_->detail().s3info();
-    ::curve::common::S3InfoOption fsS3Option;
-    ::curvefs::client::common::S3Info2FsS3Option(s3Info, &fsS3Option);
-    SetFuseClientS3Option(&opt, fsS3Option);
-
-    auto s3Client = std::make_shared<S3ClientImpl>();
-    s3Client->Init(opt.s3Opt.s3AdaptrOpt);
-
-    const uint64_t writeCacheMaxByte =
-        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
-    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
-        LOG(ERROR) << "writeCacheMaxByte is too small"
-                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
-                      ", writeCacheMaxByte = " << writeCacheMaxByte;
-        return CURVEFS_ERROR::CACHETOOSMALL;
+    // init storage adaptor
+    ret = InitStorageAdaptor(option);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(INFO) << "init storage adaptor failed, " << ret;
+        return ret;
    }

-    auto fsCacheManager = std::make_shared<FsCacheManager>(
-        dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte, writeCacheMaxByte,
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheThreads, kvClientManager_);
-    if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
-        DiskCacheType::Disable) {
-        auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
-        s3DiskCacheClient->Init(opt.s3Opt.s3AdaptrOpt);
-        auto wrapper = std::make_shared<PosixWrapper>();
-        auto diskCacheRead = std::make_shared<DiskCacheRead>();
-        auto diskCacheWrite = std::make_shared<DiskCacheWrite>();
-        auto diskCacheManager = std::make_shared<DiskCacheManager>(
-            wrapper, diskCacheWrite, diskCacheRead);
-        auto diskCacheManagerImpl = std::make_shared<DiskCacheManagerImpl>(
-            diskCacheManager, s3DiskCacheClient);
-        ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
-                               inodeManager_, mdsClient_, fsCacheManager,
-                               diskCacheManagerImpl, kvClientManager_, true);
-    } else {
-        ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
-                               inodeManager_, mdsClient_, fsCacheManager,
-                               nullptr, kvClientManager_, true);
-    }
+    LOG(INFO) << "fuse init success.";
    return ret;
 }

-
 bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
     // get kvcache cluster
    MemcacheClusterInfo kvcachecluster;
@ -136,6 +103,50 @@ bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
    return true;
 }

+CURVEFS_ERROR FuseS3Client::InitStorageAdaptor(const FuseClientOption &option) {
+    auto opt(option);
+    auto ret = CURVEFS_ERROR::OK;
+
+    // set fs S3Option
+    const auto& s3Info = fsInfo_->detail().s3info();
+    ::curve::common::S3InfoOption fsS3Option;
+    ::curvefs::client::common::S3Info2FsS3Option(s3Info, &fsS3Option);
+    // update s3 info: s3 adress, bucket, etc
+    SetFuseClientS3Option(&opt, fsS3Option);
+
+    auto s3Client = std::make_shared<S3ClientImpl>();
+    s3Client->Init(opt.s3Opt.s3AdaptrOpt);
+
+    auto fsCacheManager = std::make_shared<FsCacheManager>(
+        dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
+        option.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
+        option.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte,
+        option.s3Opt.s3ClientAdaptorOpt.readCacheThreads,
+        kvClientManager_);
+
+    if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
+        DiskCacheType::Disable) {
+        LOG(INFO) << "has disk cache.";
+        auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
+        s3DiskCacheClient->Init(opt.s3Opt.s3AdaptrOpt);
+        auto wrapper = std::make_shared<PosixWrapper>();
+        auto diskCacheRead = std::make_shared<DiskCacheRead>();
+        auto diskCacheWrite = std::make_shared<DiskCacheWrite>();
+        auto diskCacheManager = std::make_shared<DiskCacheManager>(
+            wrapper, diskCacheWrite, diskCacheRead);
+        auto diskCacheManagerImpl = std::make_shared<DiskCacheManagerImpl>(
+            diskCacheManager, s3DiskCacheClient);
+
+        ret = s3Adaptor_->Init(opt, inodeManager_, mdsClient_, fsCacheManager,
+                               diskCacheManagerImpl, kvClientManager_, fsInfo_);
+    } else {
+        ret = s3Adaptor_->Init(opt, inodeManager_, mdsClient_, fsCacheManager,
+                               nullptr, kvClientManager_, fsInfo_);
+    }
+    LOG(INFO) << "init storage adaptor success.";
+    return ret;
+}
+
 void FuseS3Client::UnInit() {
    FuseClient::UnInit();
    s3Adaptor_->Stop();
@ -144,10 +155,15 @@ void FuseS3Client::UnInit() {

 CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
                                       struct fuse_conn_info *conn) {
-    CURVEFS_ERROR ret = FuseClient::FuseOpInit(userdata, conn);
-    if (init_) {
-        s3Adaptor_->SetFsId(fsInfo_->fsid());
-        s3Adaptor_->InitMetrics(fsInfo_->fsname());
+    auto ret = FuseClient::FuseOpInit(userdata, conn);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "fuse op init failed.";
+        return ret;
+    }
+    ret = s3Adaptor_->FuseOpInit(userdata, conn);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "fuse op init failed.";
+        return ret;
    }
    return ret;
 }
@ -155,8 +171,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
 CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                        const char *buf, size_t size, off_t off,
                                        struct fuse_file_info *fi,
-                                        FileOut* fileOut) {
-    size_t *wSize = &fileOut->nwritten;
+                                        size_t *wSize) {
+    (void)req;
    // check align
    if (fi->flags & O_DIRECT) {
        if (!(is_aligned(off, DirectIOAlignment) &&
@ -218,8 +234,6 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
            }
        }
    }
-
-    inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
    return ret;
 }

@ -282,56 +296,33 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                         const char *name, mode_t mode,
                                         struct fuse_file_info *fi,
-                                         EntryOut* entryOut) {
+                                         fuse_entry_param *e) {
    VLOG(1) << "FuseOpCreate, parent: " << parent << ", name: " << name
            << ", mode: " << mode;
-
-    std::shared_ptr<InodeWrapper> inode;
    CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, inode);
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, e);
    if (ret != CURVEFS_ERROR::OK) {
        return ret;
    }
-
-    auto openFiles = fs_->BorrowMember().openFiles;
-    openFiles->Open(inode->GetInodeId(), inode);
-
-    inode->GetInodeAttr(&entryOut->attr);
-    return CURVEFS_ERROR::OK;
+    return FuseOpOpen(req, e->ino, fi);
 }

-CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req,
-                                        fuse_ino_t parent,
-                                        const char* name,
-                                        mode_t mode,
-                                        dev_t rdev,
-                                        EntryOut* entryOut) {
+CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+                                        const char *name, mode_t mode,
+                                        dev_t rdev, fuse_entry_param *e) {
    VLOG(1) << "FuseOpMkNod, parent: " << parent << ", name: " << name
            << ", mode: " << mode << ", rdev: " << rdev;
-
-    std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
-                                FsFileType::TYPE_S3, rdev, false,
-                                inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    InodeAttr attr;
-    inode->GetInodeAttr(&attr);
-    *entryOut = EntryOut(attr);
-    return CURVEFS_ERROR::OK;
+    return MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, rdev, false,
+                    e);
 }

-CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req,
-                                       fuse_ino_t ino,
-                                       fuse_ino_t newparent,
-                                       const char* newname,
-                                       EntryOut* entryOut) {
+CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char *newname,
+                                     fuse_entry_param *e) {
    VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
            << ", newname: " << newname;
    return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_S3, entryOut);
+        req, ino, newparent, newname, FsFileType::TYPE_S3, e);
 }

 CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
--- a/curvefs/src/client/s3/fuse_s3_client.h
+++ b/curvefs/src/client/s3/fuse_s3_client.h
@ -21,8 +21,8 @@
 * Author: xuchaojie
 */

-#ifndef CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
-#define CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
+#ifndef CURVEFS_SRC_CLIENT_S3_FUSE_S3_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_S3_FUSE_S3_CLIENT_H_

 #include <memory>
 #include <string>
@ -31,7 +31,7 @@
 #include <utility>

 #include "curvefs/src/client/fuse_client.h"
-#include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
 #include "curvefs/src/volume/common.h"
 #include "src/common/s3_adapter.h"
@ -47,11 +47,13 @@ class WarmupManager;
 class WarmupManagerS3Impl;
 }  // namespace warmup

-
+// s3 client
 class FuseS3Client : public FuseClient {
 public:
    FuseS3Client()
-        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {
+        : FuseClient(),
+          s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()),
+          kvClientManager_(nullptr) {
        auto readFunc = [this](fuse_req_t req, fuse_ino_t ino, size_t size,
                               off_t off, struct fuse_file_info *fi,
                               char *buffer, size_t *rSize) {
@ -66,22 +68,25 @@ class FuseS3Client : public FuseClient {
                 const std::shared_ptr<MetaServerClient> &metaClient,
                 const std::shared_ptr<InodeCacheManager> &inodeManager,
                 const std::shared_ptr<DentryCacheManager> &dentryManager,
-                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor,
+                 const std::shared_ptr<StorageAdaptor> &s3Adaptor,
                 const std::shared_ptr<warmup::WarmupManager> &warmupManager)
        : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
                     warmupManager),
-          s3Adaptor_(s3Adaptor) {}
+          s3Adaptor_(s3Adaptor),
+          kvClientManager_(nullptr) {}

    CURVEFS_ERROR Init(const FuseClientOption &option) override;

    void UnInit() override;

+/*** fuse op ***/
+
    CURVEFS_ERROR FuseOpInit(
        void *userdata, struct fuse_conn_info *conn) override;

    CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
        const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, FileOut* fileOut) override;
+        struct fuse_file_info *fi, size_t *wSize) override;

    CURVEFS_ERROR FuseOpRead(fuse_req_t req,
        fuse_ino_t ino, size_t size, off_t off,
@ -89,25 +94,17 @@ class FuseS3Client : public FuseClient {
        char *buffer,
        size_t *rSize) override;

-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
-                               fuse_ino_t parent,
-                               const char* name,
-                               mode_t mode,
-                               struct fuse_file_info *fi,
-                               EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, struct fuse_file_info *fi,
+        fuse_entry_param *e) override;

-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
-                              fuse_ino_t parent,
-                              const char* name,
-                              mode_t mode,
-                              dev_t rdev,
-                              EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, dev_t rdev,
+        fuse_entry_param *e) override;

-    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
-                             fuse_ino_t ino,
-                             fuse_ino_t newparent,
-                             const char *newname,
-                             EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+        fuse_ino_t newparent, const char *newname,
+        fuse_entry_param *e) override;

    CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
        const char *name) override;
@ -118,23 +115,22 @@ class FuseS3Client : public FuseClient {
    CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
        struct fuse_file_info *fi) override;

-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
 private:
    bool InitKVCache(const KVClientManagerOpt &opt);
+    CURVEFS_ERROR InitStorageAdaptor(const FuseClientOption &option);
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;

    void FlushData() override;

 private:
-    // s3 adaptor
-    std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
+    // s3 storage adaptor
+    std::shared_ptr<StorageAdaptor> s3Adaptor_;
+    // kv client manager
    std::shared_ptr<KVClientManager> kvClientManager_;
-
-    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };


 }  // namespace client
 }  // namespace curvefs

-#endif  // CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
+#endif  // CURVEFS_SRC_CLIENT_S3_FUSE_S3_CLIENT_H_
--- a/curvefs/src/client/volume/client_volume_adaptor.cpp
+++ b/curvefs/src/client/volume/client_volume_adaptor.cpp
@ -0,0 +1,182 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*
+ * Project: curve
+ * Created Date: Thur March 14 2023
+ * Author: wuhongsong
+ */
+
+#include <utility>
+
+#include "curvefs/src/client/volume/client_volume_adaptor.h"
+
+#include "curvefs/src/client/rpcclient/mds_client.h"
+
+namespace curvefs {
+namespace client {
+
+#define VOLUME_BLOCK_SIZE 4194304
+#define VOLUME_CHUNK_SIZE 67108864
+
+CURVEFS_ERROR VolumeClientAdaptorImpl::Init(const FuseClientOption &option,
+         std::shared_ptr<InodeCacheManager> inodeManager,
+         std::shared_ptr<MdsClient> mdsClient,
+         std::shared_ptr<FsCacheManager> fsCacheManager,
+         std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+         std::shared_ptr<KVClientManager> kvClientManager,
+         std::shared_ptr<FsInfo> fsInfo) {
+    LOG(INFO) << "volume adaptor init start.";
+    volOpts_ = option.volumeOpt;
+    SetBlockSize(VOLUME_BLOCK_SIZE);
+    SetChunkSize(VOLUME_CHUNK_SIZE);
+    auto ret = StorageAdaptor::Init(option, inodeManager,
+      mdsClient, fsCacheManager, diskCacheManagerImpl, kvClientManager, fsInfo);
+    if (ret != CURVEFS_ERROR::OK) {
+        return ret;
+    }
+    BlockDeviceClientOptions opts;
+    opts.configPath = option.bdevOpt.configPath;
+    auto ret2 = blockDeviceClient_->Init(opts);
+    if (!ret2) {
+        LOG(ERROR) << "Init block device client failed, " << ret2;
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    LOG(INFO) << "volume adaptor init sucess.";
+    return ret;
+}
+
+CURVEFS_ERROR VolumeClientAdaptorImpl::FuseOpInit(void *userdata,
+  struct fuse_conn_info *conn) {
+    LOG(INFO) << "volume fuse op init start!";
+    const auto &vol = fsInfo_->detail().volume();
+    const auto &volName = vol.volumename();
+    const auto &user = vol.user();
+    auto ret = blockDeviceClient_->Open(volName, user);
+    if (!ret) {
+        LOG(ERROR) << "BlockDeviceClientImpl open failed, ret = " << ret
+                   << ", volName = " << volName << ", user = " << user;
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    SpaceManagerOption option;
+    option.blockGroupManagerOption.fsId = fsInfo_->fsid();
+
+    option.blockGroupManagerOption.owner = GetMountOwner();
+
+    option.blockGroupManagerOption.blockGroupAllocateOnce =
+        volOpts_.allocatorOption.blockGroupOption.allocateOnce;
+    option.blockGroupManagerOption.blockGroupSize =
+        fsInfo_->detail().volume().blockgroupsize();
+    option.blockGroupManagerOption.blockSize =
+        fsInfo_->detail().volume().blocksize();
+
+    option.allocatorOption.type = volOpts_.allocatorOption.type;
+    option.allocatorOption.bitmapAllocatorOption.sizePerBit =
+        volOpts_.allocatorOption.bitmapAllocatorOption.sizePerBit;
+    option.allocatorOption.bitmapAllocatorOption.smallAllocProportion =
+        volOpts_.allocatorOption.bitmapAllocatorOption.smallAllocProportion;
+
+    option.threshold = volOpts_.threshold;
+    option.releaseInterSec = volOpts_.releaseInterSec;
+
+    spaceManager_ = absl::make_unique<SpaceManagerImpl>(
+        option, mdsClient_, blockDeviceClient_);
+    spaceManager_->Run();
+
+    storage_ = absl::make_unique<DefaultVolumeStorage>(spaceManager_.get(),
+      blockDeviceClient_.get(), GetInodeCacheManager().get());
+
+    ExtentCacheOption extentOpt;
+    extentOpt.blockSize = vol.blocksize();
+    extentOpt.sliceSize = vol.slicesize();
+
+    ExtentCache::SetOption(extentOpt);
+    LOG(INFO) << "volume fuse op init sucess.";
+    return CURVEFS_ERROR::OK;
+}
+
+int VolumeClientAdaptorImpl::Stop() {
+    LOG(INFO) << "volume adaptor stop...";
+    StorageAdaptor::Stop();
+    if (nullptr != storage_) {
+        storage_->Shutdown();
+    }
+    if (nullptr != spaceManager_) {
+        spaceManager_->Shutdown();
+    }
+    blockDeviceClient_->UnInit();
+    LOG(INFO) << "volume adaptor stop sucess.";
+    return 0;
+}
+
+CURVEFS_ERROR VolumeClientAdaptorImpl::FlushDataCache(
+  const UperFlushRequest& req, uint64_t* writeOffset) {
+    uint64_t inodeId = req.inodeId;
+    uint64_t inodeOffset = req.offset;
+    uint64_t len = req.length;
+    const char* data = req.buf;
+    VLOG(9) << "volume flush dataCache, inode: "
+       << inodeId<< ", offset: " << inodeOffset
+       << ", length: " << len;
+
+    CURVEFS_ERROR ret = storage_->Write(inodeId, inodeOffset, len, data);
+    if (ret != CURVEFS_ERROR::OK) {
+        VLOG(0) << "volume flush dataCache err, inode: "
+            << inodeId<< ", offset: " << inodeOffset
+            << ", length: " << len;
+        return ret;
+    }
+
+    VLOG(9) << "volume flush dataCache end, inode: "
+       << inodeId<< ", offset: " << inodeOffset
+       << ", length: " << len;
+// 这个正确么？
+    *writeOffset = len;
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR VolumeClientAdaptorImpl::ReadFromLowlevel(
+  UperReadRequest uperRequest) {
+    uint64_t inodeId = uperRequest.inodeId;
+    VLOG(9) << "read lowlevel start, inodeId is: " << inodeId;
+    std::vector<ReadRequest> requests;
+    char *buf = uperRequest.buf;;
+    requests = std::move(uperRequest.requests);
+    uint64_t chunkSize = GetChunkSize();
+    CURVEFS_ERROR ret;
+    for (auto req : requests) {
+        VLOG(9) << "read from storage " << req.DebugString();
+        uint64_t len = req.len;
+        uint64_t readOffset = chunkSize * req.index + req.chunkPos;
+        ret = storage_->Read(
+          inodeId, readOffset, len, buf + req.bufOffset);
+        if (ret != CURVEFS_ERROR::OK) {
+            VLOG(0) << "volume flush dataCache err, inode: "
+                << inodeId << ", offset: " << readOffset
+                << ", length: " << len;
+            return ret;
+        }
+    }
+
+    VLOG(9) << "read lowlevel end, inodeId is: "<< inodeId;
+    return ret;
+}
+
+}  // namespace client
+}  // namespace curvefs
--- a/curvefs/src/client/volume/client_volume_adaptor.h
+++ b/curvefs/src/client/volume/client_volume_adaptor.h
@ -0,0 +1,145 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Created Date: Thur March 14 2023
+ * Author: wuhongsong
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_VOLUME_CLIENT_VOLUME_ADAPTOR_H_
+#define CURVEFS_SRC_CLIENT_VOLUME_CLIENT_VOLUME_ADAPTOR_H_
+
+#include <bthread/execution_queue.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "curvefs/proto/common.pb.h"
+#include "curvefs/proto/mds.pb.h"
+#include "curvefs/proto/metaserver.pb.h"
+
+#include "src/common/wait_interval.h"
+#include "curvefs/src/client/common/common.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/client_storage_adaptor.h"
+#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/rpcclient/mds_client.h"
+#include "curvefs/src/client/s3/client_s3.h"
+
+#include "curvefs/src/client/volume/default_volume_storage.h"
+#include "curvefs/src/client/volume/volume_storage.h"
+#include "curvefs/src/volume/block_device_client.h"
+#include "curvefs/src/volume/space_manager.h"
+
+namespace curvefs {
+namespace client {
+
+using common::VolumeOption;
+using ::curvefs::volume::BlockDeviceClient;
+using ::curvefs::volume::BlockDeviceClientImpl;
+using ::curvefs::volume::BlockDeviceClientOptions;
+using ::curvefs::volume::SpaceManager;
+using ::curvefs::volume::SpaceManagerImpl;
+using ::curvefs::volume::SpaceManagerOption;
+
+// client use volume internal interface
+class VolumeClientAdaptorImpl : public StorageAdaptor {
+ public:
+    VolumeClientAdaptorImpl() : StorageAdaptor(),
+      blockDeviceClient_(std::make_shared<BlockDeviceClientImpl>()) {}
+
+    explicit VolumeClientAdaptorImpl(const std::shared_ptr<
+      BlockDeviceClient> &blockDeviceClient) : StorageAdaptor(),
+      blockDeviceClient_(blockDeviceClient) {}
+
+    virtual ~VolumeClientAdaptorImpl() {
+        LOG(INFO) << "delete VolumeClientAdaptorImpl";
+    }
+
+ public:
+    /// @brief init volume storage adaptor
+    /// @param option fuse client option
+    /// @param inodeManager inode cache manager
+    /// @param mdsClient mds client
+    /// @param fsCacheManager fscache manager
+    /// @param diskCacheManagerImpl disk cache manager
+    /// @param kvClientManager kv client manager
+    /// @param fsInfo file system information
+    /// @return error code
+    CURVEFS_ERROR Init(const FuseClientOption &option,
+      std::shared_ptr<InodeCacheManager> inodeManager,
+      std::shared_ptr<MdsClient> mdsClient,
+      std::shared_ptr<FsCacheManager> fsCacheManager,
+      std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+      std::shared_ptr<KVClientManager> kvClientManager,
+      std::shared_ptr<FsInfo> fsInfo) override;
+
+    /// @brief fuse op init
+    /// @param userdata fuse user data
+    /// @param conn fuse connect info
+    /// @param fsid  fs id
+    /// @param fsname fs name
+    /// @return error code
+    CURVEFS_ERROR FuseOpInit(void *userdata,
+      struct fuse_conn_info *conn) override;
+
+    int Stop() override;
+
+    /// @brief read data from volume storage
+    /// @param request read request
+    /// @return error code
+    CURVEFS_ERROR FlushDataCache(const UperFlushRequest& req,
+      uint64_t* writeOffset) override;
+
+    /// @brief read data from volume storage
+    /// @param request read request
+    /// @return error code
+    CURVEFS_ERROR ReadFromLowlevel(UperReadRequest request) override;
+
+    // TODO(@hzwuhongsong)
+    virtual CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper,
+      uint64_t size) { return CURVEFS_ERROR::OK;}
+
+    /// @brief get volume storage
+    /// @return volume storage
+    std::shared_ptr<VolumeStorage> getUnderStorage() {
+        return storage_;
+    }
+
+    /// @brief get space manager
+    /// @return space manager
+    std::shared_ptr<SpaceManager> getSpaceManager() {
+        return spaceManager_;
+    }
+
+ private:
+    // block device client(write/read data from volume)
+    std::shared_ptr<BlockDeviceClient> blockDeviceClient_;
+    // volume space manager
+    std::shared_ptr<SpaceManager> spaceManager_;
+    // volume storage adaptor
+    std::shared_ptr<VolumeStorage> storage_;
+    // volume option
+    VolumeOption volOpts_;
+};
+
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_VOLUME_CLIENT_VOLUME_ADAPTOR_H_
--- a/curvefs/src/client/volume/default_volume_storage.cpp
+++ b/curvefs/src/client/volume/default_volume_storage.cpp
@ -30,7 +30,7 @@
 #include <vector>

 #include "absl/meta/type_traits.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/volume/extent_cache.h"
@ -74,8 +74,7 @@ std::ostream& operator<<(std::ostream& os, const std::vector<IOPart>& iov) {
 CURVEFS_ERROR DefaultVolumeStorage::Write(uint64_t ino,
                                          off_t offset,
                                          size_t len,
-                                          const char* data,
-                                          FileOut* fileOut) {
+                                          const char* data) {
    std::shared_ptr<InodeWrapper> inodeWrapper;
    LatencyUpdater updater(&metric_.writeLatency);
    auto ret = inodeCacheManager_->GetInode(ino, inodeWrapper);
@ -117,7 +116,6 @@ CURVEFS_ERROR DefaultVolumeStorage::Write(uint64_t ino,
        }

        inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
-        inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
    }

    inodeCacheManager_->ShipToFlush(inodeWrapper);
@ -188,6 +186,8 @@ CURVEFS_ERROR DefaultVolumeStorage::Flush(uint64_t ino) {
        return CURVEFS_ERROR::OK;
    }

+    VLOG(9) << "volume storage flush: " << ino;
+
    LatencyUpdater updater(&metric_.flushLatency);
    std::shared_ptr<InodeWrapper> inodeWrapper;
    auto ret = inodeCacheManager_->GetInode(ino, inodeWrapper);
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
ilixiaocui	3527ef9491	fix bugs 1. fuse_volume_client just flush inode but do not flush data in Flush 2. Optimize blockgroup allocation logic: - fix the available group will always decrease in the case of insufficient allocation on the mds side - the blockgroup assigned by mds to the client is allowed to be smaller than the count required by the client - mds does not allocate blockgroups exceeding a certain water level 3. fix the issue where the cache of metaserver_client could only correspond to a single file system 4. Optimize log: add some vlog to print more details Signed-off-by: ilixiaocui <ilixiaocui@163.com>	2023-07-12 18:35:19 +08:00
hzwuhongsong	9f6244f29a	curvefs/client: fix s3 object will not be removed	2023-07-12 18:35:19 +08:00
hzwuhongsong	6669a1b3a4	curvefs/client: opt of bs for fs storage backend	2023-06-05 09:43:23 +08:00
ilixiaocui	65fae138b5	[WIP] Some bug fixes during testing Signed-off-by: ilixiaocui <ilixiaocui@163.com>	2023-05-31 16:35:59 +08:00
ilixiaocui	057cc60f5c	curvefs: support space deallocate for curvebs volume as backend Signed-off-by: ilixiaocui <ilixiaocui@163.com>	2023-05-31 16:35:59 +08:00