[feat] tools-v2: add snapshot copyset --all

Signed-off-by: montaguelhz <1443171175@qq.com>

.bazelrc

@@ -1,7 +1,10 @@
+build --verbose_failures
+
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
 build --incompatible_blacklisted_protos_requires_proto_info=false
 build --copt=-fdiagnostics-color=always
+
 run --copt=-fdiagnostics-color=always

.gitignore

@@ -147,3 +147,22 @@ tools-v2/proto/curvefs/*
 tools-v2/*/*.test
 tools-v2/__debug_bin
 tools-v2/vendor/
+
+.test
+.note
+.playground
+.dumpfile
+metastore_test.dat
+GPATH
+GRTAGS
+GTAGS
+core.*
+
+test/integration/*.conf
+test/integration/client/config/client.conf*
+test/integration/snapshotcloneserver/config/*.conf
+
+.pre-commit-config.yaml
+
+*.deb
+*.whl

.obm.cfg

@@ -0,0 +1,2 @@
+container_name: curve-build-playground-master
+container_image: opencurvedocker/curve-base:build-debian9

Makefile

@@ -1,6 +1,6 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build dep install image
+.PHONY: list build dep install image playground check test
 
 stor?=""
 prefix?= "$(PWD)/projects"
@@ -70,3 +70,12 @@ install:
 
 image:
 	@bash util/image.sh $(stor) $(tag) $(os)
+
+playground:
+	@bash util/playground.sh
+
+check:
+	@bash util/check.sh $(stor)
+
+test:
+	@bash util/test.sh $(stor) $(only)

WORKSPACE

@@ -75,11 +75,14 @@ bind(
 )
 
 #import the  gtest files.
-new_git_repository(
+http_archive(
     name = "com_google_googletest",
-    build_file = "//:thirdparties/gmock.BUILD",
-    remote = "https://github.com/google/googletest",
-    tag = "release-1.8.0",
+    urls = [
+        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
+        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
+    ],
+    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
+    strip_prefix = "googletest-release-1.12.1",
 )
 
 bind(
@@ -228,6 +231,24 @@ http_archive(
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
 
+# fmt
+http_archive(
+  name = "fmt",
+  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
+  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
+  strip_prefix = "fmt-9.1.0",
+  build_file = "//:thirdparties/fmt.BUILD",
+)
+
+# spdlog
+http_archive(
+  name = "spdlog",
+  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
+  strip_prefix = "spdlog-1.11.0",
+  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
+  build_file = "//:thirdparties/spdlog.BUILD",
+)
+
 # Bazel platform rules.
 http_archive(
     name = "platforms",
@@ -248,14 +269,14 @@ new_local_repository(
 http_archive(
     name = "hedron_compile_commands",
 
-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. 
+    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
     # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
     urls = [
         "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
         "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
     ],
     strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." 
+    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()

buildfs.sh

@@ -7,12 +7,22 @@ then
     exit
 fi
 
+if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
+then
+    bazelflags=''
+else
+    bazelflags='--copt -faligned-new'
+fi
+
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi
 
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
+--define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
+-DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
+
 if [ $? -ne 0 ]
 then
     echo "build curvefs failed"
@@ -34,4 +44,5 @@ then
     echo "mds_test failed"
     exit
 fi
-fi
+fi
+echo "end compile"

conf/chunkserver.conf

@@ -9,13 +9,20 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
+global.block_size=4096
+
 # clone chunk允许的最长location长度
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +115,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/chunkserver.conf.example

@@ -13,9 +13,12 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
+# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
+# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
+# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
+# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
+global.block_size=4096
 global.location_limit=3000
-# minimum alignment for io request
-global.min_io_alignment=512
 
 #
 # MDS settings
@@ -108,7 +111,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=false
+copyset.enable_odsync_when_open_chunkfile=true
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/client.conf

@@ -168,13 +168,6 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
 
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
-
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@@ -188,4 +181,3 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
-

conf/cs_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/mds.conf

@@ -197,6 +197,8 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
+# smallest read/write unit for volume, support |512| and |4096|
+mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -237,3 +239,16 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
+
+#
+## poolset rules
+#
+# for backward compatibility, rules are applied for select poolset when creating file
+#
+# for example
+#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
+#
+# when creating file reqeust doesn't have poolset, above rules are used to select poolset
+# - if filename is /dir1/file, then poolset1 is select
+# - if filename is /dir1/sub/file, then sub is select
+mds.poolset.rules=

conf/py_client.conf

@@ -153,10 +153,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

conf/snap_client.conf

@@ -159,10 +159,3 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096

curve-ansible/roles/generate_config/defaults/main.yml

@@ -164,7 +164,6 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
-chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -244,8 +243,6 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
-client_alignment_common: 512
-client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -14,8 +14,6 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
-# minimum alignment for io request
-global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -175,10 +175,3 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume={{ client_alignment_common }}
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume={{ client_alignment_clone }}

curvefs/conf/client.conf

@@ -58,6 +58,10 @@ executorOpt.maxRetryTimesBeforeConsiderSuspend=20
 # batch limit of get inode attr and xattr
 executorOpt.batchInodeAttrLimit=10000
 
+#### spaceserver
+spaceServer.spaceAddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
+spaceServer.rpcTimeoutMs=1000
+
 #### bdev
 # curve client's config file
 bdev.confPath=/etc/curve/client.conf
@@ -73,17 +77,7 @@ rpc.healthCheckIntervalSec=0
 
 #### fuseClient
 # TODO(xuchaojie): add unit
-fuseClient.attrTimeOut=1.0
-fuseClient.entryTimeOut=1.0
 fuseClient.listDentryLimit=65536
-fuseClient.flushPeriodSec=5
-fuseClient.maxNameLength=255
-fuseClient.iCacheLruSize=65536
-fuseClient.dCacheLruSize=1000000
-fuseClient.enableICacheMetrics=true
-fuseClient.enableDCacheMetrics=true
-fuseClient.lruTimeOutSec=60
-fuseClient.cto=true
 fuseClient.downloadMaxRetryTimes=3
 
 ### kvcache opt
@@ -110,7 +104,6 @@ fuseClient.maxDataSize=1024
 fuseClient.refreshDataIntervalSec=30
 fuseClient.warmupThreadsNum=10
 
-
 # the write throttle bps of fuseClient, default no limit
 fuseClient.throttle.avgWriteBytes=0
 # the write burst bps of fuseClient, default no limit
@@ -139,6 +132,32 @@ fuseClient.throttle.burstReadIops=0
 # the times that read burst Iops can continue, default 180s
 fuseClient.throttle.burstReadIopsSecs=180
 
+#### filesystem metadata
+# {
+# fs.disableXattr:
+#   if you want to get better metadata performance,
+#   you can mount fs with |fs.disableXattr| is true
+#
+# fs.lookupCache.negativeTimeoutSec:
+#   entry which not found will be cached if |timeout| > 0
+fs.cto=true
+fs.maxNameLength=255
+fs.disableXattr=false
+fs.accessLogging=true
+fs.kernelCache.attrTimeoutSec=3600
+fs.kernelCache.dirAttrTimeoutSec=3600
+fs.kernelCache.entryTimeoutSec=3600
+fs.kernelCache.dirEntryTimeoutSec=3600
+fs.lookupCache.negativeTimeoutSec=0
+fs.lookupCache.minUses=1
+fs.lookupCache.lruSize=100000
+fs.dirCache.lruSize=5000000
+fs.openFile.lruSize=65536
+fs.attrWatcher.lruSize=5000000
+fs.rpc.listDentryLimit=65536
+fs.deferSync.delay=3
+fs.deferSync.deferDirMtime=false
+# }
 
 #### volume
 volume.bigFileSize=1048576
@@ -158,15 +177,6 @@ volume.bitmapAllocator.smallAllocProportion=0.2
 # number of block groups that allocated once
 volume.blockGroup.allocateOnce=4
 
-## spaceserver
-# the space used by the blockgroup exceeds this percentage and can
-# be returned to mds [0.8-1]
-volume.space.useThreshold=0.95
-
-# the background thread calculates the time interval for returning
-# the blockgroup to mds
-volume.space.releaseInterSec=300
-
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false

curvefs/conf/curvebs_client.conf

@@ -1,191 +0,0 @@
-#
-################### mds一侧配置信息 ##################
-#
-
-# mds的地址信息，对于mds集群，地址以逗号隔开
-mds.listen.addr=127.0.0.1:6666
-
-# 初始化阶段向mds注册开关，默认为开
-mds.registerToMDS=true
-
-# 与mds通信的rpc超时时间
-mds.rpcTimeoutMS=500
-
-# 与mds通信rpc最大的超时时间, 指数退避的超时间不能超过这个值
-mds.maxRPCTimeoutMS=2000
-
-# 与mds通信重试总时间
-mds.maxRetryMS=8000
-
-# 在当前mds上连续重试次数超过该限制就切换, 这个失败次数包含超时重试次数
-mds.maxFailedTimesBeforeChangeMDS=2
-
-# 与MDS一侧保持一个lease时间内多少次续约
-mds.refreshTimesPerLease=4
-
-# mds RPC接口每次重试之前需要先睡眠一段时间
-mds.rpcRetryIntervalUS=100000
-
-# The normal retry times for trigger wait strategy
-mds.normalRetryTimesBeforeTriggerWait=3
-
-# Max retry time for IO-Path request
-mds.maxRetryMsInIOPath=86400000
-
-# Sleep interval for wait
-mds.waitSleepMs=10000
-
-#
-################# metacache配置信息 ################
-#
-
-# 获取leader的rpc超时时间
-metacache.getLeaderTimeOutMS=500
-
-# 获取leader的重试次数
-metacache.getLeaderRetry=5
-
-# 获取leader接口每次重试之前需要先睡眠一段时间
-metacache.rpcRetryIntervalUS=100000
-
-#
-############### 调度层的配置信息 #############
-#
-
-# 调度层队列大小，每个文件对应一个队列
-# 调度队列的深度会影响client端整体吞吐，这个队列存放的是异步IO任务。。
-schedule.queueCapacity=1000000
-
-# 队列的执行线程数量
-# 执行线程所要做的事情就是将IO取出，然后发到网络就返回取下一个网络任务。一个任务从
-# 队列取出到发送完rpc请求大概在(20us-100us)，20us是正常情况下不需要获取leader的时候
-# 如果在发送的时候需要获取leader，时间会在100us左右，一个线程的吞吐在10w-50w
-# 性能已经满足需求
-schedule.threadpoolSize=2
-
-# 为隔离qemu侧线程引入的任务队列，因为qemu一侧只有一个IO线程
-# 当qemu一侧调用aio接口的时候直接将调用push到任务队列就返回，
-# 这样libcurve不占用qemu的线程，不阻塞其异步调用
-isolation.taskQueueCapacity=1000000
-
-# 隔离qemu线程的任务队列线程池大小, 默认值为1个线程
-isolation.taskThreadPoolSize=1
-
-
-#
-################ 与chunkserver通信相关配置 #############
-#
-# 读写接口失败的OP之间重试睡眠
-chunkserver.opRetryIntervalUS=100000
-
-# 失败的OP重试次数
-chunkserver.opMaxRetry=2500000
-
-# 与chunkserver通信的rpc超时时间
-chunkserver.rpcTimeoutMS=1000
-
-# 开启基于appliedindex的读，用于性能优化
-chunkserver.enableAppliedIndexRead=1
-
-# 重试请求之间睡眠最长时间
-# 因为当网络拥塞的时候或者chunkserver出现过载的时候，需要增加睡眠时间
-# 这个时间最大为maxRetrySleepIntervalUs
-chunkserver.maxRetrySleepIntervalUS=8000000
-
-# 重试请求的超时rpc时间最大值，超时时间会遵循指数退避策略
-# 因为当网络拥塞的时候出现超时，需要增加RPC超时时间
-# 这个时间最大为maxTimeoutMS
-chunkserver.maxRPCTimeoutMS=8000
-
-# 同一个chunkserver连续超时上限次数
-# 如果超过这个值，就会进行健康检查，健康检查失败后，会标记为unstable
-chunkserver.maxStableTimeoutTimes=10
-# chunkserver上rpc连续超时后，健康检查请求的超时间
-chunkserver.checkHealthTimeoutMs=100
-# 同一个server上unstable的chunkserver数量超过这个值之后
-# 所有的chunkserver都会标记为unstable
-chunkserver.serverStableThreshold=3
-
-# 当底层chunkserver压力大时，可能也会触发unstable
-# 由于copyset leader may change，会导致请求超时时间设置为默认值，从而导致IO hang
-# 真正宕机的情况下，请求重试一定次数后会处理完成
-# 如果一直重试，则不是宕机情况，这时候超时时间还是要进入指数退避逻辑
-# 当一个请求重试次数超过这个值时，其超时时间一定进入指数退避
-chunkserver.minRetryTimesForceTimeoutBackoff=5
-
-# 当一个rpc重试超过次数maxRetryTimesBeforeConsiderSuspend的时候
-# 记为悬挂IO，metric会报警
-chunkserver.maxRetryTimesBeforeConsiderSuspend=20
-
-#
-################# 文件级别配置项 #############
-#
-# libcurve底层rpc调度允许最大的未返回rpc数量，每个文件的inflight RPC独立
-global.fileMaxInFlightRPCNum=128
-
-# 文件IO下发到底层chunkserver最大的分片KB
-global.fileIOSplitMaxSizeKB=64
-
-#
-################# log相关配置 ###############
-#
-# enable logging or not
-global.logging.enable=True
-#
-# log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
-global.logLevel=0
-# 设置log的路径
-global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ /curvebs/client/logs __CURVEADM_TEMPLATE__
-# 单元测试情况下
-# logpath=./runlog/
-
-#
-################# 读源卷相关配置 ###############
-#
-# 读取源卷时打开的fd超时关闭时间300s
-closefd.timeout=300
-# 读取源卷时打开的fd后台线程每600s扫描一遍fdMap，关闭超时fd
-closefd.timeInterval=600
-
-#
-############### metric 配置信息 #############
-#
-global.metricDummyServerStartPort=9000
-
-# 是否关闭健康检查: true/关闭 false/不关闭
-global.turnOffHealthCheck=true
-
-#
-### throttle config
-#
-throttle.enable=false
-
-##### discard configurations #####
-# enable/disable discard
-discard.enable=true
-# discard granularity
-discard.granularity=4096
-# discard cleanup task delay times in millisecond
-discard.taskDelayMs=60000
-
-##### alignment #####
-# default alignment
-global.alignment.commonVolume=512
-# alignment for clone volume
-# default is 4096, because lazy clone chunk bitmap granularity is 4096
-global.alignment.cloneVolume=4096
-
-##### chunkserver client option #####
-# chunkserver client rpc timeout time
-csClientOpt.rpcTimeoutMs=500
-# chunkserver client rpc max try
-csClientOpt.rpcMaxTry=86400000
-# chunkserver client rpc retry interval
-csClientOpt.rpcIntervalUs=100000
-# chunkserver client rpc max timeout time
-csClientOpt.rpcMaxTimeoutMs=8000
-
-##### chunkserver broadcaster option #####
-# broad cast max machine num
-csBroadCasterOpt.broadCastMaxNum=200
-

curvefs/conf/mds.conf

@@ -6,6 +6,9 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
 mds.loglevel=0
+# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+mds.server.idleTimeoutSec=-1
 
 #
 # space options
@@ -159,6 +162,3 @@ bs.mds.maxFailedTimesBeforeChangeMDS=2
 bs.mds.normalRetryTimesBeforeTriggerWait=3
 # sleep interval in ms for wait
 bs.mds.waitSleepMs=1000
-
-#### Options for volume space deallcatable setting
-mds.space.calIntervalSec=60

curvefs/conf/metaserver.conf

@@ -128,16 +128,25 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000
 
-### apply queue options for each copyset
-### apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding worker queue by certain rules
-# number of apply queue workers for each, each worker will start a indepent thread
-applyqueue.worker_count=3
+#
+# Concurrent apply queue
+### concurrent apply queue options for each copyset
+### concurrent apply queue is used to isolate raft threads, each worker has its own queue
+### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
 
-# apply queue depth for each copyset
+# worker_count: number of apply queue workers for each, each worker will start a indepent thread
+# queue_depth: apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-applyqueue.queue_depth=1
+# write apply queue workers count
+applyqueue.write_worker_count=3
+# write apply queue depth
+applyqueue.write_queue_depth=1
+# read apply queue workers count
+applyqueue.read_worker_count=2
+# read apply queue depth
+applyqueue.read_queue_depth=1
+
 
 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@@ -145,6 +154,10 @@ applyqueue.queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto
 
+# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
+# Default value is -1 which disables the feature.
+server.idleTimeoutSec=-1
+
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need
@@ -307,9 +320,3 @@ metaCacheOpt.metacacheGetLeaderRetry=3
 metaCacheOpt.metacacheRPCRetryIntervalUS=100000
 # RPC timeout of get leader
 metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000
-
-#### volume deallocate
-volume.deallocate.enable=true
-volume.deallocate.workerNum=5
-volume.deallocate.batchClean=10
-volume.sdk.confPath=/etc/curvefs/curvebs_client.conf

curvefs/proto/BUILD

@@ -113,7 +113,6 @@ proto_library(
     name = "curvefs_heartbeat_proto",
     srcs = ["heartbeat.proto"],
     deps = [":curvefs_common_proto",
-            ":metaserver_proto",
             "//proto:heartbeat_proto"],
 )
 

curvefs/proto/common.proto

@@ -31,13 +31,6 @@ enum BitmapLocation {
     AtEnd = 2;
 }
 
-message EmptyMsg {}
-
-message BlockGroupID {
-    required uint64 fsId = 1;
-    required uint64 offset = 2;
-}
-
 // When creating fs, `volumeSize` and `extendAlignment` are fetched from the bs cluster
 message Volume {
     optional uint64 volumeSize = 1;

curvefs/proto/heartbeat.proto

@@ -16,7 +16,6 @@
 
 syntax = "proto2";
 import "curvefs/proto/common.proto";
-import "curvefs/proto/metaserver.proto";
 import "proto/heartbeat.proto";
 package curvefs.mds.heartbeat;
 option cc_generic_services = true;
@@ -28,17 +27,6 @@ option go_package = "curvefs/proto/heartbeat";
 //     required uint64 diskUsedByte = 2;  // the disk this copyset used
 // };
 
-enum BlockGroupDeallcateStatusCode {
-    BGDP_PROCESSING = 0;
-    BGDP_DONE = 1;
-}
-
-message BlockGroupStatInfo {
-    required uint32 fsId = 1;
-    repeated metaserver.DeallocatableBlockGroup deallocatableBlockGroups = 2;
-    map<uint64, BlockGroupDeallcateStatusCode> blockGroupDeallocateStatus = 3;
-}
-
 message CopySetInfo {
     required uint32 poolId = 1;
     required uint32 copysetId = 2;
@@ -93,7 +81,6 @@ message MetaServerHeartbeatRequest {
     required uint32 leaderCount = 7;
     required uint32 copysetCount = 8;
     required MetaServerSpaceStatus spaceStatus = 9;
-    repeated BlockGroupStatInfo blockGroupStatInfos = 10;
 };
 
 message CopySetConf {
@@ -120,14 +107,11 @@ enum HeartbeatStatusCode {
     hbMetaServerIpPortNotMatch = 2;
     hbMetaServerTokenNotMatch = 3;
     hbAnalyseCopysetError = 4;
-    hbMetaServerFSUnkown = 5;
 }
 
 message MetaServerHeartbeatResponse {
     required HeartbeatStatusCode statusCode = 1;
     repeated CopySetConf needUpdateCopysets = 2;
-    // key is fsid, value is blockgroup offset
-    map<uint64, uint64> issuedBlockGroups = 3;
 };
 
 service HeartbeatService {

curvefs/proto/mds.proto

@@ -60,7 +60,6 @@ enum FSStatusCode {
     INSERT_MANAGE_INODE_FAIL = 35;
     DELETE_DENTRY_FAIL = 36;
     UPDATE_FS_FAIL = 37;
-    SPACE_RELEASE_FAIL = 38;
 }
 
 // fs interface

curvefs/proto/metaserver.proto

@@ -167,46 +167,6 @@ enum FsFileType {
     TYPE_S3 = 4;
 };
 
-message DeallocatableBlockGroup {
-    required uint64 blockGroupOffset = 1;
-    optional uint64 deallocatableSize = 2;
-    repeated uint64 inodeIdlist = 3;
-    repeated uint64 inodeIdUnderDeallocate = 4;
-
-    oneof type {
-        IncreaseDeallocatableBlockGroup increase = 5;
-        DecreaseDeallocatableBlockGroup decrease = 6;
-        MarkDeallocatableBlockGroup mark = 7;
-    }
-}
-
-message IncreaseDeallocatableBlockGroup {
-    required uint64 increaseDeallocatableSize = 1;
-    repeated uint64 inodeIdlistAdd = 2;
-}
-
-message DecreaseDeallocatableBlockGroup {
-    required uint64 decreaseDeallocatableSize = 1;
-    repeated uint64 inodedDeallocated = 2;
-}
-
-message MarkDeallocatableBlockGroup {
-    repeated uint64 inodeIdUnderDeallocate = 2;
-}
-
-message UpdateDeallocatableBlockGroupRequest {
-    required uint32 poolId = 1;
-    required uint32 copysetId = 2;
-    required uint32 partitionId = 3;
-    required uint64 fsId = 4;
-    repeated DeallocatableBlockGroup update = 5;
-}
-
-message UpdateDeallocatableBlockGroupResponse {
-    required MetaStatusCode statusCode = 1; 
-    optional uint64 appliedIndex = 2;
-}
-
 message VolumeExtent {
     required uint64 fsOffset = 1;
     required uint64 volumeOffset = 2;
@@ -219,7 +179,7 @@ message VolumeExtentSlice {
     repeated VolumeExtent extents = 2;
 }
 
-message VolumeExtentSliceList {
+message VolumeExtentList {
     repeated VolumeExtentSlice slices = 1;
 }
 
@@ -357,7 +317,7 @@ message UpdateInodeRequest {
     map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
-    optional VolumeExtentSliceList volumeExtents = 23;
+    optional VolumeExtentList volumeExtents = 23;
 }
 
 message UpdateInodeResponse {
@@ -504,7 +464,7 @@ message GetVolumeExtentRequest {
 message GetVolumeExtentResponse {
     required MetaStatusCode statusCode = 1;
     optional uint64 appliedIndex = 2;
-    optional VolumeExtentSliceList slices = 3;
+    optional VolumeExtentList slices = 3;
 }
 
 message UpdateVolumeExtentRequest {
@@ -513,7 +473,7 @@ message UpdateVolumeExtentRequest {
     required uint32 partitionId = 3;
     required uint32 fsId = 4;
     required uint64 inodeId = 5;
-    required VolumeExtentSliceList extents = 6;
+    required VolumeExtentList extents = 6;
 }
 
 message UpdateVolumeExtentResponse {
@@ -548,7 +508,4 @@ service MetaServerService {
     // volume extent interface
     rpc GetVolumeExtent(GetVolumeExtentRequest) returns (GetVolumeExtentResponse);
     rpc UpdateVolumeExtent(UpdateVolumeExtentRequest) returns (UpdateVolumeExtentResponse);
-
-    // block group with deallocatable inode list interface
-    rpc UpdateDeallocatableBlockGroup(UpdateDeallocatableBlockGroupRequest) returns (UpdateDeallocatableBlockGroupResponse);
 }

curvefs/proto/space.proto

@@ -34,7 +34,6 @@ enum SpaceErrCode {
     SpaceErrParam = 10;
     SpaceErrNotSupport = 11;
     SpaceErrExtendVolumeError = 12;
-    SpaceErrRelease = 13;
 }
 
 message BlockGroup {
@@ -49,10 +48,6 @@ message BlockGroup {
     required common.BitmapLocation bitmaplocation = 4;
     // owner, who owns this block group
     optional string owner = 5;
-    // deallocating, metaserver who is deallocate this block group
-    repeated uint32 deallocating = 6;
-    // deallocated, metaserver who finish deallocate this block group
-    repeated uint32 deallocated = 7;
 }
 
 message AllocateBlockGroupRequest {

curvefs/src/client/BUILD

@@ -43,13 +43,10 @@ cc_library(
             "s3/*.h",
             "volume/*.cpp",
             "volume/*.h",
+            "filesystem/*.cpp",
+            "filesystem/*.h",
             "warmup/*.h",
             "warmup/*.cpp",
-            "cache/*.h",
-            "cache/*.cpp",
-            "cache/diskcache/*.h",
-            "cache/diskcache/*.cpp",
-
         ],
         exclude = ["main.cpp"],
     ),
@@ -76,11 +73,14 @@ cc_library(
         "//curvefs/src/volume",
         "//curvefs/src/common:metric_utils",
         "//curvefs/src/common:dynamic_vlog",
+        "//curvefs/src/common:threading",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/synchronization",
+        "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",
         "@com_google_googletest//:gtest_prod",
+        "@spdlog//:spdlog",
     ],
 )

curvefs/src/client/async_request_closure.cpp

@@ -28,7 +28,7 @@
 #include <mutex>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 namespace curvefs {
@@ -63,26 +63,19 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
         cond_.wait(lk);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(GetStatusCode());
+    return ToFSError(GetStatusCode());
 }
 
 void UpdateVolumeExtentClosure::Run() {
     auto st = GetStatusCode();
     if (!IsOK(st)) {
-        if (inode_ != nullptr) {
-            inode_->MarkInodeError();
-            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
-                       << MetaStatusCode_Name(st)
-                       << ", inodeid: " << inode_->GetInodeId();
-        } else {
-            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
-                       << MetaStatusCode_Name(st);
-        }
+        LOG(ERROR) << "UpdateVolumeExtent failed, error: "
+                   << MetaStatusCode_Name(st)
+                   << ", inodeid: " << inode_->GetInodeId();
+        inode_->MarkInodeError();
     }
 
-    if (inode_ != nullptr) {
-        inode_->syncingVolumeExtentsMtx_.unlock();
-    }
+    inode_->syncingVolumeExtentsMtx_.unlock();
 
     if (sync_) {
         std::lock_guard<bthread::Mutex> lk(mtx_);

curvefs/src/client/async_request_closure.h

@@ -28,12 +28,15 @@
 
 #include <memory>
 
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
+using ::curvefs::client::filesystem::ToFSError;
+
 class InodeWrapper;
 
 namespace internal {

curvefs/src/client/client_operator.cpp

@@ -24,13 +24,14 @@
 
 #include "src/common/uuid.h"
 #include "curvefs/src/client/client_operator.h"
-
+#include "curvefs/src/client/filesystem/error.h"
 namespace curvefs {
 namespace client {
 
 using ::curve::common::UUIDGenerator;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::mds::topology::PartitionTxId;
+using ::curvefs::client::filesystem::ToFSError;
 
 #define LOG_ERROR(action, rc) \
     LOG(ERROR) << action << " failed, retCode = " << rc \
@@ -95,7 +96,7 @@ CURVEFS_ERROR RenameOperator::GetTxId(uint32_t fsId,
     if (rc != MetaStatusCode::OK) {
         LOG_ERROR("GetTxId", rc);
     }
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::GetLatestTxIdWithLock() {
@@ -204,7 +205,7 @@ CURVEFS_ERROR RenameOperator::PrepareRenameTx(
         LOG_ERROR("PrepareRenameTx", rc);
     }
 
-    return MetaStatusCodeToCurvefsErrCode(rc);
+    return ToFSError(rc);
 }
 
 CURVEFS_ERROR RenameOperator::PrepareTx() {
@@ -418,8 +419,6 @@ CURVEFS_ERROR RenameOperator::UpdateInodeCtime() {
 }
 
 void RenameOperator::UpdateCache() {
-    dentryManager_->DeleteCache(parentId_, name_);
-    dentryManager_->InsertOrReplaceCache(newDentry_);
     SetTxId(srcPartitionId_, srcTxId_ + 1);
     SetTxId(dstPartitionId_, dstTxId_ + 1);
 }

curvefs/src/client/client_storage_adaptor.cpp

@@ -1,301 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-/*
- * Project: curve
- * Created Date: Thur March 14 2023
- * Author: wuhongsong
- */
-
-#include "curvefs/src/client/client_storage_adaptor.h"
-
-#include <brpc/channel.h>
-#include <brpc/controller.h>
-#include <algorithm>
-#include <list>
-
-#include "absl/memory/memory.h"
-
-
-#include "curvefs/src/common/s3util.h"
-
-namespace curvefs {
-namespace client {
-
-CURVEFS_ERROR
-StorageAdaptor::Init(const FuseClientOption &fuseOption,
-    std::shared_ptr<InodeCacheManager> inodeManager,
-    std::shared_ptr<MdsClient> mdsClient,
-    std::shared_ptr<FsCacheManager> fsCacheManager,
-    std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
-    std::shared_ptr<KVClientManager> kvClientManager,
-    std::shared_ptr<FsInfo> fsInfo) {
-        const S3ClientAdaptorOption option =
-          fuseOption.s3Opt.s3ClientAdaptorOpt;
-        LOG(INFO) << "StorageAdaptor Init. block size:" << blockSize_
-            << ", chunk size: " << chunkSize_
-            << ", intervalSec: " << option.intervalSec
-            << ", flushIntervalSec: " << option.flushIntervalSec
-            << ", writeCacheMaxByte: " << option.writeCacheMaxByte
-            << ", readCacheMaxByte: " << option.readCacheMaxByte
-            << ", nearfullRatio: " << option.nearfullRatio
-            << ", baseSleepUs: " << option.baseSleepUs;
-        pageSize_ = option.pageSize;
-        if (chunkSize_ % blockSize_ != 0) {
-            LOG(ERROR) << "chunkSize:" << chunkSize_
-                    << " is not integral multiple for the blockSize:"
-                    << blockSize_;
-            return CURVEFS_ERROR::INVALIDPARAM;
-        }
-        diskCacheType_ = option.diskCacheOpt.diskCacheType;
-        memCacheNearfullRatio_ = option.nearfullRatio;
-        throttleBaseSleepUs_ = option.baseSleepUs;
-        flushIntervalSec_ = option.flushIntervalSec;
-        chunkFlushThreads_ = option.chunkFlushThreads;
-        inodeManager_ = inodeManager;
-        mdsClient_ = mdsClient;
-        fsCacheManager_ = fsCacheManager;
-        waitInterval_.Init(option.intervalSec * 1000);
-        fsInfo_ = fsInfo;
-
-        if (nullptr != diskCacheManagerImpl) {
-            diskCacheManagerImpl_ = diskCacheManagerImpl;
-            if (diskCacheManagerImpl_->Init(option) < 0) {
-                LOG(ERROR) << "Init disk cache failed";
-                return CURVEFS_ERROR::INTERNAL;
-            }
-        }
-
-        if (enableBgFlush_) {        // start background flush thread
-            toStop_.store(false, std::memory_order_release);
-            bgFlushThread_ = Thread(&StorageAdaptor::BackGroundFlush, this);
-        }
-
-        // start chunk flush threads
-        taskPool_.Start(chunkFlushThreads_);
-        LOG(INFO) << "storage Adaptor init success.";
-        return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR StorageAdaptor::FuseOpInit(void *userdata,
-  struct fuse_conn_info *conn) {
-    LOG(INFO) << " storage adaptor fuse init start.";
-    SetFsId(fsInfo_->fsid());
-    InitMetrics(fsInfo_->fsname());
-    return CURVEFS_ERROR::OK;
-}
-
-int StorageAdaptor::Stop() {
-    LOG(INFO) << " stop storage adaptor start.";
-    waitInterval_.StopWait();
-    toStop_.store(true, std::memory_order_release);
-    FsSyncSignal();
-    if (bgFlushThread_.joinable()) {
-        bgFlushThread_.join();
-    }
-    if (HasDiskCache()) {
-        diskCacheManagerImpl_->UmountDiskCache();
-    }
-    taskPool_.Stop();
-    LOG(INFO) << " stop storage adaptor success.";
-    return 0;
-}
-
-int StorageAdaptor::Write(uint64_t inodeId, uint64_t offset,
-  uint64_t length, const char *buf) {
-    VLOG(6) << "write start offset:" << offset << ", len:" << length
-            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
-    uint64_t start = butil::cpuwide_time_us();
-    FileCacheManagerPtr fileCacheManager =
-        fsCacheManager_->FindOrCreateFileCacheManager(fsId_, inodeId);
-    {
-        std::lock_guard<std::mutex> lockguard(ioMtx_);
-        fsCacheManager_->DataCacheByteInc(length);
-        uint64_t size = fsCacheManager_->GetDataCacheSize();
-        uint64_t maxSize = fsCacheManager_->GetDataCacheMaxSize();
-        if (size >= maxSize) {
-            VLOG(6) << "write cache is full, wait flush. size: " << size
-                    << ", maxSize:" << maxSize;
-            // offer to do flush
-            waitInterval_.StopWait();
-            fsCacheManager_->WaitFlush();
-        }
-    }
-    uint64_t memCacheRatio = fsCacheManager_->MemCacheRatio();
-    int64_t exceedRatio = memCacheRatio - memCacheNearfullRatio_;
-    if (exceedRatio > 0) {
-        // offer to do flush
-        waitInterval_.StopWait();
-        // upload to s3 derectly or cache disk full
-        bool needSleep =
-            (DisableDiskCache() || IsReadCache()) ||
-            (IsReadWriteCache() && diskCacheManagerImpl_->IsDiskCacheFull());
-        if (needSleep) {
-            uint32_t exponent = pow(2, (exceedRatio) / 10);
-            bthread_usleep(throttleBaseSleepUs_ * exceedRatio * exponent);
-            VLOG(6) << "write cache nearfull and use ratio is: "
-                    << memCacheRatio << ", exponent is: " << exponent;
-        }
-    }
-    int ret = fileCacheManager->Write(offset, length, buf);
-    fsCacheManager_->DataCacheByteDec(length);
-    if (nullptr != GetMetric()) {
-        CollectMetrics(&ioMetric_->adaptorWrite, ret, start);
-    }
-    VLOG(6) << "write end inodeId:" << inodeId << ",ret:" << ret;
-    return ret;
-}
-
-int StorageAdaptor::Read(uint64_t inodeId, uint64_t offset,
-  uint64_t length, char *buf) {
-    VLOG(6) << "read start offset:" << offset << ", len:" << length
-            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
-    uint64_t start = butil::cpuwide_time_us();
-    FileCacheManagerPtr fileCacheManager =
-        fsCacheManager_->FindOrCreateFileCacheManager(fsId_, inodeId);
-
-    int ret = fileCacheManager->Read(inodeId, offset, length, buf);
-    VLOG(6) << "read end inodeId:" << inodeId << ",ret:" << ret;
-    if (ret < 0) {
-        return ret;
-    }
-    if (nullptr != GetMetric()) {
-        CollectMetrics(&ioMetric_->adaptorRead, ret, start);
-    }
-    VLOG(6) << "read end offset:" << offset << ", len:" << length
-            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
-    return ret;
-}
-
-void StorageAdaptor::BackGroundFlush() {
-    while (!toStop_.load(std::memory_order_acquire)) {
-        {
-            std::unique_lock<std::mutex> lck(mtx_);
-            if (fsCacheManager_->GetDataCacheNum() == 0) {
-                VLOG(3) << "BackGroundFlush has no write cache, so wait";
-                cond_.wait(lck);
-            }
-        }
-        uint64_t start = butil::cpuwide_time_us();
-        if (fsCacheManager_->MemCacheRatio() > memCacheNearfullRatio_) {
-            VLOG(3) << "BackGroundFlush radically, write cache num is: "
-                    << fsCacheManager_->GetDataCacheNum()
-                    << "cache ratio is: " << fsCacheManager_->MemCacheRatio();
-            fsCacheManager_->FsSync(true);
-
-        } else {
-            waitInterval_.WaitForNextExcution();
-            VLOG(6) << "BackGroundFlush, write cache num is:"
-                    << fsCacheManager_->GetDataCacheNum()
-                    << "cache ratio is: " << fsCacheManager_->MemCacheRatio();
-            fsCacheManager_->FsSync(false);
-            VLOG(6) << "background fssync end";
-        }
-        if (nullptr != GetMetric()) {
-            CollectMetrics(&ioMetric_->adaptorBgFlush, 1, start);
-        }
-    }
-    return;
-}
-
-
-CURVEFS_ERROR StorageAdaptor::Flush(uint64_t inodeId) {
-    FileCacheManagerPtr fileCacheManager =
-        fsCacheManager_->FindFileCacheManager(inodeId);
-    if (nullptr == fileCacheManager) {
-        return CURVEFS_ERROR::OK;
-    }
-    VLOG(6) << "Flush data of inodeId:" << inodeId;
-    return fileCacheManager->Flush(true, false);
-}
-
-CURVEFS_ERROR StorageAdaptor::FlushAllCache(uint64_t inodeId) {
-    VLOG(6) << "FlushAllCache, inodeId:" << inodeId;
-    FileCacheManagerPtr fileCacheManager =
-        fsCacheManager_->FindFileCacheManager(inodeId);
-    if (!fileCacheManager) {
-        return CURVEFS_ERROR::OK;
-    }
-
-    // force flush data in memory to s3
-    VLOG(6) << "FlushAllCache, flush memory data of inodeId:" << inodeId;
-    CURVEFS_ERROR ret = fileCacheManager->Flush(true, false);
-    if (ret != CURVEFS_ERROR::OK) {
-        return ret;
-    }
-
-    // force flush data in diskcache to s3
-    if (HasDiskCache()) {
-        VLOG(6) << "FlushAllCache, wait inodeId:" << inodeId
-                << "related chunk upload to s3";
-        if (ClearDiskCache(inodeId) < 0) {
-            return CURVEFS_ERROR::INTERNAL;
-        }
-    }
-
-    return ret;
-}
-
-CURVEFS_ERROR StorageAdaptor::FsSync() {
-    return fsCacheManager_->FsSync(true);
-}
-
-int StorageAdaptor::ClearDiskCache(int64_t inodeId) {
-    // flush disk cache. read cache do not need clean
-    int ret =
-        diskCacheManagerImpl_->UploadWriteCacheByInode(std::to_string(inodeId));
-    LOG_IF(ERROR, ret < 0) << "FlushAllCache, inode:" << inodeId
-                           << ", upload write cache fail";
-    return ret;
-}
-
-void StorageAdaptor::ReleaseCache(uint64_t inodeId) {
-    FileCacheManagerPtr fileCacheManager =
-        fsCacheManager_->FindFileCacheManager(inodeId);
-    if (!fileCacheManager) {
-        return;
-    }
-    VLOG(9) << "ReleaseCache inode:" << inodeId;
-    fileCacheManager->ReleaseCache();
-    fsCacheManager_->ReleaseFileCacheManager(inodeId);
-    return;
-}
-
-void StorageAdaptor::Enqueue(
-  std::shared_ptr<FlushChunkCacheContext> context) {
-    auto task = [this, context]() {
-        this->FlushChunkClosure(context);
-    };
-    taskPool_.Enqueue(task);
-}
-
-int StorageAdaptor::FlushChunkClosure(
-  std::shared_ptr<FlushChunkCacheContext> context) {
-    VLOG(9) << "FlushChunkCacheClosure start: " << context->inode;
-    CURVEFS_ERROR ret = context->chunkCacheManptr->Flush(
-      context->inode, context->force);
-    // set the returned value
-    // it is need in FlushChunkCacheCallBack
-    context->retCode = ret;
-    context->cb(context);
-    VLOG(9) << "FlushChunkCacheClosure end: " << context->inode;
-    return 0;
-}
-
-}  // namespace client
-}  // namespace curvefs
-

curvefs/src/client/client_storage_adaptor.h

@@ -1,332 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-/*
- * Project: curve
- * Created Date: Thur March 14 2023
- * Author: wuhongsong
- */
-
-#ifndef CURVEFS_SRC_CLIENT_CLIENT_STORAGE_ADAPTOR_H_
-#define CURVEFS_SRC_CLIENT_CLIENT_STORAGE_ADAPTOR_H_
-
-#include <memory>
-#include <string>
-
-#include "curvefs/proto/common.pb.h"
-#include "curvefs/proto/mds.pb.h"
-#include "curvefs/proto/metaserver.pb.h"
-
-#include "curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h"
-#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
-#include "curvefs/src/client/common/common.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/error_code.h"
-#include "curvefs/src/common/define.h"
-#include "curvefs/src/client/inode_cache_manager.h"
-#include "curvefs/src/client/metric/client_metric.h"
-#include "curvefs/src/client/rpcclient/mds_client.h"
-#include "curvefs/src/client/s3/client_s3.h"
-
-using ::curve::common::Thread;
-using ::curve::common::TaskThreadPool;
-using curvefs::client::common::FuseClientOption;
-using curvefs::client::common::DiskCacheType;
-using curvefs::client::metric::IoMetric;
-using curvefs::client::rpcclient::MdsClient;
-using curvefs::metaserver::Inode;
-
-namespace curvefs {
-namespace client {
-
-class DiskCacheManagerImpl;
-class FlushChunkCacheContext;
-class ChunkCacheManager;
-
-// callback function for FlushChunkCache
-using FlushChunkCacheCallBack = std::function<
-  void(const std::shared_ptr<FlushChunkCacheContext>&)>;
-
-
-/// @brief flush chunk cache context
-/// @param inode inode id
-/// @param chunkCacheManptr chunk cache manager
-/// @param force force flush memory cache
-/// @param cb callback function
-/// @param retCode error code
-struct FlushChunkCacheContext {
-    uint64_t inode;
-    ChunkCacheManagerPtr chunkCacheManptr;
-    bool force;
-    FlushChunkCacheCallBack cb;
-    CURVEFS_ERROR retCode;
-};
-
-// the base class of the underlying storage adaptation layer
-class StorageAdaptor {
- public:
-    StorageAdaptor() {}
-    virtual ~StorageAdaptor() {
-        Stop();
-    }
-
-    /// @brief init storage adaptor
-    /// @param option fuse client option
-    /// @param inodeManager inode cache manager
-    /// @param mdsClient mds client
-    /// @param fsCacheManager fscache manager
-    /// @param diskCacheManagerImpl disk cache manager
-    /// @param kvClientManager kv client manager
-    /// @param fsInfo file system information
-    /// @return error code
-    virtual CURVEFS_ERROR
-    Init(const FuseClientOption &option,
-        std::shared_ptr<InodeCacheManager> inodeManager,
-        std::shared_ptr<MdsClient> mdsClient,
-        std::shared_ptr<FsCacheManager> fsCacheManager,
-        std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
-        std::shared_ptr<KVClientManager> kvClientManager,
-        std::shared_ptr<FsInfo> fsInfo);
-
-    virtual CURVEFS_ERROR FuseOpInit(void *userdata,
-        struct fuse_conn_info *conn);
-
-    virtual int Stop();
-
-    /// @brief flush data cache to backend storage(s3 or volume)
-    /// @param req flush request
-    /// @param writeOffset length that has been written to backend storage
-    /// @return error code
-    virtual CURVEFS_ERROR FlushDataCache(const UperFlushRequest& req,
-      uint64_t* writeOffset) = 0;
-
-    /// @brief read data from backend storage(s3 or volume)
-    /// @param request read request
-    /// @return error code
-    virtual CURVEFS_ERROR ReadFromLowlevel(UperReadRequest request) = 0;
-
-    /// @brief wrire data to memory cache
-    /// @param inodeId file inode id
-    /// @param offset offset of data in file
-    /// @param length length of data
-    /// @param buf data need to write
-    /// @return length of data that has been written to memory cache
-    virtual int Write(uint64_t inodeId, uint64_t offset,
-      uint64_t length, const char *buf);
-
-    /// @brief read data from memory cache
-    /// @param inodeId file inode id
-    /// @param offset offset of data in file
-    /// @param length length need to read
-    /// @param buf data buffer
-    /// @return length of data that has been read from memory cache
-    virtual int Read(uint64_t inodeId, uint64_t offset,
-      uint64_t length, char *buf);
-
-    virtual CURVEFS_ERROR Truncate(InodeWrapper* inodeWrapper,
-      uint64_t size) = 0;
-
-    /// @brief allocate chunk id from mds
-    /// @param fsId file system id
-    /// @param idNum chunks number need to allocate
-    /// @param chunkId chunk id that has been allocated
-    /// @return error code
-    FSStatusCode AllocChunkId(uint32_t fsId,
-      uint32_t idNum, uint64_t *chunkId) {
-        return mdsClient_->AllocS3ChunkId(fsId, idNum, chunkId);
-    }
-
-    /// @brief release all cache of file
-    /// @param inodeId file inode id
-    void ReleaseCache(uint64_t inodeId);
-
-    virtual CURVEFS_ERROR Flush(uint64_t inodeId);
-    virtual CURVEFS_ERROR FlushAllCache(uint64_t inodeId);
-    virtual CURVEFS_ERROR FsSync();
-
-    /// @brief enqueue flush chunk cache context
-    void Enqueue(std::shared_ptr<FlushChunkCacheContext> context);
-
-    void FsSyncSignal() {
-        std::lock_guard<std::mutex> lk(mtx_);
-        VLOG(3) << "fs sync signal";
-        cond_.notify_one();
-    }
-
-    void FsSyncSignalAndDataCacheInc() {
-        std::lock_guard<std::mutex> lk(mtx_);
-        fsCacheManager_->DataCacheNumInc();
-        VLOG(3) << "fs sync signal";
-        cond_.notify_one();
-    }
-
-/*** get and set element ***/
-
-    void SetFsId(uint32_t fsId) {
-        fsId_ = fsId;
-    }
-
-    uint64_t GetBlockSize() {
-        return blockSize_;
-    }
-
-    uint64_t GetChunkSize() {
-        return chunkSize_;
-    }
-
-    void SetBlockSize(const uint64_t& blockSize) {
-        blockSize_ = blockSize;
-    }
-
-    void SetChunkSize(const uint64_t& chunkSize) {
-        chunkSize_ = chunkSize;
-    }
-
-    std::shared_ptr<FsCacheManager> GetFsCacheManager() {
-        return fsCacheManager_;
-    }
-
-    uint32_t GetFlushInterval() { return flushIntervalSec_; }
-
-    uint32_t GetDiskCacheType() {
-        return diskCacheType_;
-    }
-
-    bool DisableDiskCache() {
-        return diskCacheType_ == DiskCacheType::Disable;
-    }
-
-    bool HasDiskCache() {
-        return diskCacheType_ != DiskCacheType::Disable;
-    }
-
-    bool IsReadCache() {
-        return diskCacheType_ == DiskCacheType::OnlyRead;
-    }
-
-    bool IsReadWriteCache() {
-        return diskCacheType_ == DiskCacheType::ReadWrite;
-    }
-    virtual std::shared_ptr<InodeCacheManager> GetInodeCacheManager() {
-        return inodeManager_;
-    }
-
-    std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() {
-        return diskCacheManagerImpl_;
-    }
-
-    uint32_t GetFsId() {
-        return fsId_;
-    }
-
-    uint32_t GetPageSize() {
-        return pageSize_;
-    }
-
-    void DisableBgFlush() {
-        enableBgFlush_ = false;
-    }
-
-    std::shared_ptr<IoMetric> GetMetric() {
-      return ioMetric_;
-    }
-
-    void CollectMetrics(InterfaceMetric *interface,
-      int count, uint64_t start) {
-        interface->bps.count << count;
-        interface->qps.count << 1;
-        interface->latency << (butil::cpuwide_time_us() - start);
-    }
-
-    void SetDiskCache(DiskCacheType type) {
-       diskCacheType_ = type;
-    }
-
-    CachePolicy GetCachePolicy(bool sync) {
-      const bool mayCache =
-        HasDiskCache() && !GetDiskCacheManager()->IsDiskCacheFull() && !sync;
-        if (IsReadCache() && mayCache) {
-            return CachePolicy::RCache;
-        } else if (IsReadWriteCache() && mayCache) {
-            return CachePolicy::WRCache;
-        } else {
-            return CachePolicy::NCache;
-        }
-    }
-
-    std::shared_ptr<MdsClient> GetMdsClient() {
-        return mdsClient_;
-    }
-
-    std::string GetMountOwner() {
-        return mountOwner_;
-    }
-
-    void SetMountOwner(const std::string& mountOwner) {
-        mountOwner_ = mountOwner;
-    }
-
- private:
-    int FlushChunkClosure(std::shared_ptr<FlushChunkCacheContext> context);
-    int ClearDiskCache(int64_t inodeId);
-    void BackGroundFlush();
-
-    void InitMetrics(const std::string &fsName) {
-        fsName_ = fsName;
-        ioMetric_ = std::make_shared<IoMetric>(fsName);
-        // init disk cache metrics(needed)
-        if (HasDiskCache()) {
-        diskCacheManagerImpl_->InitMetrics(fsName);
-        }
-    }
-
- protected:
-  std::shared_ptr<FsCacheManager> fsCacheManager_;
-  std::shared_ptr<MdsClient> mdsClient_;
-  std::shared_ptr<FsInfo> fsInfo_;
-
- private:
-    uint64_t blockSize_;
-    uint64_t chunkSize_;
-    uint32_t pageSize_;
-    uint32_t flushIntervalSec_;
-    uint32_t chunkFlushThreads_;
-    uint32_t memCacheNearfullRatio_;
-    uint32_t throttleBaseSleepUs_;
-    Thread bgFlushThread_;
-    std::atomic<bool> toStop_;
-    std::mutex mtx_;
-    std::mutex ioMtx_;
-    std::condition_variable cond_;
-    curve::common::WaitInterval waitInterval_;
-    std::shared_ptr<InodeCacheManager> inodeManager_;
-    std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl_;
-    DiskCacheType diskCacheType_;
-    uint32_t fsId_;
-    std::string fsName_;
-    std::string mountOwner_;
-    std::shared_ptr<IoMetric> ioMetric_;
-    bool enableBgFlush_ = true;
-
-    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
-        taskPool_;
-};
-
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_CLIENT_STORAGE_ADAPTOR_H_

curvefs/src/client/common/common.h

@@ -60,7 +60,6 @@ enum class MetaServerOpType {
     GetVolumeExtent,
     UpdateVolumeExtent,
     CreateManageInode,
-    UpdateDeallocatableBlockGroup,
 };
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);

curvefs/src/client/common/config.cpp

@@ -50,6 +50,7 @@ DEFINE_bool(enableCto, true, "acheieve cto consistency");
 DEFINE_bool(useFakeS3, false,
             "Use fake s3 to inject more metadata for testing metaserver");
 DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing");
+DEFINE_bool(access_logging, true, "enable access log");
 
 /**
  * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true
@@ -204,7 +205,6 @@ void InitDiskCacheOption(Configuration *conf,
                               &diskCacheOption->avgReadFileIops);
 }
 
-// TODO(@hzwuhongsong)
 void InitS3Option(Configuration *conf, S3Option *s3Opt) {
     conf->GetValueFatalIfFail("s3.fakeS3", &FLAGS_useFakeS3);
     conf->GetValueFatalIfFail("s3.pageSize",
@@ -245,10 +245,6 @@ void InitVolumeOption(Configuration *conf, VolumeOption *volumeOpt) {
     conf->GetValueFatalIfFail("volume.fsBlockSize", &volumeOpt->fsBlockSize);
     conf->GetValueFatalIfFail("volume.allocator.type",
                               &volumeOpt->allocatorOption.type);
-    conf->GetValueFatalIfFail("volume.space.useThreshold",
-                              &volumeOpt->threshold);
-    conf->GetValueFatalIfFail("volume.space.releaseInterSec",
-                              &volumeOpt->releaseInterSec);
 
     conf->GetValueFatalIfFail(
         "volume.blockGroup.allocateOnce",
@@ -297,6 +293,55 @@ void InitKVClientManagerOpt(Configuration *conf,
                               &config->getThreadPooln);
 }
 
+void InitFileSystemOption(Configuration* c, FileSystemOption* option) {
+    c->GetValueFatalIfFail("fs.cto", &option->cto);
+    c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto);
+    c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr);
+    c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength);
+    c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging);
+    {  // kernel cache option
+        auto o = &option->kernelCacheOption;
+        c->GetValueFatalIfFail("fs.kernelCache.attrTimeoutSec",
+                               &o->attrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeoutSec",
+                               &o->dirAttrTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.entryTimeoutSec",
+                               &o->entryTimeoutSec);
+        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeoutSec",
+                               &o->dirEntryTimeoutSec);
+    }
+    {  // lookup cache option
+        auto o = &option->lookupCacheOption;
+        c->GetValueFatalIfFail("fs.lookupCache.lruSize",
+                               &o->lruSize);
+        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeoutSec",
+                               &o->negativeTimeoutSec);
+        c->GetValueFatalIfFail("fs.lookupCache.minUses",
+                               &o->minUses);
+    }
+    {  // dir cache option
+        auto o = &option->dirCacheOption;
+        c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize);
+    }
+    {  // open file option
+        auto o = &option->openFilesOption;
+        c->GetValueFatalIfFail("fs.openFile.lruSize", &o->lruSize);
+    }
+    {  // attr watcher option
+        auto o = &option->attrWatcherOption;
+        c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize);
+    }
+    {  // rpc option
+        auto o = &option->rpcOption;
+        c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit);
+    }
+    {  // defer sync option
+        auto o = &option->deferSyncOption;
+        c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay);
+        c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime);
+    }
+}
+
 void SetBrpcOpt(Configuration *conf) {
     curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy;
     dummy.Load(conf, "defer_close_second", "rpc.defer.close.second",
@@ -317,36 +362,16 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
     InitLeaseOpt(conf, &clientOption->leaseOpt);
     InitRefreshDataOpt(conf, &clientOption->refreshDataOption);
     InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt);
+    InitFileSystemOption(conf, &clientOption->fileSystemOption);
 
-    conf->GetValueFatalIfFail("fuseClient.attrTimeOut",
-                              &clientOption->attrTimeOut);
-    conf->GetValueFatalIfFail("fuseClient.entryTimeOut",
-                              &clientOption->entryTimeOut);
     conf->GetValueFatalIfFail("fuseClient.listDentryLimit",
                               &clientOption->listDentryLimit);
     conf->GetValueFatalIfFail("fuseClient.listDentryThreads",
                               &clientOption->listDentryThreads);
-    conf->GetValueFatalIfFail("fuseClient.flushPeriodSec",
-                              &clientOption->flushPeriodSec);
-    conf->GetValueFatalIfFail("fuseClient.maxNameLength",
-                              &clientOption->maxNameLength);
-    conf->GetValueFatalIfFail("fuseClient.iCacheLruSize",
-                              &clientOption->iCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.dCacheLruSize",
-                              &clientOption->dCacheLruSize);
-    conf->GetValueFatalIfFail("fuseClient.enableICacheMetrics",
-                              &clientOption->enableICacheMetrics);
-    conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
-                              &clientOption->enableDCacheMetrics);
-    conf->GetValueFatalIfFail("fuseClient.lruTimeOutSec",
-                              &clientOption->lruTimeOutSec);
     conf->GetValueFatalIfFail("client.dummyServer.startPort",
                               &clientOption->dummyServerStartPort);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                               &clientOption->enableMultiMountPointRename);
-    conf->GetValueFatalIfFail("fuseClient.disableXattr",
-                              &clientOption->disableXattr);
-    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
     conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
                               &clientOption->downloadMaxRetryTimes);
     conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
@@ -356,12 +381,6 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
         << "Not found `fuseClient.enableSplice` in conf, use default value `"
         << std::boolalpha << clientOption->enableFuseSplice << '`';
 
-    // if enableCto, attr and entry cache must invalid
-    if (FLAGS_enableCto) {
-        clientOption->attrTimeOut = 0;
-        clientOption->entryTimeOut = 0;
-    }
-
     conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteBytes",
                               &FLAGS_fuseClientAvgWriteBytes);
     conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytes",
@@ -401,9 +420,6 @@ void SetFuseClientS3Option(FuseClientOption *clientOption,
     clientOption->s3Opt.s3AdaptrOpt.ak = fsS3Opt.ak;
     clientOption->s3Opt.s3AdaptrOpt.sk = fsS3Opt.sk;
     clientOption->s3Opt.s3AdaptrOpt.bucketName = fsS3Opt.bucketName;
-    VLOG(1) << "fuse s3 option, chunk size is: " << fsS3Opt.chunkSize
-            << "block size is: " << fsS3Opt.blockSize
-            << "s3 adress is: " << fsS3Opt.s3Address;
 }
 
 void S3Info2FsS3Option(const curvefs::common::S3Info& s3,

curvefs/src/client/common/config.h

@@ -73,6 +73,11 @@ struct LeaseOpt {
     uint32_t leaseTimeUs = 20000000;
 };
 
+struct SpaceAllocServerOption {
+    std::string spaceaddr;
+    uint64_t rpcTimeoutMs;
+};
+
 struct KVClientManagerOpt {
     int setThreadPooln = 4;
     int getThreadPooln = 4;
@@ -159,9 +164,6 @@ struct VolumeOption {
     uint64_t volBlockSize;
     uint64_t fsBlockSize;
     VolumeAllocatorOption allocatorOption;
-
-    double threshold{1.0};
-    uint64_t releaseInterSec{300};
 };
 
 struct ExtentManagerOption {
@@ -172,11 +174,65 @@ struct RefreshDataOption {
     uint64_t maxDataSize = 1024;
     uint32_t refreshDataIntervalSec = 30;
 };
+
+// { filesystem option
+struct KernelCacheOption {
+    uint32_t entryTimeoutSec;
+    uint32_t dirEntryTimeoutSec;
+    uint32_t attrTimeoutSec;
+    uint32_t dirAttrTimeoutSec;
+};
+
+struct LookupCacheOption {
+    uint64_t lruSize;
+    uint32_t negativeTimeoutSec;
+    uint32_t minUses;
+};
+
+struct DirCacheOption {
+    uint64_t lruSize;
+    uint32_t timeoutSec;
+};
+
+struct AttrWatcherOption {
+    uint64_t lruSize;
+};
+
+struct OpenFilesOption {
+    uint64_t lruSize;
+    uint32_t deferSyncSecond;
+};
+
+struct RPCOption {
+    uint32_t listDentryLimit;
+};
+
+struct DeferSyncOption {
+    uint32_t delay;
+    bool deferDirMtime;
+};
+
+struct FileSystemOption {
+    bool cto;
+    bool disableXattr;
+    uint32_t maxNameLength;
+    uint32_t blockSize = 0x10000u;
+    KernelCacheOption kernelCacheOption;
+    LookupCacheOption lookupCacheOption;
+    DirCacheOption dirCacheOption;
+    OpenFilesOption openFilesOption;
+    AttrWatcherOption attrWatcherOption;
+    RPCOption rpcOption;
+    DeferSyncOption deferSyncOption;
+};
+// }
+
 struct FuseClientOption {
     MdsOption mdsOpt;
     MetaCacheOpt metaCacheOpt;
     ExcutorOpt excutorOpt;
     ExcutorOpt excutorInternalOpt;
+    SpaceAllocServerOption spaceOpt;
     BlockDeviceClientOptions bdevOpt;
     S3Option s3Opt;
     ExtentManagerOption extentManagerOpt;
@@ -184,22 +240,13 @@ struct FuseClientOption {
     LeaseOpt leaseOpt;
     RefreshDataOption refreshDataOption;
     KVClientManagerOpt kvClientManagerOpt;
+    FileSystemOption fileSystemOption;
 
-    double attrTimeOut;
-    double entryTimeOut;
     uint32_t listDentryLimit;
     uint32_t listDentryThreads;
-    uint32_t flushPeriodSec;
-    uint32_t maxNameLength;
-    uint64_t iCacheLruSize;
-    uint64_t dCacheLruSize;
-    bool enableICacheMetrics;
-    bool enableDCacheMetrics;
-    uint32_t lruTimeOutSec;
     uint32_t dummyServerStartPort;
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;
-    bool disableXattr = false;
     uint32_t downloadMaxRetryTimes;
     uint32_t warmupThreadsNum = 10;
 };

curvefs/src/client/curve_fuse_op.h

@@ -37,7 +37,7 @@
 extern "C" {
 #endif
 
-int InitGlog(const char *confPath, const char *argv0);
+int InitLog(const char *confPath, const char *argv0);
 
 int InitFuseClient(const struct MountOption *mountOption);
 

curvefs/src/client/dentry_cache_manager.cpp

@@ -44,31 +44,13 @@ namespace client {
 
 using curve::common::WriteLockGuard;
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
-
-void DentryCacheManagerImpl::InsertOrReplaceCache(const Dentry &dentry) {
-    std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name());
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        NameLockGuard lock(nameLock_, key);
-        dCache_->Put(key, dentry);
-    }
-}
-
-void DentryCacheManagerImpl::DeleteCache(uint64_t parentId,
-                                         const std::string &name) {
-    std::string key = GetDentryCacheKey(parentId, name);
-    NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
-}
+using ::curvefs::client::filesystem::ToFSError;
 
 CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
                                                 const std::string &name,
                                                 Dentry *out) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    bool ok = dCache_->Get(key, out);
-    if (ok) {
-        return CURVEFS_ERROR::OK;
-    }
 
     MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out);
     if (ret != MetaStatusCode::OK) {
@@ -76,10 +58,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
             << "metaClient_ GetDentry failed, MetaStatusCode = " << ret
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
             << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
-    }
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, *out);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -94,12 +73,9 @@ CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) {
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << dentry.parentinodeid()
                    << ", name = " << dentry.name();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
-    if (!curvefs::client::common::FLAGS_enableCto) {
-        dCache_->Put(key, dentry);
-    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -108,14 +84,13 @@ CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent,
                                                    FsFileType type) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
-    dCache_->Remove(key);
 
     MetaStatusCode ret = metaClient_->DeleteDentry(fsId_, parent, name, type);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << parent << ", name = " << name;
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -149,7 +124,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", parent = " << parent << ", last = " << last
                        << ", count = " << limit << ", onlyDir = " << onlyDir;
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
 
         if (!onlyDir) {

curvefs/src/client/dentry_cache_manager.h

@@ -33,20 +33,18 @@
 #include <utility>
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
-#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
-#include "src/common/lru_cache.h"
 #include "src/common/concurrent/name_lock.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Dentry;
-using ::curve::common::TimedLRUCache;
-using ::curve::common::CacheMetrics;
 
 namespace curvefs {
 namespace client {
 
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 static const char* kDentryKeyDelimiter = ":";
 
@@ -59,13 +57,6 @@ class DentryCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-        uint32_t cacheTimeOutSec) = 0;
-
-    virtual void InsertOrReplaceCache(const Dentry& dentry) = 0;
-
-    virtual void DeleteCache(uint64_t parentId, const std::string& name) = 0;
-
     virtual CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) = 0;
 
@@ -86,30 +77,11 @@ class DentryCacheManager {
 class DentryCacheManagerImpl : public DentryCacheManager {
  public:
     DentryCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()),
-        dCache_(nullptr) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
 
     explicit DentryCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient),
-        dCache_(nullptr) {}
-
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-        uint32_t cacheTimeOutSec) override {
-        if (enableCacheMetrics) {
-            dCache_ = std::make_shared<
-                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize,
-                    std::make_shared<CacheMetrics>("dcache"));
-        } else {
-            dCache_ = std::make_shared<
-                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize);
-        }
-        return CURVEFS_ERROR::OK;
-    }
-
-    void InsertOrReplaceCache(const Dentry& dentry) override;
-
-    void DeleteCache(uint64_t parentId, const std::string& name) override;
+      : metaClient_(metaClient) {}
 
     CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) override;
@@ -130,8 +102,6 @@ class DentryCacheManagerImpl : public DentryCacheManager {
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    // key is parentId + name
-    std::shared_ptr<TimedLRUCache<std::string, Dentry>> dCache_;
     curve::common::GenericNameLock<Mutex> nameLock_;
 };
 

curvefs/src/client/error_code.cpp

@@ -1,98 +0,0 @@
-/*
- *  Copyright (c) 2021 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-/*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
- */
-
-#include "curvefs/src/client/error_code.h"
-
-#include <map>
-#include <string>
-
-namespace curvefs {
-namespace client {
-
-using ::curvefs::metaserver::MetaStatusCode;
-
-static const std::map<CURVEFS_ERROR, std::string> err2Msg = {
-    {CURVEFS_ERROR::OK, "OK"},
-    {CURVEFS_ERROR::INTERNAL, "internal error"},
-    {CURVEFS_ERROR::UNKNOWN, "unknown"},
-    {CURVEFS_ERROR::EXISTS, "inode or dentry already exist"},
-    {CURVEFS_ERROR::NOTEXIST, "inode or dentry not exist"},
-    {CURVEFS_ERROR::NO_SPACE, "no space to alloc"},
-    {CURVEFS_ERROR::BAD_FD, "bad fd"},
-    {CURVEFS_ERROR::INVALIDPARAM, "invalid param"},
-    {CURVEFS_ERROR::NOPERMISSION, "no permission"},
-    {CURVEFS_ERROR::NOTEMPTY, "dir not empty"},
-    {CURVEFS_ERROR::NOFLUSH, "no flush"},
-    {CURVEFS_ERROR::NOTSUPPORT, "not support"},
-    {CURVEFS_ERROR::NAMETOOLONG, "name too long"},
-    {CURVEFS_ERROR::MOUNT_POINT_EXIST, "mount point already exist"},
-    {CURVEFS_ERROR::MOUNT_FAILED, "mount failed"},
-};
-
-std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
-    os << static_cast<int>(code) << "[" << [code]() {
-        auto it = err2Msg.find(code);
-        if (it != err2Msg.end()) {
-            return it->second;
-        }
-
-        return std::string{"Unknown"};
-    }() << "]";
-
-    return os;
-}
-
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    MetaStatusCode code) {
-    CURVEFS_ERROR ret = CURVEFS_ERROR::UNKNOWN;
-    switch (code) {
-        case MetaStatusCode::OK:
-            ret = CURVEFS_ERROR::OK;
-            break;
-
-        case MetaStatusCode::NOT_FOUND:
-            ret = CURVEFS_ERROR::NOTEXIST;
-            break;
-
-        case MetaStatusCode::PARAM_ERROR:
-            ret = CURVEFS_ERROR::INVALIDPARAM;
-            break;
-
-        case MetaStatusCode::INODE_EXIST:
-        case MetaStatusCode::DENTRY_EXIST:
-            ret = CURVEFS_ERROR::EXISTS;
-            break;
-
-        case MetaStatusCode::SYM_LINK_EMPTY:
-        case MetaStatusCode::RPC_ERROR:
-            ret = CURVEFS_ERROR::INTERNAL;
-            break;
-
-        default:
-            ret = CURVEFS_ERROR::UNKNOWN;
-    }
-    return ret;
-}
-
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/README.md

@@ -0,0 +1,78 @@
+CurveFS Client Metadata Cache Design
+===
+
+```
++----------------+
++  Kernel Cache  |
++----------------+
+    |       ^
+(1) |       | (4)
+    v       |
++----------------+
++  CurveFS Fuse  |
++----------------+
+    |       ^
+(2) |       | (3)
+    v       |
++----------------+
++   MetaServer   |
++----------------+
+```
+
+(1) kernel cache timeout
+---
+
+* requests are always sent to fuse layer, except `lookup` and `getattr` request which affected by caching.
+* if entry or attribute cache timeout, vfs layer will launch request to fuse layer.
+
+(2) revalidate cache
+---
+
+* for `open` and `opendir` request, fuse layer should revalidate cache by comparing mtime, if modified, fuse layer should:
+    * `open`: return **ESTALE** to trigger vfs layer to invoke the `open` again with ignoring cache.
+    * `opendir`: drop all directory cache.
+* others, proxy request to metaserver directly.
+
+
+(3) retrive fresh metadata
+---
+
+* for `readdir` request, fuse layer should cache direcoty entries and their attributes.
+* others, no caching.
+
+(4) reply with timeout
+---
+
+* reply entry or attribute to vfs layer with corresponding cache timeout.
+* four type timeouts provided: `entryTimeout`, `dirEntryTimeout`, `attrTimeout`, `dirAtttTimeout`.
+* fuse layer should remeber the `mtime` of attribute while reply it to vfs layer.
+
+Cache Layer Level
+===
+
+```
++-------------------------------------------------------+
+|                       Kernel Cache                    |  (L1 Cache)
++-------------------------------------------------------+
+                            ^  (update length)
+                            |
++--------------------------------------------------------+
+|                       Open File Cache                  |  (L2 Cache)
+|                   (only length and chunk)              |
++--------------------------------------------------------+
+         ^                            ^
+         | (reply entry/attr)         |  (reply entry/attr)
++-----------------+                   |
+| Directory Cache |  (L3 Cache)       |
++-----------------+                   |
+         ^                            |
+         |                            |
++-----------------+ +------------------------------------+
+|     readdir     | | lookup、getattr、setattr、create... |
++-----------------+ +------------------------------------+
+         ^                            ^
+         |                            |
++--------------------------------------------------------+
+|                        MetaServer                      |
++--------------------------------------------------------+
+```

curvefs/src/client/filesystem/access_log.h

@@ -0,0 +1,89 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-17
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <unistd.h>
+#include <butil/time.h>
+#include <spdlog/spdlog.h>
+#include <spdlog/sinks/basic_file_sink.h>
+#include <spdlog/sinks/daily_file_sink.h>
+
+#include <string>
+#include <memory>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/common/config.h"
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
+
+namespace curvefs {
+namespace client {
+namespace common {
+
+DECLARE_bool(access_logging);
+
+}
+namespace filesystem {
+
+using ::absl::StrFormat;
+using ::curvefs::client::common::FLAGS_access_logging;
+using MessageHandler = std::function<std::string()>;
+
+static std::shared_ptr<spdlog::logger> Logger;
+
+bool InitAccessLog(const std::string& prefix) {
+    std::string filename = StrFormat("%s/access.%d.log", prefix, getpid());
+    Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0);
+    spdlog::flush_every(std::chrono::seconds(1));
+    return true;
+}
+
+struct AccessLogGuard {
+    explicit AccessLogGuard(MessageHandler handler)
+        : enable(FLAGS_access_logging),
+          handler(handler) {
+        if (!enable) {
+            return;
+        }
+
+        timer.start();
+    }
+
+    ~AccessLogGuard() {
+        if (!enable) {
+            return;
+        }
+
+        timer.stop();
+        Logger->info("{0} <{1:.6f}>", handler(), timer.u_elapsed() / 1e6);
+    }
+
+    bool enable;
+    MessageHandler handler;
+    butil::Timer timer;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_

curvefs/src/client/filesystem/attr_watcher.cpp

@@ -0,0 +1,117 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+AttrWatcher::AttrWatcher(AttrWatcherOption option,
+                         std::shared_ptr<OpenFiles> openFiles,
+                         std::shared_ptr<DirCache> dirCache)
+    : modifiedAt_(std::make_shared<LRUType>(option.lruSize)),
+      openFiles_(openFiles),
+      dirCache_(dirCache) {}
+
+void AttrWatcher::RemeberMtime(const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    modifiedAt_->Put(attr.inodeid(), AttrMtime(attr));
+}
+
+bool AttrWatcher::GetMtime(Ino ino, TimeSpec* time) {
+    ReadLockGuard lk(rwlock_);
+    return modifiedAt_->Get(ino, time);
+}
+
+void AttrWatcher::UpdateDirEntryAttr(Ino ino, const InodeAttr& attr) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : attr.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateAttr(ino, attr);
+
+        VLOG(1) << "Write back attribute to dir entry cache: ino = " << ino
+                << ", attr = " << attr.ShortDebugString();
+    }
+}
+
+void AttrWatcher::UpdateDirEntryLength(Ino ino, const InodeAttr& open) {
+    std::shared_ptr<DirEntryList> entries;
+    for (const auto parent : open.parent()) {
+        bool yes = dirCache_->Get(parent, &entries);
+        if (!yes) {
+            continue;
+        }
+
+        entries->UpdateLength(ino, open);
+
+        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
+                << ", attr = " << open.ShortDebugString();
+    }
+}
+
+AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                                   InodeAttr* attr,
+                                   ReplyType type,
+                                   bool writeBack)
+    : watcher(watcher), attr(attr), type(type), writeBack(writeBack) {
+    InodeAttr open;
+    Ino ino = attr->inodeid();
+    bool yes = watcher->openFiles_->GetFileAttr(ino, &open);
+    if (!yes) {
+        return;
+    }
+
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+}
+
+AttrWatcherGuard::~AttrWatcherGuard() {
+    switch (type) {
+        case ReplyType::ATTR:
+            watcher->RemeberMtime(*attr);
+            if (writeBack) {
+                watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
+            }
+            break;
+
+        case ReplyType::ONLY_LENGTH:
+            if (writeBack) {
+                watcher->UpdateDirEntryLength(attr->inodeid(), *attr);
+            }
+            break;
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/attr_watcher.h

@@ -0,0 +1,105 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
+
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::AttrWatcherOption;
+
+class AttrWatcher {
+ public:
+    using LRUType = LRUCache<Ino, struct TimeSpec>;
+
+ public:
+    AttrWatcher(AttrWatcherOption option,
+                std::shared_ptr<OpenFiles> openFiles,
+                std::shared_ptr<DirCache> dirCache);
+
+    void RemeberMtime(const InodeAttr& attr);
+
+    bool GetMtime(Ino ino, TimeSpec* time);
+
+    void UpdateDirEntryAttr(Ino ino, const InodeAttr& attr);
+
+    void UpdateDirEntryLength(Ino ino, const InodeAttr& open);
+
+ private:
+    friend class AttrWatcherGuard;
+
+ private:
+    RWLock rwlock_;
+    std::shared_ptr<LRUType> modifiedAt_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<DirCache> dirCache_;
+};
+
+
+enum class ReplyType {
+    ATTR,
+    ONLY_LENGTH
+};
+
+/*
+ * each attribute reply to kernel, the watcher will:
+ *  before reply:
+ *    1) set attibute length if the corresponding file is opened
+ *  after reply:
+ *    1) remeber attribute modified time.
+ *    2) write back attribute to dir entry cache if |writeBack| is true,
+ *       because the dir-entry attribute maybe stale.
+ */
+struct AttrWatcherGuard {
+ public:
+    AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
+                     InodeAttr* attr,
+                     ReplyType type,
+                     bool writeBack);
+
+    ~AttrWatcherGuard();
+
+ private:
+    std::shared_ptr<AttrWatcher> watcher;
+    InodeAttr* attr;
+    ReplyType type;
+    bool writeBack;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_

curvefs/src/client/filesystem/defer_sync.cpp

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <vector>
+#include <memory>
+
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DeferSync::DeferSync(DeferSyncOption option)
+    : option_(option),
+      mutex_(),
+      running_(false),
+      thread_(),
+      sleeper_(),
+      inodes_() {
+}
+
+void DeferSync::Start() {
+    if (!running_.exchange(true)) {
+        thread_ = std::thread(&DeferSync::SyncTask, this);
+        LOG(INFO) << "Defer sync thread start success";
+    }
+}
+
+void DeferSync::Stop() {
+    if (running_.exchange(false)) {
+        LOG(INFO) << "Stop defer sync thread...";
+        sleeper_.interrupt();
+        thread_.join();
+        LOG(INFO) << "Defer sync thread stopped";
+    }
+}
+
+void DeferSync::SyncTask() {
+    std::vector<std::shared_ptr<InodeWrapper>> inodes;
+    for ( ;; ) {
+        bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay));
+
+        {
+            LockGuard lk(mutex_);
+            inodes.swap(inodes_);
+        }
+        for (const auto& inode : inodes) {
+            UniqueLock lk(inode->GetUniqueLock());
+            inode->Async(nullptr, true);
+        }
+        inodes.clear();
+
+        if (!running) {
+            break;
+        }
+    }
+}
+
+void DeferSync::Push(const std::shared_ptr<InodeWrapper>& inode) {
+    LockGuard lk(mutex_);
+    inodes_.emplace_back(inode);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/defer_sync.h

@@ -0,0 +1,70 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
+
+#include <atomic>
+#include <vector>
+#include <memory>
+
+#include "src/common/interruptible_sleeper.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::DeferSyncOption;
+
+using ::curve::common::Mutex;
+using ::curve::common::LockGuard;
+using ::curve::common::InterruptibleSleeper;
+
+class DeferSync {
+ public:
+    explicit DeferSync(DeferSyncOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Push(const std::shared_ptr<InodeWrapper>& inode);
+
+ private:
+    void SyncTask();
+
+ private:
+    DeferSyncOption option_;
+    Mutex mutex_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    InterruptibleSleeper sleeper_;
+    std::vector<std::shared_ptr<InodeWrapper>> inodes_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_

curvefs/src/client/filesystem/dir_cache.cpp

@@ -0,0 +1,189 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <utility>
+
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+DirEntryList::DirEntryList()
+    : rwlock_(),
+      mtime_(),
+      entries_(),
+      attrs_() {}
+
+size_t DirEntryList::Size() {
+    ReadLockGuard lk(rwlock_);
+    return entries_.size();
+}
+
+void DirEntryList::Add(const DirEntry& dirEntry) {
+    WriteLockGuard lk(rwlock_);
+    entries_.push_back(std::move(dirEntry));
+    attrs_[dirEntry.ino] = &entries_.back();
+}
+
+bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) {
+    ReadLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+    *dirEntry = *iter->second;
+    return true;
+}
+
+bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    DirEntry* dirEntry = iter->second;
+    dirEntry->attr = std::move(attr);
+    return true;
+}
+
+bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) {
+    WriteLockGuard lk(rwlock_);
+    auto iter = attrs_.find(ino);
+    if (iter == attrs_.end()) {
+        return false;
+    }
+
+    InodeAttr* attr = &iter->second->attr;
+    attr->set_length(open.length());
+    attr->set_mtime(open.mtime());
+    attr->set_mtime_ns(open.mtime_ns());
+    if (AttrCtime(open) > AttrCtime(*attr)) {
+        attr->set_ctime(open.ctime());
+        attr->set_ctime_ns(open.ctime_ns());
+    }
+    return true;
+}
+
+void DirEntryList::Iterate(IterateHandler handler) {
+    ReadLockGuard lk(rwlock_);
+    for (auto iter = entries_.begin(); iter != entries_.end(); iter++) {
+        handler(&(*iter));
+    }
+}
+
+void DirEntryList::Clear() {
+    WriteLockGuard lk(rwlock_);
+    entries_.clear();
+    attrs_.clear();
+}
+
+void DirEntryList::SetMtime(TimeSpec mtime) {
+    WriteLockGuard lk(rwlock_);
+    mtime_ = mtime;
+}
+
+TimeSpec DirEntryList::GetMtime() {
+    ReadLockGuard lk(rwlock_);
+    return mtime_;
+}
+
+DirCache::DirCache(DirCacheOption option)
+    : rwlock_(),
+      nentries_(0),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(0);  // control size by ourself
+    mq_ = std::make_shared<MessageQueueType>("dircache", 10000);
+    mq_->Subscribe([&](const std::shared_ptr<DirEntryList>& entries){
+        entries->Clear();
+    });
+
+    LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize;
+}
+
+void DirCache::Start() {
+    mq_->Start();
+}
+
+void DirCache::Stop() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+    mq_->Stop();
+}
+
+void DirCache::Delete(Ino parent,
+                      std::shared_ptr<DirEntryList> entries,
+                      bool evit) {
+    nentries_ -= entries->Size();
+    mq_->Publish(entries);  // clear entries in background
+    lru_->Remove(parent);
+
+    VLOG(1) << "Delete directory cache (evit=" << evit << "): "
+            << "parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
+}
+
+void DirCache::Evit(size_t size) {
+    Ino parent;
+    std::shared_ptr<DirEntryList> entries;
+    while (nentries_ + size >= option_.lruSize) {
+        bool yes = lru_->GetLast(&parent, &entries);
+        if (!yes) {
+            break;
+        }
+        Delete(parent, entries, true);
+    }
+}
+
+void DirCache::Put(Ino parent, std::shared_ptr<DirEntryList> entries) {
+    WriteLockGuard lk(rwlock_);
+    Evit(entries->Size());  // it guarantee put entries success
+    lru_->Put(parent, entries);
+    nentries_ += entries->Size();
+
+    VLOG(1) << "Insert directory cache: parent = " << parent
+            << ", mtime = " << entries->GetMtime()
+            << ", size = " << entries->Size()
+            << ", nentries = " << nentries_;
+}
+
+bool DirCache::Get(Ino parent, std::shared_ptr<DirEntryList>* entries) {
+    ReadLockGuard lk(rwlock_);
+    return lru_->Get(parent, entries);
+}
+
+void DirCache::Drop(Ino parent) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = lru_->Get(parent, &entries);
+    if (yes) {
+        Delete(parent, entries, false);
+    }
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/dir_cache.h

@@ -0,0 +1,114 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
+
+#include <map>
+#include <list>
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/message_queue.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::DirCacheOption;
+
+class DirEntryList {
+ public:
+    using IterateHandler = std::function<void(DirEntry* dirEntry)>;
+
+ public:
+    DirEntryList();
+
+    size_t Size();
+
+    void Add(const DirEntry& dirEntry);
+
+    void Iterate(IterateHandler handler);
+
+    bool Get(Ino ino, DirEntry* dirEntry);
+
+    bool UpdateAttr(Ino ino, const InodeAttr& attr);
+
+    bool UpdateLength(Ino ino, const InodeAttr& open);
+
+    void Clear();
+
+    void SetMtime(TimeSpec mtime);
+
+    TimeSpec GetMtime();
+
+ private:
+    RWLock rwlock_;
+    TimeSpec mtime_;
+    std::list<DirEntry> entries_;
+    std::map<Ino, DirEntry*> attrs_;
+};
+
+class DirCache {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<DirEntryList>>;
+    using MessageType = std::shared_ptr<DirEntryList>;
+    using MessageQueueType = MessageQueue<MessageType>;
+
+ public:
+    explicit DirCache(DirCacheOption option);
+
+    void Start();
+
+    void Stop();
+
+    void Put(Ino parent, std::shared_ptr<DirEntryList> entries);
+
+    bool Get(Ino parent, std::shared_ptr<DirEntryList>* entries);
+
+    void Drop(Ino parent);
+
+ private:
+    void Delete(Ino parent, std::shared_ptr<DirEntryList> entries, bool evit);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    size_t nentries_;
+    DirCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+    std::shared_ptr<MessageQueueType> mq_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_

curvefs/src/client/filesystem/error.cpp

@@ -0,0 +1,106 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <utility>
+#include <ostream>
+
+#include "curvefs/src/client/filesystem/error.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+static const std::map<CURVEFS_ERROR, std::pair<int, std::string>> errors = {
+    { CURVEFS_ERROR::OK, { 0, "OK"} },
+    { CURVEFS_ERROR::INTERNAL,  { EIO, "internal error" } },
+    { CURVEFS_ERROR::UNKNOWN,  { -1, "unknown" } },
+    { CURVEFS_ERROR::EXISTS,  { EEXIST, "inode or dentry already exist" } },
+    { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } },
+    { CURVEFS_ERROR::NO_SPACE,  { ENOSPC, "no space to alloc" } },
+    { CURVEFS_ERROR::BAD_FD,  { EBADF, "bad file number" } },
+    { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } },
+    { CURVEFS_ERROR::NOPERMISSION,  { EACCES, "permission denied" } },
+    { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } },
+    { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } },
+    { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } },
+    { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } },
+    { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } },
+    { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } },
+    { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } },
+    { CURVEFS_ERROR::NODATA, { ENODATA, "no data available" } },
+    { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } },
+    { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } },
+    { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } },
+};
+
+std::string StrErr(CURVEFS_ERROR code) {
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        return it->second.second;
+    }
+    return "unknown";
+}
+
+int SysErr(CURVEFS_ERROR code) {
+    int syscode = -1;
+    auto it = errors.find(code);
+    if (it != errors.end()) {
+        syscode = it->second.first;
+    }
+    return (syscode == -1) ? EIO : syscode;
+}
+
+std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
+    os << static_cast<int>(code) << "[" << [code]() {
+        auto it = errors.find(code);
+        if (it != errors.end()) {
+            return it->second.second;
+        }
+
+        return std::string{"Unknown"};
+    }() << "]";
+
+    return os;
+}
+
+CURVEFS_ERROR ToFSError(MetaStatusCode code) {
+    static std::map<MetaStatusCode, CURVEFS_ERROR> errs = {
+        { MetaStatusCode::OK, CURVEFS_ERROR::OK },
+        { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST },
+        { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM },
+        { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS },
+        { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL },
+        { MetaStatusCode::RPC_ERROR, CURVEFS_ERROR::INTERNAL },
+    };
+
+    auto it = errs.find(code);
+    if (it != errs.end()) {
+        return it->second;
+    }
+    return CURVEFS_ERROR::UNKNOWN;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/error.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2021 NetEase Inc.
+ *  Copyright (c) 2023 NetEase Inc.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -14,22 +14,27 @@
  *  limitations under the License.
  */
 
-
 /*
- * Project: curve
- * Created Date: Thur May 27 2021
- * Author: xuchaojie
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
  */
 
-#ifndef CURVEFS_SRC_CLIENT_ERROR_CODE_H_
-#define CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#include <map>
+#include <string>
+#include <iostream>
 
 #include "curvefs/proto/metaserver.pb.h"
 
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+
 namespace curvefs {
 namespace client {
+namespace filesystem {
+
+using ::curvefs::metaserver::MetaStatusCode;
 
-// notice : the error code should be negative.
 enum class CURVEFS_ERROR {
     OK = 0,
     INTERNAL = -1,
@@ -50,15 +55,20 @@ enum class CURVEFS_ERROR {
     NODATA = -16,
     IO_ERROR = -17,
     CACHETOOSMALL = -18,
+    STALE = -19,
+    NOSYS = -20,
 };
 
+std::string StrErr(CURVEFS_ERROR code);
+
+int SysErr(CURVEFS_ERROR code);
 
 std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code);
 
-CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
-    ::curvefs::metaserver::MetaStatusCode code);
+CURVEFS_ERROR ToFSError(MetaStatusCode code);
 
+}  // namespace filesystem
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_

curvefs/src/client/filesystem/filesystem.cpp

@@ -0,0 +1,353 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+
+#include "curvefs/src/client/filesystem/filesystem.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
+    : option_(option), member(member) {
+    deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
+    negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
+    dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
+    openFiles_ = std::make_shared<OpenFiles>(option_.openFilesOption,
+                                             deferSync_);
+    attrWatcher_ = std::make_shared<AttrWatcher>(option_.attrWatcherOption,
+                                                 openFiles_, dirCache_);
+    handlerManager_ = std::make_shared<HandlerManager>();
+    rpc_ = std::make_shared<RPCClient>(option.rpcOption, member);
+}
+
+void FileSystem::Run() {
+    deferSync_->Start();
+    dirCache_->Start();
+}
+
+void FileSystem::Destory() {
+    openFiles_->CloseAll();
+    deferSync_->Stop();
+    dirCache_->Stop();
+}
+
+void FileSystem::Attr2Stat(InodeAttr* attr, struct stat* stat) {
+    std::memset(stat, 0, sizeof(struct stat));
+    stat->st_ino = attr->inodeid();  //  inode number
+    stat->st_mode = attr->mode();  // permission mode
+    stat->st_nlink = attr->nlink();  // number of links
+    stat->st_uid = attr->uid();  // user ID of owner
+    stat->st_gid = attr->gid();  // group ID of owner
+    stat->st_size = attr->length();  // total size, in bytes
+    stat->st_rdev = attr->rdev();  // device ID (if special file)
+    stat->st_atim.tv_sec = attr->atime();  // time of last access
+    stat->st_atim.tv_nsec = attr->atime_ns();
+    stat->st_mtim.tv_sec = attr->mtime();  // time of last modification
+    stat->st_mtim.tv_nsec = attr->mtime_ns();
+    stat->st_ctim.tv_sec = attr->ctime();  // time of last status change
+    stat->st_ctim.tv_nsec = attr->ctime_ns();
+    stat->st_blksize = option_.blockSize;  // blocksize for file system I/O
+    stat->st_blocks = 0;  // number of 512B blocks allocated
+    if (IsS3File(*attr)) {
+        stat->st_blocks = (attr->length() + 511) / 512;
+    }
+}
+
+void FileSystem::Entry2Param(EntryOut* entryOut,
+                             fuse_entry_param* e) {
+    std::memset(e, 0, sizeof(fuse_entry_param));
+    e->ino = entryOut->attr.inodeid();
+    e->generation = 0;
+    Attr2Stat(&entryOut->attr, &e->attr);
+    e->entry_timeout = entryOut->entryTimeout;
+    e->attr_timeout = entryOut->attrTimeout;
+}
+
+void FileSystem::SetEntryTimeout(EntryOut* entryOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(entryOut->attr)) {
+        entryOut->entryTimeout = option.dirEntryTimeoutSec;
+        entryOut->attrTimeout = option.dirAttrTimeoutSec;
+    } else {
+        entryOut->entryTimeout = option.entryTimeoutSec;
+        entryOut->attrTimeout = option.attrTimeoutSec;
+    }
+}
+
+void FileSystem::SetAttrTimeout(AttrOut* attrOut) {
+    auto option = option_.kernelCacheOption;
+    if (IsDir(attrOut->attr)) {
+        attrOut->attrTimeout = option.dirAttrTimeoutSec;
+    } else {
+        attrOut->attrTimeout = option.attrTimeoutSec;
+    }
+}
+
+// fuse reply*
+void FileSystem::ReplyError(Request req, CURVEFS_ERROR code) {
+    fuse_reply_err(req, SysErr(code));
+}
+
+void FileSystem::ReplyEntry(Request req,
+                            EntryOut* entryOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_entry(req, &e);
+}
+
+void FileSystem::ReplyAttr(Request req,
+                           AttrOut* attrOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr,
+                             ReplyType::ATTR, true);
+    struct stat stat;
+    SetAttrTimeout(attrOut);
+    Attr2Stat(&attrOut->attr, &stat);
+    fuse_reply_attr(req, &stat, attrOut->attrTimeout);
+}
+
+void FileSystem::ReplyReadlink(Request req, const std::string& link) {
+    fuse_reply_readlink(req, link.c_str());
+}
+
+void FileSystem::ReplyOpen(Request req, FileInfo* fi) {
+    fuse_reply_open(req, fi);
+}
+
+void FileSystem::ReplyOpen(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_open(req, fileOut->fi);
+}
+
+void FileSystem::ReplyData(Request req,
+                           struct fuse_bufvec *bufv,
+                           enum fuse_buf_copy_flags flags) {
+    fuse_reply_data(req, bufv, flags);
+}
+
+void FileSystem::ReplyWrite(Request req, FileOut* fileOut) {
+    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
+                             ReplyType::ONLY_LENGTH, true);
+    fuse_reply_write(req, fileOut->nwritten);
+}
+
+void FileSystem::ReplyBuffer(Request req, const char *buf, size_t size) {
+    fuse_reply_buf(req, buf, size);
+}
+
+void FileSystem::ReplyStatfs(Request req, const struct statvfs *stbuf) {
+    fuse_reply_statfs(req, stbuf);
+}
+
+void FileSystem::ReplyXattr(Request req, size_t size) {
+    fuse_reply_xattr(req, size);
+}
+
+void FileSystem::ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi) {
+    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
+                             ReplyType::ATTR, true);
+    fuse_entry_param e;
+    SetEntryTimeout(entryOut);
+    Entry2Param(entryOut, &e);
+    fuse_reply_create(req, &e, fi);
+}
+
+void FileSystem::AddDirEntry(Request req,
+                             DirBufferHead* buffer,
+                             DirEntry* dirEntry) {
+    struct stat stat;
+    std::memset(&stat, 0, sizeof(stat));
+    stat.st_ino = dirEntry->ino;
+
+    // add a directory entry to the buffer
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry(req,
+                      buffer->p + oldsize,  // char* buf
+                      buffer->size - oldsize,  // size_t bufisze
+                      name, &stat, buffer->size);
+}
+
+void FileSystem::AddDirEntryPlus(Request req,
+                                 DirBufferHead* buffer,
+                                 DirEntry* dirEntry) {
+    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr,
+                             ReplyType::ATTR, false);
+    struct fuse_entry_param e;
+    EntryOut entryOut(dirEntry->attr);
+    SetEntryTimeout(&entryOut);
+    Entry2Param(&entryOut, &e);
+
+    // add a directory entry to the buffer with the attributes
+    size_t oldsize = buffer->size;
+    const char* name = dirEntry->name.c_str();
+    buffer->size += fuse_add_direntry_plus(req, NULL, 0, name, NULL, 0);
+    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
+    fuse_add_direntry_plus(req,
+                           buffer->p + oldsize,  // char* buf
+                           buffer->size - oldsize,  // size_t bufisze
+                           name, &e, buffer->size);
+}
+
+// handler*
+std::shared_ptr<FileHandler> FileSystem::NewHandler() {
+    return handlerManager_->NewHandler();
+}
+
+std::shared_ptr<FileHandler> FileSystem::FindHandler(uint64_t fh) {
+    return handlerManager_->FindHandler(fh);
+}
+
+void FileSystem::ReleaseHandler(uint64_t fh) {
+    return handlerManager_->ReleaseHandler(fh);
+}
+
+FileSystemMember FileSystem::BorrowMember() {
+    return FileSystemMember(deferSync_, openFiles_, attrWatcher_);
+}
+
+// fuse request*
+CURVEFS_ERROR FileSystem::Lookup(Request req,
+                                 Ino parent,
+                                 const std::string& name,
+                                 EntryOut* entryOut) {
+    if (name.size() > option_.maxNameLength) {
+        return CURVEFS_ERROR::NAMETOOLONG;
+    }
+
+    bool yes = negative_->Get(parent, name);
+    if (yes) {
+        return CURVEFS_ERROR::NOTEXIST;
+    }
+
+    auto rc = rpc_->Lookup(parent, name, entryOut);
+    if (rc == CURVEFS_ERROR::OK) {
+        negative_->Delete(parent, name);
+    } else if (rc == CURVEFS_ERROR::NOTEXIST) {
+        negative_->Put(parent, name);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) {
+    InodeAttr attr;
+    auto rc = rpc_->GetAttr(ino, &attr);
+    if (rc == CURVEFS_ERROR::OK) {
+        *attrOut = AttrOut(attr);
+    }
+    return rc;
+}
+
+CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) {
+    InodeAttr attr;
+    CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    // revalidate directory cache
+    std::shared_ptr<DirEntryList> entries;
+    bool yes = dirCache_->Get(ino, &entries);
+    if (yes) {
+        if (entries->GetMtime() != AttrMtime(attr)) {
+            dirCache_->Drop(ino);
+        }
+    }
+
+    auto handler = NewHandler();
+    handler->mtime = AttrMtime(attr);
+    fi->fh = handler->fh;
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReadDir(Request req,
+                                  Ino ino,
+                                  FileInfo* fi,
+                                  std::shared_ptr<DirEntryList>* entries) {
+    bool yes = dirCache_->Get(ino, entries);
+    if (yes) {
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->ReadDir(ino, entries);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    (*entries)->SetMtime(FindHandler(fi->fh)->mtime);
+    dirCache_->Put(ino, *entries);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) {
+    ReleaseHandler(fi->fh);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) {
+    std::shared_ptr<InodeWrapper> inode;
+    bool yes = openFiles_->IsOpened(ino, &inode);
+    if (yes) {
+        openFiles_->Open(ino, inode);
+        // fi->keep_cache = 1;
+        return CURVEFS_ERROR::OK;
+    }
+
+    CURVEFS_ERROR rc = rpc_->Open(ino, &inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    TimeSpec mtime;
+    yes = attrWatcher_->GetMtime(ino, &mtime);
+    if (!yes) {
+        // It is rare which only arise when attribute evited for attr-watcher.
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ": attribute not found in wacther";
+        return CURVEFS_ERROR::STALE;
+    } else if (mtime != InodeMtime(inode)) {
+        LOG(WARNING) << "open(" << ino << "): stale file handler"
+                     << ", cache(" << mtime << ") vs remote("
+                     << InodeMtime(inode) << ")";
+        return CURVEFS_ERROR::STALE;
+    }
+
+    openFiles_->Open(ino, inode);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) {
+    openFiles_->Close(ino);
+    return CURVEFS_ERROR::OK;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/filesystem.h

@@ -0,0 +1,167 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-08
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
+
+#include <gtest/gtest_prod.h>
+
+#include <memory>
+#include <string>
+
+#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/attr_watcher.h"
+#include "curvefs/src/client/filesystem/rpc_client.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::FileSystemOption;
+
+struct FileSystemMember {
+    FileSystemMember(std::shared_ptr<DeferSync> deferSync,
+                     std::shared_ptr<OpenFiles> openFiles,
+                     std::shared_ptr<AttrWatcher> attrWatcher)
+        : deferSync(deferSync),
+          openFiles(openFiles),
+          attrWatcher(attrWatcher) {}
+
+    std::shared_ptr<DeferSync> deferSync;
+    std::shared_ptr<OpenFiles> openFiles;
+    std::shared_ptr<AttrWatcher> attrWatcher;
+};
+
+class FileSystem {
+ public:
+    FileSystem(struct FileSystemOption option, ExternalMember member);
+
+    void Run();
+
+    void Destory();
+
+    // fuse request
+    CURVEFS_ERROR Lookup(Request req,
+                         Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut);
+
+    CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR ReadDir(Request req,
+                          Ino ino,
+                          FileInfo* fi,
+                          std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi);
+
+    CURVEFS_ERROR Release(Request req, Ino ino);
+
+    // fuse reply: we control all replies to vfs layer in same entrance.
+    void ReplyError(Request req, CURVEFS_ERROR code);
+
+    void ReplyEntry(Request req, EntryOut* entryOut);
+
+    void ReplyAttr(Request req, AttrOut* attrOut);
+
+    void ReplyReadlink(Request req, const std::string& link);
+
+    void ReplyOpen(Request req, FileInfo *fi);
+
+    void ReplyOpen(Request req, FileOut* fileOut);
+
+    void ReplyData(Request req,
+                   struct fuse_bufvec *bufv,
+                   enum fuse_buf_copy_flags flags);
+
+    void ReplyWrite(Request req, FileOut* fileOut);
+
+    void ReplyBuffer(Request req, const char *buf, size_t size);
+
+    void ReplyStatfs(Request req, const struct statvfs *stbuf);
+
+    void ReplyXattr(Request req, size_t size);
+
+    void ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi);
+
+    void AddDirEntry(Request req,
+                     DirBufferHead* buffer,
+                     DirEntry* dirEntry);
+
+    void AddDirEntryPlus(Request req,
+                         DirBufferHead* buffer,
+                         DirEntry* dirEntry);
+
+    // utility: file handler
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t fh);
+
+    void ReleaseHandler(uint64_t fh);
+
+    // utility: others
+    FileSystemMember BorrowMember();
+
+ private:
+    FRIEND_TEST(FileSystemTest, Attr2Stat);
+    FRIEND_TEST(FileSystemTest, Entry2Param);
+    FRIEND_TEST(FileSystemTest, SetEntryTimeout);
+    FRIEND_TEST(FileSystemTest, SetAttrTimeout);
+
+    // utility: convert to system type.
+    void Attr2Stat(InodeAttr* attr, struct stat* stat);
+
+    void Entry2Param(EntryOut* entryOut, fuse_entry_param* e);
+
+    // utility: set entry/attribute timeout
+    void SetEntryTimeout(EntryOut* entryOut);
+
+    void SetAttrTimeout(AttrOut* attrOut);
+
+ private:
+    FileSystemOption option_;
+    ExternalMember member;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<LookupCache> negative_;
+    std::shared_ptr<DirCache> dirCache_;
+    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<AttrWatcher> attrWatcher_;
+    std::shared_ptr<HandlerManager> handlerManager_;
+    std::shared_ptr<RPCClient> rpc_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_

curvefs/src/client/filesystem/lookup_cache.cpp

@@ -0,0 +1,104 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include <ctime>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/utils.h"
+#include "curvefs/src/client/filesystem/lookup_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+#define RETURN_FALSE_IF_DISABLED() \
+    do {                           \
+        if (!enable_) {            \
+            return false;          \
+        }                          \
+    } while (0)
+
+
+LookupCache::LookupCache(LookupCacheOption option)
+    : enable_(option.negativeTimeoutSec > 0),
+      rwlock_(),
+      option_(option) {
+    lru_ = std::make_shared<LRUType>(option.lruSize);
+    if (enable_) {
+        LOG(INFO) << "Using lookup negative lru cache"
+                  << ", timeout = " << option.negativeTimeoutSec
+                  << ", capacity = " << option.lruSize;
+    }
+}
+
+std::string LookupCache::CacheKey(Ino parent, const std::string& name) {
+    return absl::StrFormat("%d:%s", parent, name);
+}
+
+bool LookupCache::Get(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    ReadLockGuard lk(rwlock_);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
+    if (!yes) {
+        VLOG(1) << absl::StrFormat("Lookup cache not found: key(%d,%s)",
+                                   parent, name);
+        return false;
+    } else if (entry.uses < option_.minUses) {
+        return false;
+    } else if (entry.expireTime < Now()) {
+        return false;
+    }
+    return true;
+}
+
+bool LookupCache::Put(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    WriteLockGuard lk(rwlock_);
+    CacheEntry entry;
+    auto key = CacheKey(parent, name);
+    bool yes = lru_->Get(key, &entry);
+    if (yes) {
+        entry.uses++;
+    } else {
+        entry.uses = 0;
+    }
+
+    entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0);
+    lru_->Put(key, entry);
+    return true;
+}
+
+bool LookupCache::Delete(Ino parent, const std::string& name) {
+    RETURN_FALSE_IF_DISABLED();
+    WriteLockGuard lk(rwlock_);
+    auto key = CacheKey(parent, name);
+    lru_->Remove(key);
+    return true;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/lookup_cache.h

@@ -0,0 +1,77 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-31
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
+
+#include <memory>
+#include <string>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::RWLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::common::LookupCacheOption;
+
+// memory cache for lookup result, now we only support cache negative result,
+// and other positive entry will be cached in kernel.
+class LookupCache {
+ public:
+    struct CacheEntry {
+        uint32_t uses;
+        TimeSpec expireTime;
+    };
+
+    using LRUType = LRUCache<std::string, CacheEntry>;
+
+ public:
+    explicit LookupCache(LookupCacheOption option);
+
+    bool Get(Ino parent, const std::string& name);
+
+    bool Put(Ino parent, const std::string& name);
+
+    bool Delete(Ino parent, const std::string& name);
+
+ private:
+    std::string CacheKey(Ino parent, const std::string& name);
+
+ private:
+    bool enable_;
+    RWLock rwlock_;
+    LookupCacheOption option_;
+    std::shared_ptr<LRUType> lru_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_

curvefs/src/client/filesystem/message_queue.h

@@ -0,0 +1,118 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
+
+#include <glog/logging.h>
+
+#include <string>
+#include <atomic>
+#include <thread>
+
+#include "src/common/concurrent/task_queue.h"
+#include "curvefs/src/common/threading.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::TaskQueue;
+using ::curvefs::common::SetThreadName;
+
+template <typename MessageT>
+class MessageQueue {
+ public:
+    using MessageHandler = std::function<void(const MessageT& message)>;
+
+ public:
+    MessageQueue(const std::string& name, size_t bufferSize)
+        : name_(name),
+          running_(false),
+          thread_(),
+          handler_(),
+          queue_(bufferSize) {}
+
+    void Start() {
+        if (running_.exchange(true)) {
+            return;
+        }
+
+        thread_ = std::thread(&MessageQueue::Consumer, this);
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread start success";
+    }
+
+    void Stop() {
+        if (!running_.exchange(false)) {
+            return;
+        }
+
+        auto wakeup = []() {};
+        queue_.Push(wakeup);
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stoping...";
+
+        thread_.join();
+
+        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
+                  << "consumer thread stopped";
+    }
+
+    void Publish(MessageT message) {
+        if (handler_ != nullptr) {
+            queue_.Push([this, message](){
+                this->handler_(message);
+            });
+        }
+    }
+
+    void Subscribe(MessageHandler handler) {
+        handler_ = handler;
+    }
+
+ private:
+    void Consumer() {
+        SetThreadName(name_.c_str());
+        while (running_.load(std::memory_order_relaxed)) {
+            queue_.Pop()();
+        }
+
+        while (queue_.Size() > 0) {
+            queue_.Pop()();
+        }
+    }
+
+ private:
+    std::string name_;
+    std::atomic<bool> running_;
+    std::thread thread_;
+    MessageHandler handler_;
+    TaskQueue queue_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_

curvefs/src/client/filesystem/meta.cpp

@@ -0,0 +1,129 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <sys/stat.h>
+
+#include "absl/strings/str_format.h"
+#include "curvefs/src/client/filesystem/meta.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+HandlerManager::HandlerManager()
+    : mutex_(),
+      dirBuffer_(std::make_shared<DirBuffer>()),
+      handlers_() {}
+
+HandlerManager::~HandlerManager() {
+    dirBuffer_->DirBufferFreeAll();
+}
+
+std::shared_ptr<FileHandler> HandlerManager::NewHandler() {
+    UniqueLock lk(mutex_);
+    auto handler = std::make_shared<FileHandler>();
+    handler->fh = dirBuffer_->DirBufferNew();
+    handler->buffer = dirBuffer_->DirBufferGet(handler->fh);
+    handler->padding = false;
+    handlers_.emplace(handler->fh, handler);
+    return handler;
+}
+
+std::shared_ptr<FileHandler> HandlerManager::FindHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    auto iter = handlers_.find(fh);
+    if (iter == handlers_.end()) {
+        return nullptr;
+    }
+    return iter->second;
+}
+
+void HandlerManager::ReleaseHandler(uint64_t fh) {
+    UniqueLock lk(mutex_);
+    dirBuffer_->DirBufferRelease(fh);
+    handlers_.erase(fh);
+}
+
+std::string StrMode(uint16_t mode) {
+    static std::map<uint16_t, char> type2char = {
+        { S_IFSOCK, 's' },
+        { S_IFLNK, 'l' },
+        { S_IFREG, '-' },
+        { S_IFBLK, 'b' },
+        { S_IFDIR, 'd' },
+        { S_IFCHR, 'c' },
+        { S_IFIFO, 'f' },
+        { 0, '?' },
+    };
+
+    std::string s("?rwxrwxrwx");
+    s[0] = type2char[mode & (S_IFMT & 0xffff)];
+    if (mode & S_ISUID) {
+        s[3] = 's';
+    }
+    if (mode & S_ISGID) {
+        s[6] = 's';
+    }
+    if (mode & S_ISVTX) {
+        s[9] = 't';
+    }
+
+    for (auto i = 0; i < 9; i++) {
+        if ((mode & (1 << i)) == 0) {
+            if ((s[9-i] == 's') || (s[9-i] == 't')) {
+                s[9-i] &= 0xDF;
+            } else {
+                s[9-i] = '-';
+            }
+        }
+    }
+    return s;
+}
+
+namespace {
+
+std::string Attr2Str(const InodeAttr& attr) {
+    if (!attr.IsInitialized()) {
+        return "";
+    }
+
+    std::string smode;
+    return absl::StrFormat(" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])",
+        attr.inodeid(), StrMode(attr.mode()).c_str(), attr.mode(), attr.nlink(),
+        attr.uid(), attr.gid(),
+        attr.atime(), attr.mtime(), attr.ctime(),
+        attr.length());
+}
+
+}  // namespace
+
+std::string StrEntry(EntryOut entryOut) {
+    return Attr2Str(entryOut.attr);
+}
+
+std::string StrAttr(AttrOut attrOut) {
+    return Attr2Str(attrOut.attr);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/meta.h

@@ -0,0 +1,178 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <iostream>
+
+#include "src/common/concurrent/concurrent.h"
+#include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/client/fuse_common.h"
+#include "curvefs/src/client/dir_buffer.h"
+#include "curvefs/src/client/inode_wrapper.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::Mutex;
+using ::curve::common::RWLock;
+using ::curve::common::UniqueLock;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::metaserver::XAttr;
+using ::curvefs::metaserver::Dentry;
+using ::curvefs::metaserver::InodeAttr;
+using ::curvefs::metaserver::FsFileType;
+using ::curvefs::client::DirBufferHead;
+using ::curvefs::client::InodeWrapper;
+
+using Ino = fuse_ino_t;
+using Request = fuse_req_t;
+using FileInfo = struct fuse_file_info;
+
+struct EntryOut {
+    EntryOut() = default;
+
+    explicit EntryOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double entryTimeout;
+    double attrTimeout;
+};
+
+struct AttrOut {
+    AttrOut() = default;
+
+    explicit AttrOut(InodeAttr attr) : attr(attr) {}
+
+    InodeAttr attr;
+    double attrTimeout;
+};
+
+struct DirEntry {
+    DirEntry() = default;
+
+    DirEntry(Ino ino, const std::string& name, InodeAttr attr)
+        : ino(ino), name(name), attr(attr) {}
+
+    Ino ino;
+    std::string name;
+    InodeAttr attr;
+};
+
+struct FileOut {
+    FileOut() = default;
+
+    FileOut(FileInfo* fi, InodeAttr attr)
+        : fi(fi), attr(attr), nwritten(0) {}
+
+    FileOut(InodeAttr attr, size_t nwritten)
+        : fi(nullptr), attr(attr), nwritten(nwritten) {}
+
+    FileInfo* fi;
+    InodeAttr attr;
+    size_t nwritten;
+};
+
+struct TimeSpec {
+    TimeSpec() : seconds(0), nanoSeconds(0) {}
+
+    TimeSpec(uint64_t seconds, uint32_t nanoSeconds)
+        : seconds(seconds), nanoSeconds(nanoSeconds) {}
+
+    TimeSpec(const TimeSpec& time)
+        : seconds(time.seconds), nanoSeconds(time.nanoSeconds) {}
+
+    TimeSpec& operator=(const TimeSpec& time) = default;
+
+    TimeSpec operator+(const TimeSpec& time) const {
+        return TimeSpec(seconds + time.seconds, nanoSeconds + time.nanoSeconds);
+    }
+
+    uint64_t seconds;
+    uint32_t nanoSeconds;
+};
+
+struct FileHandler {
+    uint64_t fh;
+    DirBufferHead* buffer;
+    TimeSpec mtime;
+    bool padding;  // padding buffer
+};
+
+class HandlerManager {
+ public:
+    HandlerManager();
+
+    ~HandlerManager();
+
+    std::shared_ptr<FileHandler> NewHandler();
+
+    std::shared_ptr<FileHandler> FindHandler(uint64_t id);
+
+    void ReleaseHandler(uint64_t id);
+
+ private:
+    Mutex mutex_;
+    std::shared_ptr<DirBuffer> dirBuffer_;
+    std::map<uint64_t, std::shared_ptr<FileHandler>> handlers_;
+};
+
+inline bool operator==(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds == rhs.seconds) &&
+        (lhs.nanoSeconds == rhs.nanoSeconds);
+}
+
+inline bool operator!=(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return !(lhs == rhs);
+}
+
+inline bool operator<(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds < rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds < rhs.nanoSeconds);
+}
+
+inline bool operator>(const TimeSpec& lhs, const TimeSpec& rhs) {
+    return (lhs.seconds > rhs.seconds) ||
+        (lhs.seconds == rhs.seconds && lhs.nanoSeconds > rhs.nanoSeconds);
+}
+
+inline std::ostream &operator<<(std::ostream &os, const TimeSpec& time) {
+    return os << time.seconds << "." << time.nanoSeconds;
+}
+
+std::string StrMode(uint16_t mode);
+
+std::string StrEntry(EntryOut entryOut);
+
+std::string StrAttr(AttrOut attrOut);
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_

curvefs/src/client/filesystem/openfile.cpp

@@ -0,0 +1,141 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+OpenFiles::OpenFiles(OpenFilesOption option,
+                     std::shared_ptr<DeferSync> deferSync)
+    : rwlock_(),
+      option_(option),
+      deferSync_(deferSync),
+      files_(std::make_shared<LRUType>(option.lruSize)) {
+    LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize;
+}
+
+/*
+ * Delete(...) does:
+ *   1) publish to message queue which will flush file to server with async
+ *   2) delete file from lru cache
+ */
+void OpenFiles::Delete(Ino ino,
+                       const std::shared_ptr<OpenFile>& file,
+                       bool flush) {
+    if (flush) {
+        deferSync_->Push(file->inode);
+    }
+    files_->Remove(ino);
+
+    VLOG(1) << "Delete open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+void OpenFiles::Evit(size_t size) {
+    Ino ino;
+    std::shared_ptr<OpenFile> file;
+    while (files_->Size() + size >= option_.lruSize) {
+        bool yes = files_->GetLast(&ino, &file);
+        if (!yes) {
+            break;
+        }
+        Delete(ino, file, true);
+    }
+}
+
+void OpenFiles::Open(Ino ino, std::shared_ptr<InodeWrapper> inode) {
+    WriteLockGuard lk(rwlock_);
+
+    Evit(1);
+
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (yes) {
+        file->refs++;
+        return;
+    }
+
+    file = std::make_shared<OpenFile>(inode);
+    file->refs++;
+    files_->Put(ino, file);
+
+    VLOG(1) << "Insert open file cache: ino = " << ino
+            << ", refs = " << file->refs
+            << ", mtime = " << InodeMtime(file->inode);
+}
+
+bool OpenFiles::IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+    *inode = file->inode;
+    return true;
+}
+
+void OpenFiles::Close(Ino ino) {
+    WriteLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return;
+    }
+
+    if (file->refs > 0) {
+        file->refs--;
+    }
+
+    if (file->refs == 0) {
+        Delete(ino, file, false);  // file already flushed before close
+    }
+}
+
+/*
+ * CloseAll() does:
+ *   flush all file to server and delete from LRU cache
+ */
+void OpenFiles::CloseAll() {
+    WriteLockGuard lk(rwlock_);
+    Evit(option_.lruSize);
+}
+
+bool OpenFiles::GetFileAttr(Ino ino, InodeAttr* attr) {
+    ReadLockGuard lk(rwlock_);
+    std::shared_ptr<OpenFile> file;
+    bool yes = files_->Get(ino, &file);
+    if (!yes) {
+        return false;
+    }
+
+    file->inode->GetInodeAttr(attr);
+    return true;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/openfile.h

@@ -0,0 +1,86 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-09
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
+
+#include <memory>
+
+#include "src/common/lru_cache.h"
+#include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curve::common::LRUCache;
+using ::curve::common::ReadLockGuard;
+using ::curve::common::WriteLockGuard;
+using ::curvefs::client::InodeWrapper;
+using ::curvefs::client::common::OpenFilesOption;
+
+struct OpenFile {
+    explicit OpenFile(std::shared_ptr<InodeWrapper> inode)
+        : inode(inode), refs(0) {}
+
+    std::shared_ptr<InodeWrapper> inode;
+    uint64_t refs;
+};
+
+class OpenFiles {
+ public:
+    using LRUType = LRUCache<Ino, std::shared_ptr<OpenFile>>;
+
+ public:
+    explicit OpenFiles(OpenFilesOption option,
+                       std::shared_ptr<DeferSync> deferSync);
+
+    void Open(Ino ino, std::shared_ptr<InodeWrapper> inode);
+
+    bool IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+    void Close(Ino ino);
+
+    void CloseAll();
+
+    bool GetFileAttr(Ino ino, InodeAttr* inode);
+
+ private:
+    void Delete(Ino ino, const std::shared_ptr<OpenFile>& file, bool flush);
+
+    void Evit(size_t size);
+
+ private:
+    RWLock rwlock_;
+    OpenFilesOption option_;
+    std::shared_ptr<DeferSync> deferSync_;
+    std::shared_ptr<LRUType> files_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_

curvefs/src/client/filesystem/package.h

@@ -0,0 +1,51 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
+
+#include <memory>
+
+#include "curvefs/src/client/dentry_cache_manager.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/xattr_manager.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+struct ExternalMember {  // external member depended by FileSystem
+    ExternalMember() = delete;
+    ExternalMember(std::shared_ptr<DentryCacheManager> dentryManager,
+                   std::shared_ptr<InodeCacheManager> inodeManager)
+        : dentryManager(dentryManager),
+          inodeManager(inodeManager) {}
+
+    std::shared_ptr<DentryCacheManager> dentryManager;
+    std::shared_ptr<InodeCacheManager> inodeManager;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_

curvefs/src/client/filesystem/rpc_client.cpp

@@ -0,0 +1,128 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <map>
+#include <set>
+#include <list>
+#include <string>
+#include <memory>
+#include <utility>
+
+#include "curvefs/src/client/filesystem/rpc_client.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+RPCClient::RPCClient(RPCOption option,
+                     ExternalMember member)
+    : option_(option),
+      inodeManager_(member.inodeManager),
+      dentryManager_(member.dentryManager) {}
+
+CURVEFS_ERROR RPCClient::GetAttr(Ino ino, InodeAttr* attr) {
+    CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(ino, attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(getattr::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::Lookup(Ino parent,
+                                const std::string& name,
+                                EntryOut* entryOut) {
+    Dentry dentry;
+    CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry);
+    if (rc != CURVEFS_ERROR::OK) {
+        if (rc != CURVEFS_ERROR::NOTEXIST) {
+            LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc
+                       << ", parent = " << parent << ", name = " << name;
+        }
+        return rc;
+    }
+
+    Ino ino = dentry.inodeid();
+    rc = inodeManager_->GetInodeAttr(ino, &entryOut->attr);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(lookup::GetInodeAttr) failed, retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
+                                 std::shared_ptr<DirEntryList>* entries) {
+    uint32_t limit = option_.listDentryLimit;
+
+    std::list<Dentry> dentries;
+    CURVEFS_ERROR rc = dentryManager_->ListDentry(ino, &dentries, limit);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc
+                   << ", ino = " << ino;
+        return rc;
+    }
+
+    std::set<uint64_t> inos;
+    std::map<uint64_t, InodeAttr> attrs;
+    std::for_each(dentries.begin(), dentries.end(), [&](Dentry& dentry){
+        inos.emplace(dentry.inodeid());
+    });
+    rc = inodeManager_->BatchGetInodeAttrAsync(ino, &inos, &attrs);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(readdir::BatchGetInodeAttrAsync) failed"
+                   << ", retCode = " << rc << ", ino = " << ino;
+        return rc;
+    }
+
+    DirEntry dirEntry;
+    for (const auto& dentry : dentries) {
+        Ino ino = dentry.inodeid();
+        auto iter = attrs.find(ino);
+        if (iter == attrs.end()) {
+            LOG(WARNING) << "rpc(readdir::BatchGetInodeAttrAsync) "
+                         << "missing attribute, ino = " << ino;
+            continue;
+        }
+
+        // NOTE: we can't use std::move() for attribute for hard link
+        // which will sharing inode attribute.
+        dirEntry.ino = ino;
+        dirEntry.name = std::move(dentry.name());
+        dirEntry.attr = iter->second;
+        (*entries)->Add(dirEntry);
+    }
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR RPCClient::Open(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
+    CURVEFS_ERROR rc = inodeManager_->GetInode(ino, *inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "rpc(open/GetInode) failed" << ", retCode = " << rc
+                   << ", ino = " << ino;
+    }
+    return rc;
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/rpc_client.h

@@ -0,0 +1,65 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-07
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
+
+#include <memory>
+#include <string>
+
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+#include "curvefs/src/client/filesystem/dir_cache.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+using ::curvefs::client::common::RPCOption;
+
+class RPCClient {
+ public:
+    RPCClient(RPCOption option,
+              ExternalMember member);
+
+    CURVEFS_ERROR GetAttr(Ino ino, InodeAttr* attr);
+
+    CURVEFS_ERROR Lookup(Ino parent,
+                         const std::string& name,
+                         EntryOut* entryOut);
+
+    CURVEFS_ERROR ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries);
+
+    CURVEFS_ERROR Open(Ino ino, std::shared_ptr<InodeWrapper>* inode);
+
+ private:
+    RPCOption option_;
+    std::shared_ptr<InodeCacheManager> inodeManager_;
+    std::shared_ptr<DentryCacheManager> dentryManager_;
+};
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_

curvefs/src/client/filesystem/utils.cpp

@@ -0,0 +1,69 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#include <glog/logging.h>
+
+#include "curvefs/src/client/filesystem/utils.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+bool IsDir(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_DIRECTORY;
+}
+
+bool IsS3File(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_S3;
+}
+
+bool IsVolmeFile(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_FILE;
+}
+
+bool IsSymLink(const InodeAttr& attr) {
+    return attr.type() == FsFileType::TYPE_SYM_LINK;
+}
+
+TimeSpec AttrMtime(const InodeAttr& attr) {
+    return TimeSpec(attr.mtime(), attr.mtime_ns());
+}
+
+TimeSpec AttrCtime(const InodeAttr& attr) {
+    return TimeSpec(attr.ctime(), attr.ctime_ns());
+}
+
+TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode) {
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    return AttrMtime(attr);
+}
+
+TimeSpec Now() {
+    struct timespec now;
+    clock_gettime(CLOCK_REALTIME, &now);
+    return TimeSpec(now.tv_sec, now.tv_nsec);
+}
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/filesystem/utils.h

@@ -0,0 +1,59 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: Curve
+ * Created Date: 2023-03-06
+ * Author: Jingli Chen (Wine93)
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+#define CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
+
+#include <memory>
+
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/package.h"
+
+namespace curvefs {
+namespace client {
+namespace filesystem {
+
+// directory
+bool IsDir(const InodeAttr& attr);
+
+// file which data is stored in s3
+bool IsS3File(const InodeAttr& attr);
+
+// file which data is stored in volume
+bool IsVolmeFile(const InodeAttr& attr);
+
+// symbol link
+bool IsSymLink(const InodeAttr& attr);
+
+struct TimeSpec AttrMtime(const InodeAttr& attr);
+
+struct TimeSpec AttrCtime(const InodeAttr& attr);
+
+struct TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode);
+
+struct TimeSpec Now();
+
+}  // namespace filesystem
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_

curvefs/src/client/fuse_client.cpp

@@ -35,11 +35,10 @@
 
 #include "curvefs/proto/mds.pb.h"
 #include "curvefs/src/client/common/common.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/fuse_common.h"
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/inode_wrapper.h"
-#include "curvefs/src/client/kvclient/memcache_client.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
 #include "curvefs/src/client/xattr_manager.h"
 #include "curvefs/src/common/define.h"
@@ -50,9 +49,16 @@
 
 #define PORT_LIMIT 65535
 
+using ::curvefs::common::S3Info;
+using ::curvefs::common::Volume;
+using ::curvefs::mds::topology::PartitionTxId;
 using ::curvefs::mds::FSStatusCode_Name;
 using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH;
 using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
+using ::curvefs::client::filesystem::ExternalMember;
+using ::curvefs::client::filesystem::DirEntry;
+using ::curvefs::client::filesystem::DirEntryList;
+using ::curvefs::client::filesystem::FileOut;
 
 #define RETURN_IF_UNSUCCESS(action)                                            \
     do {                                                                       \
@@ -111,25 +117,16 @@ using common::FLAGS_fuseClientAvgReadBytes;
 using common::FLAGS_fuseClientBurstReadBytes;
 using common::FLAGS_fuseClientBurstReadBytesSecs;
 
-// static void on_throttle_timer(void *arg) {
-//     FuseClient *fuseClient = reinterpret_cast<FuseClient *>(arg);
-//     fuseClient->InitQosParam();
-// }
+static void on_throttle_timer(void *arg) {
+    FuseClient *fuseClient = reinterpret_cast<FuseClient *>(arg);
+    fuseClient->InitQosParam();
+}
 
 CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
-    LOG(INFO) << "fuse client init start.";
     option_ = option;
-    const uint64_t writeCacheMaxByte =
-      option_.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
-    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
-        LOG(ERROR) << "writeCacheMaxByte is too small"
-                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
-                      ", writeCacheMaxByte = " << writeCacheMaxByte;
-        return CURVEFS_ERROR::CACHETOOSMALL;
-    }
 
     mdsBase_ = new MDSBaseClient();
-    auto ret = mdsClient_->Init(option.mdsOpt, mdsBase_);
+    FSStatusCode ret = mdsClient_->Init(option.mdsOpt, mdsBase_);
     if (ret != FSStatusCode::OK) {
         return CURVEFS_ERROR::INTERNAL;
     }
@@ -160,25 +157,26 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     curve::client::ClientDummyServerInfo::GetInstance().SetPort(listenPort);
     curve::client::ClientDummyServerInfo::GetInstance().SetIP(localIp);
 
-    auto ret2 =
+    {
+        ExternalMember member(dentryManager_, inodeManager_);
+        fs_ = std::make_shared<FileSystem>(option_.fileSystemOption, member);
+    }
+
+    MetaStatusCode ret2 =
         metaClient_->Init(option.excutorOpt, option.excutorInternalOpt,
                           metaCache, channelManager);
     if (ret2 != MetaStatusCode::OK) {
         return CURVEFS_ERROR::INTERNAL;
     }
 
-    auto ret3 =
-        inodeManager_->Init(option.iCacheLruSize, option.enableICacheMetrics,
-                            option.flushPeriodSec, option.refreshDataOption,
-                            option.lruTimeOutSec);
-    if (ret3 != CURVEFS_ERROR::OK) {
-        return ret3;
-    }
-    ret3 =
-        dentryManager_->Init(option.dCacheLruSize, option.enableDCacheMetrics,
-            option.lruTimeOutSec);
-    if (ret3 != CURVEFS_ERROR::OK) {
-        return ret3;
+    {  // init inode manager
+        auto member = fs_->BorrowMember();
+        CURVEFS_ERROR rc = inodeManager_->Init(option.refreshDataOption,
+                                               member.openFiles,
+                                               member.deferSync);
+        if (rc != CURVEFS_ERROR::OK) {
+            return rc;
+        }
     }
 
     if (warmupManager_ != nullptr) {
@@ -187,8 +185,8 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     }
 
     InitQosParam();
-    LOG(INFO) << "fuse client init success.";
-    return ret3;
+
+    return CURVEFS_ERROR::OK;
 }
 
 void FuseClient::UnInit() {
@@ -199,15 +197,13 @@ void FuseClient::UnInit() {
     delete mdsBase_;
     mdsBase_ = nullptr;
 
-    // while (bthread_timer_del(throttleTimer_) == 1) {
-    //     bthread_usleep(1000);
-    // }
-    // s3Adaptor_->Stop();
+    while (bthread_timer_del(throttleTimer_) == 1) {
+        bthread_usleep(1000);
+    }
 }
 
 CURVEFS_ERROR FuseClient::Run() {
     if (isStop_.exchange(false)) {
-        inodeManager_->Run();
         return CURVEFS_ERROR::OK;
     }
     return CURVEFS_ERROR::INTERNAL;
@@ -215,7 +211,6 @@ CURVEFS_ERROR FuseClient::Run() {
 
 void FuseClient::Fini() {
     if (!isStop_.exchange(true)) {
-        inodeManager_->Stop();
         xattrManager_->Stop();
     }
 }
@@ -224,6 +219,7 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                      struct fuse_conn_info *conn) {
     (void)userdata;
     (void)conn;
+    fs_->Run();
     return CURVEFS_ERROR::OK;
 }
 
@@ -233,7 +229,7 @@ void FuseClient::FuseOpDestroy(void *userdata) {
     }
 
     FlushAll();
-    dirBuf_->DirBufferFreeAll();
+    fs_->Destory();
 
     // stop lease before umount fs, otherwise, lease request after umount fs
     // will add a mountpoint entry.
@@ -265,91 +261,38 @@ void FuseClient::FuseOpDestroy(void *userdata) {
               << " success!";
 }
 
-void InodeAttr2ParamAttr(const InodeAttr &inodeAttr, struct stat *attr) {
-    attr->st_ino = inodeAttr.inodeid();
-    attr->st_mode = inodeAttr.mode();
-    attr->st_nlink = inodeAttr.nlink();
-    attr->st_uid = inodeAttr.uid();
-    attr->st_gid = inodeAttr.gid();
-    attr->st_size = inodeAttr.length();
-    attr->st_rdev = inodeAttr.rdev();
-    attr->st_atim.tv_sec = inodeAttr.atime();
-    attr->st_atim.tv_nsec = inodeAttr.atime_ns();
-    attr->st_mtim.tv_sec = inodeAttr.mtime();
-    attr->st_mtim.tv_nsec = inodeAttr.mtime_ns();
-    attr->st_ctim.tv_sec = inodeAttr.ctime();
-    attr->st_ctim.tv_nsec = inodeAttr.ctime_ns();
-    attr->st_blksize = kOptimalIOBlockSize;
-    switch (inodeAttr.type()) {
-        // TODO(@hzwuhongsong): type_volume?
-        case metaserver::TYPE_S3:
-            attr->st_blocks = (inodeAttr.length() + 511) / 512;
-            break;
-        default:
-            attr->st_blocks = 0;
-            break;
+CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       EntryOut* entryOut) {
+    CURVEFS_ERROR rc = fs_->Lookup(req, parent, name, entryOut);
+    if (rc != CURVEFS_ERROR::OK && rc != CURVEFS_ERROR::NOTEXIST) {
+        LOG(ERROR) << "Lookup() failed, retCode = " << rc
+                   << ", parent = " << parent << ", name = " << name;
     }
+    return rc;
 }
 
-void GetDentryParamFromInodeAttr(
-    const FuseClientOption &option,
-    const InodeAttr &inodeAttr,
-    fuse_entry_param *param) {
-    memset(param, 0, sizeof(fuse_entry_param));
-    param->ino = inodeAttr.inodeid();
-    param->generation = 0;
-    InodeAttr2ParamAttr(inodeAttr, &param->attr);
-    param->attr_timeout = option.attrTimeOut;
-    param->entry_timeout = option.entryTimeOut;
-}
-
-CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
-                                       const char *name, fuse_entry_param *e) {
-    (void)req;
-    VLOG(1) << "FuseOpLookup parent: " << parent
-            << ", name: " << name;
-    if (strlen(name) > option_.maxNameLength) {
-        return CURVEFS_ERROR::NAMETOOLONG;
-    }
-
-    Dentry dentry;
-    CURVEFS_ERROR ret = dentryManager_->GetDentry(parent, name, &dentry);
-    if (ret != CURVEFS_ERROR::OK) {
-        if (ret != CURVEFS_ERROR::NOTEXIST) {
-            LOG(WARNING) << "dentryManager_ get dentry fail, ret = " << ret
-                         << ", parent inodeid = " << parent
-                         << ", name = " << name;
-        }
-        return ret;
-    }
-
-    fuse_ino_t ino = dentry.inodeid();
-    InodeAttr attr;
-    ret = inodeManager_->GetInodeAttr(ino, &attr);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
-    }
-
-    GetDentryParamFromInodeAttr(option_, attr, e);
-    return ret;
-}
-
-CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
-                                     struct fuse_file_info *fi) {
-    (void)req;
-    VLOG(1) << "FuseOpOpen, ino: " << ino;
+CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
+                                          fuse_ino_t ino,
+                                          struct fuse_file_info* fi,
+                                          FileOut* fileOut) {
     std::shared_ptr<InodeWrapper> inodeWrapper;
+    // alredy opened
     CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
                    << ", inodeid = " << ino;
         return ret;
     }
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+
+    fileOut->fi = fi;
+    inodeWrapper->GetInodeAttr(&fileOut->attr);
+
     if (fi->flags & O_TRUNC) {
         if (fi->flags & O_WRONLY || fi->flags & O_RDWR) {
+            ::curve::common::UniqueLock lgGuard =
+                inodeWrapper->GetUniqueLock();
             uint64_t length = inodeWrapper->GetLengthLocked();
             CURVEFS_ERROR tRet = Truncate(inodeWrapper.get(), 0);
             if (tRet != CURVEFS_ERROR::OK) {
@@ -384,18 +327,29 @@ CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
                     }
                 }
             }
+            inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
         } else {
             return CURVEFS_ERROR::NOPERMISSION;
         }
     }
-    if (FLAGS_enableCto) {
-        inodeManager_->AddOpenedInode(ino);
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     struct fuse_file_info* fi,
+                                     FileOut* fileOut) {
+    CURVEFS_ERROR rc = fs_->Open(req, ino, fi);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "open(" << ino << ") failed, retCode = " << rc;
+        return rc;
     }
-    return ret;
+    return HandleOpenFlags(req, ino, fi, fileOut);
 }
 
 CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
     fuse_ino_t parent, FsFileType type, NlinkChange nlink) {
+
     std::shared_ptr<InodeWrapper> parentInodeWrapper;
     auto ret = inodeManager_->GetInode(parent, parentInodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
@@ -411,16 +365,27 @@ CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
         if (FsFileType::TYPE_DIRECTORY == type) {
             parentInodeWrapper->UpdateNlinkLocked(nlink);
         }
+
+        if (option_.fileSystemOption.deferSyncOption.deferDirMtime) {
+            inodeManager_->ShipToFlush(parentInodeWrapper);
+        } else {
+            return parentInodeWrapper->SyncAttr();
+        }
     }
-    inodeManager_->ShipToFlush(parentInodeWrapper);
+
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
-                                   const char *name, mode_t mode,
-                                   FsFileType type, dev_t rdev, bool internal,
-                                   fuse_entry_param *e) {
-    if (strlen(name) > option_.maxNameLength) {
+CURVEFS_ERROR FuseClient::MakeNode(
+    fuse_req_t req,
+    fuse_ino_t parent,
+    const char* name,
+    mode_t mode,
+    FsFileType type,
+    dev_t rdev,
+    bool internal,
+    std::shared_ptr<InodeWrapper>& inodeWrapper) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -451,7 +416,6 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
     param.rdev = rdev;
     param.parent = parent;
 
-    std::shared_ptr<InodeWrapper> inodeWrapper;
     CURVEFS_ERROR ret = inodeManager_->CreateInode(param, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager CreateInode fail, ret = " << ret
@@ -522,20 +486,26 @@ CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttr(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
-                                      const char *name, mode_t mode,
-                                      fuse_entry_param *e) {
+CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char *name,
+                                      mode_t mode,
+                                      EntryOut* entryOut) {
     VLOG(1) << "FuseOpMkDir, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
     bool internal = false;
-    return MakeNode(req, parent, name, S_IFDIR | mode,
-                    FsFileType::TYPE_DIRECTORY, 0, internal, e);
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, S_IFDIR | mode,
+                                FsFileType::TYPE_DIRECTORY, 0, internal, inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
@@ -594,8 +564,6 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent,
                          << ", xattr = " << xattr.DebugString();
         }
     }
-
-    inodeManager_->ClearInodeCache(ino);
     return ret;
 }
 
@@ -608,10 +576,13 @@ std::string GetRecycleTimeDirName() {
     return now;
 }
 
-CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, uint64_t parent,
-                    const char *name, mode_t mode, ManageInodeType manageType,
-                    fuse_entry_param *e) {
-    if (strlen(name) > option_.maxNameLength) {
+CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req,
+                                           uint64_t parent,
+                                           const char* name,
+                                           mode_t mode,
+                                           ManageInodeType manageType,
+                                           EntryOut* entryOut) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -695,9 +666,7 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, uint64_t parent,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttrLocked(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
+    inodeWrapper->GetInodeAttrLocked(&entryOut->attr);
     return ret;
 }
 
@@ -711,9 +680,10 @@ CURVEFS_ERROR FuseClient::GetOrCreateRecycleDir(fuse_req_t req, Dentry *out) {
     } else if (ret == CURVEFS_ERROR::NOTEXIST) {
         LOG(INFO) << "recycle dir is not exist, create " << RECYCLENAME
                   << ", parentid = " << ROOTINODEID;
-        fuse_entry_param param;
+        EntryOut entryOut;
         ret = CreateManageNode(req, ROOTINODEID, RECYCLENAME,
-                       S_IFDIR | 0755, ManageInodeType::TYPE_RECYCLE, &param);
+                               S_IFDIR | 0755, ManageInodeType::TYPE_RECYCLE,
+                               &entryOut);
         if (ret != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "CreateManageNode failed, ret = " << ret
                        << ", inode = " << ROOTINODEID
@@ -750,17 +720,17 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
                    << ", name = " << recycleTimeDirName;
         return ret;
     } else if (ret == CURVEFS_ERROR::NOTEXIST) {
-        fuse_entry_param param;
+        std::shared_ptr<InodeWrapper> inode;
         bool internal = true;
         ret = MakeNode(req, RECYCLEINODEID, recycleTimeDirName.c_str(),
-              S_IFDIR | 0755, FsFileType::TYPE_DIRECTORY, 0, internal, &param);
+              S_IFDIR | 0755, FsFileType::TYPE_DIRECTORY, 0, internal, inode);
         if (ret != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "MakeNode failed, ret = " << ret
                        << ", inode = " << RECYCLEINODEID
                        << ", name = " << recycleTimeDirName;
             return ret;
         }
-        recycleTimeDirIno = param.ino;
+        recycleTimeDirIno = inode->GetInodeId();
     } else {
         recycleTimeDirIno = dentry.inodeid();
     }
@@ -770,7 +740,7 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
 
     // 4. move inode to recycle time dir
     ret = FuseOpRename(req, parent, name,
-                                     recycleTimeDirIno, newName.c_str());
+                       recycleTimeDirIno, newName.c_str(), 0);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "MoveToRecycle failed, ret = " << ret
                    << ", inodeid = " << ino
@@ -807,7 +777,7 @@ bool FuseClient::ShouldMoveToRecycle(fuse_ino_t parent) {
 
 CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
                                      const char *name, FsFileType type) {
-    if (strlen(name) > option_.maxNameLength) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -863,146 +833,87 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
-                                        struct fuse_file_info *fi) {
-    (void)req;
-    VLOG(1) << "FuseOpOpenDir ino = " << ino;
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
+CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        struct fuse_file_info* fi) {
+    CURVEFS_ERROR rc = fs_->OpenDir(req, ino, fi);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "opendir() failed, retCode = " << rc
+                   << ", ino = " << ino;
     }
-
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-
-    uint64_t dindex = dirBuf_->DirBufferNew();
-    fi->fh = dindex;
-    VLOG(1) << "FuseOpOpenDir, ino: " << ino << ", dindex: " << dindex;
-    return ret;
+    return rc;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
-                                           struct fuse_file_info *fi) {
-    (void)req;
-    uint64_t dindex = fi->fh;
-    VLOG(1) << "FuseOpReleaseDir, ino: " << ino << ", dindex: " << dindex;
-    dirBuf_->DirBufferRelease(dindex);
+CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        size_t size,
+                                        off_t off,
+                                        struct fuse_file_info *fi,
+                                        char** bufferOut,
+                                        size_t* rSize,
+                                        bool plus) {
+    auto handler = fs_->FindHandler(fi->fh);
+    DirBufferHead* buffer = handler->buffer;
+    if (!handler->padding) {
+        auto entries = std::make_shared<DirEntryList>();
+        CURVEFS_ERROR rc = fs_->ReadDir(req, ino, fi, &entries);
+        if (rc != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "readdir() failed, retCode = " << rc
+                       << ", ino = " << ino << ", fh = " << fi->fh;
+            return rc;
+        }
 
-    // release inodeAttr cache
-    inodeManager_->ReleaseCache(ino);
+        entries->Iterate([&](DirEntry* dirEntry){
+            if (plus) {
+                fs_->AddDirEntryPlus(req, buffer, dirEntry);
+            } else {
+                fs_->AddDirEntry(req, buffer, dirEntry);
+            }
+        });
+        handler->padding = true;
+    }
+
+    if (off < buffer->size) {
+        *bufferOut = buffer->p + off;
+        *rSize = std::min(buffer->size - off, size);
+    } else {
+        *bufferOut = nullptr;
+        *rSize = 0;
+    }
     return CURVEFS_ERROR::OK;
 }
 
-static void dirbuf_add(fuse_req_t req, struct DirBufferHead *b,
-                       const Dentry &dentry,
-                       const FuseClientOption &option,
-                       bool cacheDir,
-                       InodeAttr *attr = nullptr) {
-    struct stat stbuf;
-    struct fuse_entry_param param;
-    size_t oldsize = b->size;
-    if (!cacheDir) {
-        b->size += fuse_add_direntry(req, NULL, 0, dentry.name().c_str(),
-                                     NULL, 0);
-        b->p = static_cast<char *>(realloc(b->p, b->size));
-        memset(&stbuf, 0, sizeof(stbuf));
-        stbuf.st_ino = dentry.inodeid();
-        fuse_add_direntry(req, b->p + oldsize, b->size - oldsize,
-                          dentry.name().c_str(), &stbuf, b->size);
-    } else {
-        b->size += fuse_add_direntry_plus(req, NULL, 0, dentry.name().c_str(),
-                                          NULL, 0);
-        b->p = static_cast<char *>(realloc(b->p, b->size));
-        GetDentryParamFromInodeAttr(option, *attr, &param);
-        fuse_add_direntry_plus(req, b->p + oldsize, b->size - oldsize,
-                               dentry.name().c_str(), &param, b->size);
+CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req,
+                                           fuse_ino_t ino,
+                                           struct fuse_file_info* fi) {
+    CURVEFS_ERROR rc = fs_->ReleaseDir(req, ino, fi);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "releasedir() failed, retCode = " << rc
+                   << ", ino = " << ino;
     }
-}
-
-CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
-                                            size_t size, off_t off,
-                                            struct fuse_file_info *fi,
-                                            char **buffer, size_t *rSize,
-                                            bool cacheDir) {
-    VLOG(1) << "FuseOpReadDirPlus ino: " << ino << ", size: " << size
-            << ", off = " << off << ", cacheDir = " << cacheDir;
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
-    }
-
-    uint64_t dindex = fi->fh;
-    DirBufferHead *bufHead = dirBuf_->DirBufferGet(dindex);
-    if (!bufHead->wasRead) {
-        std::list<Dentry> dentryList;
-        std::set<uint64_t> inodeIds;
-        std::map<uint64_t, InodeAttr> inodeAttrMap;
-        auto limit = option_.listDentryLimit;
-        ret = dentryManager_->ListDentry(ino, &dentryList, limit);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
-                       << ", parent = " << ino;
-            return ret;
-        }
-
-        if (!cacheDir) {
-            for (const auto &dentry : dentryList) {
-                dirbuf_add(req, bufHead, dentry, option_, cacheDir);
-            }
-        } else {
-            for (const auto &dentry : dentryList) {
-                inodeIds.emplace(dentry.inodeid());
-            }
-            VLOG(3) << "batch get inode size = " << inodeIds.size();
-            ret = inodeManager_->BatchGetInodeAttrAsync(ino, &inodeIds,
-                                                        &inodeAttrMap);
-            if (ret != CURVEFS_ERROR::OK) {
-                LOG(ERROR) << "BatchGetInodeAttr failed when FuseOpReadDir"
-                           << ", parentId = " << ino;
-                return ret;
-            }
-
-            for (const auto &dentry : dentryList) {
-                auto iter = inodeAttrMap.find(dentry.inodeid());
-                if (iter != inodeAttrMap.end()) {
-                    dirbuf_add(req, bufHead, dentry, option_,
-                               cacheDir, &iter->second);
-                } else {
-                    LOG(WARNING) << "BatchGetInodeAttr missing some inodes,"
-                                 << " inodeId = " << dentry.inodeid();
-                }
-            }
-        }
-        bufHead->wasRead = true;
-    }
-
-    if (off < static_cast<ssize_t>(bufHead->size)) {
-        *buffer = bufHead->p + off;
-        *rSize = std::min(bufHead->size - off, size);
-    } else {
-        *buffer = nullptr;
-        *rSize = 0;
-    }
-    return ret;
+    return rc;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
                                        const char *name, fuse_ino_t newparent,
-                                       const char *newname) {
-    (void)req;
+                                       const char *newname,
+                                       unsigned int flags) {
     VLOG(1) << "FuseOpRename from (" << parent << ", " << name << ") to ("
             << newparent << ", " << newname << ")";
-    if (strlen(name) > option_.maxNameLength ||
-        strlen(newname) > option_.maxNameLength) {
+
+    // TODO(Wine93): the flag RENAME_EXCHANGE and RENAME_NOREPLACE
+    // is only used in linux interface renameat(), not required by posix,
+    // we can ignore it now
+    if (flags != 0) {
+        return CURVEFS_ERROR::INVALIDPARAM;
+    }
+
+    uint64_t maxNameLength = option_.fileSystemOption.maxNameLength;
+    if (strlen(name) > maxNameLength || strlen(newname) > maxNameLength) {
         LOG(WARNING) << "FuseOpRename name too long, name = " << name
                      << ", name len = " << strlen(name) << ", new name = "
                      << newname << ", new name len = " << strlen(newname)
-                     << ", maxNameLength = " << option_.maxNameLength;
+                     << ", maxNameLength = " << maxNameLength;
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -1041,30 +952,24 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
     return rc;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info *fi,
-                                        struct stat *attr) {
-    (void)req;
-    (void)fi;
-    VLOG(1) << "FuseOpGetAttr ino = " << ino;
-    InodeAttr inodeAttr;
-    CURVEFS_ERROR ret =
-        inodeManager_->GetInodeAttr(ino, &inodeAttr);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
-                   << ", inodeid = " << ino;
-        return ret;
+                                        AttrOut* attrOut) {
+    CURVEFS_ERROR rc = fs_->GetAttr(req, ino, attrOut);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "getattr() fail, retCode = " << rc
+                   << ", ino = " << ino;
     }
-    InodeAttr2ParamAttr(inodeAttr, attr);
-    return ret;
+    return rc;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
-                                        struct stat *attr, int to_set,
-                                        struct fuse_file_info *fi,
-                                        struct stat *attrOut) {
-    (void)req;
-    (void)fi;
+CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        struct stat* attr,
+                                        int to_set,
+                                        struct fuse_file_info* fi,
+                                        struct AttrOut* attrOut) {
     VLOG(1) << "FuseOpSetAttr to_set: " << to_set << ", ino: " << ino
             << ", attr: " << *attr;
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -1121,9 +1026,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
         if (ret != CURVEFS_ERROR::OK) {
             return ret;
         }
-        InodeAttr inodeAttr;
-        inodeWrapper->GetInodeAttrLocked(&inodeAttr);
-        InodeAttr2ParamAttr(inodeAttr, attrOut);
+        inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
 
         if (enableSumInDir_.load() && changeSize != 0) {
             // update parent summary info
@@ -1148,9 +1051,7 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    InodeAttr inodeAttr;
-    inodeWrapper->GetInodeAttrLocked(&inodeAttr);
-    InodeAttr2ParamAttr(inodeAttr, attrOut);
+    inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
     return ret;
 }
 
@@ -1160,8 +1061,8 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
     (void)req;
     VLOG(9) << "FuseOpGetXattr, ino: " << ino
             << ", name: " << name << ", size = " << size;
-    if (option_.disableXattr) {
-        return CURVEFS_ERROR::NOTSUPPORT;
+    if (option_.fileSystemOption.disableXattr) {
+        return CURVEFS_ERROR::NOSYS;
     }
 
     InodeAttr inodeAttr;
@@ -1198,12 +1099,6 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, const char* value,
                                          size_t size, int flags) {
-    (void)req;
-    (void)flags;
-    if (option_.disableXattr) {
-        return CURVEFS_ERROR::NOTSUPPORT;
-    }
-
     std::string strname(name);
     std::string strvalue(value, size);
     VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
@@ -1286,10 +1181,12 @@ CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
     return CURVEFS_ERROR::OUT_OF_RANGE;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
-                                        fuse_ino_t parent, const char *name,
-                                        fuse_entry_param *e) {
-    if (strlen(name) > option_.maxNameLength) {
+CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req,
+                                        const char* link,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        EntryOut* entryOut) {
+    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     const struct fuse_ctx *ctx = fuse_req_ctx(req);
@@ -1358,18 +1255,17 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttr(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
+    inodeWrapper->GetInodeAttr(&entryOut->attr);
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
+CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     fuse_ino_t newparent,
+                                     const char* newname,
                                      FsFileType type,
-                                     fuse_entry_param *e) {
-    (void)req;
-    if (strlen(newname) > option_.maxNameLength) {
+                                     EntryOut* entryOut) {
+    if (strlen(newname) > option_.fileSystemOption.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -1430,9 +1326,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
         }
     }
 
-    InodeAttr attr;
-    inodeWrapper->GetInodeAttr(&attr);
-    GetDentryParamFromInodeAttr(option_, attr, e);
+    inodeWrapper->GetInodeAttr(&entryOut->attr);
     return ret;
 }
 
@@ -1451,24 +1345,18 @@ CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
-    (void)req;
-    (void)fi;
-    VLOG(1) << "FuseOpRelease, ino: " << ino;
-    if (FLAGS_enableCto) {
-        inodeManager_->RemoveOpenedInode(ino);
+    CURVEFS_ERROR rc = fs_->Release(req, ino);
+    if (rc != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "release() failed, ino = " << ino;
     }
-    return CURVEFS_ERROR::OK;
+    return rc;
 }
 
-void FuseClient::FlushInode() { inodeManager_->FlushInodeOnce(); }
-
-void FuseClient::FlushInodeAll() { inodeManager_->FlushAll(); }
-
 void FuseClient::FlushAll() {
     FlushData();
-    FlushInodeAll();
 }
 
 CURVEFS_ERROR
@@ -1543,11 +1431,11 @@ void FuseClient::InitQosParam() {
 
     throttle_.UpdateThrottleParams(params);
 
-    // int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1),
-    //                             on_throttle_timer, this);
-    // if (ret != 0) {
-    //     LOG(ERROR) << "Create fuse client throttle timer failed!";
-    // }
+    int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1),
+                                on_throttle_timer, this);
+    if (ret != 0) {
+        LOG(ERROR) << "Create fuse client throttle timer failed!";
+    }
 }
 
 }  // namespace client

curvefs/src/client/fuse_client.h

@@ -45,7 +45,6 @@
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
-#include "curvefs/src/client/volume/client_volume_adaptor.h"
 #include "curvefs/src/common/fast_align.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/concurrent/concurrent.h"
@@ -57,6 +56,8 @@
 #include "curvefs/src/client/xattr_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
 #include "src/common/throttle.h"
+#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/filesystem/filesystem.h"
 
 #define DirectIOAlignment 512
 
@@ -67,11 +68,11 @@ using ::curvefs::common::FSType;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::metaserver::ManageInodeType;
 using ::curvefs::client::metric::FSMetric;
-using curvefs::volume::kMiB;
 
 namespace curvefs {
 namespace client {
 
+
 namespace warmup {
 class WarmupManager;
 }
@@ -82,6 +83,10 @@ using rpcclient::MdsClient;
 using rpcclient::MdsClientImpl;
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
+using ::curvefs::client::filesystem::FileSystem;
+using ::curvefs::client::filesystem::EntryOut;
+using ::curvefs::client::filesystem::AttrOut;
+using ::curvefs::client::filesystem::FileOut;
 
 using curvefs::common::is_aligned;
 
@@ -92,17 +97,16 @@ using mds::Mountpoint;
 class FuseClient {
  public:
     FuseClient()
-    : mdsClient_(std::make_shared<MdsClientImpl>()),
-      metaClient_(std::make_shared<MetaServerClientImpl>()),
-      inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
-      dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
-      dirBuf_(std::make_shared<DirBuffer>()),
-      fsInfo_(nullptr),
-      init_(false),
-      enableSumInDir_(false),
-      warmupManager_(nullptr),
-      mdsBase_(nullptr),
-      isStop_(true) {}
+      : mdsClient_(std::make_shared<MdsClientImpl>()),
+        metaClient_(std::make_shared<MetaServerClientImpl>()),
+        inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
+        dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
+        fsInfo_(nullptr),
+        init_(false),
+        enableSumInDir_(false),
+        warmupManager_(nullptr),
+        mdsBase_(nullptr),
+        isStop_(true) {}
 
     virtual ~FuseClient() {}
 
@@ -115,7 +119,6 @@ class FuseClient {
             metaClient_(metaClient),
             inodeManager_(inodeManager),
             dentryManager_(dentryManager),
-            dirBuf_(std::make_shared<DirBuffer>()),
             fsInfo_(nullptr),
             init_(false),
             enableSumInDir_(false),
@@ -131,8 +134,6 @@ class FuseClient {
 
     virtual void Fini();
 
-/*** fuse op***/
-
     virtual CURVEFS_ERROR FuseOpInit(
         void* userdata, struct fuse_conn_info* conn);
 
@@ -141,31 +142,42 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                       const char* buf, size_t size, off_t off,
                                       struct fuse_file_info* fi,
-                                      size_t* wSize) = 0;
+                                      FileOut* fileOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                      size_t size, off_t off,
                                      struct fuse_file_info* fi, char* buffer,
                                      size_t* rSize) = 0;
 
-    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
-                                     struct fuse_file_info* fi);
+    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     struct fuse_file_info* fi,
+                                     FileOut* fileOut);
 
-    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, mode_t mode,
+    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       mode_t mode,
                                        struct fuse_file_info* fi,
-                                       fuse_entry_param* e) = 0;
+                                       EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode, dev_t rdev,
-                                      fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      dev_t rdev,
+                                      EntryOut* entryOut) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
-                                      const char* name, mode_t mode,
-                                      fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req,
+                                      fuse_ino_t parent,
+                                      const char* name,
+                                      mode_t mode,
+                                      EntryOut* entryOut);
 
     virtual CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                        const char* name) = 0;
@@ -173,30 +185,38 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
                                       const char* name);
 
-    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req,
+                                        fuse_ino_t ino,
                                         struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req,
+                                           fuse_ino_t ino,
                                            struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
-                                            size_t size, off_t off,
-                                            struct fuse_file_info* fi,
-                                            char** buffer, size_t* rSize,
-                                            bool cacheDir);
+    virtual CURVEFS_ERROR FuseOpReadDir(fuse_req_t req,
+                                        fuse_ino_t ino,
+                                        size_t size,
+                                        off_t off,
+                                        struct fuse_file_info* fi,
+                                        char** bufferOut,
+                                        size_t* rSize,
+                                        bool plus);
 
-    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req, fuse_ino_t parent,
-                                       const char* name, fuse_ino_t newparent,
-                                       const char* newname);
+    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req,
+                                       fuse_ino_t parent,
+                                       const char* name,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       unsigned int flags);
 
     virtual CURVEFS_ERROR FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info* fi,
-                                        struct stat* attr);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct stat* attr, int to_set,
                                         struct fuse_file_info* fi,
-                                        struct stat* attrOut);
+                                        struct AttrOut* out);
 
     virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, std::string* value,
@@ -209,13 +229,17 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 
-    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req, const char* link,
-                                        fuse_ino_t parent, const char* name,
-                                        fuse_entry_param* e);
+    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req,
+                                        const char* link,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        EntryOut* entryOut);
 
-    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char* newname,
-                                     fuse_entry_param* e) = 0;
+    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                                     fuse_ino_t ino,
+                                     fuse_ino_t newparent,
+                                     const char* newname,
+                                     EntryOut* entryOut) = 0;
 
     virtual CURVEFS_ERROR FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string* linkStr);
@@ -226,9 +250,13 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                       int datasync,
                                       struct fuse_file_info* fi) = 0;
-
     virtual CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
-                                      struct fuse_file_info *fi) = 0;
+                                      struct fuse_file_info *fi) {
+        (void)req;
+        (void)ino;
+        (void)fi;
+        return CURVEFS_ERROR::OK;
+    }
 
     virtual CURVEFS_ERROR FuseOpStatFs(fuse_req_t req, fuse_ino_t ino,
                                        struct statvfs* stbuf) {
@@ -245,19 +273,11 @@ class FuseClient {
         stbuf->f_fsid = fsInfo_->fsid();
 
         stbuf->f_flag = 0;
-        stbuf->f_namemax = option_.maxNameLength;
+        stbuf->f_namemax = option_.fileSystemOption.maxNameLength;
         return CURVEFS_ERROR::OK;
     }
 
-/*** flush inode ***/
-
-    virtual void FlushInode();
-
-    virtual void FlushInodeAll();
-
-    virtual void FlushAll();
-
-/*** get or set some elements***/
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
 
     void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
         fsInfo_ = fsInfo;
@@ -274,6 +294,12 @@ class FuseClient {
         return fsInfo_;
     }
 
+    std::shared_ptr<FileSystem> GetFileSystem() {
+        return fs_;
+    }
+
+    virtual void FlushAll();
+
     // for unit test
     void SetEnableSumInDir(bool enable) {
         enableSumInDir_ = enable;
@@ -306,22 +332,26 @@ class FuseClient {
     void Add(bool isRead, size_t size) { throttle_.Add(isRead, size); }
 
     void InitQosParam();
-    Mountpoint& GetMountPoint() {
-        return mountpoint_;
-    }
 
  protected:
-    CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
-                           mode_t mode, FsFileType type, dev_t rdev,
-                           bool internal, fuse_entry_param* e);
+    CURVEFS_ERROR MakeNode(fuse_req_t req,
+                           fuse_ino_t parent,
+                           const char* name,
+                           mode_t mode,
+                           FsFileType type,
+                           dev_t rdev,
+                           bool internal,
+                           std::shared_ptr<InodeWrapper>& InodeWrapper);  // NOLINT
 
     CURVEFS_ERROR RemoveNode(fuse_req_t req, fuse_ino_t parent,
                              const char* name, FsFileType type);
 
-    CURVEFS_ERROR CreateManageNode(fuse_req_t req, uint64_t parent,
-                                   const char *name, mode_t mode,
+    CURVEFS_ERROR CreateManageNode(fuse_req_t req,
+                                   uint64_t parent,
+                                   const char* name,
+                                   mode_t mode,
                                    ManageInodeType manageType,
-                                   fuse_entry_param *e);
+                                   EntryOut* entryOut);
 
     CURVEFS_ERROR GetOrCreateRecycleDir(fuse_req_t req, Dentry *out);
 
@@ -334,10 +364,17 @@ class FuseClient {
 
     bool ShouldMoveToRecycle(fuse_ino_t parent);
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                             fuse_ino_t newparent, const char* newname,
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char* newname,
                              FsFileType type,
-                             fuse_entry_param* e);
+                             EntryOut* entryOut);
+
+    CURVEFS_ERROR HandleOpenFlags(fuse_req_t req,
+                                  fuse_ino_t ino,
+                                  struct fuse_file_info* fi,
+                                  FileOut* fileOut);
 
     int SetHostPortInMountPoint(Mountpoint* out) {
         char hostname[kMaxHostNameLength];
@@ -352,9 +389,8 @@ class FuseClient {
         return 0;
     }
 
- private:
-    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
 
+ private:
     virtual void FlushData() = 0;
 
     CURVEFS_ERROR UpdateParentMCTimeAndNlink(
@@ -365,8 +401,8 @@ class FuseClient {
         std::string newName(name);
         newName = std::to_string(parent) + "_" + std::to_string(ino)
                     + "_" + newName;
-        if (newName.length() > option_.maxNameLength) {
-            newName = newName.substr(0, option_.maxNameLength);
+        if (newName.length() > option_.fileSystemOption.maxNameLength) {
+            newName = newName.substr(0, option_.fileSystemOption.maxNameLength);
         }
 
         return newName;
@@ -390,9 +426,6 @@ class FuseClient {
 
     std::shared_ptr<LeaseExecutor> leaseExecutor_;
 
-    // dir buffer
-    std::shared_ptr<DirBuffer> dirBuf_;
-
     // filesystem info
     std::shared_ptr<FsInfo> fsInfo_;
 
@@ -411,6 +444,8 @@ class FuseClient {
     // warmup manager
     std::shared_ptr<warmup::WarmupManager> warmupManager_;
 
+    std::shared_ptr<FileSystem> fs_;
+
  private:
     MDSBaseClient* mdsBase_;
 
@@ -421,8 +456,6 @@ class FuseClient {
     Throttle throttle_;
 
     bthread_timer_t throttleTimer_;
-
-    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };
 
 }  // namespace client

curvefs/src/client/fuse_s3_client.cpp

@@ -24,7 +24,7 @@
 #include <memory>
 #include <vector>
 
-#include "curvefs/src/client/s3/fuse_s3_client.h"
+#include "curvefs/src/client/fuse_s3_client.h"
 #include "curvefs/src/client/kvclient/memcache_client.h"
 
 namespace curvefs {
@@ -47,30 +47,63 @@ using curvefs::mds::topology::MemcacheClusterInfo;
 using curvefs::mds::topology::MemcacheServerInfo;
 
 CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
-    LOG(INFO) << "fuse init start.";
-    auto ret = FuseClient::Init(option);
+    FuseClientOption opt(option);
+
+    CURVEFS_ERROR ret = FuseClient::Init(opt);
     if (ret != CURVEFS_ERROR::OK) {
-        LOG(INFO) << "fuse init failed, " << ret;
         return ret;
     }
 
     // init kvcache
     if (FLAGS_supportKVcache && !InitKVCache(option.kvClientManagerOpt)) {
-        LOG(INFO) << "init kv cache failed.";
         return CURVEFS_ERROR::INTERNAL;
     }
 
-    // init storage adaptor
-    ret = InitStorageAdaptor(option);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(INFO) << "init storage adaptor failed, " << ret;
-        return ret;
+    // set fs S3Option
+    const auto& s3Info = fsInfo_->detail().s3info();
+    ::curve::common::S3InfoOption fsS3Option;
+    ::curvefs::client::common::S3Info2FsS3Option(s3Info, &fsS3Option);
+    SetFuseClientS3Option(&opt, fsS3Option);
+
+    auto s3Client = std::make_shared<S3ClientImpl>();
+    s3Client->Init(opt.s3Opt.s3AdaptrOpt);
+
+    const uint64_t writeCacheMaxByte =
+        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
+    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
+        LOG(ERROR) << "writeCacheMaxByte is too small"
+                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
+                      ", writeCacheMaxByte = " << writeCacheMaxByte;
+        return CURVEFS_ERROR::CACHETOOSMALL;
     }
 
-    LOG(INFO) << "fuse init success.";
+    auto fsCacheManager = std::make_shared<FsCacheManager>(
+        dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte, writeCacheMaxByte,
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheThreads, kvClientManager_);
+    if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
+        DiskCacheType::Disable) {
+        auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
+        s3DiskCacheClient->Init(opt.s3Opt.s3AdaptrOpt);
+        auto wrapper = std::make_shared<PosixWrapper>();
+        auto diskCacheRead = std::make_shared<DiskCacheRead>();
+        auto diskCacheWrite = std::make_shared<DiskCacheWrite>();
+        auto diskCacheManager = std::make_shared<DiskCacheManager>(
+            wrapper, diskCacheWrite, diskCacheRead);
+        auto diskCacheManagerImpl = std::make_shared<DiskCacheManagerImpl>(
+            diskCacheManager, s3DiskCacheClient);
+        ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
+                               inodeManager_, mdsClient_, fsCacheManager,
+                               diskCacheManagerImpl, kvClientManager_, true);
+    } else {
+        ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
+                               inodeManager_, mdsClient_, fsCacheManager,
+                               nullptr, kvClientManager_, true);
+    }
     return ret;
 }
 
+
 bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
      // get kvcache cluster
     MemcacheClusterInfo kvcachecluster;
@@ -103,50 +136,6 @@ bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
     return true;
 }
 
-CURVEFS_ERROR FuseS3Client::InitStorageAdaptor(const FuseClientOption &option) {
-    auto opt(option);
-    auto ret = CURVEFS_ERROR::OK;
-
-    // set fs S3Option
-    const auto& s3Info = fsInfo_->detail().s3info();
-    ::curve::common::S3InfoOption fsS3Option;
-    ::curvefs::client::common::S3Info2FsS3Option(s3Info, &fsS3Option);
-    // update s3 info: s3 adress, bucket, etc
-    SetFuseClientS3Option(&opt, fsS3Option);
-
-    auto s3Client = std::make_shared<S3ClientImpl>();
-    s3Client->Init(opt.s3Opt.s3AdaptrOpt);
-
-    auto fsCacheManager = std::make_shared<FsCacheManager>(
-        dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
-        option.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
-        option.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte,
-        option.s3Opt.s3ClientAdaptorOpt.readCacheThreads,
-        kvClientManager_);
-
-    if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
-        DiskCacheType::Disable) {
-        LOG(INFO) << "has disk cache.";
-        auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
-        s3DiskCacheClient->Init(opt.s3Opt.s3AdaptrOpt);
-        auto wrapper = std::make_shared<PosixWrapper>();
-        auto diskCacheRead = std::make_shared<DiskCacheRead>();
-        auto diskCacheWrite = std::make_shared<DiskCacheWrite>();
-        auto diskCacheManager = std::make_shared<DiskCacheManager>(
-            wrapper, diskCacheWrite, diskCacheRead);
-        auto diskCacheManagerImpl = std::make_shared<DiskCacheManagerImpl>(
-            diskCacheManager, s3DiskCacheClient);
-
-        ret = s3Adaptor_->Init(opt, inodeManager_, mdsClient_, fsCacheManager,
-                               diskCacheManagerImpl, kvClientManager_, fsInfo_);
-    } else {
-        ret = s3Adaptor_->Init(opt, inodeManager_, mdsClient_, fsCacheManager,
-                               nullptr, kvClientManager_, fsInfo_);
-    }
-    LOG(INFO) << "init storage adaptor success.";
-    return ret;
-}
-
 void FuseS3Client::UnInit() {
     FuseClient::UnInit();
     s3Adaptor_->Stop();
@@ -155,15 +144,10 @@ void FuseS3Client::UnInit() {
 
 CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
                                        struct fuse_conn_info *conn) {
-    auto ret = FuseClient::FuseOpInit(userdata, conn);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "fuse op init failed.";
-        return ret;
-    }
-    ret = s3Adaptor_->FuseOpInit(userdata, conn);
-    if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "fuse op init failed.";
-        return ret;
+    CURVEFS_ERROR ret = FuseClient::FuseOpInit(userdata, conn);
+    if (init_) {
+        s3Adaptor_->SetFsId(fsInfo_->fsid());
+        s3Adaptor_->InitMetrics(fsInfo_->fsname());
     }
     return ret;
 }
@@ -171,8 +155,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
 CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                         const char *buf, size_t size, off_t off,
                                         struct fuse_file_info *fi,
-                                        size_t *wSize) {
-    (void)req;
+                                        FileOut* fileOut) {
+    size_t *wSize = &fileOut->nwritten;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -234,6 +218,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
             }
         }
     }
+
+    inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
     return ret;
 }
 
@@ -296,33 +282,56 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                          const char *name, mode_t mode,
                                          struct fuse_file_info *fi,
-                                         fuse_entry_param *e) {
+                                         EntryOut* entryOut) {
     VLOG(1) << "FuseOpCreate, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
+
+    std::shared_ptr<InodeWrapper> inode;
     CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, e);
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    return FuseOpOpen(req, e->ino, fi);
+
+    auto openFiles = fs_->BorrowMember().openFiles;
+    openFiles->Open(inode->GetInodeId(), inode);
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                        const char *name, mode_t mode,
-                                        dev_t rdev, fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req,
+                                        fuse_ino_t parent,
+                                        const char* name,
+                                        mode_t mode,
+                                        dev_t rdev,
+                                        EntryOut* entryOut) {
     VLOG(1) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, rdev, false,
-                    e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
+                                FsFileType::TYPE_S3, rdev, false,
+                                inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    *entryOut = EntryOut(attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
-                                     fuse_entry_param *e) {
+CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req,
+                                       fuse_ino_t ino,
+                                       fuse_ino_t newparent,
+                                       const char* newname,
+                                       EntryOut* entryOut) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_S3, e);
+        req, ino, newparent, newname, FsFileType::TYPE_S3, entryOut);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,

curvefs/src/client/fuse_s3_client.h

@@ -21,8 +21,8 @@
  * Author: xuchaojie
  */
 
-#ifndef CURVEFS_SRC_CLIENT_S3_FUSE_S3_CLIENT_H_
-#define CURVEFS_SRC_CLIENT_S3_FUSE_S3_CLIENT_H_
+#ifndef CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
 
 #include <memory>
 #include <string>
@@ -31,7 +31,7 @@
 #include <utility>
 
 #include "curvefs/src/client/fuse_client.h"
-#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
+#include "curvefs/src/client/s3/client_s3_cache_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
 #include "curvefs/src/volume/common.h"
 #include "src/common/s3_adapter.h"
@@ -47,13 +47,11 @@ class WarmupManager;
 class WarmupManagerS3Impl;
 }  // namespace warmup
 
-// s3 client
+
 class FuseS3Client : public FuseClient {
  public:
     FuseS3Client()
-        : FuseClient(),
-          s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()),
-          kvClientManager_(nullptr) {
+        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {
         auto readFunc = [this](fuse_req_t req, fuse_ino_t ino, size_t size,
                                off_t off, struct fuse_file_info *fi,
                                char *buffer, size_t *rSize) {
@@ -68,25 +66,22 @@ class FuseS3Client : public FuseClient {
                  const std::shared_ptr<MetaServerClient> &metaClient,
                  const std::shared_ptr<InodeCacheManager> &inodeManager,
                  const std::shared_ptr<DentryCacheManager> &dentryManager,
-                 const std::shared_ptr<StorageAdaptor> &s3Adaptor,
+                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor,
                  const std::shared_ptr<warmup::WarmupManager> &warmupManager)
         : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
                      warmupManager),
-          s3Adaptor_(s3Adaptor),
-          kvClientManager_(nullptr) {}
+          s3Adaptor_(s3Adaptor) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
 
     void UnInit() override;
 
-/*** fuse op ***/
-
     CURVEFS_ERROR FuseOpInit(
         void *userdata, struct fuse_conn_info *conn) override;
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, size_t *wSize) override;
+        struct fuse_file_info *fi, FileOut* fileOut) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
         fuse_ino_t ino, size_t size, off_t off,
@@ -94,17 +89,25 @@ class FuseS3Client : public FuseClient {
         char *buffer,
         size_t *rSize) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, struct fuse_file_info *fi,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                               fuse_ino_t parent,
+                               const char* name,
+                               mode_t mode,
+                               struct fuse_file_info *fi,
+                               EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, dev_t rdev,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                              fuse_ino_t parent,
+                              const char* name,
+                              mode_t mode,
+                              dev_t rdev,
+                              EntryOut* entryOut) override;
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-        fuse_ino_t newparent, const char *newname,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
+                             fuse_ino_t ino,
+                             fuse_ino_t newparent,
+                             const char *newname,
+                             EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
         const char *name) override;
@@ -115,22 +118,23 @@ class FuseS3Client : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
         struct fuse_file_info *fi) override;
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
  private:
     bool InitKVCache(const KVClientManagerOpt &opt);
-    CURVEFS_ERROR InitStorageAdaptor(const FuseClientOption &option);
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
 
     void FlushData() override;
 
  private:
-    // s3 storage adaptor
-    std::shared_ptr<StorageAdaptor> s3Adaptor_;
-    // kv client manager
+    // s3 adaptor
+    std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
     std::shared_ptr<KVClientManager> kvClientManager_;
+
+    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };
 
 
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_S3_FUSE_S3_CLIENT_H_
+#endif  // CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_

curvefs/src/client/fuse_volume_client.cpp

@@ -21,7 +21,7 @@
  * Author: xuchaojie
  */
 
-#include "curvefs/src/client/volume/fuse_volume_client.h"
+#include "curvefs/src/client/fuse_volume_client.h"
 
 #include <butil/time.h>
 #include <bvar/bvar.h>
@@ -32,7 +32,7 @@
 #include "absl/cleanup/cleanup.h"
 #include "absl/memory/memory.h"
 #include "curvefs/proto/mds.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/volume/default_volume_storage.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/volume/common.h"
@@ -41,38 +41,38 @@
 namespace curvefs {
 namespace client {
 
-namespace common {
-DECLARE_bool(enableCto);
-}  // namespace common
-
 using ::curvefs::volume::SpaceManagerImpl;
 using ::curvefs::volume::SpaceManagerOption;
 using ::curvefs::volume::BlockDeviceClientOptions;
 using ::curvefs::volume::BlockDeviceClientImpl;
-using ::curvefs::client::common::FLAGS_enableCto;
 
 CURVEFS_ERROR FuseVolumeClient::Init(const FuseClientOption &option) {
+    volOpts_ = option.volumeOpt;
+
     CURVEFS_ERROR ret = FuseClient::Init(option);
+
     if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "Init failed: " << ret;
         return ret;
     }
 
-    auto fsCacheManager = std::make_shared<FsCacheManager>(
-        dynamic_cast<StorageAdaptor*>(storageAdaptor_.get()),
-        option.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
-        option.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte,
-        option.s3Opt.s3ClientAdaptorOpt.readCacheThreads,
-        nullptr);  // bs no need cache cluster
-    ret = storageAdaptor_->Init(option,
-        inodeManager_, mdsClient_, fsCacheManager,
-        nullptr, nullptr, fsInfo_);  // no need cache cluster and diskcache
+    BlockDeviceClientOptions opts;
+    opts.configPath = option.bdevOpt.configPath;
+
+    bool ret2 = blockDeviceClient_->Init(opts);
+
+    if (!ret2) {
+        LOG(ERROR) << "Init block device client failed";
+        return CURVEFS_ERROR::INTERNAL;
+    }
 
     return ret;
 }
 
 void FuseVolumeClient::UnInit() {
-    storageAdaptor_->Stop();
+    storage_->Shutdown();
+    spaceManager_->Shutdown();
+    blockDeviceClient_->UnInit();
+
     FuseClient::UnInit();
 }
 
@@ -83,13 +83,47 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpInit(void *userdata,
         LOG(ERROR) << "fuse op init failed, error: " << ret;
         return ret;
     }
-    Mountpoint mountPoint = GetMountPoint();
-    std::string mountOwner = mountPoint.hostname() + ":" +
-        std::to_string(mountPoint.port()) +
-        ":" + mountPoint.path();
-    storageAdaptor_->SetMountOwner(mountOwner);
-    storageAdaptor_->FuseOpInit(userdata, conn);
-    LOG(INFO) << "fuse op Init success: " << mountOwner;
+
+    const auto &vol = fsInfo_->detail().volume();
+    const auto &volName = vol.volumename();
+    const auto &user = vol.user();
+    auto ret2 = blockDeviceClient_->Open(volName, user);
+    if (!ret2) {
+        LOG(ERROR) << "BlockDeviceClientImpl open failed, ret = " << ret
+                   << ", volName = " << volName << ", user = " << user;
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    SpaceManagerOption option;
+    option.blockGroupManagerOption.fsId = fsInfo_->fsid();
+    option.blockGroupManagerOption.owner = mountpoint_.hostname() + ":" +
+                                           std::to_string(mountpoint_.port()) +
+                                           ":" + mountpoint_.path();
+    option.blockGroupManagerOption.blockGroupAllocateOnce =
+        volOpts_.allocatorOption.blockGroupOption.allocateOnce;
+    option.blockGroupManagerOption.blockGroupSize =
+        fsInfo_->detail().volume().blockgroupsize();
+    option.blockGroupManagerOption.blockSize =
+        fsInfo_->detail().volume().blocksize();
+
+    option.allocatorOption.type = volOpts_.allocatorOption.type;
+    option.allocatorOption.bitmapAllocatorOption.sizePerBit =
+        volOpts_.allocatorOption.bitmapAllocatorOption.sizePerBit;
+    option.allocatorOption.bitmapAllocatorOption.smallAllocProportion =
+        volOpts_.allocatorOption.bitmapAllocatorOption.smallAllocProportion;
+
+    spaceManager_ = absl::make_unique<SpaceManagerImpl>(option, mdsClient_,
+                                                        blockDeviceClient_);
+
+    storage_ = absl::make_unique<DefaultVolumeStorage>(
+        spaceManager_.get(), blockDeviceClient_.get(), inodeManager_.get());
+
+    ExtentCacheOption extentOpt;
+    extentOpt.blockSize = vol.blocksize();
+    extentOpt.sliceSize = vol.slicesize();
+
+    ExtentCache::SetOption(extentOpt);
+
     return CURVEFS_ERROR::OK;
 }
 
@@ -99,72 +133,54 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
                                             size_t size,
                                             off_t off,
                                             struct fuse_file_info *fi,
-                                            size_t *wSize) {
-    (void)req;
+                                            FileOut* fileOut) {
     VLOG(9) << "write start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
+    size_t* wSize = &fileOut->nwritten;
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
-              is_aligned(size, DirectIOAlignment)))
+              is_aligned(size, DirectIOAlignment))) {
+            fsMetric_->userWrite.eps.count << 1;
             return CURVEFS_ERROR::INVALIDPARAM;
-    }
-    uint64_t start = butil::cpuwide_time_us();
-    int wRet = storageAdaptor_->Write(ino, off, size, buf);
-    if (wRet < 0) {
-        LOG(ERROR) << "storageAdaptor_ write failed, ret = " << wRet;
-        return CURVEFS_ERROR::INTERNAL;
+        }
     }
 
-    if (fsMetric_.get() != nullptr) {
-        fsMetric_->userWrite.bps.count << wRet;
-        fsMetric_->userWrite.qps.count << 1;
-        uint64_t duration = butil::cpuwide_time_us() - start;
-        fsMetric_->userWrite.latency << duration;
-        fsMetric_->userWriteIoSize.set_value(wRet);
-    }
+    butil::Timer timer;
+    timer.start();
 
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf, fileOut);
     if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                   << ", inodeid = " << ino;
+        if (fsMetric_) {
+            fsMetric_->userWrite.eps.count << 1;
+        }
+        LOG(ERROR) << "write error, ino: " << ino << ", offset: " << off
+                   << ", len: " << size
+                   << ", error: " << ret;
         return ret;
     }
 
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-
-    *wSize = wRet;
-    size_t changeSize = 0;
-    // update file len
-    if (inodeWrapper->GetLengthLocked() < off + *wSize) {
-        changeSize = off + *wSize - inodeWrapper->GetLengthLocked();
-        inodeWrapper->SetLengthLocked(off + *wSize);
-    }
-
-    inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
-
-    inodeManager_->ShipToFlush(inodeWrapper);
+    *wSize = size;
 
+    // NOTE: O_DIRECT/O_SYNC/O_DSYNC have simillar semantic, but not exactly the
+    // same, see `man 2 open` for more details
     if (fi->flags & O_DIRECT || fi->flags & O_SYNC || fi->flags & O_DSYNC) {
         // Todo: do some cache flush later
     }
 
-    if (enableSumInDir_ && changeSize != 0) {
-        const Inode* inode = inodeWrapper->GetInodeLocked();
-        XAttr xattr;
-        xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
-            std::to_string(changeSize)});
-        for (const auto &it : inode->parent()) {
-            auto tret = xattrManager_->UpdateParentInodeXattr(it, xattr, true);
-            if (tret != CURVEFS_ERROR::OK) {
-                LOG(ERROR) << "UpdateParentInodeXattr failed,"
-                           << " inodeId = " << it
-                           << ", xattr = " << xattr.DebugString();
-            }
-        }
+    timer.stop();
+
+    if (fsMetric_) {
+        fsMetric_->userWrite.bps.count << size;
+        fsMetric_->userWrite.qps.count << 1;
+        fsMetric_->userWrite.latency << timer.u_elapsed();
+        fsMetric_->userWriteIoSize.set_value(size);
     }
-    return ret;
+
+    VLOG(9) << "write end, ino: " << ino << ", offset: " << off
+            << ", length: " << size << ", written: " << *wSize;
+
+    return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
@@ -175,90 +191,101 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
                                            char *buffer,
                                            size_t *rSize) {
     (void)req;
-    VLOG(9) << "read start, ino: " << ino << ", offset: " << off
+    VLOG(3) << "read start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
-              is_aligned(size, DirectIOAlignment)))
+              is_aligned(size, DirectIOAlignment))) {
+            fsMetric_->userRead.eps.count << 1;
+
             return CURVEFS_ERROR::INVALIDPARAM;
+        }
     }
 
-    uint64_t start = butil::cpuwide_time_us();
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    butil::Timer timer;
+    timer.start();
+
+    CURVEFS_ERROR ret = storage_->Read(ino, off, size, buffer);
     if (ret != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
-                   << ", inodeid = " << ino;
+        if (fsMetric_) {
+            fsMetric_->userRead.eps.count << 1;
+        }
+        LOG(ERROR) << "read error, ino: " << ino << ", offset: " << off
+                   << ", len: " << size << ", error: " << ret;
         return ret;
     }
-    uint64_t fileSize = inodeWrapper->GetLength();
 
-    size_t len = 0;
-    if (fileSize <= off) {
-        *rSize = 0;
-        return CURVEFS_ERROR::OK;
-    } else if (fileSize < off + size) {
-        len = fileSize - off;
-    } else {
-        len = size;
-    }
-
-    int rRet = storageAdaptor_->Read(ino, off, len, buffer);
-    if (rRet < 0) {
-        LOG(ERROR) << "storageAdaptor_ read failed, ret = " << rRet;
-        return CURVEFS_ERROR::INTERNAL;
-    }
-    *rSize = rRet;
-
-    if (fsMetric_.get() != nullptr) {
-        fsMetric_->userRead.bps.count << rRet;
+    if (fsMetric_) {
+        fsMetric_->userRead.bps.count << size;
         fsMetric_->userRead.qps.count << 1;
-        uint64_t duration = butil::cpuwide_time_us() - start;
-        fsMetric_->userRead.latency << duration;
-        fsMetric_->userReadIoSize.set_value(rRet);
+        fsMetric_->userRead.latency << timer.u_elapsed();
+        fsMetric_->userReadIoSize.set_value(size);
     }
 
-    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-    inodeWrapper->UpdateTimestampLocked(kAccessTime);
-    inodeManager_->ShipToFlush(inodeWrapper);
+    *rSize = size;
 
-    VLOG(9) << "read end, read size = " << *rSize;
-    return ret;
+    VLOG(3) << "read end, ino: " << ino << ", offset: " << off
+            << ", length: " << size << ", rsize: " << *rSize;
+
+    return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                              const char *name, mode_t mode,
                                              struct fuse_file_info *fi,
-                                             fuse_entry_param *e) {
+                                             EntryOut* entryOut) {
     VLOG(3) << "FuseOpCreate, parent: " << parent
               << ", name: " << name
               << ", mode: " << mode;
-    CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, false, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR ret = MakeNode(
+        req, parent, name, mode, FsFileType::TYPE_FILE, 0, false, inode);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    return FuseOpOpen(req, e->ino, fi);
+
+    auto openFiles = fs_->BorrowMember().openFiles;
+    openFiles->Open(inode->GetInodeId(), inode);
+
+    inode->GetInodeAttr(&entryOut->attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-                                            const char *name, mode_t mode,
-                                            dev_t rdev, fuse_entry_param *e) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req,
+                                            fuse_ino_t parent,
+                                            const char* name,
+                                            mode_t mode,
+                                            dev_t rdev,
+                                            EntryOut* entryOut) {
     VLOG(3) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-    return MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, rdev,
-                    false, e);
+
+    std::shared_ptr<InodeWrapper> inode;
+    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
+                                FsFileType::TYPE_FILE, rdev,
+                                false, inode);
+    if (rc != CURVEFS_ERROR::OK) {
+        return rc;
+    }
+
+    InodeAttr attr;
+    inode->GetInodeAttr(&attr);
+    *entryOut = EntryOut(attr);
+    return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                                     fuse_ino_t newparent, const char *newname,
-                                     fuse_entry_param *e) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req,
+                                           fuse_ino_t ino,
+                                           fuse_ino_t newparent,
+                                           const char* newname,
+                                           EntryOut* entryOut) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_FILE, e);
+        req, ino, newparent, newname, FsFileType::TYPE_FILE, entryOut);
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
@@ -273,9 +300,8 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
     (void)req;
     (void)fi;
     VLOG(3) << "FuseOpFsync start, ino: " << ino << ", datasync: " << datasync;
-    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
-    ret = dynamic_cast<VolumeClientAdaptorImpl*>(
-      storageAdaptor_.get())->getUnderStorage()->Flush(ino);
+
+    CURVEFS_ERROR ret = storage_->Flush(ino);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "Storage flush ino: " << ino << " failed, error: " << ret;
         return ret;
@@ -311,20 +337,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
     (void)fi;
     VLOG(9) << "FuseOpFlush, ino: " << ino;
 
-    if (FLAGS_enableCto) {
-        auto ret = storageAdaptor_->FlushAllCache(ino);
-        if (ret != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "FuseOpFlush, flush all cache fail, ret = " << ret
-                       << ", ino: " << ino;
-            return ret;
-        }
-        VLOG(3) << "FuseOpFlush, ino: " << ino << " flush to volume ok";
-    }
-
-    CURVEFS_ERROR ret =
-        dynamic_cast<VolumeClientAdaptorImpl *>(storageAdaptor_.get())
-            ->getUnderStorage()
-            ->Flush(ino);
+    CURVEFS_ERROR ret = storage_->Flush(ino);
     LOG_IF(ERROR, ret != CURVEFS_ERROR::OK)
         << "Flush error, ino: " << ino << ", error: " << ret;
 
@@ -336,13 +349,11 @@ void FuseVolumeClient::FlushData() {
 }
 
 void FuseVolumeClient::SetSpaceManagerForTesting(SpaceManager *manager) {
-    dynamic_cast< VolumeClientAdaptorImpl *>(
-      storageAdaptor_.get())->getSpaceManager().reset(manager);
+    spaceManager_.reset(manager);
 }
 
 void FuseVolumeClient::SetVolumeStorageForTesting(VolumeStorage *storage) {
-    dynamic_cast< VolumeClientAdaptorImpl *>(
-      storageAdaptor_.get())->getUnderStorage().reset(storage);
+    storage_.reset(storage);
 }
 
 }  // namespace client

curvefs/src/client/fuse_volume_client.h

@@ -21,13 +21,13 @@
  * Author: xuchaojie
  */
 
-#ifndef CURVEFS_SRC_CLIENT_VOLUME_FUSE_VOLUME_CLIENT_H_
-#define CURVEFS_SRC_CLIENT_VOLUME_FUSE_VOLUME_CLIENT_H_
+#ifndef CURVEFS_SRC_CLIENT_FUSE_VOLUME_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_FUSE_VOLUME_CLIENT_H_
 
 #include <memory>
 
 #include "curvefs/src/client/fuse_client.h"
-#include "curvefs/src/client/volume/client_volume_adaptor.h"
+#include "curvefs/src/client/volume/volume_storage.h"
 #include "curvefs/src/volume/block_device_client.h"
 #include "curvefs/src/volume/space_manager.h"
 
@@ -35,18 +35,15 @@ namespace curvefs {
 namespace client {
 
 using common::VolumeOption;
-using mds::Mountpoint;
 using ::curvefs::volume::BlockDeviceClient;
 using ::curvefs::volume::BlockDeviceClientImpl;
 using ::curvefs::volume::SpaceManager;
 
-// volume client
 class FuseVolumeClient : public FuseClient {
  public:
     FuseVolumeClient()
-        : FuseClient() {
-        storageAdaptor_ = std::make_shared<VolumeClientAdaptorImpl>();
-    }
+        : FuseClient(),
+          blockDeviceClient_(std::make_shared<BlockDeviceClientImpl>()) {}
 
     // for UNIT_TEST
     FuseVolumeClient(
@@ -54,43 +51,46 @@ class FuseVolumeClient : public FuseClient {
         const std::shared_ptr<MetaServerClient> &metaClient,
         const std::shared_ptr<InodeCacheManager> &inodeManager,
         const std::shared_ptr<DentryCacheManager> &dentryManager,
-        const std::shared_ptr<BlockDeviceClient> &blockDeviceClient,
-        const std::shared_ptr<VolumeClientAdaptorImpl> &storageAdaptor)
+        const std::shared_ptr<BlockDeviceClient> &blockDeviceClient)
         : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
-                     nullptr) {
-            storageAdaptor_ = storageAdaptor;
-        }
+                     nullptr),
+          blockDeviceClient_(blockDeviceClient) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
 
     void UnInit() override;
 
-/*** fuse op ***/
-
     CURVEFS_ERROR FuseOpInit(
         void *userdata, struct fuse_conn_info *conn) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, struct fuse_file_info *fi,
-        fuse_entry_param *e) override;
-
-    CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
-        size_t size, off_t off,
-        struct fuse_file_info *fi, char *buffer,
-        size_t *rSize) override;
-
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi,
-        size_t *wSize) override;
+        struct fuse_file_info *fi, FileOut* fileOut) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
-        const char *name, mode_t mode, dev_t rdev,
-        fuse_entry_param *e) override;
+    CURVEFS_ERROR FuseOpRead(fuse_req_t req,
+            fuse_ino_t ino, size_t size, off_t off,
+            struct fuse_file_info *fi,
+            char *buffer,
+            size_t *rSize) override;
+
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
+                               fuse_ino_t parent,
+                               const char* name,
+                               mode_t mode,
+                               struct fuse_file_info* fi,
+                               EntryOut* entryOut) override;
+
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
+                              fuse_ino_t parent,
+                              const char* name,
+                              mode_t mode,
+                              dev_t rdev,
+                              EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-        fuse_ino_t newparent, const char *newname,
-        fuse_entry_param *e) override;
+                             fuse_ino_t newparent,
+                             const char *newname,
+                             EntryOut* entryOut) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                const char *name) override;
@@ -101,20 +101,24 @@ class FuseVolumeClient : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                               struct fuse_file_info *fi) override;
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
     void SetSpaceManagerForTesting(SpaceManager *manager);
 
     void SetVolumeStorageForTesting(VolumeStorage *storage);
 
  private:
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
     void FlushData() override;
 
  private:
-    std::shared_ptr<StorageAdaptor> storageAdaptor_;
+    std::shared_ptr<BlockDeviceClient> blockDeviceClient_;
+    std::unique_ptr<SpaceManager> spaceManager_;
+    std::unique_ptr<VolumeStorage> storage_;
+
+    VolumeOption volOpts_;
 };
 
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_VOLUME_FUSE_VOLUME_CLIENT_H_
+#endif  // CURVEFS_SRC_CLIENT_FUSE_VOLUME_CLIENT_H_

curvefs/src/client/inode_cache_manager.cpp

@@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 using ::curvefs::metaserver::Inode;
@@ -46,36 +46,11 @@ DECLARE_bool(enableCto);
 namespace curvefs {
 namespace client {
 
+using ::curvefs::client::filesystem::ToFSError;
+
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
 using curvefs::client::common::FLAGS_enableCto;
 
-class TrimICacheAsyncDone : public MetaServerClientDone {
- public:
-    explicit TrimICacheAsyncDone(
-        const std::shared_ptr<InodeWrapper> &inodeWrapper,
-        const std::shared_ptr<InodeCacheManagerImpl> &inodeCacheManager)
-        : inodeWrapper_(inodeWrapper), inodeCacheManager_(inodeCacheManager) {}
-
-    void Run() override {
-        std::unique_ptr<TrimICacheAsyncDone> self_guard(this);
-        MetaStatusCode ret = GetStatusCode();
-        if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
-            LOG(ERROR) << "metaClient_ UpdateInode failed, "
-                       << "MetaStatusCode: " << ret
-                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
-                       << ", inodeid: " << inodeWrapper_->GetInodeId();
-            inodeWrapper_->MarkInodeError();
-        }
-        VLOG(9) << "Trim inode " << inodeWrapper_->GetInodeId()
-                << " async success.";
-        inodeCacheManager_->RemoveICache(inodeWrapper_);
-    };
-
- private:
-    std::shared_ptr<InodeWrapper> inodeWrapper_;
-    std::shared_ptr<InodeCacheManagerImpl> inodeCacheManager_;
-};
-
 #define GET_INODE_REMOTE(FSID, INODEID, OUT, STREAMING)                        \
     MetaStatusCode ret = metaClient_->GetInode(FSID, INODEID, OUT, STREAMING); \
     if (ret != MetaStatusCode::OK) {                                           \
@@ -83,17 +58,7 @@ class TrimICacheAsyncDone : public MetaServerClientDone {
             << "metaClient_ GetInode failed, MetaStatusCode = " << ret         \
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)          \
             << ", inodeid = " << INODEID;                                      \
-        return MetaStatusCodeToCurvefsErrCode(ret);                            \
-    }
-
-#define PUT_INODE_CACHE(INODEID, INODEWRAPPER)                                 \
-    std::shared_ptr<InodeWrapper> eliminatedOne;                               \
-    bool eliminated = iCache_->Put(INODEID, INODEWRAPPER, &eliminatedOne);     \
-    if (eliminated) {                                                          \
-        VLOG(3) << "GetInode eliminate one inode, ino: "                       \
-                << eliminatedOne->GetInodeId()                                 \
-                << ", iCache does not evict inodes via put interface ";        \
-        assert(0);                                                             \
+        return ToFSError(ret);                            \
     }
 
 #define REFRESH_DATA_REMOTE(OUT, STREAMING)                                    \
@@ -106,16 +71,8 @@ CURVEFS_ERROR
 InodeCacheManagerImpl::GetInode(uint64_t inodeId,
                                 std::shared_ptr<InodeWrapper> &out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    // get inode from cache
-    bool ok = iCache_->Get(inodeId, &out);
-    if (ok && NeedUseCache(inodeId, out, false)) {
-        curve::common::UniqueLock lgGuard = out->GetUniqueLock();
-        if (out->GetType() == FsFileType::TYPE_FILE) {
-            return CURVEFS_ERROR::OK;
-        }
-        VLOG(9) << "get inode: " << inodeId
-                << " from icache ok, need refresh data";
-        REFRESH_DATA_REMOTE(out, out->NeedRefreshData());
+    bool yes = openFiles_->IsOpened(inodeId, &out);
+    if (yes) {
         return CURVEFS_ERROR::OK;
     }
 
@@ -128,27 +85,14 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
         option_.refreshDataIntervalSec);
 
     // refresh data
-    VLOG(9) << "get inode: " << inodeId << " from icache fail, get from remote";
     REFRESH_DATA_REMOTE(out, streaming);
 
-    // put to cache
-    PUT_INODE_CACHE(inodeId, out);
-
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
                                                   InodeAttr *out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    // 1. find in icache
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    bool ok = iCache_->Get(inodeId, &inodeWrapper);
-    if (ok && NeedUseCache(inodeId, inodeWrapper, true)) {
-        inodeWrapper->GetInodeAttr(out);
-        return CURVEFS_ERROR::OK;
-    }
-
-    // 2. get form metaserver
     std::set<uint64_t> inodeIds;
     std::list<InodeAttr> attrs;
     inodeIds.emplace(inodeId);
@@ -158,7 +102,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed"
                    << ", inodeId = " << inodeId << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
     if (attrs.size() != 1) {
@@ -175,21 +119,6 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
     std::set<uint64_t> *inodeIds,
     std::list<InodeAttr> *attrs) {
-    // get some inode attr in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
-            InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace_back(std::move(tmpAttr));
-            iter = inodeIds->erase(iter);
-            continue;
-        }
-        ++iter;
-    }
-
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -201,7 +130,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return MetaStatusCodeToCurvefsErrCode(ret);
+    return ToFSError(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
@@ -209,26 +138,6 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     std::set<uint64_t> *inodeIds,
     std::map<uint64_t, InodeAttr> *attrs) {
     NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    std::map<uint64_t, InodeAttr> cachedAttr;
-    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
-
-    // get some inode attr in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
-            InodeAttr tmpAttr;
-            inodeWrapper->GetInodeAttr(&tmpAttr);
-            attrs->emplace(*iter, std::move(tmpAttr));
-            iter = inodeIds->erase(iter);
-        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
-            attrs->emplace(*iter, cachedAttr[*iter]);
-            iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
-        }
-    }
 
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
@@ -240,12 +149,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
         return CURVEFS_ERROR::NOTEXIST;
     }
 
+    ::curve::common::Mutex mutex;
     std::shared_ptr<CountDownEvent> cond =
         std::make_shared<CountDownEvent>(inodeGroups.size());
     for (const auto& it : inodeGroups) {
         VLOG(3) << "BatchGetInodeAttrAsync Send " << it.size();
-        auto* done = new BatchGetInodeAttrAsyncDone(shared_from_this(),
-                                                    cond, parentId);
+        auto* done = new BatchGetInodeAttrAsyncDone(attrs, &mutex, cond);
         MetaStatusCode ret = metaClient_->BatchGetInodeAttrAsync(fsId_, it,
                                                                  done);
         if (MetaStatusCode::OK != ret) {
@@ -258,30 +167,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
 
     // wait for all sudrequest finished
     cond->Wait();
-
-    bool ok  = iAttrCache_->Get(parentId, attrs);
-    if (!ok) {
-        LOG(WARNING) << "get attrs form iAttrCache_ failed.";
-    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
     std::set<uint64_t> *inodeIds,
     std::list<XAttr> *xattr) {
-    // get some inode in icache
-    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
-        std::shared_ptr<InodeWrapper> inodeWrapper;
-        NameLockGuard lock(nameLock_, std::to_string(*iter));
-        bool ok = iCache_->Get(*iter, &inodeWrapper);
-        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
-            xattr->emplace_back(inodeWrapper->GetXattr());
-            iter = inodeIds->erase(iter);
-        } else {
-            ++iter;
-        }
-    }
-
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -292,7 +183,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return MetaStatusCodeToCurvefsErrCode(ret);
+    return ToFSError(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
@@ -303,23 +194,11 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ CreateInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-    uint64_t inodeid = inode.inodeid();
     out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
         s3ChunkInfoMetric_, option_.maxDataSize,
         option_.refreshDataIntervalSec);
-
-    std::shared_ptr<InodeWrapper> eliminatedOne;
-    bool eliminated = false;
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeid));
-        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
-    }
-    if (eliminated) {
-        /* iCache does not evict inodes via put interface */
-        assert(0);
-    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -332,159 +211,29 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateManageInode(
         LOG(ERROR) << "metaClient_ CreateManageInode failed, MetaStatusCode = "
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-    uint64_t inodeid = inode.inodeid();
     out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
         s3ChunkInfoMetric_, option_.maxDataSize,
         option_.refreshDataIntervalSec);
-
-    std::shared_ptr<InodeWrapper> eliminatedOne;
-    bool eliminated = false;
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeid));
-        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
-    }
-    if (eliminated) {
-        /* iCache does not evict inodes via put interface */
-        assert(0);
-    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::DeleteInode(uint64_t inodeId) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    iCache_->Remove(inodeId);
     MetaStatusCode ret = metaClient_->DeleteInode(fsId_, inodeId);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeId = " << inodeId;
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
-
-    curve::common::LockGuard lg2(dirtyMapMutex_);
-    dirtyMap_.erase(inodeId);
     return CURVEFS_ERROR::OK;
 }
 
-void InodeCacheManagerImpl::AddInodeAttrs(
-    uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
-    iAttrCache_->Set(parentId, inodeAttrs);
-}
-
-void InodeCacheManagerImpl::ClearInodeCache(uint64_t inodeId) {
-    {
-        NameLockGuard lock(nameLock_, std::to_string(inodeId));
-        iCache_->Remove(inodeId);
-    }
-    curve::common::LockGuard lg2(dirtyMapMutex_);
-    dirtyMap_.erase(inodeId);
-}
-
 void InodeCacheManagerImpl::ShipToFlush(
-    const std::shared_ptr<InodeWrapper> &inodeWrapper) {
-    curve::common::LockGuard lg(dirtyMapMutex_);
-    dirtyMap_.emplace(inodeWrapper->GetInodeId(), inodeWrapper);
-}
-
-void InodeCacheManagerImpl::FlushAll() {
-    while (!dirtyMap_.empty()) {
-        FlushInodeOnce();
-    }
-}
-
-void InodeCacheManagerImpl::FlushInodeOnce() {
-    std::map<uint64_t, std::shared_ptr<InodeWrapper>> temp_;
-    {
-        curve::common::LockGuard lg(dirtyMapMutex_);
-        temp_.swap(dirtyMap_);
-    }
-    for (auto it = temp_.begin(); it != temp_.end(); it++) {
-        curve::common::UniqueLock ulk = it->second->GetUniqueLock();
-        it->second->Async(nullptr, true);
-    }
-}
-
-void InodeCacheManagerImpl::ReleaseCache(uint64_t parentId) {
-    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
-    iAttrCache_->Release(parentId);
-}
-
-void InodeCacheManagerImpl::FlushInodeBackground() {
-    LOG(INFO) << "flush thread is start.";
-    while (!isStop_.load()) {
-        if (iCache_->Size() > maxCacheSize_) {
-            TrimIcache(iCache_->Size() - maxCacheSize_);
-        }
-        FlushInodeOnce();
-        sleeper_.wait_for(std::chrono::seconds(flushPeriodSec_));
-    }
-    LOG(INFO) << "flush thread is stop.";
-}
-
-namespace {
-// Wether an inode is dirty or not.
-// if |needLock| is true, we acquire the lock firstly and do the test,
-// otherwise, we assume the lock is already held.
-bool IsDirtyInode(InodeWrapper *ino, bool needLock) {
-    auto check = [ino]() {
-        return ino->IsDirty() || !ino->S3ChunkInfoEmptyNolock() ||
-               ino->GetMutableExtentCacheLocked()->HasDirtyExtents();
-    };
-
-    if (needLock) {
-        auto lk = ino->GetUniqueLock();
-        return check();
-    }
-
-    return check();
-}
-}  // namespace
-
-void InodeCacheManagerImpl::RemoveICache(
-    const std::shared_ptr<InodeWrapper> &inode) {
-    if (!IsDirtyInode(inode.get(), true)) {
-        uint64_t inodeId = inode->GetInodeId();
-        NameLockGuard lock(nameLock_, std::to_string(inodeId));
-        ::curve::common::UniqueLock lgGuard = inode->GetUniqueLock();
-        iCache_->Remove(inodeId);
-    }
-}
-
-void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
-    std::shared_ptr<InodeWrapper> inodeWrapper;
-    uint64_t inodeId;
-    VLOG(3) << "TrimIcache trimSize " << trimSize;
-    while (trimSize > 0) {
-        bool ok = iCache_->GetLast(&inodeId, &inodeWrapper);
-        if (ok) {
-            NameLockGuard lock(nameLock_, std::to_string(inodeId));
-            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
-            if (IsDirtyInode(inodeWrapper.get(), false)) {
-                VLOG(9) << "TrimIcache sync dirty inode " << inodeId;
-                dirtyMapMutex_.lock();
-                dirtyMap_.erase(inodeId);
-                dirtyMapMutex_.unlock();
-                auto *done =
-                    new TrimICacheAsyncDone(inodeWrapper, shared_from_this());
-                inodeWrapper->Async(done);
-            } else {
-                VLOG(9) << "TrimIcache remove inode " << inodeId
-                        << " from iCache";
-                iCache_->Remove(inodeId);
-            }
-            trimSize--;
-            // remove the attr of the inode in iattrcache
-            auto parents = inodeWrapper->GetParentLocked();
-            for (uint64_t parent : parents) {
-                iAttrCache_->Remove(parent, inodeId);
-            }
-        } else {
-            LOG(ERROR) << "icache size " << iCache_->Size();
-            assert(0);
-        }
-    }
+    const std::shared_ptr<InodeWrapper>& inode) {
+    deferSync_->Push(inode);
 }
 
 CURVEFS_ERROR
@@ -521,60 +270,5 @@ InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
     return rc;
 }
 
-void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
-    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
-    curve::common::LockGuard lg(openInodesMutex_);
-    openedInodes_.emplace(inodeId);
-}
-
-void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
-    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
-    curve::common::LockGuard lg(openInodesMutex_);
-    auto iter = openedInodes_.find(inodeId);
-    if (iter != openedInodes_.end()) {
-        openedInodes_.erase(iter);
-    }
-}
-
-bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
-    curve::common::LockGuard lg(openInodesMutex_);
-    auto iter = openedInodes_.find(inodeId);
-    return iter != openedInodes_.end();
-}
-
-bool InodeCacheManagerImpl::NeedUseCache(uint64_t inodeId,
-    const std::shared_ptr<InodeWrapper> &inodeWrapper,
-    bool onlyAttr) {
-    auto lock = inodeWrapper->GetUniqueLock();
-
-    if (onlyAttr) {
-        if (inodeWrapper->IsDirty()) {
-            return true;
-        }
-    } else {
-        if (IsDirtyInode(inodeWrapper.get(), false)) {
-            return true;
-        }
-    }
-
-    if (((FLAGS_enableCto && OpenInodeCached(inodeId)) || !FLAGS_enableCto)
-        && !IsTimeOut(inodeWrapper)) {
-        return true;
-    }
-
-    VLOG(9) << "inodeId " << inodeId << " is not need use cache";
-    return false;
-}
-
-bool InodeCacheManagerImpl::IsTimeOut(
-    const std::shared_ptr<InodeWrapper> &inodeWrapper) const {
-    uint32_t time = inodeWrapper->GetCachedTime();
-    if (cacheTimeOutSec_ > 0 &&
-        TimeUtility::GetTimeofDaySec() - time >= cacheTimeOutSec_) {
-        return true;
-    }
-    return false;
-}
-
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_cache_manager.h

@@ -38,12 +38,14 @@
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "src/common/concurrent/name_lock.h"
 #include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/filesystem/openfile.h"
+#include "curvefs/src/client/filesystem/defer_sync.h"
 
 using ::curve::common::LRUCache;
 using ::curve::common::CacheMetrics;
@@ -64,57 +66,8 @@ using rpcclient::BatchGetInodeAttrDone;
 using curve::common::CountDownEvent;
 using metric::S3ChunkInfoMetric;
 using common::RefreshDataOption;
-
-class InodeAttrCache {
- public:
-    InodeAttrCache() {}
-    ~InodeAttrCache() {}
-
-    bool Get(uint64_t parentId, std::map<uint64_t, InodeAttr> *imap) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            imap->insert(iter->second.begin(), iter->second.end());
-            return true;
-        }
-        return false;
-    }
-
-    void Set(uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        VLOG(1) << "parentId = " << parentId
-                << ", iAttrCache set size = " << inodeAttrs.size();
-        auto& inner = iAttrCache_[parentId];
-        for (const auto &it : inodeAttrs) {
-            inner.emplace(it.inodeid(), it);
-        }
-    }
-
-    void Release(uint64_t parentId) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto size = iAttrCache_.size();
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            iAttrCache_.erase(iter);
-        }
-        VLOG(1) << "inodeId = " << parentId
-                << ", inode attr cache release, before = "
-                << size << ", after = " << iAttrCache_.size();
-    }
-
-    void Remove(uint64_t parentId, uint64_t inodeId) {
-        curve::common::LockGuard lg(iAttrCacheMutex_);
-        auto iter = iAttrCache_.find(parentId);
-        if (iter != iAttrCache_.end()) {
-            iter->second.erase(inodeId);
-        }
-    }
-
- private:
-    // inodeAttr cache; <parentId, <inodeId, inodeAttr>>
-    std::map<uint64_t, std::map<uint64_t, InodeAttr>> iAttrCache_;
-    curve::common::Mutex iAttrCacheMutex_;
-};
+using ::curvefs::client::filesystem::OpenFiles;
+using ::curvefs::client::filesystem::DeferSync;
 
 class InodeCacheManager {
  public:
@@ -126,14 +79,9 @@ class InodeCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-                               uint32_t flushPeriodSec,
-                               RefreshDataOption option,
-                               uint32_t cacheTimeOutSec) = 0;
-
-    virtual void Run() = 0;
-
-    virtual void Stop() = 0;
+    virtual CURVEFS_ERROR Init(RefreshDataOption option,
+                               std::shared_ptr<OpenFiles> openFiles,
+                               std::shared_ptr<DeferSync> deferSync) = 0;
 
     virtual CURVEFS_ERROR
     GetInode(uint64_t inodeId,
@@ -161,24 +109,9 @@ class InodeCacheManager {
 
     virtual CURVEFS_ERROR DeleteInode(uint64_t inodeId) = 0;
 
-    virtual void AddInodeAttrs(uint64_t parentId,
-        const RepeatedPtrField<InodeAttr>& inodeAttrs) = 0;
-
-    virtual void ClearInodeCache(uint64_t inodeId) = 0;
-
     virtual void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) = 0;
 
-    virtual void FlushAll() = 0;
-
-    virtual void FlushInodeOnce() = 0;
-
-    virtual void ReleaseCache(uint64_t parentId) = 0;
-
-    virtual void AddOpenedInode(uint64_t inodeId) = 0;
-
-    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
-
  protected:
     uint32_t fsId_;
 };
@@ -187,59 +120,22 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     public std::enable_shared_from_this<InodeCacheManagerImpl> {
  public:
     InodeCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()),
-        iCache_(nullptr),
-        iAttrCache_(nullptr),
-        isStop_(true),
-        cacheTimeOutSec_(0) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
 
     explicit InodeCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient),
-        iCache_(nullptr),
-        iAttrCache_(nullptr),
-        cacheTimeOutSec_(0) {}
+      : metaClient_(metaClient) {}
 
-    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
-                       uint32_t flushPeriodSec,
-                       RefreshDataOption option,
-                       uint32_t cacheTimeOutSec) override {
-        if (enableCacheMetrics) {
-            iCache_ = std::make_shared<
-                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0,
-                    std::make_shared<CacheMetrics>("icache"));
-        } else {
-            iCache_ = std::make_shared<
-                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0);
-        }
-        maxCacheSize_ = cacheSize;
+    CURVEFS_ERROR Init(RefreshDataOption option,
+                       std::shared_ptr<OpenFiles> openFiles,
+                       std::shared_ptr<DeferSync> deferSync) override {
         option_ = option;
-        flushPeriodSec_ = flushPeriodSec;
-        cacheTimeOutSec_ = cacheTimeOutSec;
-        iAttrCache_ = std::make_shared<InodeAttrCache>();
         s3ChunkInfoMetric_ = std::make_shared<S3ChunkInfoMetric>();
+        openFiles_ =  openFiles;
+        deferSync_ = deferSync;
         return CURVEFS_ERROR::OK;
     }
 
-    void Run() {
-        isStop_.exchange(false);
-        flushThread_ =
-            Thread(&InodeCacheManagerImpl::FlushInodeBackground, this);
-        LOG(INFO) << "Start inodeManager flush thread ok.";
-    }
-
-    void Stop() {
-        isStop_.exchange(true);
-        LOG(INFO) << "stop inodeManager flush thread ...";
-        sleeper_.interrupt();
-        flushThread_.join();
-    }
-
-    bool IsDirtyMapExist(uint64_t inodeId) {
-        curve::common::LockGuard lg(dirtyMapMutex_);
-        return dirtyMap_.count(inodeId) > 0;
-    }
-
     CURVEFS_ERROR GetInode(uint64_t inodeId,
                            std::shared_ptr<InodeWrapper> &out) override;
 
@@ -263,78 +159,37 @@ class InodeCacheManagerImpl : public InodeCacheManager,
 
     CURVEFS_ERROR DeleteInode(uint64_t inodeId) override;
 
-    void AddInodeAttrs(uint64_t parentId,
-        const RepeatedPtrField<InodeAttr>& inodeAttrs) override;
-
-    void ClearInodeCache(uint64_t inodeId) override;
-
     void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) override;
 
-    void FlushAll() override;
-
-    void FlushInodeOnce() override;
-
-    void ReleaseCache(uint64_t parentId) override;
-
-    void RemoveICache(const std::shared_ptr<InodeWrapper> &inode);
-
-    void AddOpenedInode(uint64_t inodeId) override;
-
-    void RemoveOpenedInode(uint64_t inodeId) override;
-
-    bool NeedUseCache(uint64_t inodeId,
-        const std::shared_ptr<InodeWrapper> &inodeWrapper,
-        bool onlyAttr);
-
-    bool IsTimeOut(const std::shared_ptr<InodeWrapper> &inodeWrapper) const;
-
  private:
-    virtual void FlushInodeBackground();
-    void TrimIcache(uint64_t trimSize);
     CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
                               bool streaming = true);
-    bool OpenInodeCached(uint64_t inodeId);
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
-    std::shared_ptr<LRUCache<uint64_t,
-        std::shared_ptr<InodeWrapper>>> iCache_;
     std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric_;
 
-    std::shared_ptr<InodeAttrCache> iAttrCache_;
+    std::shared_ptr<OpenFiles> openFiles_;
 
-    // dirty map, key is inodeid
-    std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
-    curve::common::Mutex dirtyMapMutex_;
-
-    // record opened inode
-    std::multiset<uint64_t> openedInodes_;
-    curve::common::Mutex openInodesMutex_;
+    std::shared_ptr<DeferSync> deferSync_;
 
     curve::common::GenericNameLock<Mutex> nameLock_;
 
     curve::common::GenericNameLock<Mutex> asyncNameLock_;
 
-    uint64_t maxCacheSize_;
     RefreshDataOption option_;
-    uint32_t flushPeriodSec_;
-    Thread flushThread_;
-    InterruptibleSleeper sleeper_;
-    Atomic<bool> isStop_;
-    // cache timeout seconds, 0 means never timeout
-    uint32_t cacheTimeOutSec_;
 };
 
 class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
  public:
-    BatchGetInodeAttrAsyncDone(
-        const std::shared_ptr<InodeCacheManager> &inodeCacheManager,
-        std::shared_ptr<CountDownEvent> cond,
-        uint64_t parentId):
-        inodeCacheManager_(inodeCacheManager),
-        cond_(cond),
-        parentId_(parentId) {}
+    BatchGetInodeAttrAsyncDone(std::map<uint64_t, InodeAttr>* attrs,
+                               ::curve::common::Mutex* mutex,
+                               std::shared_ptr<CountDownEvent> cond):
+        attrs_(attrs),
+        mutex_(mutex),
+        cond_(cond) {}
+
     ~BatchGetInodeAttrAsyncDone() {}
 
     void Run() override {
@@ -342,22 +197,25 @@ class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
         MetaStatusCode ret = GetStatusCode();
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "BatchGetInodeAttrAsync failed, "
-                       << "parentId = " << parentId_
                        << ", MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret);
         } else {
             auto inodeAttrs = GetInodeAttrs();
             VLOG(3) << "BatchGetInodeAttrAsyncDone update inodeAttrCache"
                     << " size = " << inodeAttrs.size();
-            inodeCacheManager_->AddInodeAttrs(parentId_, inodeAttrs);
+
+            curve::common::LockGuard lk(*mutex_);
+            for (const auto& attr : inodeAttrs) {
+                attrs_->emplace(attr.inodeid(), attr);
+            }
         }
         cond_->Signal();
     };
 
  private:
-    std::shared_ptr<InodeCacheManager> inodeCacheManager_;
+    ::curve::common::Mutex* mutex_;
+    std::map<uint64_t, InodeAttr>* attrs_;
     std::shared_ptr<CountDownEvent> cond_;
-    uint64_t parentId_;
 };
 
 }  // namespace client

curvefs/src/client/inode_wrapper.cpp

@@ -33,7 +33,6 @@
 
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/async_request_closure.h"
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
@@ -47,6 +46,7 @@ namespace client {
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::DataIndices;
+using ::curvefs::client::filesystem::ToFSError;
 
 bvar::Adder<int64_t> g_alive_inode_count{"alive_inode_count"};
 
@@ -193,7 +193,7 @@ CURVEFS_ERROR InodeWrapper::SyncAttr(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
 
         dirty_ = false;
@@ -213,7 +213,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3ChunkInfo(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         ClearS3ChunkInfoAdd();
     }
@@ -251,8 +251,6 @@ void InodeWrapper::FlushS3ChunkInfoAsync() {
 CURVEFS_ERROR InodeWrapper::FlushVolumeExtent() {
     std::lock_guard<::curve::common::Mutex> guard(syncingVolumeExtentsMtx_);
     if (!extentCache_.HasDirtyExtents()) {
-        VLOG(9) << "FlushVolumeExtent, ino: " << inode_.inodeid()
-                << ", no dirty extents";
         return CURVEFS_ERROR::OK;
     }
 
@@ -296,7 +294,7 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
                    << "MetaStatusCode: " << ret
                    << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                    << ", inodeid: " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     auto before = s3ChunkInfoSize_;
     inode_.mutable_s3chunkinfomap()->swap(s3ChunkInfoMap);
@@ -330,7 +328,7 @@ CURVEFS_ERROR InodeWrapper::Link(uint64_t parent) {
                    <<", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
 
     dirty_ = false;
@@ -398,7 +396,7 @@ CURVEFS_ERROR InodeWrapper::UnLink(uint64_t parent) {
                        << ", MetaStatusCode = " << ret
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", inodeid = " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         dirty_ = false;
         dirtyAttr_.Clear();
@@ -429,7 +427,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParent(
                    << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return MetaStatusCodeToCurvefsErrCode(ret);
+        return ToFSError(ret);
     }
     dirty_ = false;
     dirtyAttr_.Clear();
@@ -478,18 +476,12 @@ void InodeWrapper::Async(MetaServerClientDone *done, bool internal) {
 
 void InodeWrapper::AsyncFlushAttrAndExtents(MetaServerClientDone *done,
                                             bool /*internal*/) {
-    VLOG(9) << "async inode: " << inode_.ShortDebugString()
-            << ", is dirty: " << dirty_
-            << ", has dirty extents: " << extentCache_.HasDirtyExtents();
     if (dirty_ || extentCache_.HasDirtyExtents()) {
         LockSyncingInode();
         syncingVolumeExtentsMtx_.lock();
         DataIndices indices;
         if (extentCache_.HasDirtyExtents()) {
             indices.volumeExtents = extentCache_.GetDirtyExtents();
-            VLOG(9) << "aync inode: " << inode_.ShortDebugString()
-                    << ", volume extents: "
-                    << indices.volumeExtents->ShortDebugString();
         }
 
         metaClient_->UpdateInodeWithOutNlinkAsync(
@@ -522,7 +514,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return MetaStatusCodeToCurvefsErrCode(ret);
+            return ToFSError(ret);
         }
         ClearS3ChunkInfoAdd();
         dirty_ = false;
@@ -591,7 +583,7 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
-    VolumeExtentSliceList extents;
+    VolumeExtentList extents;
     auto st = metaClient_->GetVolumeExtent(inode_.fsid(), inode_.inodeid(),
                                            true, &extents);
     VLOG(9) << "RefreshVolumeExtent, ino: " << inode_.inodeid()
@@ -605,7 +597,7 @@ CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
         LOG(ERROR) << "GetVolumeExtent failed, inodeid: " << inode_.inodeid();
     }
 
-    return MetaStatusCodeToCurvefsErrCode(st);
+    return ToFSError(st);
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshNlink() {

curvefs/src/client/inode_wrapper.h

@@ -34,12 +34,12 @@
 
 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/timeutility.h"
+#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Inode;
 using ::curvefs::metaserver::S3ChunkInfoList;
@@ -52,16 +52,14 @@ constexpr int kAccessTime = 1 << 0;
 constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;
 
-using ::curvefs::metaserver::VolumeExtentSliceList;
+using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 enum class InodeStatus {
     kNormal = 0,
     kError = -1,
 };
 
-// TODO(xuchaojie) : get from conf maybe?
-const uint32_t kOptimalIOBlockSize = 0x10000u;
-
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;

curvefs/src/client/kvclient/kvclient_manager.cpp

@@ -41,7 +41,6 @@ namespace client {
 
 bool KVClientManager::Init(const KVClientManagerOpt &config,
                            const std::shared_ptr<KVClient> &kvclient) {
-    LOG(INFO) << "KVClientManager init.";
     client_ = kvclient;
     return threadPool_.Start(config.setThreadPooln) == 0;
 }

curvefs/src/client/kvclient/kvclient_manager.h

@@ -57,7 +57,6 @@ struct SetKVCacheTask {
     std::string key;
     const char *value;
     uint64_t length;
-    uint64_t startTime;
     SetKVCacheDone done;
     SetKVCacheTask() = default;
     SetKVCacheTask(
@@ -72,7 +71,6 @@ struct GetKVCacheTask {
     uint64_t offset;
     uint64_t length;
     bool res;
-    uint64_t startTime;
     GetKVCacheDone done;
     GetKVCacheTask(const std::string &k, char *v, uint64_t off, uint64_t len)
         : key(k), value(v), offset(off), length(len), res(false) {

curvefs/src/client/main.cpp

@@ -51,13 +51,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     release : FuseOpRelease,
     fsync : FuseOpFsync,
     opendir : FuseOpOpenDir,
-    // TODO(wuhongsong): readdirplus is problematic,
-    // resulting in inconsistent metadata
-    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    // readdir : 0,
-    // #else
+    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    readdir : 0,
+    #else
     readdir : FuseOpReadDir,
-    // #endif
+    #endif
     releasedir : FuseOpReleaseDir,
     fsyncdir : 0,
     statfs : FuseOpStatFs,
@@ -81,11 +79,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     flock : 0,
     fallocate : 0,
     #endif
-    // TODO(wuhongsong): The current implementation is problematic,
-    // resulting in inconsistent metadata
-    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    readdirplus : FuseOpReadDirPlus,
+    #else
     readdirplus : 0,
-    // #endif
+    #endif
     #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 4)
     copy_file_range : 0,
     #endif
@@ -222,8 +220,8 @@ int main(int argc, char *argv[]) {
 
     fuse_daemonize(opts.foreground);
 
-    if (InitGlog(mOpts.conf, argv[0]) < 0) {
-        printf("Init glog failed, confpath = %s\n", mOpts.conf);
+    if (InitLog(mOpts.conf, argv[0]) < 0) {
+        printf("Init log failed, confpath = %s\n", mOpts.conf);
     }
 
     ret = InitFuseClient(&mOpts);

curvefs/src/client/metric/client_metric.cpp

@@ -31,7 +31,7 @@ const std::string MetaServerClientMetric::prefix = "curvefs_metaserver_client";
 const std::string ClientOpMetric::prefix = "curvefs_client";  // NOLINT
 const std::string S3MultiManagerMetric::prefix = "curvefs_client_manager";  // NOLINT
 const std::string FSMetric::prefix = "curvefs_client";  // NOLINT
-const std::string IoMetric::prefix = "curvefs_s3";  // NOLINT
+const std::string S3Metric::prefix = "curvefs_s3";  // NOLINT
 const std::string DiskCacheMetric::prefix = "curvefs_disk_cache";  // NOLINT
 const std::string KVClientMetric::prefix = "curvefs_kvclient";  // NOLINT
 const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info";  // NOLINT

curvefs/src/client/metric/client_metric.h

@@ -92,12 +92,13 @@ struct MetaServerClientMetric {
     // volume extent
     InterfaceMetric updateVolumeExtent;
     InterfaceMetric getVolumeExtent;
-    InterfaceMetric updateDeallocatableBlockGroup;
 
     MetaServerClientMetric()
-        : getDentry(prefix, "getDentry"), listDentry(prefix, "listDentry"),
+        : getDentry(prefix, "getDentry"),
+          listDentry(prefix, "listDentry"),
           createDentry(prefix, "createDentry"),
-          deleteDentry(prefix, "deleteDentry"), getInode(prefix, "getInode"),
+          deleteDentry(prefix, "deleteDentry"),
+          getInode(prefix, "getInode"),
           batchGetInodeAttr(prefix, "batchGetInodeAttr"),
           batchGetXattr(prefix, "batchGetXattr"),
           createInode(prefix, "createInode"),
@@ -106,9 +107,7 @@ struct MetaServerClientMetric {
           appendS3ChunkInfo(prefix, "appendS3ChunkInfo"),
           prepareRenameTx(prefix, "prepareRenameTx"),
           updateVolumeExtent(prefix, "updateVolumeExtent"),
-          getVolumeExtent(prefix, "getVolumeExtent"),
-          updateDeallocatableBlockGroup(prefix,
-                                        "updateDeallocatableBlockGroup") {}
+          getVolumeExtent(prefix, "getVolumeExtent") {}
 };
 
 struct InflightGuard {
@@ -228,35 +227,27 @@ struct FSMetric {
           userReadIoSize(prefix, fsName + "_userReadIoSize", 0) {}
 };
 
-
-struct IoMetric {
+struct S3Metric {
     static const std::string prefix;
+
     std::string fsName;
     InterfaceMetric adaptorWrite;
     InterfaceMetric adaptorRead;
-    InterfaceMetric adaptorBgFlush;
-    InterfaceMetric adaptorFlushBackend;
     InterfaceMetric adaptorWriteS3;
+    InterfaceMetric adaptorWriteDiskCache;
     InterfaceMetric adaptorReadS3;
-    InterfaceMetric adaptorWriteKvCache;
-    InterfaceMetric adaptorReadKvCache;
-    InterfaceMetric adaptorReadBackend;
     InterfaceMetric adaptorReadDiskCache;
     bvar::Status<uint32_t> readSize;
     bvar::Status<uint32_t> writeSize;
 
-    explicit IoMetric(const std::string &name = "")
+    explicit S3Metric(const std::string &name = "")
         : fsName(!name.empty() ? name
                                : prefix + curve::common::ToHexString(this)),
           adaptorWrite(prefix, fsName + "_adaptor_write"),
           adaptorRead(prefix, fsName + "_adaptor_read"),
-          adaptorBgFlush(prefix, fsName + "_adaptor_bg_flush"),
-          adaptorFlushBackend(prefix, fsName + "_adaptor_flush_backend"),
-          adaptorReadBackend(prefix, fsName + "_adaptor_read_backend"),
           adaptorWriteS3(prefix, fsName + "_adaptor_write_s3"),
+          adaptorWriteDiskCache(prefix, fsName + "_adaptor_write_disk_cache"),
           adaptorReadS3(prefix, fsName + "_adaptor_read_s3"),
-          adaptorWriteKvCache(prefix, fsName + "_adaptor_write_kv_cache"),
-          adaptorReadKvCache(prefix, fsName + "_adaptor_read_kv_cache"),
           adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache"),
           readSize(prefix, fsName + "_adaptor_read_size", 0),
           writeSize(prefix, fsName + "_adaptor_write_size", 0) {}
@@ -266,16 +257,12 @@ struct DiskCacheMetric {
     static const std::string prefix;
 
     std::string fsName;
-    InterfaceMetric adaptorWriteDiskCache;
-    InterfaceMetric adaptorReadDiskCache;
     InterfaceMetric writeS3;
     bvar::Status<uint64_t> diskUsedBytes;
 
     explicit DiskCacheMetric(const std::string &name = "")
         : fsName(!name.empty() ? name
                                : prefix + curve::common::ToHexString(this)),
-          adaptorWriteDiskCache(prefix, fsName + "_adaptor_write_disk_cache"),
-          adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache"),
           writeS3(prefix, fsName + "_write_s3"),
           diskUsedBytes(prefix, fsName + "_diskcache_usedbytes", 0) {}
 };

curvefs/src/client/rpcclient/base_client.cpp

@@ -180,6 +180,7 @@ void MDSBaseClient::AllocateVolumeBlockGroup(
     request.set_fsid(fsId);
     request.set_count(count);
     request.set_owner(owner);
+
     SpaceService_Stub stub(channel);
     stub.AllocateBlockGroup(cntl, &request, response, nullptr);
 }

curvefs/src/client/rpcclient/mds_client.cpp

@@ -710,17 +710,14 @@ SpaceErrCode MdsClientImpl::AllocateVolumeBlockGroup(
         AllocateBlockGroupResponse response;
         mdsbasecli_->AllocateVolumeBlockGroup(fsId, count, owner, &response,
                                               cntl, channel);
-        VLOG(9) << "AllocateVolumeBlockGroup, response: "
-                    << response.DebugString();
 
         CHECK_RPC_AND_RETRY_IF_ERROR("AllocateVolumeBlockGroup");
 
         auto status = response.status();
-        LOG_IF(WARNING, status != SpaceErrCode::SpaceOk)
-            << "Allocate volume block group failed, err: "
-            << SpaceErrCode_Name(status);
-
-        if (response.blockgroups_size() == 0) {
+        if (status != SpaceErrCode::SpaceOk) {
+            LOG(WARNING) << "Allocate volume block group failed, err: "
+                         << SpaceErrCode_Name(status);
+        } else if (response.blockgroups_size() == 0) {
             LOG(WARNING) << "Allocate volume block group failed, no block "
                             "group allcoated";
             return SpaceErrCode::SpaceErrNoSpace;

curvefs/src/client/rpcclient/metaserver_client.cpp

@@ -65,7 +65,6 @@ using BatchGetXAttrExcutor = TaskExecutor;
 using GetOrModifyS3ChunkInfoExcutor = TaskExecutor;
 using UpdateVolumeExtentExecutor = TaskExecutor;
 using GetVolumeExtentExecutor = TaskExecutor;
-using UpdateDeallocatableBlockGroupExcutor = TaskExecutor;
 
 using ::curvefs::common::LatencyUpdater;
 using ::curvefs::common::StreamConnection;
@@ -971,7 +970,6 @@ void MetaServerClientImpl::UpdateInodeAsync(const UpdateInodeRequest &request,
         req.set_poolid(poolID);
         req.set_copysetid(copysetID);
         req.set_partitionid(partitionID);
-        VLOG(9) << "update inode async req: " << req.ShortDebugString();
 
         auto *rpcDone = new UpdateInodeRpcDone(taskExecutorDone, &metric_);
         curvefs::metaserver::MetaServerService_Stub stub(channel);
@@ -1357,8 +1355,8 @@ MetaStatusCode MetaServerClientImpl::CreateManageInode(const InodeParam &param,
 
     auto taskCtx = std::make_shared<TaskContext>(
         MetaServerOpType::CreateManageInode, task, param.fsId, 0);
-    CreateManagerInodeExcutor excutor(opt_, metaCache_, channelManager_,
-                                      std::move(taskCtx));
+    CreateInodeExcutor excutor(opt_, metaCache_, channelManager_,
+                               std::move(taskCtx));
     return ConvertToMetaStatusCode(excutor.DoRPCTask());
 }
 
@@ -1467,7 +1465,7 @@ void UpdateVolumeExtentRpcDone::Run() {
     } while (0)
 
 void MetaServerClientImpl::AsyncUpdateVolumeExtent(
-    uint32_t fsId, uint64_t inodeId, const VolumeExtentSliceList &extents,
+    uint32_t fsId, uint64_t inodeId, const VolumeExtentList &extents,
     MetaServerClientDone *done) {
     auto task = AsyncRPCTask {
         (void)txId;
@@ -1475,7 +1473,7 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
         metric_.updateVolumeExtent.qps.count << 1;
         metaserver::UpdateVolumeExtentRequest request;
         SET_COMMON_FIELDS;
-        request.set_allocated_extents(new VolumeExtentSliceList{extents});
+        request.set_allocated_extents(new VolumeExtentList{extents});
 
         auto *rpcDone =
             new UpdateVolumeExtentRpcDone(taskExecutorDone, &metric_);
@@ -1495,8 +1493,7 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
 namespace {
 
 struct ParseVolumeExtentCallBack {
-    explicit ParseVolumeExtentCallBack(VolumeExtentSliceList *ext)
-        : extents(ext) {}
+    explicit ParseVolumeExtentCallBack(VolumeExtentList *ext) : extents(ext) {}
 
     bool operator()(butil::IOBuf *data) const {
         metaserver::VolumeExtentSlice slice;
@@ -1509,7 +1506,7 @@ struct ParseVolumeExtentCallBack {
         return true;
     }
 
-    VolumeExtentSliceList *extents;
+    VolumeExtentList *extents;
 };
 
 }  // namespace
@@ -1517,7 +1514,7 @@ struct ParseVolumeExtentCallBack {
 MetaStatusCode
 MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
                                       bool streaming,
-                                      VolumeExtentSliceList *extents) {
+                                      VolumeExtentList *extents) {
     auto task = RPCTask {
         (void)txId;
         (void)applyIndex;
@@ -1599,66 +1596,9 @@ MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
     return ConvertToMetaStatusCode(executor.DoRPCTask());
 }
 
-MetaStatusCode MetaServerClientImpl::UpdateDeallocatableBlockGroup(
-    uint32_t fsId, uint64_t inodeId, DeallocatableBlockGroupMap *statistic) {
-    auto task = RPCTask {
-        metric_.updateDeallocatableBlockGroup.qps.count << 1;
-        LatencyUpdater updater(&metric_.updateDeallocatableBlockGroup.latency);
-
-        metaserver::UpdateDeallocatableBlockGroupRequest request;
-        metaserver::UpdateDeallocatableBlockGroupResponse response;
-        request.set_poolid(poolID);
-        request.set_copysetid(copysetID);
-        request.set_partitionid(partitionID);
-        request.set_fsid(fsId);
-        auto *upadte = request.mutable_update();
-        for (auto &it : *statistic) {
-            upadte->Add()->CopyFrom(std::move(it.second));
-        }
-
-        curvefs::metaserver::MetaServerService_Stub stub(channel);
-        stub.UpdateDeallocatableBlockGroup(cntl, &request, &response, nullptr);
-
-        if (cntl->Failed()) {
-            metric_.updateDeallocatableBlockGroup.eps.count << 1;
-            LOG(WARNING) << "UpdateDeallocatableBlockGroup failed"
-                         << ", errorCode = " << cntl->ErrorCode()
-                         << ", errorText = " << cntl->ErrorText()
-                         << ", logId = " << cntl->log_id();
-            return -cntl->ErrorCode();
-        }
-
-        auto rc = response.statuscode();
-        if (rc != MetaStatusCode::OK) {
-            metric_.updateDeallocatableBlockGroup.eps.count << 1;
-            LOG(WARNING) << "UpdateDeallocatableBlockGroup: retCode = " << rc
-                         << ", message = " << MetaStatusCode_Name(rc);
-        } else if (response.has_appliedindex()) {
-            metaCache_->UpdateApplyIndex(CopysetGroupID(poolID, copysetID),
-                                         response.appliedindex());
-        } else {
-            LOG(WARNING) << "UpdateDeallocatableBlockGroup OK"
-                         << ", but applyIndex not set in response:"
-                         << response.DebugString();
-            return -1;
-        }
-
-        VLOG(6) << "UpdateDeallocatableBlockGroup done, request: "
-                << request.DebugString()
-                << "response: " << response.DebugString();
-        return rc;
-    };
-
-    auto taskCtx = std::make_shared<TaskContext>(
-        MetaServerOpType::UpdateDeallocatableBlockGroup, task, fsId, inodeId);
-
-    UpdateDeallocatableBlockGroupExcutor excutor(
-        opt_, metaCache_, channelManager_, std::move(taskCtx));
-    return ConvertToMetaStatusCode(excutor.DoRPCTask());
-}
-
-MetaStatusCode MetaServerClientImpl::GetInodeAttr(
-    uint32_t fsId, uint64_t inodeid, InodeAttr *attr) {
+MetaStatusCode MetaServerClientImpl::GetInodeAttr(uint32_t fsId,
+                                                  uint64_t inodeid,
+                                                  InodeAttr *attr) {
     std::set<uint64_t> inodeIds;
     inodeIds.insert(inodeid);
     std::list<InodeAttr> attrs;

curvefs/src/client/rpcclient/metaserver_client.h

@@ -30,7 +30,6 @@
 #include <set>
 #include <unordered_map>
 #include <utility>
-#include <map>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/metaserver.pb.h"
@@ -50,24 +49,21 @@ using ::curvefs::metaserver::InodeAttr;
 using ::curvefs::metaserver::XAttr;
 using ::curvefs::metaserver::MetaStatusCode;
 using ::curvefs::metaserver::S3ChunkInfoList;
-using ::curvefs::metaserver::DeallocatableBlockGroup;
 using ::curvefs::common::StreamStatus;
 using ::curvefs::common::StreamClient;
-using ::curvefs::metaserver::Time;
-
-using DeallocatableBlockGroupMap = std::map<uint64_t, DeallocatableBlockGroup>;
 using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+using ::curvefs::metaserver::Time;
 
 namespace curvefs {
 namespace client {
 namespace rpcclient {
 
 using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
-using ::curvefs::metaserver::VolumeExtentSliceList;
+using ::curvefs::metaserver::VolumeExtentList;
 
 struct DataIndices {
     absl::optional<S3ChunkInfoMap> s3ChunkInfoMap;
-    absl::optional<VolumeExtentSliceList> volumeExtents;
+    absl::optional<VolumeExtentList> volumeExtents;
 };
 
 class MetaServerClient {
@@ -165,17 +161,13 @@ class MetaServerClient {
 
     virtual void AsyncUpdateVolumeExtent(uint32_t fsId,
                                          uint64_t inodeId,
-                                         const VolumeExtentSliceList &extents,
+                                         const VolumeExtentList &extents,
                                          MetaServerClientDone *done) = 0;
 
     virtual MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                            uint64_t inodeId,
                                            bool streaming,
-                                           VolumeExtentSliceList *extents) = 0;
-
-    virtual MetaStatusCode
-    UpdateDeallocatableBlockGroup(uint32_t fsId, uint64_t inodeId,
-                                  DeallocatableBlockGroupMap *statistic) = 0;
+                                           VolumeExtentList *extents) = 0;
 };
 
 class MetaServerClientImpl : public MetaServerClient {
@@ -273,17 +265,13 @@ class MetaServerClientImpl : public MetaServerClient {
 
     void AsyncUpdateVolumeExtent(uint32_t fsId,
                                  uint64_t inodeId,
-                                 const VolumeExtentSliceList &extents,
+                                 const VolumeExtentList &extents,
                                  MetaServerClientDone *done) override;
 
     MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                    uint64_t inodeId,
                                    bool streaming,
-                                   VolumeExtentSliceList *extents) override;
-
-    MetaStatusCode UpdateDeallocatableBlockGroup(
-        uint32_t fsId, uint64_t inodeId,
-        DeallocatableBlockGroupMap *statistic) override;
+                                   VolumeExtentList *extents) override;
 
  private:
     MetaStatusCode UpdateInode(const UpdateInodeRequest &request,

curvefs/src/client/rpcclient/task_excutor.cpp

@@ -27,10 +27,8 @@
 
 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/common/define.h"
 
 using ::curvefs::metaserver::MetaStatusCode;
-using ::curvefs::RECYCLEINODEID;
 
 namespace curvefs {
 namespace client {
@@ -341,17 +339,6 @@ bool CreateInodeExcutor::GetTarget() {
     return true;
 }
 
-bool CreateManagerInodeExcutor::GetTarget() {
-    if (!metaCache_->GetTarget(task_->fsID, RECYCLEINODEID, &task_->target,
-                               &task_->applyIndex)) {
-        LOG(ERROR) << "CreateManagerInodeExcutor select target for task fail, "
-                   << task_->TaskContextStr()
-                   << ", recycleInodeId = " << RECYCLEINODEID;
-        return false;
-    }
-    return true;
-}
-
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/rpcclient/task_excutor.h

@@ -275,18 +275,6 @@ class CreateInodeExcutor : public TaskExecutor {
     bool GetTarget() override;
 };
 
-class CreateManagerInodeExcutor : public TaskExecutor {
- public:
-    explicit CreateManagerInodeExcutor(
-        const ExcutorOpt &opt, const std::shared_ptr<MetaCache> &metaCache,
-        const std::shared_ptr<ChannelManager<MetaserverID>> &channelManager,
-        const std::shared_ptr<TaskContext> &task)
-        : TaskExecutor(opt, metaCache, channelManager, task) {}
-
- protected:
-    bool GetTarget() override;
-};
-
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/s3/client_s3_adaptor.h

@@ -19,15 +19,12 @@
  * Created Date: 21-5-31
  * Author: huyao
  */
-
 #ifndef CURVEFS_SRC_CLIENT_S3_CLIENT_S3_ADAPTOR_H_
 #define CURVEFS_SRC_CLIENT_S3_CLIENT_S3_ADAPTOR_H_
 
 #include <bthread/execution_queue.h>
 
 #include <memory>
-#include <utility>
-#include <set>
 #include <string>
 #include <vector>
 
@@ -36,216 +33,204 @@
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/client_storage_adaptor.h"
-#include "curvefs/src/client/kvclient/kvclient_manager.h"
-#include "curvefs/src/client/kvclient/kvclient.h"
-#include "curvefs/src/client/error_code.h"
-#include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
-#include "curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h"
+#include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "curvefs/src/client/s3/disk_cache_manager_impl.h"
 #include "src/common/wait_interval.h"
-
 namespace curvefs {
 namespace client {
 
-using curve::common::GetObjectAsyncCallBack;
-using curve::common::PutObjectAsyncCallBack;
-using curve::common::S3Adapter;
+using ::curve::common::Thread;
+using ::curve::common::TaskThreadPool;
 using curvefs::client::common::S3ClientAdaptorOption;
-using curvefs::client::metric::IoMetric;
+using curvefs::client::common::DiskCacheType;
+using curvefs::metaserver::Inode;
+using curvefs::metaserver::S3ChunkInfo;
+using curvefs::metaserver::S3ChunkInfoList;
+using rpcclient::MdsClient;
+using curvefs::client::metric::S3Metric;
 
-/// @brief s3 read request
-/// @param chunkId chunk id
-/// @param offset file offset
-/// @param len read length
-/// @param objectOffset first offset in the block
-/// @param readOffset read buf offset
-/// @param fsId file system id
-/// @param inodeId inode id
-/// @param compaction compaction flag
-struct S3ReadRequest {
-    uint64_t chunkId;
-    uint64_t offset;
-    uint64_t len;
-    uint64_t objectOffset;
-    uint64_t readOffset;
-    uint64_t fsId;
-    uint64_t inodeId;
-    uint64_t compaction;
+class DiskCacheManagerImpl;
+class FlushChunkCacheContext;
+class ChunkCacheManager;
 
-    std::string DebugString() const {
-        std::ostringstream os;
-        os << "S3ReadRequest ( chunkId = " << chunkId << ", offset = " << offset
-           << ", len = " << len << ", objectOffset = " << objectOffset
-           << ", readOffset = " << readOffset << ", fsId = " << fsId
-           << ", inodeId = " << inodeId << ", compaction = " << compaction
-           << " )";
-        return os.str();
-    }
-};
-
-inline std::string
-S3ReadRequestVecDebugString(const std::vector<S3ReadRequest> &reqs) {
-    std::ostringstream os;
-    for_each(reqs.begin(), reqs.end(),
-             [&](const S3ReadRequest &req) { os << req.DebugString() << " "; });
-    return os.str();
-}
-
-// client use s3 internal interface
-class S3ClientAdaptorImpl : public StorageAdaptor {
+class S3ClientAdaptor {
  public:
-    S3ClientAdaptorImpl() : StorageAdaptor() {}
-
-    // for unittest
-    explicit S3ClientAdaptorImpl(std::shared_ptr<
-      S3Client> client) : StorageAdaptor() {
-        client_ = client;
-    }
-
-    virtual ~S3ClientAdaptorImpl() {
-        LOG(INFO) << "delete S3ClientAdaptorImpl";
-    }
-
-    /// @brief init s3 storage adaptor
-    /// @param option fuse client option
-    /// @param inodeManager inode cache manager
-    /// @param mdsClient mds client
-    /// @param fsCacheManager fscache manager
-    /// @param diskCacheManagerImpl disk cache manager
-    /// @param kvClientManager kv client manager
-    /// @param fsInfo file system information
-    /// @return error code
-    CURVEFS_ERROR Init(const FuseClientOption &option,
+    S3ClientAdaptor() {}
+    virtual ~S3ClientAdaptor() {}
+    /**
+     * @brief Initailize s3 client
+     * @param[in] options the options for s3 client
+     */
+    virtual CURVEFS_ERROR
+    Init(const S3ClientAdaptorOption &option, std::shared_ptr<S3Client> client,
          std::shared_ptr<InodeCacheManager> inodeManager,
          std::shared_ptr<MdsClient> mdsClient,
          std::shared_ptr<FsCacheManager> fsCacheManager,
          std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
          std::shared_ptr<KVClientManager> kvClientManager,
-         std::shared_ptr<FsInfo> fsInfo) override;
+         bool startBackGround = false) = 0;
+    /**
+     * @brief write data to s3
+     * @param[in] options the options for s3 client
+     */
+    virtual int Write(uint64_t inodeId, uint64_t offset, uint64_t length,
+                      const char *buf) = 0;
+    virtual int Read(uint64_t inodeId, uint64_t offset, uint64_t length,
+                     char *buf) = 0;
+    virtual CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper,
+                                   uint64_t size) = 0;
+    virtual void ReleaseCache(uint64_t inodeId) = 0;
+    virtual CURVEFS_ERROR Flush(uint64_t inodeId) = 0;
+    virtual CURVEFS_ERROR FlushAllCache(uint64_t inodeId) = 0;
+    virtual CURVEFS_ERROR FsSync() = 0;
+    virtual int Stop() = 0;
+    virtual FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum,
+                                        uint64_t *chunkId) = 0;
+    virtual void SetFsId(uint32_t fsId) = 0;
+    virtual void InitMetrics(const std::string &fsName) = 0;
+    virtual void CollectMetrics(InterfaceMetric *interface, int count,
+                                uint64_t start) = 0;
+    virtual std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() = 0;
+    virtual std::shared_ptr<S3Client> GetS3Client() = 0;
+    virtual uint64_t GetBlockSize() = 0;
+    virtual uint64_t GetChunkSize() = 0;
+    virtual uint32_t GetObjectPrefix() = 0;
+    virtual bool HasDiskCache() = 0;
+};
 
-    int Stop() override;
+using FlushChunkCacheCallBack = std::function<
+  void(const std::shared_ptr<FlushChunkCacheContext>&)>;
 
-    /// @brief read data from s3 storage
-    /// @param request read request
-    /// @return error code
-    CURVEFS_ERROR FlushDataCache(const UperFlushRequest& req,
-      uint64_t* writeOffset) override;
-
-    /// @brief read data from s3 storage
-    /// @param request read request
-    /// @return error code
-    CURVEFS_ERROR ReadFromLowlevel(UperReadRequest request) override;
+struct FlushChunkCacheContext {
+    uint64_t inode;
+    ChunkCacheManagerPtr chunkCacheManptr;
+    bool force;
+    FlushChunkCacheCallBack cb;
+    CURVEFS_ERROR retCode;
+};
 
+// client use s3 internal interface
+class S3ClientAdaptorImpl : public S3ClientAdaptor {
+ public:
+    S3ClientAdaptorImpl() {}
+    virtual ~S3ClientAdaptorImpl() {
+        LOG(INFO) << "delete S3ClientAdaptorImpl";
+    }
+    /**
+     * @brief Initailize s3 client
+     * @param[in] options the options for s3 client
+     */
+    CURVEFS_ERROR
+    Init(const S3ClientAdaptorOption &option, std::shared_ptr<S3Client> client,
+         std::shared_ptr<InodeCacheManager> inodeManager,
+         std::shared_ptr<MdsClient> mdsClient,
+         std::shared_ptr<FsCacheManager> fsCacheManager,
+         std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+         std::shared_ptr<KVClientManager> kvClientManager,
+         bool startBackGround = false);
+    /**
+     * @brief write data to s3
+     * @param[in] options the options for s3 client
+     */
+    int Write(uint64_t inodeId, uint64_t offset, uint64_t length,
+              const char *buf);
+    int Read(uint64_t inodeId, uint64_t offset, uint64_t length, char *buf);
     CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper, uint64_t size);
-
+    void ReleaseCache(uint64_t inodeId);
+    CURVEFS_ERROR Flush(uint64_t inodeId);
+    CURVEFS_ERROR FlushAllCache(uint64_t inodeId);
+    CURVEFS_ERROR FsSync();
+    int Stop();
+    uint64_t GetBlockSize() {
+        return blockSize_;
+    }
+    uint64_t GetChunkSize() {
+        return chunkSize_;
+    }
     uint32_t GetObjectPrefix() {
         return objectPrefix_;
     }
 
+    std::shared_ptr<FsCacheManager> GetFsCacheManager() {
+        return fsCacheManager_;
+    }
+    uint32_t GetFlushInterval() { return flushIntervalSec_; }
     std::shared_ptr<S3Client> GetS3Client() { return client_; }
-
-    CURVEFS_ERROR FuseOpInit(void *userdata,
-      struct fuse_conn_info *conn) override {
-        StorageAdaptor::FuseOpInit(userdata, conn);
-        return CURVEFS_ERROR::OK;
-    }
-
- private:
-     enum class ReadStatus {
-        OK = 0,
-        S3_READ_FAIL = -1,
-        S3_NOT_EXIST = -2,
-    };
-
-    ReadStatus toReadStatus(const int retCode) {
-        ReadStatus st = ReadStatus::OK;
-        if (retCode < 0) {
-            st = (retCode == -2) ? ReadStatus::S3_NOT_EXIST
-                                 : ReadStatus::S3_READ_FAIL;
-        }
-        return st;
-    }
-
-    S3ClientAdaptorImpl::ReadStatus ReadKVRequest(
-      const std::vector<S3ReadRequest> &kvRequests,
-      char *dataBuf, uint64_t fileLen);
-
-    CURVEFS_ERROR PrepareFlushTasks(const UperFlushRequest& req,
-      std::vector<std::shared_ptr<PutObjectAsyncContext>> *s3Tasks,
-      std::vector<std::shared_ptr<SetKVCacheTask>> *kvCacheTasks,
-      uint64_t* writeOffset);
-
-    void FlushTaskExecute(CachePolicy cachePoily,
-      const std::vector<std::shared_ptr<PutObjectAsyncContext>> &s3Tasks,
-      const std::vector<std::shared_ptr<SetKVCacheTask>> &kvCacheTasks);
-
-    void PrefetchS3Objs(uint64_t inodeId,
-      const std::vector<std::pair<std::string, uint64_t>> &prefetchObjs);
-
-    void HandleReadRequest(
-      const ReadRequest &request, const S3ChunkInfo &s3ChunkInfo,
-      std::vector<ReadRequest> *addReadRequests,
-      std::vector<uint64_t> *deletingReq, std::vector<S3ReadRequest> *requests,
-      char *dataBuf, uint64_t fsId, uint64_t inodeId);
-
-    void GenerateS3Request(ReadRequest request,
-      const S3ChunkInfoList &s3ChunkInfoList,
-      char *dataBuf,
-      std::vector<S3ReadRequest> *requests,
-      uint64_t fsId,
-      uint64_t inodeId);
-
-    // miss read from memory read/write cache, need read from
-    // kv(localdisk/remote cache/s3)
-    int GenerateKVReuqest(const std::shared_ptr<InodeWrapper> &inodeWrapper,
-                          const std::vector<ReadRequest> &readRequest,
-                          char *dataBuf, std::vector<S3ReadRequest> *kvRequest);
-
-    int HandleReadS3NotExist(uint32_t retry,
-      const std::shared_ptr<InodeWrapper> &inodeWrapper);
-
-    bool ReadKVRequestFromS3(const std::string &name,
-      char *databuf, uint64_t offset, uint64_t length, int *ret);
-
-    bool ReadKVRequestFromRemoteCache(const std::string &name,
-      char *databuf, uint64_t offset, uint64_t length);
-
-    bool ReadKVRequestFromLocalCache(const std::string &name, char *databuf,
-      uint64_t offset, uint64_t len);
-
-    // thread function for ReadKVRequest
-    void ProcessKVRequest(const S3ReadRequest &req, char *dataBuf,
-        uint64_t fileLen,
-        std::once_flag &cancelFlag,     // NOLINT
-        std::atomic<bool> &isCanceled,  // NOLINT
-        std::atomic<int> &retCode);     // NOLINT
-
-    void PrefetchForBlock(const S3ReadRequest &req, uint64_t fileLen,
-      uint64_t blockSize, uint64_t chunkSize, uint64_t startBlockIndex);
-
-    void GetChunkLoc(uint64_t offset, uint64_t *index,
-      uint64_t *chunkPos, uint64_t *chunkSize);
-
-    void GetBlockLoc(uint64_t offset, uint64_t *chunkIndex, uint64_t *chunkPos,
-      uint64_t *blockIndex, uint64_t *blockPos);
-
     uint32_t GetPrefetchBlocks() {
         return prefetchBlocks_;
     }
+    uint32_t GetDiskCacheType() {
+        return diskCacheType_;
+    }
+    bool DisableDiskCache() {
+        return diskCacheType_ == DiskCacheType::Disable;
+    }
+    bool HasDiskCache() {
+        return diskCacheType_ != DiskCacheType::Disable;
+    }
+    bool IsReadCache() {
+        return diskCacheType_ == DiskCacheType::OnlyRead;
+    }
+    bool IsReadWriteCache() {
+        return diskCacheType_ == DiskCacheType::ReadWrite;
+    }
+    std::shared_ptr<InodeCacheManager> GetInodeCacheManager() {
+        return inodeManager_;
+    }
+    std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() {
+        return diskCacheManagerImpl_;
+    }
+    FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum,
+                                uint64_t *chunkId);
+    void FsSyncSignal() {
+        std::lock_guard<std::mutex> lk(mtx_);
+        VLOG(3) << "fs sync signal";
+        cond_.notify_one();
+    }
+    void FsSyncSignalAndDataCacheInc() {
+        std::lock_guard<std::mutex> lk(mtx_);
+        fsCacheManager_->DataCacheNumInc();
+        VLOG(3) << "fs sync signal";
+        cond_.notify_one();
+    }
+    void SetFsId(uint32_t fsId) {
+        fsId_ = fsId;
+    }
+    uint32_t GetFsId() {
+        return fsId_;
+    }
+    uint32_t GetPageSize() {
+        return pageSize_;
+    }
+    void InitMetrics(const std::string &fsName);
+    void CollectMetrics(InterfaceMetric *interface, int count, uint64_t start);
+    void SetDiskCache(DiskCacheType type) {
+       diskCacheType_ = type;
+    }
+
+    uint32_t GetMaxReadRetryIntervalMs() const {
+        return maxReadRetryIntervalMs_;
+    }
 
     uint32_t GetReadRetryIntervalMs() const {
         return readRetryIntervalMs_;
     }
 
+ private:
+    void BackGroundFlush();
+
     using AsyncDownloadTask = std::function<void()>;
+
     static int ExecAsyncDownloadTask(void* meta, bthread::TaskIterator<AsyncDownloadTask>& iter);  // NOLINT
 
+    int ClearDiskCache(int64_t inodeId);
+
  public:
-  void PushAsyncTask(const AsyncDownloadTask& task) {
+    void PushAsyncTask(const AsyncDownloadTask& task) {
         static thread_local unsigned int seed = time(nullptr);
 
         int idx = rand_r(&seed) % downloadTaskQueues_.size();
@@ -256,62 +241,47 @@ class S3ClientAdaptorImpl : public StorageAdaptor {
             task();
         }
     }
+    std::shared_ptr<S3Metric> s3Metric_;
+
+    void Enqueue(std::shared_ptr<FlushChunkCacheContext> context);
 
  private:
-  class AsyncPrefetchCallback {
-   public:
-      AsyncPrefetchCallback(uint64_t inode, S3ClientAdaptorImpl *s3Client)
-          : inode_(inode), s3Client_(s3Client) {}
-
-      void operator()(const S3Adapter *,
-                      const std::shared_ptr<GetObjectAsyncContext> &context) {
-          std::unique_ptr<char[]> guard(context->buf);
-
-          if (context->retCode != 0) {
-              LOG(WARNING) << "prefetch failed, key: " << context->key;
-              return;
-          }
-
-          int ret = s3Client_->GetDiskCacheManager()->WriteReadDirect(
-              context->key, context->buf, context->actualLen);
-          if (ret < 0) {
-              LOG_EVERY_SECOND(INFO) <<
-                "prefetch failed, write read directly failed, key: "
-                << context->key;
-          }
-          {
-            curve::common::LockGuard lg(s3Client_->downloadMtx_);
-            s3Client_->downloadingObj_.erase(context->key);
-          }
-          VLOG(9) << "prefetch end, objectname is: " << context->key
-                  << ", len is: " << context->len
-                  << ", actual len is: " <<  context->actualLen;
-      }
-
-   private:
-      const uint64_t inode_;
-      S3ClientAdaptorImpl *s3Client_;
-  };
-
- protected:
-  curve::common::Mutex downloadMtx_;
-  std::set<std::string> downloadingObj_;
-
- private:
-    std::vector<bthread::ExecutionQueueId<AsyncDownloadTask>>
-      downloadTaskQueues_;
-    // prefetch blocks nums
+    std::shared_ptr<S3Client> client_;
+    uint64_t blockSize_;
+    uint64_t chunkSize_;
     uint32_t prefetchBlocks_;
-    // prefetch thread nums
     uint32_t prefetchExecQueueNum_;
-    // read and max retry times when read s3 failed
+    std::string allocateServerEps_;
+    uint32_t flushIntervalSec_;
+    uint32_t chunkFlushThreads_;
+    uint32_t memCacheNearfullRatio_;
+    uint32_t throttleBaseSleepUs_;
+    uint32_t maxReadRetryIntervalMs_;
     uint32_t readRetryIntervalMs_;
     uint32_t objectPrefix_;
-    uint32_t maxReadRetryIntervalMs_;
-    // s3 client manager(put or get object form s3)
-    std::shared_ptr<S3Client> client_;
-    // kv client manager
-    std::shared_ptr<KVClientManager> kvClientManager_;
+    Thread bgFlushThread_;
+    std::atomic<bool> toStop_;
+    std::mutex mtx_;
+    std::mutex ioMtx_;
+    std::condition_variable cond_;
+    curve::common::WaitInterval waitInterval_;
+    std::shared_ptr<FsCacheManager> fsCacheManager_;
+    std::shared_ptr<InodeCacheManager> inodeManager_;
+    std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl_;
+    DiskCacheType diskCacheType_;
+    std::shared_ptr<MdsClient> mdsClient_;
+    uint32_t fsId_;
+    std::string fsName_;
+    std::vector<bthread::ExecutionQueueId<AsyncDownloadTask>>
+      downloadTaskQueues_;
+    uint32_t pageSize_;
+
+    int FlushChunkClosure(std::shared_ptr<FlushChunkCacheContext> context);
+
+    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
+        taskPool_;
+
+    std::shared_ptr<KVClientManager> kvClientManager_ = nullptr;
 };
 
 }  // namespace client

curvefs/src/client/s3/client_s3_cache_manager.h

@@ -19,8 +19,8 @@
  * Created Date: 21-8-18
  * Author: huyao
  */
-#ifndef CURVEFS_SRC_CLIENT_CACHE_FUSE_CLIENT_CACHE_MANAGER_H_
-#define CURVEFS_SRC_CLIENT_CACHE_FUSE_CLIENT_CACHE_MANAGER_H_
+#ifndef CURVEFS_SRC_CLIENT_S3_CLIENT_S3_CACHE_MANAGER_H_
+#define CURVEFS_SRC_CLIENT_S3_CLIENT_S3_CACHE_MANAGER_H_
 
 #include <algorithm>
 #include <cstring>
@@ -34,7 +34,7 @@
 #include <set>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/filesystem/error.h"
 #include "curvefs/src/client/s3/client_s3.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/concurrent/task_thread_pool.h"
@@ -48,16 +48,19 @@ using curve::common::WriteLockGuard;
 namespace curvefs {
 namespace client {
 
-class StorageAdaptor;
+class S3ClientAdaptorImpl;
 class ChunkCacheManager;
 class FileCacheManager;
 class FsCacheManager;
 class DataCache;
+class S3ReadRequest;
 using FileCacheManagerPtr = std::shared_ptr<FileCacheManager>;
 using ChunkCacheManagerPtr = std::shared_ptr<ChunkCacheManager>;
 using DataCachePtr = std::shared_ptr<DataCache>;
 using WeakDataCachePtr = std::weak_ptr<DataCache>;
-using curve::common::TaskThreadPool;
+using curve::common::GetObjectAsyncCallBack;
+using curve::common::PutObjectAsyncCallBack;
+using curve::common::S3Adapter;
 using curvefs::metaserver::Inode;
 using curvefs::metaserver::S3ChunkInfo;
 using curvefs::metaserver::S3ChunkInfoList;
@@ -84,22 +87,34 @@ struct ReadRequest {
     }
 };
 
-struct UperFlushRequest {
-    uint64_t inodeId;
-    const char *buf;
-    uint64_t length;
-    uint64_t offset;   // offset at inode
+struct S3ReadRequest {
     uint64_t chunkId;
-    uint64_t chunkPos;
-    bool sync;
+    uint64_t offset;  // file offset
+    uint64_t len;
+    uint64_t objectOffset;  // s3 object's begin in the block
+    uint64_t readOffset;    // read buf offset
+    uint64_t fsId;
+    uint64_t inodeId;
+    uint64_t compaction;
+
+    std::string DebugString() const {
+        std::ostringstream os;
+        os << "S3ReadRequest ( chunkId = " << chunkId << ", offset = " << offset
+           << ", len = " << len << ", objectOffset = " << objectOffset
+           << ", readOffset = " << readOffset << ", fsId = " << fsId
+           << ", inodeId = " << inodeId << ", compaction = " << compaction
+           << " )";
+        return os.str();
+    }
 };
 
-struct  UperReadRequest {
-    std::vector<ReadRequest> requests;
-    uint64_t inodeId;
-    std::shared_ptr<InodeWrapper> inodeWapper;
-    char *buf;
-};
+inline std::string
+S3ReadRequestVecDebugString(const std::vector<S3ReadRequest> &reqs) {
+    std::ostringstream os;
+    for_each(reqs.begin(), reqs.end(),
+             [&](const S3ReadRequest &req) { os << req.DebugString() << " "; });
+    return os.str();
+}
 
 struct ObjectChunkInfo {
     S3ChunkInfo s3ChunkInfo;
@@ -119,7 +134,7 @@ enum DataCacheStatus {
 
 class DataCache : public std::enable_shared_from_this<DataCache> {
  public:
-    DataCache(StorageAdaptor *s3ClientAdaptor,
+    DataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
               ChunkCacheManagerPtr chunkCacheManager, uint64_t chunkPos,
               uint64_t len, const char *data,
               std::shared_ptr<KVClientManager> kvClientManager);
@@ -194,10 +209,21 @@ class DataCache : public std::enable_shared_from_this<DataCache> {
                              const char *data);
     void AddDataBefore(uint64_t len, const char *data);
 
+    CURVEFS_ERROR PrepareFlushTasks(
+        uint64_t inodeId, char *data,
+        std::vector<std::shared_ptr<PutObjectAsyncContext>> *s3Tasks,
+        std::vector<std::shared_ptr<SetKVCacheTask>> *kvCacheTasks,
+        uint64_t *chunkId, uint64_t *writeOffset);
+
+    void FlushTaskExecute(
+        CachePolicy cachePolicy,
+        const std::vector<std::shared_ptr<PutObjectAsyncContext>> &s3Tasks,
+        const std::vector<std::shared_ptr<SetKVCacheTask>> &kvCacheTasks);
+
     CachePolicy GetCachePolicy(bool toS3);
 
  private:
-    StorageAdaptor *storageAdaptor_;
+    S3ClientAdaptorImpl *s3ClientAdaptor_;
     ChunkCacheManagerPtr chunkCacheManager_;
     uint64_t chunkPos_;  // useful chunkPos
     uint64_t len_;  // useful len
@@ -212,19 +238,33 @@ class DataCache : public std::enable_shared_from_this<DataCache> {
     std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
+class S3ReadResponse {
+ public:
+    explicit S3ReadResponse(char *data, uint64_t length)
+        : data_(data), len_(length) {}
+
+    char *GetDataBuf() { return data_; }
+
+    uint64_t GetBufLen() { return len_; }
+
+ private:
+    char *data_;
+    uint64_t len_;
+};
+
 class ChunkCacheManager
     : public std::enable_shared_from_this<ChunkCacheManager> {
  public:
-    ChunkCacheManager(uint64_t index, StorageAdaptor *s3ClientAdaptor,
+    ChunkCacheManager(uint64_t index, S3ClientAdaptorImpl *s3ClientAdaptor,
                       std::shared_ptr<KVClientManager> kvClientManager)
-        : index_(index), storageAdaptor_(s3ClientAdaptor),
+        : index_(index), s3ClientAdaptor_(s3ClientAdaptor),
           flushingDataCache_(nullptr),
           kvClientManager_(std::move(kvClientManager)) {}
     virtual ~ChunkCacheManager() = default;
-    virtual void ReadChunk(uint64_t index, uint64_t chunkPos, uint64_t readLen,
+    void ReadChunk(uint64_t index, uint64_t chunkPos, uint64_t readLen,
                    char *dataBuf, uint64_t dataBufOffset,
                    std::vector<ReadRequest> *requests);
-    virtual void WriteNewDataCache(StorageAdaptor *s3ClientAdaptor,
+    virtual void WriteNewDataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
                                    uint32_t chunkPos, uint32_t len,
                                    const char *data);
     virtual void AddReadDataCache(DataCachePtr dataCache);
@@ -280,7 +320,7 @@ class ChunkCacheManager
         dataRCacheMap_;  // first is pos in chunk
 
     RWLock rwLockRead_;  //  for read cache
-    StorageAdaptor *storageAdaptor_;
+    S3ClientAdaptorImpl *s3ClientAdaptor_;
     curve::common::Mutex flushMtx_;
     DataCachePtr flushingDataCache_;
     curve::common::Mutex flushingDataCacheMtx_;
@@ -291,10 +331,10 @@ class ChunkCacheManager
 class FileCacheManager {
  public:
     FileCacheManager(uint32_t fsid, uint64_t inode,
-      StorageAdaptor *s3ClientAdaptor,
-      std::shared_ptr<KVClientManager> kvClientManager,
-      std::shared_ptr<TaskThreadPool<>> threadPool)
-        : fsId_(fsid), inode_(inode), storageAdaptor_(s3ClientAdaptor),
+                     S3ClientAdaptorImpl *s3ClientAdaptor,
+                     std::shared_ptr<KVClientManager> kvClientManager,
+                     std::shared_ptr<TaskThreadPool<>> threadPool)
+        : fsId_(fsid), inode_(inode), s3ClientAdaptor_(s3ClientAdaptor),
           kvClientManager_(std::move(kvClientManager)),
           readTaskPool_(threadPool) {}
     FileCacheManager() = default;
@@ -313,8 +353,6 @@ class FileCacheManager {
     virtual int Read(uint64_t inodeId, uint64_t offset, uint64_t length,
                      char *dataBuf);
 
-    // for test
-
     bool IsEmpty() { return chunkCacheMap_.empty(); }
 
     uint64_t GetInodeId() const { return inode_; }
@@ -331,6 +369,24 @@ class FileCacheManager {
  private:
     void WriteChunk(uint64_t index, uint64_t chunkPos, uint64_t writeLen,
                     const char *dataBuf);
+    void GenerateS3Request(ReadRequest request,
+                           const S3ChunkInfoList &s3ChunkInfoList,
+                           char *dataBuf, std::vector<S3ReadRequest> *requests,
+                           uint64_t fsId, uint64_t inodeId);
+
+    void PrefetchS3Objs(
+        const std::vector<std::pair<std::string, uint64_t>> &prefetchObjs);
+
+    void HandleReadRequest(const ReadRequest &request,
+                           const S3ChunkInfo &s3ChunkInfo,
+                           std::vector<ReadRequest> *addReadRequests,
+                           std::vector<uint64_t> *deletingReq,
+                           std::vector<S3ReadRequest> *requests, char *dataBuf,
+                           uint64_t fsId, uint64_t inodeId);
+
+    int HandleReadRequest(const std::vector<S3ReadRequest> &requests,
+                          std::vector<S3ReadResponse> *responses,
+                          uint64_t fileLen);
 
     // GetChunkLoc: get chunk info according to offset
     void GetChunkLoc(uint64_t offset, uint64_t *index, uint64_t *chunkPos,
@@ -345,7 +401,12 @@ class FileCacheManager {
                           uint64_t *actualReadLen,
                           std::vector<ReadRequest> *memCacheMissRequest);
 
-/*  whs
+    // miss read from memory read/write cache, need read from
+    // kv(localdisk/remote cache/s3)
+    int GenerateKVRequest(const std::shared_ptr<InodeWrapper> &inodeWrapper,
+                          const std::vector<ReadRequest> &readRequest,
+                          char *dataBuf, std::vector<S3ReadRequest> *kvRequest);
+
     enum class ReadStatus {
         OK = 0,
         S3_READ_FAIL = -1,
@@ -360,7 +421,39 @@ class FileCacheManager {
         }
         return st;
     }
-*/
+
+    // read kv request, need
+    ReadStatus ReadKVRequest(const std::vector<S3ReadRequest> &kvRequests,
+                             char *dataBuf, uint64_t fileLen);
+
+    // thread function for ReadKVRequest
+    void ProcessKVRequest(const S3ReadRequest &req, char *dataBuf,
+                          uint64_t fileLen,
+                          std::once_flag &cancelFlag,     // NOLINT
+                          std::atomic<bool> &isCanceled,  // NOLINT
+                          std::atomic<int> &retCode);     // NOLINT
+
+    // read kv request from local disk cache
+    bool ReadKVRequestFromLocalCache(const std::string &name, char *databuf,
+                                     uint64_t offset, uint64_t len);
+
+    // read kv request from remote cache like memcached
+    bool ReadKVRequestFromRemoteCache(const std::string &name, char *databuf,
+                                      uint64_t offset, uint64_t length);
+
+    // read kv request from s3
+    bool ReadKVRequestFromS3(const std::string &name, char *databuf,
+                             uint64_t offset, uint64_t length, int *ret);
+
+    // read retry policy when read from s3 occur not exist error
+    int HandleReadS3NotExist(uint32_t retry,
+                             const std::shared_ptr<InodeWrapper> &inodeWrapper);
+
+    // prefetch for block
+    void PrefetchForBlock(const S3ReadRequest &req, uint64_t fileLen,
+                         uint64_t blockSize, uint64_t chunkSize,
+                         uint64_t startBlockIndex);
+
  private:
     friend class AsyncPrefetchCallback;
 
@@ -369,7 +462,9 @@ class FileCacheManager {
     std::map<uint64_t, ChunkCacheManagerPtr> chunkCacheMap_;  // first is index
     RWLock rwLock_;
     curve::common::Mutex mtx_;
-    StorageAdaptor *storageAdaptor_;
+    S3ClientAdaptorImpl *s3ClientAdaptor_;
+    curve::common::Mutex downloadMtx_;
+    std::set<std::string> downloadingObj_;
 
     std::shared_ptr<KVClientManager> kvClientManager_;
     std::shared_ptr<TaskThreadPool<>> readTaskPool_;
@@ -377,27 +472,19 @@ class FileCacheManager {
 
 class FsCacheManager {
  public:
-    FsCacheManager(StorageAdaptor *s3ClientAdaptor,
+    FsCacheManager(S3ClientAdaptorImpl *s3ClientAdaptor,
                    uint64_t readCacheMaxByte, uint64_t writeCacheMaxByte,
                    uint32_t readCacheThreads,
                    std::shared_ptr<KVClientManager> kvClientManager)
         : lruByte_(0), wDataCacheNum_(0), wDataCacheByte_(0),
           readCacheMaxByte_(readCacheMaxByte),
           writeCacheMaxByte_(writeCacheMaxByte),
-          storageAdaptor_(s3ClientAdaptor), isWaiting_(false),
+          s3ClientAdaptor_(s3ClientAdaptor), isWaiting_(false),
           kvClientManager_(std::move(kvClientManager)) {
-            LOG(INFO) << "whs read task pool start. " << readCacheThreads;
-            readTaskPool_->Start(readCacheThreads);
-            LOG(INFO) << "whs read task pool start end";
+        readTaskPool_->Start(readCacheThreads);
     }
-
     FsCacheManager() = default;
-    virtual ~FsCacheManager() {
-        LOG(INFO) << "whs read task pool stop";
-        readTaskPool_->Stop();
-        LOG(INFO) << "whs read task pool stop end";
-    }
-
+    virtual ~FsCacheManager() { readTaskPool_->Stop(); }
     virtual FileCacheManagerPtr FindFileCacheManager(uint64_t inodeId);
     virtual FileCacheManagerPtr FindOrCreateFileCacheManager(uint64_t fsId,
                                                      uint64_t inodeId);
@@ -460,11 +547,6 @@ class FsCacheManager {
         assert(ret.second);
         (void)ret;
     }
-
-    std::shared_ptr<TaskThreadPool<>> GetReadTaskPool() {
-        return readTaskPool_;
-    }
-
     void DataCacheNumInc();
     void DataCacheNumFetchSub(uint64_t v);
     void DataCacheByteInc(uint64_t v);
@@ -503,7 +585,7 @@ class FsCacheManager {
     std::atomic<uint64_t> wDataCacheByte_;
     uint64_t readCacheMaxByte_;
     uint64_t writeCacheMaxByte_;
-    StorageAdaptor *storageAdaptor_;
+    S3ClientAdaptorImpl *s3ClientAdaptor_;
     bool isWaiting_;
     std::mutex mutex_;
     std::condition_variable cond_;
@@ -519,4 +601,4 @@ class FsCacheManager {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_CACHE_FUSE_CLIENT_CACHE_MANAGER_H_
+#endif  // CURVEFS_SRC_CLIENT_S3_CLIENT_S3_CACHE_MANAGER_H_

curvefs/src/client/s3/disk_cache_base.cpp

@@ -28,7 +28,7 @@
 #include <dirent.h>
 #include <functional>
 
-#include "curvefs/src/client/cache/diskcache/disk_cache_base.h"
+#include "curvefs/src/client/s3/disk_cache_base.h"
 
 namespace curvefs {
 

curvefs/src/client/s3/disk_cache_base.h

@@ -20,8 +20,8 @@
  * Author: wuhongsong
  */
 
-#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_BASE_H_
-#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_BASE_H_
+#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_BASE_H_
+#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_BASE_H_
 
 #include <glog/logging.h>
 
@@ -71,4 +71,4 @@ class DiskCacheBase {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_BASE_H_
+#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_BASE_H_

curvefs/src/client/s3/disk_cache_manager.cpp

@@ -19,9 +19,6 @@
  * Created Date: 21-08-13
  * Author: hzwuhongsong
  */
-
-#include "curvefs/src/client/cache/diskcache/disk_cache_manager.h"
-
 #include <sys/vfs.h>
 #include <errno.h>
 #include <string>
@@ -30,6 +27,7 @@
 #include <list>
 
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
+#include "curvefs/src/client/s3/disk_cache_manager.h"
 #include "curvefs/src/common/s3util.h"
 
 namespace curvefs {
@@ -232,12 +230,7 @@ int DiskCacheManager::WriteDiskFile(const std::string fileName, const char *buf,
                                     uint64_t length, bool force) {
     // write throttle
     diskCacheThrottle_.Add(false, length);
-    uint64_t start = butil::cpuwide_time_us();
     int ret = cacheWrite_->WriteDiskFile(fileName, buf, length, force);
-    if ((ret >= 0) &&
-      metric_.get() != nullptr) {
-        CollectMetrics(&metric_->adaptorWriteDiskCache, length, start);
-    }
     if (ret > 0)
         AddDiskUsedBytes(ret);
     return ret;
@@ -251,26 +244,14 @@ int DiskCacheManager::ReadDiskFile(const std::string name, char *buf,
                                    uint64_t offset, uint64_t length) {
     // read throttle
     diskCacheThrottle_.Add(true, length);
-    uint64_t start = butil::cpuwide_time_us();
-    int ret = cacheRead_->ReadDiskFile(
-      name, buf, offset, length);
-    if ((ret >= 0) &&
-      metric_.get() != nullptr) {
-        CollectMetrics(&metric_->adaptorWriteDiskCache, length, start);
-    }
-    return ret;
+    return cacheRead_->ReadDiskFile(name, buf, offset, length);
 }
 
 int DiskCacheManager::WriteReadDirect(const std::string fileName,
                                       const char *buf, uint64_t length) {
     // write hrottle
     diskCacheThrottle_.Add(false, length);
-    uint64_t start = butil::cpuwide_time_us();
     int ret = cacheRead_->WriteDiskFile(fileName, buf, length);
-    if ((ret >= 0) &&
-      metric_.get() != nullptr) {
-        CollectMetrics(&metric_->adaptorWriteDiskCache, length, start);
-    }
     if (ret > 0)
         AddDiskUsedBytes(ret);
     return ret;

curvefs/src/client/s3/disk_cache_manager.h

@@ -19,9 +19,8 @@
  * Created Date: 21-08-13
  * Author: wuhongsong
  */
-
-#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_H_
-#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_H_
+#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_
+#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_
 
 #include <bthread/mutex.h>
 
@@ -39,8 +38,8 @@
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/common/utils.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/cache/diskcache/disk_cache_write.h"
-#include "curvefs/src/client/cache/diskcache/disk_cache_read.h"
+#include "curvefs/src/client/s3/disk_cache_write.h"
+#include "curvefs/src/client/s3/disk_cache_read.h"
 #include "curvefs/src/client/common/config.h"
 namespace curvefs {
 namespace client {
@@ -60,7 +59,7 @@ class DiskCacheManager {
                      std::shared_ptr<DiskCacheWrite> cacheWrite,
                      std::shared_ptr<DiskCacheRead> cacheRead);
     DiskCacheManager() {}
-    virtual ~DiskCacheManager() { TrimStop();}
+    virtual ~DiskCacheManager() { TrimStop(); }
 
     virtual int Init(std::shared_ptr<S3Client> client,
         const S3ClientAdaptorOption option);
@@ -77,48 +76,33 @@ class DiskCacheManager {
     void AddCache(const std::string &name);
 
     int CreateDir();
-
     std::string GetCacheReadFullDir();
-
     std::string GetCacheWriteFullDir();
 
     int WriteDiskFile(const std::string fileName, const char *buf,
                       uint64_t length, bool force = true);
-
+    void AsyncUploadEnqueue(const std::string objName);
     virtual int WriteReadDirect(const std::string fileName, const char *buf,
                                 uint64_t length);
-
     int ReadDiskFile(const std::string name, char *buf, uint64_t offset,
                      uint64_t length);
-
     int LinkWriteToRead(const std::string fileName,
                         const std::string fullWriteDir,
                         const std::string fullReadDir);
-
-    void AsyncUploadEnqueue(const std::string objName);
-
     int UploadAllCacheWriteFile();
-
     int UploadWriteCacheByInode(const std::string &inode);
-
     int ClearReadCache(const std::list<std::string> &files);
-
     /**
      * @brief get use ratio of cache disk
      * @return the use ratio
      */
     int64_t SetDiskFsUsedRatio();
-
     virtual bool IsDiskCacheFull();
-
     bool IsDiskCacheSafe();
-
     /**
      * @brief: start trim thread.
      */
-
     int TrimRun();
-
     /**
      * @brief: stop trim thread.
      */
@@ -145,7 +129,6 @@ class DiskCacheManager {
                 << ", now is: " << usedBytes_.load();
         return;
     }
-
     /**
      * @brief dec the used bytes of disk cache.
      * can not dec disk used bytes after file have been loaded,
@@ -160,15 +143,12 @@ class DiskCacheManager {
                 << ", now is: " << usedBytes_.load();
         return;
     }
-
     void SetDiskInitUsedBytes();
-
     uint64_t GetDiskUsedbytes() {
         return usedBytes_.load();
     }
 
     void InitQosParam();
-
     /**
      * @brief trim cache func.
      */
@@ -184,14 +164,6 @@ class DiskCacheManager {
      */
     bool IsCacheClean();
 
-    void CollectMetrics(InterfaceMetric *interface,
-      int count, uint64_t start) {
-        interface->bps.count << count;
-        interface->qps.count << 1;
-        interface->latency << (butil::cpuwide_time_us() - start);
-    }
-
- private:
     curve::common::Thread backEndThread_;
     curve::common::Atomic<bool> isRunning_;
     curve::common::InterruptibleSleeper sleeper_;
@@ -230,4 +202,4 @@ class DiskCacheManager {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_H_
+#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_

curvefs/src/client/s3/disk_cache_manager_impl.cpp

@@ -27,7 +27,7 @@
 #include <memory>
 
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
-#include "curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h"
+#include "curvefs/src/client/s3/disk_cache_manager_impl.h"
 
 namespace curvefs {
 
@@ -70,6 +70,8 @@ void DiskCacheManagerImpl::Enqueue(
     taskPool_.Enqueue(task);
 }
 
+
+
 int DiskCacheManagerImpl::WriteReadDirectClosure(
     std::shared_ptr<PutObjectAsyncContext> context) {
         VLOG(9) << "WriteReadClosure start, name: " << context->key;

curvefs/src/client/s3/disk_cache_manager_impl.h

@@ -19,8 +19,8 @@
  * Created Date: 21-08-13
  * Author: wuhongsong
  */
-#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_IMPL_H_
-#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_IMPL_H_
+#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_IMPL_H_
+#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_IMPL_H_
 
 #include <bthread/mutex.h>
 
@@ -35,9 +35,9 @@
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/cache/diskcache/disk_cache_write.h"
-#include "curvefs/src/client/cache/diskcache/disk_cache_read.h"
-#include "curvefs/src/client/cache/diskcache/disk_cache_manager.h"
+#include "curvefs/src/client/s3/disk_cache_write.h"
+#include "curvefs/src/client/s3/disk_cache_read.h"
+#include "curvefs/src/client/s3/disk_cache_manager.h"
 
 namespace curvefs {
 namespace client {
@@ -46,7 +46,6 @@ using curvefs::common::PosixWrapper;
 using curvefs::client::common::DiskCacheType;
 using curve::common::TaskThreadPool;
 
-// disk cache opt
 struct DiskCacheOption {
     DiskCacheType diskCacheType;
     uint64_t trimCheckIntervalSec;
@@ -73,15 +72,13 @@ class DiskCacheManagerImpl {
         std::shared_ptr<S3Client> client);
     DiskCacheManagerImpl() {}
     virtual ~DiskCacheManagerImpl() {}
-
     /**
      * @brief init DiskCacheManagerImpl
      * @param[in] client S3Client
      * @param[in] option config option
      * @return success: 0, fail : < 0
-    */
-    virtual int Init(const S3ClientAdaptorOption option);
-
+     */
+    int Init(const S3ClientAdaptorOption option);
     /**
      * @brief Write obj
      * @param[in] name obj name
@@ -90,14 +87,12 @@ class DiskCacheManagerImpl {
      * @return success: write length, fail : < 0
      */
     int Write(const std::string name, const char *buf, uint64_t length);
-
     /**
      * @brief whether obj is cached in cached disk
      * @param[in] name obj name
      * @return cached: true, not cached : < 0
      */
     bool IsCached(const std::string name);
-
     /**
      * @brief read obj
      * @param[in] name obj name
@@ -114,18 +109,9 @@ class DiskCacheManagerImpl {
      */
     int UmountDiskCache();
 
-    /// @brief disk cache is full or not
-    /// @return true: full, false: not full
     bool IsDiskCacheFull();
-
-    /// @brief write to cache read disk
-    /// @param fileName file name
-    /// @param buf data need to write
-    /// @param length data length need to write
-    /// @return data length write to cache read
     int WriteReadDirect(const std::string fileName, const char *buf,
                         uint64_t length);
-
     void InitMetrics(std::string fsName);
 
     virtual int UploadWriteCacheByInode(const std::string &inode);
@@ -138,17 +124,14 @@ class DiskCacheManagerImpl {
  private:
     int WriteDiskFile(const std::string name, const char *buf, uint64_t length);
 
+    std::shared_ptr<DiskCacheManager> diskCacheManager_;
+
+    bool forceFlush_;
+    std::shared_ptr<S3Client> client_;
+
     int WriteClosure(std::shared_ptr<PutObjectAsyncContext> context);
 
     int WriteReadDirectClosure(std::shared_ptr<PutObjectAsyncContext> context);
-
- private:
-    // disk cache manager
-    std::shared_ptr<DiskCacheManager> diskCacheManager_;
-    // use fdatasync or not
-    bool forceFlush_;
-    // s3 client interface(put/get object from s3)
-    std::shared_ptr<S3Client> client_;
     // threads for disk cache
     uint32_t threads_;
     TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
@@ -158,4 +141,4 @@ class DiskCacheManagerImpl {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_IMPL_H_
+#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_IMPL_H_

curvefs/src/client/s3/disk_cache_read.cpp

@@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>
 
-#include "curvefs/src/client/cache/diskcache/disk_cache_read.h"
+#include "curvefs/src/client/s3/disk_cache_read.h"
 #include "curvefs/src/common/s3util.h"
 
 namespace curvefs {

curvefs/src/client/s3/disk_cache_read.h

@@ -19,8 +19,8 @@
  * Created Date: 21-08-13
  * Author: wuhongsong
  */
-#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_READ_H_
-#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_READ_H_
+#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_READ_H_
+#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_READ_H_
 
 #include <list>
 #include <string>
@@ -32,7 +32,7 @@
 #include "src/common/interruptible_sleeper.h"
 #include "src/common/lru_cache.h"
 #include "curvefs/src/common/wrap_posix.h"
-#include "curvefs/src/client/cache/diskcache/disk_cache_base.h"
+#include "curvefs/src/client/s3/disk_cache_base.h"
 
 namespace curvefs {
 namespace client {
@@ -73,4 +73,4 @@ class DiskCacheRead : public DiskCacheBase {
 
 }  // namespace client
 }  // namespace curvefs
-#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_READ_H_
+#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_READ_H_

curvefs/src/client/s3/disk_cache_write.cpp

@@ -27,8 +27,7 @@
 #include <dirent.h>
 
 #include <vector>
-
-#include "curvefs/src/client/cache/diskcache/disk_cache_write.h"
+#include "curvefs/src/client/s3/disk_cache_write.h"
 #include "curvefs/src/common/s3util.h"
 
 namespace curvefs {

curvefs/src/client/s3/disk_cache_write.h

@@ -19,8 +19,8 @@
  * Created Date: 21-08-13
  * Author: wuhongsong
  */
-#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_WRITE_H_
-#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_WRITE_H_
+#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_
+#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_
 
 #include <sys/stat.h>
 #include <bthread/condition_variable.h>
@@ -39,8 +39,9 @@
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/common/utils.h"
 #include "curvefs/src/client/s3/client_s3.h"
+#include "curvefs/src/client/s3/disk_cache_read.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/cache/diskcache/disk_cache_base.h"
+#include "curvefs/src/client/s3/disk_cache_base.h"
 
 namespace curvefs {
 namespace client {
@@ -166,4 +167,4 @@ class DiskCacheWrite : public DiskCacheBase {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_WRITE_H_
+#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_

curvefs/src/client/volume/client_volume_adaptor.cpp

@@ -1,182 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-
-/*
- * Project: curve
- * Created Date: Thur March 14 2023
- * Author: wuhongsong
- */
-
-#include <utility>
-
-#include "curvefs/src/client/volume/client_volume_adaptor.h"
-
-#include "curvefs/src/client/rpcclient/mds_client.h"
-
-namespace curvefs {
-namespace client {
-
-#define VOLUME_BLOCK_SIZE 4194304
-#define VOLUME_CHUNK_SIZE 67108864
-
-CURVEFS_ERROR VolumeClientAdaptorImpl::Init(const FuseClientOption &option,
-         std::shared_ptr<InodeCacheManager> inodeManager,
-         std::shared_ptr<MdsClient> mdsClient,
-         std::shared_ptr<FsCacheManager> fsCacheManager,
-         std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
-         std::shared_ptr<KVClientManager> kvClientManager,
-         std::shared_ptr<FsInfo> fsInfo) {
-    LOG(INFO) << "volume adaptor init start.";
-    volOpts_ = option.volumeOpt;
-    SetBlockSize(VOLUME_BLOCK_SIZE);
-    SetChunkSize(VOLUME_CHUNK_SIZE);
-    auto ret = StorageAdaptor::Init(option, inodeManager,
-      mdsClient, fsCacheManager, diskCacheManagerImpl, kvClientManager, fsInfo);
-    if (ret != CURVEFS_ERROR::OK) {
-        return ret;
-    }
-    BlockDeviceClientOptions opts;
-    opts.configPath = option.bdevOpt.configPath;
-    auto ret2 = blockDeviceClient_->Init(opts);
-    if (!ret2) {
-        LOG(ERROR) << "Init block device client failed, " << ret2;
-        return CURVEFS_ERROR::INTERNAL;
-    }
-
-    LOG(INFO) << "volume adaptor init sucess.";
-    return ret;
-}
-
-CURVEFS_ERROR VolumeClientAdaptorImpl::FuseOpInit(void *userdata,
-  struct fuse_conn_info *conn) {
-    LOG(INFO) << "volume fuse op init start!";
-    const auto &vol = fsInfo_->detail().volume();
-    const auto &volName = vol.volumename();
-    const auto &user = vol.user();
-    auto ret = blockDeviceClient_->Open(volName, user);
-    if (!ret) {
-        LOG(ERROR) << "BlockDeviceClientImpl open failed, ret = " << ret
-                   << ", volName = " << volName << ", user = " << user;
-        return CURVEFS_ERROR::INTERNAL;
-    }
-
-    SpaceManagerOption option;
-    option.blockGroupManagerOption.fsId = fsInfo_->fsid();
-
-    option.blockGroupManagerOption.owner = GetMountOwner();
-
-    option.blockGroupManagerOption.blockGroupAllocateOnce =
-        volOpts_.allocatorOption.blockGroupOption.allocateOnce;
-    option.blockGroupManagerOption.blockGroupSize =
-        fsInfo_->detail().volume().blockgroupsize();
-    option.blockGroupManagerOption.blockSize =
-        fsInfo_->detail().volume().blocksize();
-
-    option.allocatorOption.type = volOpts_.allocatorOption.type;
-    option.allocatorOption.bitmapAllocatorOption.sizePerBit =
-        volOpts_.allocatorOption.bitmapAllocatorOption.sizePerBit;
-    option.allocatorOption.bitmapAllocatorOption.smallAllocProportion =
-        volOpts_.allocatorOption.bitmapAllocatorOption.smallAllocProportion;
-
-    option.threshold = volOpts_.threshold;
-    option.releaseInterSec = volOpts_.releaseInterSec;
-
-    spaceManager_ = absl::make_unique<SpaceManagerImpl>(
-        option, mdsClient_, blockDeviceClient_);
-    spaceManager_->Run();
-
-    storage_ = absl::make_unique<DefaultVolumeStorage>(spaceManager_.get(),
-      blockDeviceClient_.get(), GetInodeCacheManager().get());
-
-    ExtentCacheOption extentOpt;
-    extentOpt.blockSize = vol.blocksize();
-    extentOpt.sliceSize = vol.slicesize();
-
-    ExtentCache::SetOption(extentOpt);
-    LOG(INFO) << "volume fuse op init sucess.";
-    return CURVEFS_ERROR::OK;
-}
-
-int VolumeClientAdaptorImpl::Stop() {
-    LOG(INFO) << "volume adaptor stop...";
-    StorageAdaptor::Stop();
-    if (nullptr != storage_) {
-        storage_->Shutdown();
-    }
-    if (nullptr != spaceManager_) {
-        spaceManager_->Shutdown();
-    }
-    blockDeviceClient_->UnInit();
-    LOG(INFO) << "volume adaptor stop sucess.";
-    return 0;
-}
-
-CURVEFS_ERROR VolumeClientAdaptorImpl::FlushDataCache(
-  const UperFlushRequest& req, uint64_t* writeOffset) {
-    uint64_t inodeId = req.inodeId;
-    uint64_t inodeOffset = req.offset;
-    uint64_t len = req.length;
-    const char* data = req.buf;
-    VLOG(9) << "volume flush dataCache, inode: "
-       << inodeId<< ", offset: " << inodeOffset
-       << ", length: " << len;
-
-    CURVEFS_ERROR ret = storage_->Write(inodeId, inodeOffset, len, data);
-    if (ret != CURVEFS_ERROR::OK) {
-        VLOG(0) << "volume flush dataCache err, inode: "
-            << inodeId<< ", offset: " << inodeOffset
-            << ", length: " << len;
-        return ret;
-    }
-
-    VLOG(9) << "volume flush dataCache end, inode: "
-       << inodeId<< ", offset: " << inodeOffset
-       << ", length: " << len;
-// 这个正确么？
-    *writeOffset = len;
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR VolumeClientAdaptorImpl::ReadFromLowlevel(
-  UperReadRequest uperRequest) {
-    uint64_t inodeId = uperRequest.inodeId;
-    VLOG(9) << "read lowlevel start, inodeId is: " << inodeId;
-    std::vector<ReadRequest> requests;
-    char *buf = uperRequest.buf;;
-    requests = std::move(uperRequest.requests);
-    uint64_t chunkSize = GetChunkSize();
-    CURVEFS_ERROR ret;
-    for (auto req : requests) {
-        VLOG(9) << "read from storage " << req.DebugString();
-        uint64_t len = req.len;
-        uint64_t readOffset = chunkSize * req.index + req.chunkPos;
-        ret = storage_->Read(
-          inodeId, readOffset, len, buf + req.bufOffset);
-        if (ret != CURVEFS_ERROR::OK) {
-            VLOG(0) << "volume flush dataCache err, inode: "
-                << inodeId << ", offset: " << readOffset
-                << ", length: " << len;
-            return ret;
-        }
-    }
-
-    VLOG(9) << "read lowlevel end, inodeId is: "<< inodeId;
-    return ret;
-}
-
-}  // namespace client
-}  // namespace curvefs