curvefs: support space deallocate for curvebs volume as backend

Signed-off-by: ilixiaocui <ilixiaocui@163.com>

.bazelrc

@@ -1,10 +1,7 @@
-build --verbose_failures
-
 build --define=with_glog=true --define=libunwind=true
 build --copt -DHAVE_ZLIB=1 --copt -DGFLAGS_NS=google --copt -DUSE_BTHREAD_MUTEX
 build --cxxopt -Wno-error=format-security
 build:gcc7-later --cxxopt -faligned-new
 build --incompatible_blacklisted_protos_requires_proto_info=false
 build --copt=-fdiagnostics-color=always
-
-run --copt=-fdiagnostics-color=always
+run --copt=-fdiagnostics-color=always

.gitignore

@@ -147,22 +147,3 @@ tools-v2/proto/curvefs/*
 tools-v2/*/*.test
 tools-v2/__debug_bin
 tools-v2/vendor/
-
-.test
-.note
-.playground
-.dumpfile
-metastore_test.dat
-GPATH
-GRTAGS
-GTAGS
-core.*
-
-test/integration/*.conf
-test/integration/client/config/client.conf*
-test/integration/snapshotcloneserver/config/*.conf
-
-.pre-commit-config.yaml
-
-*.deb
-*.whl

.obm.cfg

@@ -1,2 +0,0 @@
-container_name: curve-build-playground-master
-container_image: opencurvedocker/curve-base:build-debian9

Makefile

@@ -1,6 +1,6 @@
 # Copyright (C) 2021 Jingli Chen (Wine93), NetEase Inc.
 
-.PHONY: list build dep install image playground check test
+.PHONY: list build dep install image
 
 stor?=""
 prefix?= "$(PWD)/projects"
@@ -70,12 +70,3 @@ install:
 
 image:
 	@bash util/image.sh $(stor) $(tag) $(os)
-
-playground:
-	@bash util/playground.sh
-
-check:
-	@bash util/check.sh $(stor)
-
-test:
-	@bash util/test.sh $(stor) $(only)

WORKSPACE

@@ -75,14 +75,11 @@ bind(
 )
 
 #import the  gtest files.
-http_archive(
+new_git_repository(
     name = "com_google_googletest",
-    urls = [
-        "https://curve-build.nos-eastchina1.126.net/googletest-release-1.12.1.tar.gz",
-        "https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz",
-    ],
-    sha256 = "81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2",
-    strip_prefix = "googletest-release-1.12.1",
+    build_file = "//:thirdparties/gmock.BUILD",
+    remote = "https://github.com/google/googletest",
+    tag = "release-1.8.0",
 )
 
 bind(
@@ -231,24 +228,6 @@ http_archive(
   sha256 = "59b862f50e710277f8ede96f083a5bb8d7c9595376146838b9580be90374ee1f",
 )
 
-# fmt
-http_archive(
-  name = "fmt",
-  url = "https://github.com/fmtlib/fmt/archive/9.1.0.tar.gz",
-  sha256 = "5dea48d1fcddc3ec571ce2058e13910a0d4a6bab4cc09a809d8b1dd1c88ae6f2",
-  strip_prefix = "fmt-9.1.0",
-  build_file = "//:thirdparties/fmt.BUILD",
-)
-
-# spdlog
-http_archive(
-  name = "spdlog",
-  urls = ["https://github.com/gabime/spdlog/archive/refs/tags/v1.11.0.tar.gz"],
-  strip_prefix = "spdlog-1.11.0",
-  sha256 = "ca5cae8d6cac15dae0ec63b21d6ad3530070650f68076f3a4a862ca293a858bb",
-  build_file = "//:thirdparties/spdlog.BUILD",
-)
-
 # Bazel platform rules.
 http_archive(
     name = "platforms",
@@ -269,14 +248,14 @@ new_local_repository(
 http_archive(
     name = "hedron_compile_commands",
 
-    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here.
+    # Replace the commit hash in both places (below) with the latest, rather than using the stale one here. 
     # Even better, set up Renovate and let it do the work for you (see "Suggestion: Updates" in the README).
     urls = [
         "https://curve-build.nos-eastchina1.126.net/bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
         "https://github.com/hedronvision/bazel-compile-commands-extractor/archive/af9af15f7bc16fc3e407e2231abfcb62907d258f.tar.gz",
     ],
     strip_prefix = "bazel-compile-commands-extractor-af9af15f7bc16fc3e407e2231abfcb62907d258f",
-    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..."
+    # When you first run this tool, it'll recommend a sha256 hash to put here with a message like: "DEBUG: Rule 'hedron_compile_commands' indicated that a canonical reproducible form can be obtained by modifying arguments sha256 = ..." 
 )
 load("@hedron_compile_commands//:workspace_setup.bzl", "hedron_compile_commands_setup")
 hedron_compile_commands_setup()

buildfs.sh

@@ -7,22 +7,12 @@ then
     exit
 fi
 
-if [ `gcc -dumpversion | awk -F'.' '{print $1}'` -le 6 ]
-then
-    bazelflags=''
-else
-    bazelflags='--copt -faligned-new'
-fi
-
 if [ "$1" = "debug" ]
 then
 DEBUG_FLAG="--compilation_mode=dbg"
 fi
 
-bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s \
---define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt \
--DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib ${bazelflags}
-
+bazel build curvefs/...  --copt -DHAVE_ZLIB=1 ${DEBUG_FLAG} -s --define=with_glog=true --define=libunwind=true --copt -DGFLAGS_NS=google --copt -Wno-error=format-security --copt -DUSE_BTHREAD_MUTEX --copt -DCURVEVERSION=${curve_version} --linkopt -L/usr/local/lib
 if [ $? -ne 0 ]
 then
     echo "build curvefs failed"
@@ -44,5 +34,4 @@ then
     echo "mds_test failed"
     exit
 fi
-fi
-echo "end compile"
+fi

conf/chunkserver.conf

@@ -9,20 +9,13 @@ global.enable_external_server=true
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__
 global.external_subnet=127.0.0.0/24
 # chunk大小，一般16MB
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
 global.meta_page_size=4096
-# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
-# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
-# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
-# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
-# it will be overwritten from chunkfilepool.meta if `chunkfilepool.enable_get_chunk_from_pool` is true
-global.block_size=4096
-
 # clone chunk允许的最长location长度
 global.location_limit=3000
+# minimum alignment for io request
+global.min_io_alignment=512
 
 #
 # MDS settings
@@ -115,7 +108,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=true
+copyset.enable_odsync_when_open_chunkfile=false
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/chunkserver.conf.example

@@ -13,12 +13,9 @@ global.chunk_size=16777216
 # chunk 元数据页大小，一般4KB
 global.meta_page_size=4096
 # clone chunk允许的最长location长度
-# chunk's block size, IO requests must align with it, supported value is |512| and |4096|
-# it should consist with `block_size` in chunkfilepool.meta_path and `mds.volume.blockSize` in MDS's configurations
-# for clone chunk and snapshot chunk, it's also the minimum granularity that each bit represents
-# if set to |512|, we need 4096 bytes bitmap for each chunk, so meta_page_size should be 8192 or larger.
-global.block_size=4096
 global.location_limit=3000
+# minimum alignment for io request
+global.min_io_alignment=512
 
 #
 # MDS settings
@@ -111,7 +108,7 @@ copyset.scan_rpc_retry_times=3
 # the follower send scanmap to leader rpc retry interval
 copyset.scan_rpc_retry_interval_us=100000
 # enable O_DSYNC when open chunkfile
-copyset.enable_odsync_when_open_chunkfile=true
+copyset.enable_odsync_when_open_chunkfile=false
 # sync trigger seconds
 copyset.sync_trigger_seconds=25
 # sync chunk limit default = 2MB

conf/client.conf

@@ -168,6 +168,13 @@ discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
 
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
+
 ##### chunkserver client option #####
 # chunkserver client rpc timeout time
 csClientOpt.rpcTimeoutMs=500
@@ -181,3 +188,4 @@ csClientOpt.rpcMaxTimeoutMs=8000
 ##### chunkserver broadcaster option #####
 # broad cast max machine num
 csBroadCasterOpt.broadCastMaxNum=200
+

conf/cs_client.conf

@@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

conf/mds.conf

@@ -197,8 +197,6 @@ mds.curvefs.defaultSegmentSize=1073741824
 mds.curvefs.minFileLength=10737418240
 # curvefs的默认最大文件大小，20TB = 20*1024*1024*1024*1024 = 21990232555520
 mds.curvefs.maxFileLength=21990232555520
-# smallest read/write unit for volume, support |512| and |4096|
-mds.curvefs.blockSize=4096
 
 #
 # chunkseverclient config
@@ -239,16 +237,3 @@ mds.throttle.iopsPerGB=30
 mds.throttle.bpsMinInMB=120
 mds.throttle.bpsMaxInMB=260
 mds.throttle.bpsPerGBInMB=0.3
-
-#
-## poolset rules
-#
-# for backward compatibility, rules are applied for select poolset when creating file
-#
-# for example
-#   mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
-#
-# when creating file reqeust doesn't have poolset, above rules are used to select poolset
-# - if filename is /dir1/file, then poolset1 is select
-# - if filename is /dir1/sub/file, then sub is select
-mds.poolset.rules=

conf/py_client.conf

@@ -153,3 +153,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

conf/snap_client.conf

@@ -159,3 +159,10 @@ discard.enable=false
 discard.granularity=4096
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096

curve-ansible/roles/generate_config/defaults/main.yml

@@ -164,6 +164,7 @@ chunkserver_walfilepool_retry_times: 5
 chunkserver_trash_expire_after_sec: 300
 chunkserver_trash_scan_period_sec: 120
 chunkserver_common_log_dir: ./runlog/
+chunkserver_min_io_alignment: 512
 
 # 快照克隆配置默认值
 snap_client_config_path: /etc/curve/snap_client.conf
@@ -243,6 +244,8 @@ client_throttle_enable: false
 client_discard_enable: true
 client_discard_granularity: 4096
 client_discard_task_delay_ms: 60000
+client_alignment_common: 512
+client_alignment_clone: 4096
 
 # nebd默认配置
 client_config_path: /etc/curve/client.conf

curve-ansible/roles/generate_config/templates/chunkserver.conf.j2

@@ -14,6 +14,8 @@ global.chunk_size={{ chunk_size }}
 global.meta_page_size={{ chunkserver_meta_page_size }}
 # clone chunk允许的最长location长度
 global.location_limit={{ chunkserver_location_limit }}
+# minimum alignment for io request
+global.min_io_alignment={{ chunkserver_min_io_alignment }}
 
 #
 # MDS settings

curve-ansible/roles/generate_config/templates/client.conf.j2

@@ -175,3 +175,10 @@ discard.enable={{ client_discard_enable }}
 discard.granularity={{ client_discard_granularity }}
 # discard cleanup task delay times in millisecond
 discard.taskDelayMs={{ client_discard_task_delay_ms }}
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume={{ client_alignment_common }}
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume={{ client_alignment_clone }}

curvefs/conf/client.conf

@@ -58,10 +58,6 @@ executorOpt.maxRetryTimesBeforeConsiderSuspend=20
 # batch limit of get inode attr and xattr
 executorOpt.batchInodeAttrLimit=10000
 
-#### spaceserver
-spaceServer.spaceAddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
-spaceServer.rpcTimeoutMs=1000
-
 #### bdev
 # curve client's config file
 bdev.confPath=/etc/curve/client.conf
@@ -77,7 +73,17 @@ rpc.healthCheckIntervalSec=0
 
 #### fuseClient
 # TODO(xuchaojie): add unit
+fuseClient.attrTimeOut=1.0
+fuseClient.entryTimeOut=1.0
 fuseClient.listDentryLimit=65536
+fuseClient.flushPeriodSec=5
+fuseClient.maxNameLength=255
+fuseClient.iCacheLruSize=65536
+fuseClient.dCacheLruSize=1000000
+fuseClient.enableICacheMetrics=true
+fuseClient.enableDCacheMetrics=true
+fuseClient.lruTimeOutSec=60
+fuseClient.cto=true
 fuseClient.downloadMaxRetryTimes=3
 
 ### kvcache opt
@@ -104,61 +110,6 @@ fuseClient.maxDataSize=1024
 fuseClient.refreshDataIntervalSec=30
 fuseClient.warmupThreadsNum=10
 
-# the write throttle bps of fuseClient, default no limit
-fuseClient.throttle.avgWriteBytes=0
-# the write burst bps of fuseClient, default no limit
-fuseClient.throttle.burstWriteBytes=0
-# the times that write burst bps can continue, default 180s
-fuseClient.throttle.burstWriteBytesSecs=180
-
-# the write throttle iops of fuseClient, default no limit
-fuseClient.throttle.avgWriteIops=0
-# the write burst iops of fuseClient, default no limit
-fuseClient.throttle.burstWriteIops=0
-# the times that write burst Iops can continue, default 180s
-fuseClient.throttle.burstWriteIopsSecs=180
-
-# the read throttle bps of fuseClient, default no limit
-fuseClient.throttle.avgReadBytes=0
-# the read burst bps of fuseClient, default no limit
-fuseClient.throttle.burstReadBytes=0
-# the times that read burst bps can continue, default 180s
-fuseClient.throttle.burstReadBytesSecs=180
-
-# the read throttle iops of fuseClient, default no limit
-fuseClient.throttle.avgReadIops=0
-# the read burst Iops of fuseClient, default no limit
-fuseClient.throttle.burstReadIops=0
-# the times that read burst Iops can continue, default 180s
-fuseClient.throttle.burstReadIopsSecs=180
-
-#### filesystem metadata
-# {
-# fs.disableXattr:
-#   if you want to get better metadata performance,
-#   you can mount fs with |fs.disableXattr| is true
-#
-# fs.lookupCache.negativeTimeoutSec:
-#   entry which not found will be cached if |timeout| > 0
-fs.cto=true
-fs.maxNameLength=255
-fs.disableXattr=false
-fs.accessLogging=true
-fs.kernelCache.attrTimeoutSec=3600
-fs.kernelCache.dirAttrTimeoutSec=3600
-fs.kernelCache.entryTimeoutSec=3600
-fs.kernelCache.dirEntryTimeoutSec=3600
-fs.lookupCache.negativeTimeoutSec=0
-fs.lookupCache.minUses=1
-fs.lookupCache.lruSize=100000
-fs.dirCache.lruSize=5000000
-fs.openFile.lruSize=65536
-fs.attrWatcher.lruSize=5000000
-fs.rpc.listDentryLimit=65536
-fs.deferSync.delay=3
-fs.deferSync.deferDirMtime=false
-# }
-
 #### volume
 volume.bigFileSize=1048576
 volume.volBlockSize=4096
@@ -177,9 +128,20 @@ volume.bitmapAllocator.smallAllocProportion=0.2
 # number of block groups that allocated once
 volume.blockGroup.allocateOnce=4
 
+## spaceserver
+# the space used by the blockgroup exceeds this percentage and can 
+# be returned to mds [0.8-1]
+volume.space.useThreshold=0.95
+
+# the background thread calculates the time interval for returning
+# the blockgroup to mds
+volume.space.releaseInterSec=300
+
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false
+# the max size that fuse send
+s3.fuseMaxSize=131072
 s3.pageSize=65536
 # prefetch blocks that disk cache use
 s3.prefetchBlocks=1
@@ -194,7 +156,6 @@ s3.baseSleepUs=500
 s3.threadScheduleInterval=3
 # data cache flush wait time
 s3.cacheFlushIntervalSec=5
-# write cache < 8,388,608 (8MB) is not allowed
 s3.writeCacheMaxByte=838860800
 s3.readCacheMaxByte=209715200
 # file cache read thread num

curvefs/conf/mds.conf

@@ -6,9 +6,6 @@ mds.listen.addr=127.0.0.1:6700  #__CURVEADM_TEMPLATE__ ${service_addr}:${service
 mds.dummy.port=7700  # __CURVEADM_TEMPLATE__ ${service_dummy_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_mds_listen_dummy_port }} __ANSIBLE_TEMPLATE__
 mds.common.logDir=/tmp/curvefs/mds  # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ /tmp/{{ inventory_hostname }}/curvefs/mds __ANSIBLE_TEMPLATE__
 mds.loglevel=0
-# If a connection does not read or write,it's treated as "idle" and will be closed by server soon.
-# Default value is -1 which disables the feature.
-mds.server.idleTimeoutSec=-1
 
 #
 # space options
@@ -162,3 +159,6 @@ bs.mds.maxFailedTimesBeforeChangeMDS=2
 bs.mds.normalRetryTimesBeforeTriggerWait=3
 # sleep interval in ms for wait
 bs.mds.waitSleepMs=1000
+
+#### Options for volume space deallcatable setting
+mds.space.calIntervalSec=60

curvefs/conf/metaserver.conf

@@ -128,25 +128,16 @@ copyset.trash.scan_periodsec=120
 # this config item should be tuned according cpu/memory/disk
 service.max_inflight_request=5000
 
-#
-# Concurrent apply queue
-### concurrent apply queue options for each copyset
-### concurrent apply queue is used to isolate raft threads, each worker has its own queue
-### when a task can be applied it's been pushed into a corresponding read/write worker queue by certain rules
+### apply queue options for each copyset
+### apply queue is used to isolate raft threads, each worker has its own queue
+### when a task can be applied it's been pushed into a corresponding worker queue by certain rules
+# number of apply queue workers for each, each worker will start a indepent thread
+applyqueue.worker_count=3
 
-# worker_count: number of apply queue workers for each, each worker will start a indepent thread
-# queue_depth: apply queue depth for each copyset
+# apply queue depth for each copyset
 # all tasks in queue must be done when do raft snapshot, and raft apply and raft snapshot are executed in same thread
 # so, if queue depth is too large, it will cause other tasks to wait too long for apply
-# write apply queue workers count
-applyqueue.write_worker_count=3
-# write apply queue depth
-applyqueue.write_queue_depth=1
-# read apply queue workers count
-applyqueue.read_worker_count=2
-# read apply queue depth
-applyqueue.read_queue_depth=1
-
+applyqueue.queue_depth=1
 
 # number of worker threads that created by brpc::Server
 # if set to |auto|, threads create by brpc::Server is equal to `getconf _NPROCESSORS_ONLN` + 1
@@ -154,10 +145,6 @@ applyqueue.read_queue_depth=1
 # it is recommended to set it to |auto| unless there is a significant performance improvement
 bthread.worker_count=auto
 
-# If a connection does not read or write, it's treated as "idle" and will be closed by server soon.
-# Default value is -1 which disables the feature.
-server.idleTimeoutSec=-1
-
 ### Braft related flags
 ### These configurations are ignored if the command line startup options are set
 # Call fsync when need
@@ -320,3 +307,9 @@ metaCacheOpt.metacacheGetLeaderRetry=3
 metaCacheOpt.metacacheRPCRetryIntervalUS=100000
 # RPC timeout of get leader
 metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000
+
+#### volume deallocate
+volume.deallocate.enable=true
+volume.deallocate.workerNum=5
+volume.deallocate.batchClean=10
+volume.sdk.confPath=/etc/curvefs/curvebs_client.conf

curvefs/proto/BUILD

@@ -113,6 +113,7 @@ proto_library(
     name = "curvefs_heartbeat_proto",
     srcs = ["heartbeat.proto"],
     deps = [":curvefs_common_proto",
+            ":metaserver_proto",
             "//proto:heartbeat_proto"],
 )
 

curvefs/proto/common.proto

@@ -31,6 +31,13 @@ enum BitmapLocation {
     AtEnd = 2;
 }
 
+message EmptyMsg {}
+
+message BlockGroupID {
+    required uint64 fsId = 1;
+    required uint64 offset = 2;
+}
+
 // When creating fs, `volumeSize` and `extendAlignment` are fetched from the bs cluster
 message Volume {
     optional uint64 volumeSize = 1;

curvefs/proto/heartbeat.proto

@@ -16,6 +16,7 @@
 
 syntax = "proto2";
 import "curvefs/proto/common.proto";
+import "curvefs/proto/metaserver.proto";
 import "proto/heartbeat.proto";
 package curvefs.mds.heartbeat;
 option cc_generic_services = true;
@@ -27,6 +28,17 @@ option go_package = "curvefs/proto/heartbeat";
 //     required uint64 diskUsedByte = 2;  // the disk this copyset used
 // };
 
+enum BlockGroupDeallcateStatusCode {
+    BGDP_PROCESSING = 0;
+    BGDP_DONE = 1;
+}
+
+message BlockGroupStatInfo {
+    required uint32 fsId = 1;
+    repeated metaserver.DeallocatableBlockGroup deallocatableBlockGroups = 2;
+    map<uint64, BlockGroupDeallcateStatusCode> blockGroupDeallocateStatus = 3;
+}
+
 message CopySetInfo {
     required uint32 poolId = 1;
     required uint32 copysetId = 2;
@@ -81,6 +93,7 @@ message MetaServerHeartbeatRequest {
     required uint32 leaderCount = 7;
     required uint32 copysetCount = 8;
     required MetaServerSpaceStatus spaceStatus = 9;
+    repeated BlockGroupStatInfo blockGroupStatInfos = 10;
 };
 
 message CopySetConf {
@@ -107,11 +120,14 @@ enum HeartbeatStatusCode {
     hbMetaServerIpPortNotMatch = 2;
     hbMetaServerTokenNotMatch = 3;
     hbAnalyseCopysetError = 4;
+    hbMetaServerFSUnkown = 5;
 }
 
 message MetaServerHeartbeatResponse {
     required HeartbeatStatusCode statusCode = 1;
     repeated CopySetConf needUpdateCopysets = 2;
+    // key is fsid, value is blockgroup offset
+    map<uint64, uint64> issuedBlockGroups = 3;
 };
 
 service HeartbeatService {

curvefs/proto/mds.proto

@@ -191,7 +191,6 @@ message RefreshSessionRequest {
 message RefreshSessionResponse {
     required FSStatusCode statusCode = 1;
     repeated topology.PartitionTxId latestTxIdList = 2;
-    optional bool enableSumInDir = 3;
 }
 
 message DLockValue {

curvefs/proto/metaserver.proto

@@ -167,6 +167,46 @@ enum FsFileType {
     TYPE_S3 = 4;
 };
 
+message DeallocatableBlockGroup {
+    required uint64 blockGroupOffset = 1;
+    optional uint64 deallocatableSize = 2;
+    repeated uint64 inodeIdlist = 3;
+    repeated uint64 inodeIdUnderDeallocate = 4;
+
+    oneof type {
+        IncreaseDeallocatableBlockGroup increase = 5;
+        DecreaseDeallocatableBlockGroup decrease = 6;
+        MarkDeallocatableBlockGroup mark = 7;
+    }
+}
+
+message IncreaseDeallocatableBlockGroup {
+    required uint64 increaseDeallocatableSize = 1;
+    repeated uint64 inodeIdlistAdd = 2;
+}
+
+message DecreaseDeallocatableBlockGroup {
+    required uint64 decreaseDeallocatableSize = 1;
+    repeated uint64 inodedDeallocated = 2;
+}
+
+message MarkDeallocatableBlockGroup {
+    repeated uint64 inodeIdUnderDeallocate = 2;
+}
+
+message UpdateDeallocatableBlockGroupRequest {
+    required uint32 poolId = 1;
+    required uint32 copysetId = 2;
+    required uint32 partitionId = 3;
+    required uint64 fsId = 4;
+    repeated DeallocatableBlockGroup update = 5;
+}
+
+message UpdateDeallocatableBlockGroupResponse {
+    required MetaStatusCode statusCode = 1; 
+    optional uint64 appliedIndex = 2;
+}
+
 message VolumeExtent {
     required uint64 fsOffset = 1;
     required uint64 volumeOffset = 2;
@@ -179,7 +219,7 @@ message VolumeExtentSlice {
     repeated VolumeExtent extents = 2;
 }
 
-message VolumeExtentList {
+message VolumeExtentSliceList {
     repeated VolumeExtentSlice slices = 1;
 }
 
@@ -317,7 +357,7 @@ message UpdateInodeRequest {
     map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
-    optional VolumeExtentList volumeExtents = 23;
+    optional VolumeExtentSliceList volumeExtents = 23;
 }
 
 message UpdateInodeResponse {
@@ -464,7 +504,7 @@ message GetVolumeExtentRequest {
 message GetVolumeExtentResponse {
     required MetaStatusCode statusCode = 1;
     optional uint64 appliedIndex = 2;
-    optional VolumeExtentList slices = 3;
+    optional VolumeExtentSliceList slices = 3;
 }
 
 message UpdateVolumeExtentRequest {
@@ -473,7 +513,7 @@ message UpdateVolumeExtentRequest {
     required uint32 partitionId = 3;
     required uint32 fsId = 4;
     required uint64 inodeId = 5;
-    required VolumeExtentList extents = 6;
+    required VolumeExtentSliceList extents = 6;
 }
 
 message UpdateVolumeExtentResponse {
@@ -508,4 +548,7 @@ service MetaServerService {
     // volume extent interface
     rpc GetVolumeExtent(GetVolumeExtentRequest) returns (GetVolumeExtentResponse);
     rpc UpdateVolumeExtent(UpdateVolumeExtentRequest) returns (UpdateVolumeExtentResponse);
+
+    // block group with deallocatable inode list interface
+    rpc UpdateDeallocatableBlockGroup(UpdateDeallocatableBlockGroupRequest) returns (UpdateDeallocatableBlockGroupResponse);
 }

curvefs/proto/space.proto

@@ -48,6 +48,10 @@ message BlockGroup {
     required common.BitmapLocation bitmaplocation = 4;
     // owner, who owns this block group
     optional string owner = 5;
+    // deallocating, metaserver who is deallocate this block group
+    repeated uint32 deallocating = 6;
+    // deallocated, metaserver who finish deallocate this block group
+    repeated uint32 deallocated = 7;
 }
 
 message AllocateBlockGroupRequest {

curvefs/src/client/BUILD

@@ -43,8 +43,6 @@ cc_library(
             "s3/*.h",
             "volume/*.cpp",
             "volume/*.h",
-            "filesystem/*.cpp",
-            "filesystem/*.h",
             "warmup/*.h",
             "warmup/*.cpp",
         ],
@@ -73,14 +71,11 @@ cc_library(
         "//curvefs/src/volume",
         "//curvefs/src/common:metric_utils",
         "//curvefs/src/common:dynamic_vlog",
-        "//curvefs/src/common:threading",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/strings",
         "@com_google_absl//absl/synchronization",
-        "@com_google_absl//absl/strings:str_format",
         "@com_google_absl//absl/meta:type_traits",
         "@com_google_absl//absl/types:optional",
         "@com_google_googletest//:gtest_prod",
-        "@spdlog//:spdlog",
     ],
 )

curvefs/src/client/async_request_closure.cpp

@@ -28,7 +28,7 @@
 #include <mutex>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 namespace curvefs {
@@ -63,19 +63,26 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
         cond_.wait(lk);
     }
 
-    return ToFSError(GetStatusCode());
+    return MetaStatusCodeToCurvefsErrCode(GetStatusCode());
 }
 
 void UpdateVolumeExtentClosure::Run() {
     auto st = GetStatusCode();
     if (!IsOK(st)) {
-        LOG(ERROR) << "UpdateVolumeExtent failed, error: "
-                   << MetaStatusCode_Name(st)
-                   << ", inodeid: " << inode_->GetInodeId();
-        inode_->MarkInodeError();
+        if (inode_ != nullptr) {
+            inode_->MarkInodeError();
+            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
+                       << MetaStatusCode_Name(st)
+                       << ", inodeid: " << inode_->GetInodeId();
+        } else {
+            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
+                       << MetaStatusCode_Name(st);
+        }
     }
 
-    inode_->syncingVolumeExtentsMtx_.unlock();
+    if (inode_ != nullptr) {
+        inode_->syncingVolumeExtentsMtx_.unlock();
+    }
 
     if (sync_) {
         std::lock_guard<bthread::Mutex> lk(mtx_);

curvefs/src/client/async_request_closure.h

@@ -28,15 +28,12 @@
 
 #include <memory>
 
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
-#include "curvefs/src/client/filesystem/error.h"
 
 namespace curvefs {
 namespace client {
 
-using ::curvefs::client::filesystem::CURVEFS_ERROR;
-using ::curvefs::client::filesystem::ToFSError;
-
 class InodeWrapper;
 
 namespace internal {

curvefs/src/client/client_operator.cpp

@@ -24,14 +24,13 @@
 
 #include "src/common/uuid.h"
 #include "curvefs/src/client/client_operator.h"
-#include "curvefs/src/client/filesystem/error.h"
+
 namespace curvefs {
 namespace client {
 
 using ::curve::common::UUIDGenerator;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::mds::topology::PartitionTxId;
-using ::curvefs::client::filesystem::ToFSError;
 
 #define LOG_ERROR(action, rc) \
     LOG(ERROR) << action << " failed, retCode = " << rc \
@@ -96,7 +95,7 @@ CURVEFS_ERROR RenameOperator::GetTxId(uint32_t fsId,
     if (rc != MetaStatusCode::OK) {
         LOG_ERROR("GetTxId", rc);
     }
-    return ToFSError(rc);
+    return MetaStatusCodeToCurvefsErrCode(rc);
 }
 
 CURVEFS_ERROR RenameOperator::GetLatestTxIdWithLock() {
@@ -205,7 +204,7 @@ CURVEFS_ERROR RenameOperator::PrepareRenameTx(
         LOG_ERROR("PrepareRenameTx", rc);
     }
 
-    return ToFSError(rc);
+    return MetaStatusCodeToCurvefsErrCode(rc);
 }
 
 CURVEFS_ERROR RenameOperator::PrepareTx() {
@@ -419,6 +418,8 @@ CURVEFS_ERROR RenameOperator::UpdateInodeCtime() {
 }
 
 void RenameOperator::UpdateCache() {
+    dentryManager_->DeleteCache(parentId_, name_);
+    dentryManager_->InsertOrReplaceCache(newDentry_);
     SetTxId(srcPartitionId_, srcTxId_ + 1);
     SetTxId(dstPartitionId_, dstTxId_ + 1);
 }

curvefs/src/client/common/common.h

@@ -60,6 +60,7 @@ enum class MetaServerOpType {
     GetVolumeExtent,
     UpdateVolumeExtent,
     CreateManageInode,
+    UpdateDeallocatableBlockGroup,
 };
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);

curvefs/src/client/common/config.cpp

@@ -50,56 +50,6 @@ DEFINE_bool(enableCto, true, "acheieve cto consistency");
 DEFINE_bool(useFakeS3, false,
             "Use fake s3 to inject more metadata for testing metaserver");
 DEFINE_bool(supportKVcache, false, "use kvcache to speed up sharing");
-DEFINE_bool(access_logging, true, "enable access log");
-
-/**
- * use curl -L fuseclient:port/flags/fuseClientAvgWriteBytes?setvalue=true
- * for dynamic parameter configuration
- */
-static bool pass_uint64(const char *, uint64_t) { return true; }
-
-DEFINE_uint64(fuseClientAvgWriteBytes, 0,
-              "the write throttle bps of fuse client");
-DEFINE_validator(fuseClientAvgWriteBytes, &pass_uint64);
-DEFINE_uint64(fuseClientBurstWriteBytes, 0,
-              "the write burst bps of fuse client");
-DEFINE_validator(fuseClientBurstWriteBytes, &pass_uint64);
-DEFINE_uint64(fuseClientBurstWriteBytesSecs, 180,
-              "the times that write burst bps can continue");
-DEFINE_validator(fuseClientBurstWriteBytesSecs, &pass_uint64);
-
-
-DEFINE_uint64(fuseClientAvgWriteIops, 0,
-              "the write throttle iops of fuse client");
-DEFINE_validator(fuseClientAvgWriteIops, &pass_uint64);
-DEFINE_uint64(fuseClientBurstWriteIops, 0,
-              "the write burst iops of fuse client");
-DEFINE_validator(fuseClientBurstWriteIops, &pass_uint64);
-DEFINE_uint64(fuseClientBurstWriteIopsSecs, 180,
-              "the times that write burst iops can continue");
-DEFINE_validator(fuseClientBurstWriteIopsSecs, &pass_uint64);
-
-
-DEFINE_uint64(fuseClientAvgReadBytes, 0,
-              "the Read throttle bps of fuse client");
-DEFINE_validator(fuseClientAvgReadBytes, &pass_uint64);
-DEFINE_uint64(fuseClientBurstReadBytes, 0,
-              "the Read burst bps of fuse client");
-DEFINE_validator(fuseClientBurstReadBytes, &pass_uint64);
-DEFINE_uint64(fuseClientBurstReadBytesSecs, 180,
-              "the times that Read burst bps can continue");
-DEFINE_validator(fuseClientBurstReadBytesSecs, &pass_uint64);
-
-
-DEFINE_uint64(fuseClientAvgReadIops, 0,
-              "the Read throttle iops of fuse client");
-DEFINE_validator(fuseClientAvgReadIops, &pass_uint64);
-DEFINE_uint64(fuseClientBurstReadIops, 0,
-              "the Read burst iops of fuse client");
-DEFINE_validator(fuseClientBurstReadIops, &pass_uint64);
-DEFINE_uint64(fuseClientBurstReadIopsSecs, 180,
-              "the times that Read burst iops can continue");
-DEFINE_validator(fuseClientBurstReadIopsSecs, &pass_uint64);
 
 void InitMdsOption(Configuration *conf, MdsOption *mdsOpt) {
     conf->GetValueFatalIfFail("mdsOpt.mdsMaxRetryMS", &mdsOpt->mdsMaxRetryMS);
@@ -207,6 +157,8 @@ void InitDiskCacheOption(Configuration *conf,
 
 void InitS3Option(Configuration *conf, S3Option *s3Opt) {
     conf->GetValueFatalIfFail("s3.fakeS3", &FLAGS_useFakeS3);
+    conf->GetValueFatalIfFail("s3.fuseMaxSize",
+                              &s3Opt->s3ClientAdaptorOpt.fuseMaxSize);
     conf->GetValueFatalIfFail("s3.pageSize",
                               &s3Opt->s3ClientAdaptorOpt.pageSize);
     conf->GetValueFatalIfFail("s3.prefetchBlocks",
@@ -245,6 +197,10 @@ void InitVolumeOption(Configuration *conf, VolumeOption *volumeOpt) {
     conf->GetValueFatalIfFail("volume.fsBlockSize", &volumeOpt->fsBlockSize);
     conf->GetValueFatalIfFail("volume.allocator.type",
                               &volumeOpt->allocatorOption.type);
+    conf->GetValueFatalIfFail("volume.space.useThreshold",
+                              &volumeOpt->threshold);
+    conf->GetValueFatalIfFail("volume.space.releaseInterSec",
+                              &volumeOpt->releaseInterSec);
 
     conf->GetValueFatalIfFail(
         "volume.blockGroup.allocateOnce",
@@ -293,55 +249,6 @@ void InitKVClientManagerOpt(Configuration *conf,
                               &config->getThreadPooln);
 }
 
-void InitFileSystemOption(Configuration* c, FileSystemOption* option) {
-    c->GetValueFatalIfFail("fs.cto", &option->cto);
-    c->GetValueFatalIfFail("fs.cto", &FLAGS_enableCto);
-    c->GetValueFatalIfFail("fs.disableXattr", &option->disableXattr);
-    c->GetValueFatalIfFail("fs.maxNameLength", &option->maxNameLength);
-    c->GetValueFatalIfFail("fs.accessLogging", &FLAGS_access_logging);
-    {  // kernel cache option
-        auto o = &option->kernelCacheOption;
-        c->GetValueFatalIfFail("fs.kernelCache.attrTimeoutSec",
-                               &o->attrTimeoutSec);
-        c->GetValueFatalIfFail("fs.kernelCache.dirAttrTimeoutSec",
-                               &o->dirAttrTimeoutSec);
-        c->GetValueFatalIfFail("fs.kernelCache.entryTimeoutSec",
-                               &o->entryTimeoutSec);
-        c->GetValueFatalIfFail("fs.kernelCache.dirEntryTimeoutSec",
-                               &o->dirEntryTimeoutSec);
-    }
-    {  // lookup cache option
-        auto o = &option->lookupCacheOption;
-        c->GetValueFatalIfFail("fs.lookupCache.lruSize",
-                               &o->lruSize);
-        c->GetValueFatalIfFail("fs.lookupCache.negativeTimeoutSec",
-                               &o->negativeTimeoutSec);
-        c->GetValueFatalIfFail("fs.lookupCache.minUses",
-                               &o->minUses);
-    }
-    {  // dir cache option
-        auto o = &option->dirCacheOption;
-        c->GetValueFatalIfFail("fs.dirCache.lruSize", &o->lruSize);
-    }
-    {  // open file option
-        auto o = &option->openFilesOption;
-        c->GetValueFatalIfFail("fs.openFile.lruSize", &o->lruSize);
-    }
-    {  // attr watcher option
-        auto o = &option->attrWatcherOption;
-        c->GetValueFatalIfFail("fs.attrWatcher.lruSize", &o->lruSize);
-    }
-    {  // rpc option
-        auto o = &option->rpcOption;
-        c->GetValueFatalIfFail("fs.rpc.listDentryLimit", &o->listDentryLimit);
-    }
-    {  // defer sync option
-        auto o = &option->deferSyncOption;
-        c->GetValueFatalIfFail("fs.deferSync.delay", &o->delay);
-        c->GetValueFatalIfFail("fs.deferSync.deferDirMtime", &o->deferDirMtime);
-    }
-}
-
 void SetBrpcOpt(Configuration *conf) {
     curve::common::GflagsLoadValueFromConfIfCmdNotSet dummy;
     dummy.Load(conf, "defer_close_second", "rpc.defer.close.second",
@@ -362,16 +269,36 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
     InitLeaseOpt(conf, &clientOption->leaseOpt);
     InitRefreshDataOpt(conf, &clientOption->refreshDataOption);
     InitKVClientManagerOpt(conf, &clientOption->kvClientManagerOpt);
-    InitFileSystemOption(conf, &clientOption->fileSystemOption);
 
+    conf->GetValueFatalIfFail("fuseClient.attrTimeOut",
+                              &clientOption->attrTimeOut);
+    conf->GetValueFatalIfFail("fuseClient.entryTimeOut",
+                              &clientOption->entryTimeOut);
     conf->GetValueFatalIfFail("fuseClient.listDentryLimit",
                               &clientOption->listDentryLimit);
     conf->GetValueFatalIfFail("fuseClient.listDentryThreads",
                               &clientOption->listDentryThreads);
+    conf->GetValueFatalIfFail("fuseClient.flushPeriodSec",
+                              &clientOption->flushPeriodSec);
+    conf->GetValueFatalIfFail("fuseClient.maxNameLength",
+                              &clientOption->maxNameLength);
+    conf->GetValueFatalIfFail("fuseClient.iCacheLruSize",
+                              &clientOption->iCacheLruSize);
+    conf->GetValueFatalIfFail("fuseClient.dCacheLruSize",
+                              &clientOption->dCacheLruSize);
+    conf->GetValueFatalIfFail("fuseClient.enableICacheMetrics",
+                              &clientOption->enableICacheMetrics);
+    conf->GetValueFatalIfFail("fuseClient.enableDCacheMetrics",
+                              &clientOption->enableDCacheMetrics);
+    conf->GetValueFatalIfFail("fuseClient.lruTimeOutSec",
+                              &clientOption->lruTimeOutSec);
     conf->GetValueFatalIfFail("client.dummyServer.startPort",
                               &clientOption->dummyServerStartPort);
     conf->GetValueFatalIfFail("fuseClient.enableMultiMountPointRename",
                               &clientOption->enableMultiMountPointRename);
+    conf->GetValueFatalIfFail("fuseClient.disableXattr",
+                              &clientOption->disableXattr);
+    conf->GetValueFatalIfFail("fuseClient.cto", &FLAGS_enableCto);
     conf->GetValueFatalIfFail("fuseClient.downloadMaxRetryTimes",
                               &clientOption->downloadMaxRetryTimes);
     conf->GetValueFatalIfFail("fuseClient.warmupThreadsNum",
@@ -381,33 +308,12 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) {
         << "Not found `fuseClient.enableSplice` in conf, use default value `"
         << std::boolalpha << clientOption->enableFuseSplice << '`';
 
-    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteBytes",
-                              &FLAGS_fuseClientAvgWriteBytes);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytes",
-                              &FLAGS_fuseClientBurstWriteBytes);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteBytesSecs",
-                              &FLAGS_fuseClientBurstWriteBytesSecs);
+    // if enableCto, attr and entry cache must invalid
+    if (FLAGS_enableCto) {
+        clientOption->attrTimeOut = 0;
+        clientOption->entryTimeOut = 0;
+    }
 
-    conf->GetValueFatalIfFail("fuseClient.throttle.avgWriteIops",
-                              &FLAGS_fuseClientAvgWriteIops);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIops",
-                              &FLAGS_fuseClientBurstWriteIops);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstWriteIopsSecs",
-                              &FLAGS_fuseClientBurstWriteIopsSecs);
-
-    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadBytes",
-                              &FLAGS_fuseClientAvgReadBytes);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytes",
-                              &FLAGS_fuseClientBurstReadBytes);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadBytesSecs",
-                              &FLAGS_fuseClientBurstReadBytesSecs);
-
-    conf->GetValueFatalIfFail("fuseClient.throttle.avgReadIops",
-                              &FLAGS_fuseClientAvgReadIops);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIops",
-                              &FLAGS_fuseClientBurstReadIops);
-    conf->GetValueFatalIfFail("fuseClient.throttle.burstReadIopsSecs",
-                              &FLAGS_fuseClientBurstReadIopsSecs);
     SetBrpcOpt(conf);
 }
 

curvefs/src/client/common/config.h

@@ -73,11 +73,6 @@ struct LeaseOpt {
     uint32_t leaseTimeUs = 20000000;
 };
 
-struct SpaceAllocServerOption {
-    std::string spaceaddr;
-    uint64_t rpcTimeoutMs;
-};
-
 struct KVClientManagerOpt {
     int setThreadPooln = 4;
     int getThreadPooln = 4;
@@ -122,6 +117,7 @@ struct DiskCacheOption {
 struct S3ClientAdaptorOption {
     uint64_t blockSize;
     uint64_t chunkSize;
+    uint32_t fuseMaxSize;
     uint64_t pageSize;
     uint32_t prefetchBlocks;
     uint32_t prefetchExecQueueNum;
@@ -164,6 +160,9 @@ struct VolumeOption {
     uint64_t volBlockSize;
     uint64_t fsBlockSize;
     VolumeAllocatorOption allocatorOption;
+
+    double threshold{1.0};
+    uint64_t releaseInterSec{300};
 };
 
 struct ExtentManagerOption {
@@ -174,65 +173,11 @@ struct RefreshDataOption {
     uint64_t maxDataSize = 1024;
     uint32_t refreshDataIntervalSec = 30;
 };
-
-// { filesystem option
-struct KernelCacheOption {
-    uint32_t entryTimeoutSec;
-    uint32_t dirEntryTimeoutSec;
-    uint32_t attrTimeoutSec;
-    uint32_t dirAttrTimeoutSec;
-};
-
-struct LookupCacheOption {
-    uint64_t lruSize;
-    uint32_t negativeTimeoutSec;
-    uint32_t minUses;
-};
-
-struct DirCacheOption {
-    uint64_t lruSize;
-    uint32_t timeoutSec;
-};
-
-struct AttrWatcherOption {
-    uint64_t lruSize;
-};
-
-struct OpenFilesOption {
-    uint64_t lruSize;
-    uint32_t deferSyncSecond;
-};
-
-struct RPCOption {
-    uint32_t listDentryLimit;
-};
-
-struct DeferSyncOption {
-    uint32_t delay;
-    bool deferDirMtime;
-};
-
-struct FileSystemOption {
-    bool cto;
-    bool disableXattr;
-    uint32_t maxNameLength;
-    uint32_t blockSize = 0x10000u;
-    KernelCacheOption kernelCacheOption;
-    LookupCacheOption lookupCacheOption;
-    DirCacheOption dirCacheOption;
-    OpenFilesOption openFilesOption;
-    AttrWatcherOption attrWatcherOption;
-    RPCOption rpcOption;
-    DeferSyncOption deferSyncOption;
-};
-// }
-
 struct FuseClientOption {
     MdsOption mdsOpt;
     MetaCacheOpt metaCacheOpt;
     ExcutorOpt excutorOpt;
     ExcutorOpt excutorInternalOpt;
-    SpaceAllocServerOption spaceOpt;
     BlockDeviceClientOptions bdevOpt;
     S3Option s3Opt;
     ExtentManagerOption extentManagerOpt;
@@ -240,13 +185,22 @@ struct FuseClientOption {
     LeaseOpt leaseOpt;
     RefreshDataOption refreshDataOption;
     KVClientManagerOpt kvClientManagerOpt;
-    FileSystemOption fileSystemOption;
 
+    double attrTimeOut;
+    double entryTimeOut;
     uint32_t listDentryLimit;
     uint32_t listDentryThreads;
+    uint32_t flushPeriodSec;
+    uint32_t maxNameLength;
+    uint64_t iCacheLruSize;
+    uint64_t dCacheLruSize;
+    bool enableICacheMetrics;
+    bool enableDCacheMetrics;
+    uint32_t lruTimeOutSec;
     uint32_t dummyServerStartPort;
     bool enableMultiMountPointRename = false;
     bool enableFuseSplice = false;
+    bool disableXattr = false;
     uint32_t downloadMaxRetryTimes;
     uint32_t warmupThreadsNum = 10;
 };

curvefs/src/client/curve_fuse_op.h

@@ -37,7 +37,7 @@
 extern "C" {
 #endif
 
-int InitLog(const char *confPath, const char *argv0);
+int InitGlog(const char *confPath, const char *argv0);
 
 int InitFuseClient(const struct MountOption *mountOption);
 

curvefs/src/client/dentry_cache_manager.cpp

@@ -44,13 +44,31 @@ namespace client {
 
 using curve::common::WriteLockGuard;
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
-using ::curvefs::client::filesystem::ToFSError;
+
+void DentryCacheManagerImpl::InsertOrReplaceCache(const Dentry &dentry) {
+    std::string key = GetDentryCacheKey(dentry.parentinodeid(), dentry.name());
+    if (!curvefs::client::common::FLAGS_enableCto) {
+        NameLockGuard lock(nameLock_, key);
+        dCache_->Put(key, dentry);
+    }
+}
+
+void DentryCacheManagerImpl::DeleteCache(uint64_t parentId,
+                                         const std::string &name) {
+    std::string key = GetDentryCacheKey(parentId, name);
+    NameLockGuard lock(nameLock_, key);
+    dCache_->Remove(key);
+}
 
 CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
                                                 const std::string &name,
                                                 Dentry *out) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
+    bool ok = dCache_->Get(key, out);
+    if (ok) {
+        return CURVEFS_ERROR::OK;
+    }
 
     MetaStatusCode ret = metaClient_->GetDentry(fsId_, parent, name, out);
     if (ret != MetaStatusCode::OK) {
@@ -58,7 +76,10 @@ CURVEFS_ERROR DentryCacheManagerImpl::GetDentry(uint64_t parent,
             << "metaClient_ GetDentry failed, MetaStatusCode = " << ret
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
             << ", parent = " << parent << ", name = " << name;
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
+    }
+    if (!curvefs::client::common::FLAGS_enableCto) {
+        dCache_->Put(key, *out);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -73,9 +94,12 @@ CURVEFS_ERROR DentryCacheManagerImpl::CreateDentry(const Dentry &dentry) {
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << dentry.parentinodeid()
                    << ", name = " << dentry.name();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
 
+    if (!curvefs::client::common::FLAGS_enableCto) {
+        dCache_->Put(key, dentry);
+    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -84,13 +108,14 @@ CURVEFS_ERROR DentryCacheManagerImpl::DeleteDentry(uint64_t parent,
                                                    FsFileType type) {
     std::string key = GetDentryCacheKey(parent, name);
     NameLockGuard lock(nameLock_, key);
+    dCache_->Remove(key);
 
     MetaStatusCode ret = metaClient_->DeleteDentry(fsId_, parent, name, type);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", parent = " << parent << ", name = " << name;
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
     return CURVEFS_ERROR::OK;
 }
@@ -124,7 +149,7 @@ CURVEFS_ERROR DentryCacheManagerImpl::ListDentry(uint64_t parent,
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", parent = " << parent << ", last = " << last
                        << ", count = " << limit << ", onlyDir = " << onlyDir;
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
 
         if (!onlyDir) {

curvefs/src/client/dentry_cache_manager.h

@@ -33,18 +33,20 @@
 #include <utility>
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
+#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
+#include "src/common/lru_cache.h"
 #include "src/common/concurrent/name_lock.h"
-#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Dentry;
+using ::curve::common::TimedLRUCache;
+using ::curve::common::CacheMetrics;
 
 namespace curvefs {
 namespace client {
 
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
-using ::curvefs::client::filesystem::CURVEFS_ERROR;
 
 static const char* kDentryKeyDelimiter = ":";
 
@@ -57,6 +59,13 @@ class DentryCacheManager {
         fsId_ = fsId;
     }
 
+    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+        uint32_t cacheTimeOutSec) = 0;
+
+    virtual void InsertOrReplaceCache(const Dentry& dentry) = 0;
+
+    virtual void DeleteCache(uint64_t parentId, const std::string& name) = 0;
+
     virtual CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) = 0;
 
@@ -77,11 +86,30 @@ class DentryCacheManager {
 class DentryCacheManagerImpl : public DentryCacheManager {
  public:
     DentryCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()),
+        dCache_(nullptr) {}
 
     explicit DentryCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient) {}
+      : metaClient_(metaClient),
+        dCache_(nullptr) {}
+
+    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+        uint32_t cacheTimeOutSec) override {
+        if (enableCacheMetrics) {
+            dCache_ = std::make_shared<
+                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize,
+                    std::make_shared<CacheMetrics>("dcache"));
+        } else {
+            dCache_ = std::make_shared<
+                TimedLRUCache<std::string, Dentry>>(cacheTimeOutSec, cacheSize);
+        }
+        return CURVEFS_ERROR::OK;
+    }
+
+    void InsertOrReplaceCache(const Dentry& dentry) override;
+
+    void DeleteCache(uint64_t parentId, const std::string& name) override;
 
     CURVEFS_ERROR GetDentry(uint64_t parent,
         const std::string &name, Dentry *out) override;
@@ -102,6 +130,8 @@ class DentryCacheManagerImpl : public DentryCacheManager {
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
+    // key is parentId + name
+    std::shared_ptr<TimedLRUCache<std::string, Dentry>> dCache_;
     curve::common::GenericNameLock<Mutex> nameLock_;
 };
 

curvefs/src/client/error_code.cpp

@@ -0,0 +1,98 @@
+/*
+ *  Copyright (c) 2021 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*
+ * Project: curve
+ * Created Date: Thur May 27 2021
+ * Author: xuchaojie
+ */
+
+#include "curvefs/src/client/error_code.h"
+
+#include <map>
+#include <string>
+
+namespace curvefs {
+namespace client {
+
+using ::curvefs::metaserver::MetaStatusCode;
+
+static const std::map<CURVEFS_ERROR, std::string> err2Msg = {
+    {CURVEFS_ERROR::OK, "OK"},
+    {CURVEFS_ERROR::INTERNAL, "internal error"},
+    {CURVEFS_ERROR::UNKNOWN, "unknown"},
+    {CURVEFS_ERROR::EXISTS, "inode or dentry already exist"},
+    {CURVEFS_ERROR::NOTEXIST, "inode or dentry not exist"},
+    {CURVEFS_ERROR::NO_SPACE, "no space to alloc"},
+    {CURVEFS_ERROR::BAD_FD, "bad fd"},
+    {CURVEFS_ERROR::INVALIDPARAM, "invalid param"},
+    {CURVEFS_ERROR::NOPERMISSION, "no permission"},
+    {CURVEFS_ERROR::NOTEMPTY, "dir not empty"},
+    {CURVEFS_ERROR::NOFLUSH, "no flush"},
+    {CURVEFS_ERROR::NOTSUPPORT, "not support"},
+    {CURVEFS_ERROR::NAMETOOLONG, "name too long"},
+    {CURVEFS_ERROR::MOUNT_POINT_EXIST, "mount point already exist"},
+    {CURVEFS_ERROR::MOUNT_FAILED, "mount failed"},
+};
+
+std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
+    os << static_cast<int>(code) << "[" << [code]() {
+        auto it = err2Msg.find(code);
+        if (it != err2Msg.end()) {
+            return it->second;
+        }
+
+        return std::string{"Unknown"};
+    }() << "]";
+
+    return os;
+}
+
+CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
+    MetaStatusCode code) {
+    CURVEFS_ERROR ret = CURVEFS_ERROR::UNKNOWN;
+    switch (code) {
+        case MetaStatusCode::OK:
+            ret = CURVEFS_ERROR::OK;
+            break;
+
+        case MetaStatusCode::NOT_FOUND:
+            ret = CURVEFS_ERROR::NOTEXIST;
+            break;
+
+        case MetaStatusCode::PARAM_ERROR:
+            ret = CURVEFS_ERROR::INVALIDPARAM;
+            break;
+
+        case MetaStatusCode::INODE_EXIST:
+        case MetaStatusCode::DENTRY_EXIST:
+            ret = CURVEFS_ERROR::EXISTS;
+            break;
+
+        case MetaStatusCode::SYM_LINK_EMPTY:
+        case MetaStatusCode::RPC_ERROR:
+            ret = CURVEFS_ERROR::INTERNAL;
+            break;
+
+        default:
+            ret = CURVEFS_ERROR::UNKNOWN;
+    }
+    return ret;
+}
+
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/error_code.h

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2023 NetEase Inc.
+ *  Copyright (c) 2021 NetEase Inc.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -14,27 +14,22 @@
  *  limitations under the License.
  */
 
+
 /*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
+ * Project: curve
+ * Created Date: Thur May 27 2021
+ * Author: xuchaojie
  */
 
-#include <map>
-#include <string>
-#include <iostream>
+#ifndef CURVEFS_SRC_CLIENT_ERROR_CODE_H_
+#define CURVEFS_SRC_CLIENT_ERROR_CODE_H_
 
 #include "curvefs/proto/metaserver.pb.h"
 
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
-
 namespace curvefs {
 namespace client {
-namespace filesystem {
-
-using ::curvefs::metaserver::MetaStatusCode;
 
+// notice : the error code should be negative.
 enum class CURVEFS_ERROR {
     OK = 0,
     INTERNAL = -1,
@@ -54,21 +49,15 @@ enum class CURVEFS_ERROR {
     OUT_OF_RANGE = -15,
     NODATA = -16,
     IO_ERROR = -17,
-    CACHETOOSMALL = -18,
-    STALE = -19,
-    NOSYS = -20,
 };
 
-std::string StrErr(CURVEFS_ERROR code);
-
-int SysErr(CURVEFS_ERROR code);
 
 std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code);
 
-CURVEFS_ERROR ToFSError(MetaStatusCode code);
+CURVEFS_ERROR MetaStatusCodeToCurvefsErrCode(
+    ::curvefs::metaserver::MetaStatusCode code);
 
-}  // namespace filesystem
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ERROR_H_
+#endif  // CURVEFS_SRC_CLIENT_ERROR_CODE_H_

curvefs/src/client/filesystem/README.md

@@ -1,78 +0,0 @@
-CurveFS Client Metadata Cache Design
-===
-
-```
-+----------------+
-+  Kernel Cache  |
-+----------------+
-    |       ^
-(1) |       | (4)
-    v       |
-+----------------+
-+  CurveFS Fuse  |
-+----------------+
-    |       ^
-(2) |       | (3)
-    v       |
-+----------------+
-+   MetaServer   |
-+----------------+
-```
-
-(1) kernel cache timeout
----
-
-* requests are always sent to fuse layer, except `lookup` and `getattr` request which affected by caching.
-* if entry or attribute cache timeout, vfs layer will launch request to fuse layer.
-
-(2) revalidate cache
----
-
-* for `open` and `opendir` request, fuse layer should revalidate cache by comparing mtime, if modified, fuse layer should:
-    * `open`: return **ESTALE** to trigger vfs layer to invoke the `open` again with ignoring cache.
-    * `opendir`: drop all directory cache.
-* others, proxy request to metaserver directly.
-
-
-(3) retrive fresh metadata
----
-
-* for `readdir` request, fuse layer should cache direcoty entries and their attributes.
-* others, no caching.
-
-(4) reply with timeout
----
-
-* reply entry or attribute to vfs layer with corresponding cache timeout.
-* four type timeouts provided: `entryTimeout`, `dirEntryTimeout`, `attrTimeout`, `dirAtttTimeout`.
-* fuse layer should remeber the `mtime` of attribute while reply it to vfs layer.
-
-Cache Layer Level
-===
-
-```
-+-------------------------------------------------------+
-|                       Kernel Cache                    |  (L1 Cache)
-+-------------------------------------------------------+
-                            ^  (update length)
-                            |
-+--------------------------------------------------------+
-|                       Open File Cache                  |  (L2 Cache)
-|                   (only length and chunk)              |
-+--------------------------------------------------------+
-         ^                            ^
-         | (reply entry/attr)         |  (reply entry/attr)
-+-----------------+                   |
-| Directory Cache |  (L3 Cache)       |
-+-----------------+                   |
-         ^                            |
-         |                            |
-+-----------------+ +------------------------------------+
-|     readdir     | | lookup、getattr、setattr、create... |
-+-----------------+ +------------------------------------+
-         ^                            ^
-         |                            |
-+--------------------------------------------------------+
-|                        MetaServer                      |
-+--------------------------------------------------------+
-```

curvefs/src/client/filesystem/access_log.h

@@ -1,89 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-17
- * Author: Jingli Chen (Wine93)
- */
-
-#include <unistd.h>
-#include <butil/time.h>
-#include <spdlog/spdlog.h>
-#include <spdlog/sinks/basic_file_sink.h>
-#include <spdlog/sinks/daily_file_sink.h>
-
-#include <string>
-#include <memory>
-
-#include "absl/strings/str_format.h"
-#include "curvefs/src/client/common/config.h"
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_
-
-namespace curvefs {
-namespace client {
-namespace common {
-
-DECLARE_bool(access_logging);
-
-}
-namespace filesystem {
-
-using ::absl::StrFormat;
-using ::curvefs::client::common::FLAGS_access_logging;
-using MessageHandler = std::function<std::string()>;
-
-static std::shared_ptr<spdlog::logger> Logger;
-
-bool InitAccessLog(const std::string& prefix) {
-    std::string filename = StrFormat("%s/access.%d.log", prefix, getpid());
-    Logger = spdlog::daily_logger_mt("fuse_access", filename, 0, 0);
-    spdlog::flush_every(std::chrono::seconds(1));
-    return true;
-}
-
-struct AccessLogGuard {
-    explicit AccessLogGuard(MessageHandler handler)
-        : enable(FLAGS_access_logging),
-          handler(handler) {
-        if (!enable) {
-            return;
-        }
-
-        timer.start();
-    }
-
-    ~AccessLogGuard() {
-        if (!enable) {
-            return;
-        }
-
-        timer.stop();
-        Logger->info("{0} <{1:.6f}>", handler(), timer.u_elapsed() / 1e6);
-    }
-
-    bool enable;
-    MessageHandler handler;
-    butil::Timer timer;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ACCESS_LOG_H_

curvefs/src/client/filesystem/attr_watcher.cpp

@@ -1,117 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#include "curvefs/src/client/filesystem/utils.h"
-#include "curvefs/src/client/filesystem/attr_watcher.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-AttrWatcher::AttrWatcher(AttrWatcherOption option,
-                         std::shared_ptr<OpenFiles> openFiles,
-                         std::shared_ptr<DirCache> dirCache)
-    : modifiedAt_(std::make_shared<LRUType>(option.lruSize)),
-      openFiles_(openFiles),
-      dirCache_(dirCache) {}
-
-void AttrWatcher::RemeberMtime(const InodeAttr& attr) {
-    WriteLockGuard lk(rwlock_);
-    modifiedAt_->Put(attr.inodeid(), AttrMtime(attr));
-}
-
-bool AttrWatcher::GetMtime(Ino ino, TimeSpec* time) {
-    ReadLockGuard lk(rwlock_);
-    return modifiedAt_->Get(ino, time);
-}
-
-void AttrWatcher::UpdateDirEntryAttr(Ino ino, const InodeAttr& attr) {
-    std::shared_ptr<DirEntryList> entries;
-    for (const auto parent : attr.parent()) {
-        bool yes = dirCache_->Get(parent, &entries);
-        if (!yes) {
-            continue;
-        }
-
-        entries->UpdateAttr(ino, attr);
-
-        VLOG(1) << "Write back attribute to dir entry cache: ino = " << ino
-                << ", attr = " << attr.ShortDebugString();
-    }
-}
-
-void AttrWatcher::UpdateDirEntryLength(Ino ino, const InodeAttr& open) {
-    std::shared_ptr<DirEntryList> entries;
-    for (const auto parent : open.parent()) {
-        bool yes = dirCache_->Get(parent, &entries);
-        if (!yes) {
-            continue;
-        }
-
-        entries->UpdateLength(ino, open);
-
-        VLOG(1) << "Write back file length to dir entry cache: ino = " << ino
-                << ", attr = " << open.ShortDebugString();
-    }
-}
-
-AttrWatcherGuard::AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
-                                   InodeAttr* attr,
-                                   ReplyType type,
-                                   bool writeBack)
-    : watcher(watcher), attr(attr), type(type), writeBack(writeBack) {
-    InodeAttr open;
-    Ino ino = attr->inodeid();
-    bool yes = watcher->openFiles_->GetFileAttr(ino, &open);
-    if (!yes) {
-        return;
-    }
-
-    attr->set_length(open.length());
-    attr->set_mtime(open.mtime());
-    attr->set_mtime_ns(open.mtime_ns());
-    if (AttrCtime(open) > AttrCtime(*attr)) {
-        attr->set_ctime(open.ctime());
-        attr->set_ctime_ns(open.ctime_ns());
-    }
-}
-
-AttrWatcherGuard::~AttrWatcherGuard() {
-    switch (type) {
-        case ReplyType::ATTR:
-            watcher->RemeberMtime(*attr);
-            if (writeBack) {
-                watcher->UpdateDirEntryAttr(attr->inodeid(), *attr);
-            }
-            break;
-
-        case ReplyType::ONLY_LENGTH:
-            if (writeBack) {
-                watcher->UpdateDirEntryLength(attr->inodeid(), *attr);
-            }
-            break;
-    }
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/attr_watcher.h

@@ -1,105 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_
-
-#include <memory>
-
-#include "src/common/lru_cache.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/openfile.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::RWLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::common::AttrWatcherOption;
-
-class AttrWatcher {
- public:
-    using LRUType = LRUCache<Ino, struct TimeSpec>;
-
- public:
-    AttrWatcher(AttrWatcherOption option,
-                std::shared_ptr<OpenFiles> openFiles,
-                std::shared_ptr<DirCache> dirCache);
-
-    void RemeberMtime(const InodeAttr& attr);
-
-    bool GetMtime(Ino ino, TimeSpec* time);
-
-    void UpdateDirEntryAttr(Ino ino, const InodeAttr& attr);
-
-    void UpdateDirEntryLength(Ino ino, const InodeAttr& open);
-
- private:
-    friend class AttrWatcherGuard;
-
- private:
-    RWLock rwlock_;
-    std::shared_ptr<LRUType> modifiedAt_;
-    std::shared_ptr<OpenFiles> openFiles_;
-    std::shared_ptr<DirCache> dirCache_;
-};
-
-
-enum class ReplyType {
-    ATTR,
-    ONLY_LENGTH
-};
-
-/*
- * each attribute reply to kernel, the watcher will:
- *  before reply:
- *    1) set attibute length if the corresponding file is opened
- *  after reply:
- *    1) remeber attribute modified time.
- *    2) write back attribute to dir entry cache if |writeBack| is true,
- *       because the dir-entry attribute maybe stale.
- */
-struct AttrWatcherGuard {
- public:
-    AttrWatcherGuard(std::shared_ptr<AttrWatcher> watcher,
-                     InodeAttr* attr,
-                     ReplyType type,
-                     bool writeBack);
-
-    ~AttrWatcherGuard();
-
- private:
-    std::shared_ptr<AttrWatcher> watcher;
-    InodeAttr* attr;
-    ReplyType type;
-    bool writeBack;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_ATTR_WATCHER_H_

curvefs/src/client/filesystem/defer_sync.cpp

@@ -1,86 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <vector>
-#include <memory>
-
-#include "curvefs/src/client/filesystem/defer_sync.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-DeferSync::DeferSync(DeferSyncOption option)
-    : option_(option),
-      mutex_(),
-      running_(false),
-      thread_(),
-      sleeper_(),
-      inodes_() {
-}
-
-void DeferSync::Start() {
-    if (!running_.exchange(true)) {
-        thread_ = std::thread(&DeferSync::SyncTask, this);
-        LOG(INFO) << "Defer sync thread start success";
-    }
-}
-
-void DeferSync::Stop() {
-    if (running_.exchange(false)) {
-        LOG(INFO) << "Stop defer sync thread...";
-        sleeper_.interrupt();
-        thread_.join();
-        LOG(INFO) << "Defer sync thread stopped";
-    }
-}
-
-void DeferSync::SyncTask() {
-    std::vector<std::shared_ptr<InodeWrapper>> inodes;
-    for ( ;; ) {
-        bool running = sleeper_.wait_for(std::chrono::seconds(option_.delay));
-
-        {
-            LockGuard lk(mutex_);
-            inodes.swap(inodes_);
-        }
-        for (const auto& inode : inodes) {
-            UniqueLock lk(inode->GetUniqueLock());
-            inode->Async(nullptr, true);
-        }
-        inodes.clear();
-
-        if (!running) {
-            break;
-        }
-    }
-}
-
-void DeferSync::Push(const std::shared_ptr<InodeWrapper>& inode) {
-    LockGuard lk(mutex_);
-    inodes_.emplace_back(inode);
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/defer_sync.h

@@ -1,70 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_
-
-#include <atomic>
-#include <vector>
-#include <memory>
-
-#include "src/common/interruptible_sleeper.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curvefs::client::common::DeferSyncOption;
-
-using ::curve::common::Mutex;
-using ::curve::common::LockGuard;
-using ::curve::common::InterruptibleSleeper;
-
-class DeferSync {
- public:
-    explicit DeferSync(DeferSyncOption option);
-
-    void Start();
-
-    void Stop();
-
-    void Push(const std::shared_ptr<InodeWrapper>& inode);
-
- private:
-    void SyncTask();
-
- private:
-    DeferSyncOption option_;
-    Mutex mutex_;
-    std::atomic<bool> running_;
-    std::thread thread_;
-    InterruptibleSleeper sleeper_;
-    std::vector<std::shared_ptr<InodeWrapper>> inodes_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DEFER_SYNC_H_

curvefs/src/client/filesystem/dir_cache.cpp

@@ -1,189 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#include <utility>
-
-#include "curvefs/src/client/filesystem/dir_cache.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-DirEntryList::DirEntryList()
-    : rwlock_(),
-      mtime_(),
-      entries_(),
-      attrs_() {}
-
-size_t DirEntryList::Size() {
-    ReadLockGuard lk(rwlock_);
-    return entries_.size();
-}
-
-void DirEntryList::Add(const DirEntry& dirEntry) {
-    WriteLockGuard lk(rwlock_);
-    entries_.push_back(std::move(dirEntry));
-    attrs_[dirEntry.ino] = &entries_.back();
-}
-
-bool DirEntryList::Get(Ino ino, DirEntry* dirEntry) {
-    ReadLockGuard lk(rwlock_);
-    auto iter = attrs_.find(ino);
-    if (iter == attrs_.end()) {
-        return false;
-    }
-    *dirEntry = *iter->second;
-    return true;
-}
-
-bool DirEntryList::UpdateAttr(Ino ino, const InodeAttr& attr) {
-    WriteLockGuard lk(rwlock_);
-    auto iter = attrs_.find(ino);
-    if (iter == attrs_.end()) {
-        return false;
-    }
-
-    DirEntry* dirEntry = iter->second;
-    dirEntry->attr = std::move(attr);
-    return true;
-}
-
-bool DirEntryList::UpdateLength(Ino ino, const InodeAttr& open) {
-    WriteLockGuard lk(rwlock_);
-    auto iter = attrs_.find(ino);
-    if (iter == attrs_.end()) {
-        return false;
-    }
-
-    InodeAttr* attr = &iter->second->attr;
-    attr->set_length(open.length());
-    attr->set_mtime(open.mtime());
-    attr->set_mtime_ns(open.mtime_ns());
-    if (AttrCtime(open) > AttrCtime(*attr)) {
-        attr->set_ctime(open.ctime());
-        attr->set_ctime_ns(open.ctime_ns());
-    }
-    return true;
-}
-
-void DirEntryList::Iterate(IterateHandler handler) {
-    ReadLockGuard lk(rwlock_);
-    for (auto iter = entries_.begin(); iter != entries_.end(); iter++) {
-        handler(&(*iter));
-    }
-}
-
-void DirEntryList::Clear() {
-    WriteLockGuard lk(rwlock_);
-    entries_.clear();
-    attrs_.clear();
-}
-
-void DirEntryList::SetMtime(TimeSpec mtime) {
-    WriteLockGuard lk(rwlock_);
-    mtime_ = mtime;
-}
-
-TimeSpec DirEntryList::GetMtime() {
-    ReadLockGuard lk(rwlock_);
-    return mtime_;
-}
-
-DirCache::DirCache(DirCacheOption option)
-    : rwlock_(),
-      nentries_(0),
-      option_(option) {
-    lru_ = std::make_shared<LRUType>(0);  // control size by ourself
-    mq_ = std::make_shared<MessageQueueType>("dircache", 10000);
-    mq_->Subscribe([&](const std::shared_ptr<DirEntryList>& entries){
-        entries->Clear();
-    });
-
-    LOG(INFO) << "Using directory lru cache, capacity = " << option_.lruSize;
-}
-
-void DirCache::Start() {
-    mq_->Start();
-}
-
-void DirCache::Stop() {
-    WriteLockGuard lk(rwlock_);
-    Evit(option_.lruSize);
-    mq_->Stop();
-}
-
-void DirCache::Delete(Ino parent,
-                      std::shared_ptr<DirEntryList> entries,
-                      bool evit) {
-    nentries_ -= entries->Size();
-    mq_->Publish(entries);  // clear entries in background
-    lru_->Remove(parent);
-
-    VLOG(1) << "Delete directory cache (evit=" << evit << "): "
-            << "parent = " << parent
-            << ", mtime = " << entries->GetMtime()
-            << ", size = " << entries->Size()
-            << ", nentries = " << nentries_;
-}
-
-void DirCache::Evit(size_t size) {
-    Ino parent;
-    std::shared_ptr<DirEntryList> entries;
-    while (nentries_ + size >= option_.lruSize) {
-        bool yes = lru_->GetLast(&parent, &entries);
-        if (!yes) {
-            break;
-        }
-        Delete(parent, entries, true);
-    }
-}
-
-void DirCache::Put(Ino parent, std::shared_ptr<DirEntryList> entries) {
-    WriteLockGuard lk(rwlock_);
-    Evit(entries->Size());  // it guarantee put entries success
-    lru_->Put(parent, entries);
-    nentries_ += entries->Size();
-
-    VLOG(1) << "Insert directory cache: parent = " << parent
-            << ", mtime = " << entries->GetMtime()
-            << ", size = " << entries->Size()
-            << ", nentries = " << nentries_;
-}
-
-bool DirCache::Get(Ino parent, std::shared_ptr<DirEntryList>* entries) {
-    ReadLockGuard lk(rwlock_);
-    return lru_->Get(parent, entries);
-}
-
-void DirCache::Drop(Ino parent) {
-    WriteLockGuard lk(rwlock_);
-    std::shared_ptr<DirEntryList> entries;
-    bool yes = lru_->Get(parent, &entries);
-    if (yes) {
-        Delete(parent, entries, false);
-    }
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/dir_cache.h

@@ -1,114 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_
-
-#include <map>
-#include <list>
-#include <memory>
-
-#include "src/common/lru_cache.h"
-#include "src/common/concurrent/concurrent.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/message_queue.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::RWLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::common::DirCacheOption;
-
-class DirEntryList {
- public:
-    using IterateHandler = std::function<void(DirEntry* dirEntry)>;
-
- public:
-    DirEntryList();
-
-    size_t Size();
-
-    void Add(const DirEntry& dirEntry);
-
-    void Iterate(IterateHandler handler);
-
-    bool Get(Ino ino, DirEntry* dirEntry);
-
-    bool UpdateAttr(Ino ino, const InodeAttr& attr);
-
-    bool UpdateLength(Ino ino, const InodeAttr& open);
-
-    void Clear();
-
-    void SetMtime(TimeSpec mtime);
-
-    TimeSpec GetMtime();
-
- private:
-    RWLock rwlock_;
-    TimeSpec mtime_;
-    std::list<DirEntry> entries_;
-    std::map<Ino, DirEntry*> attrs_;
-};
-
-class DirCache {
- public:
-    using LRUType = LRUCache<Ino, std::shared_ptr<DirEntryList>>;
-    using MessageType = std::shared_ptr<DirEntryList>;
-    using MessageQueueType = MessageQueue<MessageType>;
-
- public:
-    explicit DirCache(DirCacheOption option);
-
-    void Start();
-
-    void Stop();
-
-    void Put(Ino parent, std::shared_ptr<DirEntryList> entries);
-
-    bool Get(Ino parent, std::shared_ptr<DirEntryList>* entries);
-
-    void Drop(Ino parent);
-
- private:
-    void Delete(Ino parent, std::shared_ptr<DirEntryList> entries, bool evit);
-
-    void Evit(size_t size);
-
- private:
-    RWLock rwlock_;
-    size_t nentries_;
-    DirCacheOption option_;
-    std::shared_ptr<LRUType> lru_;
-    std::shared_ptr<MessageQueueType> mq_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_DIR_CACHE_H_

curvefs/src/client/filesystem/error.cpp

@@ -1,106 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <map>
-#include <utility>
-#include <ostream>
-
-#include "curvefs/src/client/filesystem/error.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-static const std::map<CURVEFS_ERROR, std::pair<int, std::string>> errors = {
-    { CURVEFS_ERROR::OK, { 0, "OK"} },
-    { CURVEFS_ERROR::INTERNAL,  { EIO, "internal error" } },
-    { CURVEFS_ERROR::UNKNOWN,  { -1, "unknown" } },
-    { CURVEFS_ERROR::EXISTS,  { EEXIST, "inode or dentry already exist" } },
-    { CURVEFS_ERROR::NOTEXIST, { ENOENT, "inode or dentry not exist" } },
-    { CURVEFS_ERROR::NO_SPACE,  { ENOSPC, "no space to alloc" } },
-    { CURVEFS_ERROR::BAD_FD,  { EBADF, "bad file number" } },
-    { CURVEFS_ERROR::INVALIDPARAM , { EINVAL , "invalid argument" } },
-    { CURVEFS_ERROR::NOPERMISSION,  { EACCES, "permission denied" } },
-    { CURVEFS_ERROR::NOTEMPTY, { ENOTEMPTY, "directory not empty" } },
-    { CURVEFS_ERROR::NOFLUSH, { -1, "no flush" } },
-    { CURVEFS_ERROR::NOTSUPPORT, { EOPNOTSUPP, "operation not supported" } },
-    { CURVEFS_ERROR::NAMETOOLONG, { ENAMETOOLONG, "file name too long" } },
-    { CURVEFS_ERROR::MOUNT_POINT_EXIST, { -1, "mount point already exist" } },
-    { CURVEFS_ERROR::MOUNT_FAILED, { -1, "mount failed" } },
-    { CURVEFS_ERROR::OUT_OF_RANGE, { ERANGE, "out of range" } },
-    { CURVEFS_ERROR::NODATA, { ENODATA, "no data available" } },
-    { CURVEFS_ERROR::IO_ERROR, { EIO, "I/O error" } },
-    { CURVEFS_ERROR::STALE, { ESTALE, "stale file handler" } },
-    { CURVEFS_ERROR::NOSYS, { ENOSYS, "invalid system call" } },
-};
-
-std::string StrErr(CURVEFS_ERROR code) {
-    auto it = errors.find(code);
-    if (it != errors.end()) {
-        return it->second.second;
-    }
-    return "unknown";
-}
-
-int SysErr(CURVEFS_ERROR code) {
-    int syscode = -1;
-    auto it = errors.find(code);
-    if (it != errors.end()) {
-        syscode = it->second.first;
-    }
-    return (syscode == -1) ? EIO : syscode;
-}
-
-std::ostream &operator<<(std::ostream &os, CURVEFS_ERROR code) {
-    os << static_cast<int>(code) << "[" << [code]() {
-        auto it = errors.find(code);
-        if (it != errors.end()) {
-            return it->second.second;
-        }
-
-        return std::string{"Unknown"};
-    }() << "]";
-
-    return os;
-}
-
-CURVEFS_ERROR ToFSError(MetaStatusCode code) {
-    static std::map<MetaStatusCode, CURVEFS_ERROR> errs = {
-        { MetaStatusCode::OK, CURVEFS_ERROR::OK },
-        { MetaStatusCode::NOT_FOUND, CURVEFS_ERROR::NOTEXIST },
-        { MetaStatusCode::PARAM_ERROR, CURVEFS_ERROR::INVALIDPARAM },
-        { MetaStatusCode::INODE_EXIST, CURVEFS_ERROR::EXISTS },
-        { MetaStatusCode::DENTRY_EXIST, CURVEFS_ERROR::EXISTS },
-        { MetaStatusCode::SYM_LINK_EMPTY, CURVEFS_ERROR::INTERNAL },
-        { MetaStatusCode::RPC_ERROR, CURVEFS_ERROR::INTERNAL },
-    };
-
-    auto it = errs.find(code);
-    if (it != errs.end()) {
-        return it->second;
-    }
-    return CURVEFS_ERROR::UNKNOWN;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/filesystem.cpp

@@ -1,353 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-08
- * Author: Jingli Chen (Wine93)
- */
-
-#include <map>
-
-#include "curvefs/src/client/filesystem/filesystem.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-FileSystem::FileSystem(FileSystemOption option, ExternalMember member)
-    : option_(option), member(member) {
-    deferSync_ = std::make_shared<DeferSync>(option.deferSyncOption);
-    negative_ = std::make_shared<LookupCache>(option.lookupCacheOption);
-    dirCache_ = std::make_shared<DirCache>(option.dirCacheOption);
-    openFiles_ = std::make_shared<OpenFiles>(option_.openFilesOption,
-                                             deferSync_);
-    attrWatcher_ = std::make_shared<AttrWatcher>(option_.attrWatcherOption,
-                                                 openFiles_, dirCache_);
-    handlerManager_ = std::make_shared<HandlerManager>();
-    rpc_ = std::make_shared<RPCClient>(option.rpcOption, member);
-}
-
-void FileSystem::Run() {
-    deferSync_->Start();
-    dirCache_->Start();
-}
-
-void FileSystem::Destory() {
-    openFiles_->CloseAll();
-    deferSync_->Stop();
-    dirCache_->Stop();
-}
-
-void FileSystem::Attr2Stat(InodeAttr* attr, struct stat* stat) {
-    std::memset(stat, 0, sizeof(struct stat));
-    stat->st_ino = attr->inodeid();  //  inode number
-    stat->st_mode = attr->mode();  // permission mode
-    stat->st_nlink = attr->nlink();  // number of links
-    stat->st_uid = attr->uid();  // user ID of owner
-    stat->st_gid = attr->gid();  // group ID of owner
-    stat->st_size = attr->length();  // total size, in bytes
-    stat->st_rdev = attr->rdev();  // device ID (if special file)
-    stat->st_atim.tv_sec = attr->atime();  // time of last access
-    stat->st_atim.tv_nsec = attr->atime_ns();
-    stat->st_mtim.tv_sec = attr->mtime();  // time of last modification
-    stat->st_mtim.tv_nsec = attr->mtime_ns();
-    stat->st_ctim.tv_sec = attr->ctime();  // time of last status change
-    stat->st_ctim.tv_nsec = attr->ctime_ns();
-    stat->st_blksize = option_.blockSize;  // blocksize for file system I/O
-    stat->st_blocks = 0;  // number of 512B blocks allocated
-    if (IsS3File(*attr)) {
-        stat->st_blocks = (attr->length() + 511) / 512;
-    }
-}
-
-void FileSystem::Entry2Param(EntryOut* entryOut,
-                             fuse_entry_param* e) {
-    std::memset(e, 0, sizeof(fuse_entry_param));
-    e->ino = entryOut->attr.inodeid();
-    e->generation = 0;
-    Attr2Stat(&entryOut->attr, &e->attr);
-    e->entry_timeout = entryOut->entryTimeout;
-    e->attr_timeout = entryOut->attrTimeout;
-}
-
-void FileSystem::SetEntryTimeout(EntryOut* entryOut) {
-    auto option = option_.kernelCacheOption;
-    if (IsDir(entryOut->attr)) {
-        entryOut->entryTimeout = option.dirEntryTimeoutSec;
-        entryOut->attrTimeout = option.dirAttrTimeoutSec;
-    } else {
-        entryOut->entryTimeout = option.entryTimeoutSec;
-        entryOut->attrTimeout = option.attrTimeoutSec;
-    }
-}
-
-void FileSystem::SetAttrTimeout(AttrOut* attrOut) {
-    auto option = option_.kernelCacheOption;
-    if (IsDir(attrOut->attr)) {
-        attrOut->attrTimeout = option.dirAttrTimeoutSec;
-    } else {
-        attrOut->attrTimeout = option.attrTimeoutSec;
-    }
-}
-
-// fuse reply*
-void FileSystem::ReplyError(Request req, CURVEFS_ERROR code) {
-    fuse_reply_err(req, SysErr(code));
-}
-
-void FileSystem::ReplyEntry(Request req,
-                            EntryOut* entryOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
-                             ReplyType::ATTR, true);
-    fuse_entry_param e;
-    SetEntryTimeout(entryOut);
-    Entry2Param(entryOut, &e);
-    fuse_reply_entry(req, &e);
-}
-
-void FileSystem::ReplyAttr(Request req,
-                           AttrOut* attrOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &attrOut->attr,
-                             ReplyType::ATTR, true);
-    struct stat stat;
-    SetAttrTimeout(attrOut);
-    Attr2Stat(&attrOut->attr, &stat);
-    fuse_reply_attr(req, &stat, attrOut->attrTimeout);
-}
-
-void FileSystem::ReplyReadlink(Request req, const std::string& link) {
-    fuse_reply_readlink(req, link.c_str());
-}
-
-void FileSystem::ReplyOpen(Request req, FileInfo* fi) {
-    fuse_reply_open(req, fi);
-}
-
-void FileSystem::ReplyOpen(Request req, FileOut* fileOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
-                             ReplyType::ONLY_LENGTH, true);
-    fuse_reply_open(req, fileOut->fi);
-}
-
-void FileSystem::ReplyData(Request req,
-                           struct fuse_bufvec *bufv,
-                           enum fuse_buf_copy_flags flags) {
-    fuse_reply_data(req, bufv, flags);
-}
-
-void FileSystem::ReplyWrite(Request req, FileOut* fileOut) {
-    AttrWatcherGuard watcher(attrWatcher_, &fileOut->attr,
-                             ReplyType::ONLY_LENGTH, true);
-    fuse_reply_write(req, fileOut->nwritten);
-}
-
-void FileSystem::ReplyBuffer(Request req, const char *buf, size_t size) {
-    fuse_reply_buf(req, buf, size);
-}
-
-void FileSystem::ReplyStatfs(Request req, const struct statvfs *stbuf) {
-    fuse_reply_statfs(req, stbuf);
-}
-
-void FileSystem::ReplyXattr(Request req, size_t size) {
-    fuse_reply_xattr(req, size);
-}
-
-void FileSystem::ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi) {
-    AttrWatcherGuard watcher(attrWatcher_, &entryOut->attr,
-                             ReplyType::ATTR, true);
-    fuse_entry_param e;
-    SetEntryTimeout(entryOut);
-    Entry2Param(entryOut, &e);
-    fuse_reply_create(req, &e, fi);
-}
-
-void FileSystem::AddDirEntry(Request req,
-                             DirBufferHead* buffer,
-                             DirEntry* dirEntry) {
-    struct stat stat;
-    std::memset(&stat, 0, sizeof(stat));
-    stat.st_ino = dirEntry->ino;
-
-    // add a directory entry to the buffer
-    size_t oldsize = buffer->size;
-    const char* name = dirEntry->name.c_str();
-    buffer->size += fuse_add_direntry(req, NULL, 0, name, NULL, 0);
-    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
-    fuse_add_direntry(req,
-                      buffer->p + oldsize,  // char* buf
-                      buffer->size - oldsize,  // size_t bufisze
-                      name, &stat, buffer->size);
-}
-
-void FileSystem::AddDirEntryPlus(Request req,
-                                 DirBufferHead* buffer,
-                                 DirEntry* dirEntry) {
-    AttrWatcherGuard watcher(attrWatcher_, &dirEntry->attr,
-                             ReplyType::ATTR, false);
-    struct fuse_entry_param e;
-    EntryOut entryOut(dirEntry->attr);
-    SetEntryTimeout(&entryOut);
-    Entry2Param(&entryOut, &e);
-
-    // add a directory entry to the buffer with the attributes
-    size_t oldsize = buffer->size;
-    const char* name = dirEntry->name.c_str();
-    buffer->size += fuse_add_direntry_plus(req, NULL, 0, name, NULL, 0);
-    buffer->p = static_cast<char *>(realloc(buffer->p, buffer->size));
-    fuse_add_direntry_plus(req,
-                           buffer->p + oldsize,  // char* buf
-                           buffer->size - oldsize,  // size_t bufisze
-                           name, &e, buffer->size);
-}
-
-// handler*
-std::shared_ptr<FileHandler> FileSystem::NewHandler() {
-    return handlerManager_->NewHandler();
-}
-
-std::shared_ptr<FileHandler> FileSystem::FindHandler(uint64_t fh) {
-    return handlerManager_->FindHandler(fh);
-}
-
-void FileSystem::ReleaseHandler(uint64_t fh) {
-    return handlerManager_->ReleaseHandler(fh);
-}
-
-FileSystemMember FileSystem::BorrowMember() {
-    return FileSystemMember(deferSync_, openFiles_, attrWatcher_);
-}
-
-// fuse request*
-CURVEFS_ERROR FileSystem::Lookup(Request req,
-                                 Ino parent,
-                                 const std::string& name,
-                                 EntryOut* entryOut) {
-    if (name.size() > option_.maxNameLength) {
-        return CURVEFS_ERROR::NAMETOOLONG;
-    }
-
-    bool yes = negative_->Get(parent, name);
-    if (yes) {
-        return CURVEFS_ERROR::NOTEXIST;
-    }
-
-    auto rc = rpc_->Lookup(parent, name, entryOut);
-    if (rc == CURVEFS_ERROR::OK) {
-        negative_->Delete(parent, name);
-    } else if (rc == CURVEFS_ERROR::NOTEXIST) {
-        negative_->Put(parent, name);
-    }
-    return rc;
-}
-
-CURVEFS_ERROR FileSystem::GetAttr(Request req, Ino ino, AttrOut* attrOut) {
-    InodeAttr attr;
-    auto rc = rpc_->GetAttr(ino, &attr);
-    if (rc == CURVEFS_ERROR::OK) {
-        *attrOut = AttrOut(attr);
-    }
-    return rc;
-}
-
-CURVEFS_ERROR FileSystem::OpenDir(Request req, Ino ino, FileInfo* fi) {
-    InodeAttr attr;
-    CURVEFS_ERROR rc = rpc_->GetAttr(ino, &attr);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    // revalidate directory cache
-    std::shared_ptr<DirEntryList> entries;
-    bool yes = dirCache_->Get(ino, &entries);
-    if (yes) {
-        if (entries->GetMtime() != AttrMtime(attr)) {
-            dirCache_->Drop(ino);
-        }
-    }
-
-    auto handler = NewHandler();
-    handler->mtime = AttrMtime(attr);
-    fi->fh = handler->fh;
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::ReadDir(Request req,
-                                  Ino ino,
-                                  FileInfo* fi,
-                                  std::shared_ptr<DirEntryList>* entries) {
-    bool yes = dirCache_->Get(ino, entries);
-    if (yes) {
-        return CURVEFS_ERROR::OK;
-    }
-
-    CURVEFS_ERROR rc = rpc_->ReadDir(ino, entries);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    (*entries)->SetMtime(FindHandler(fi->fh)->mtime);
-    dirCache_->Put(ino, *entries);
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::ReleaseDir(Request req, Ino ino, FileInfo* fi) {
-    ReleaseHandler(fi->fh);
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::Open(Request req, Ino ino, FileInfo* fi) {
-    std::shared_ptr<InodeWrapper> inode;
-    bool yes = openFiles_->IsOpened(ino, &inode);
-    if (yes) {
-        openFiles_->Open(ino, inode);
-        // fi->keep_cache = 1;
-        return CURVEFS_ERROR::OK;
-    }
-
-    CURVEFS_ERROR rc = rpc_->Open(ino, &inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    TimeSpec mtime;
-    yes = attrWatcher_->GetMtime(ino, &mtime);
-    if (!yes) {
-        // It is rare which only arise when attribute evited for attr-watcher.
-        LOG(WARNING) << "open(" << ino << "): stale file handler"
-                     << ": attribute not found in wacther";
-        return CURVEFS_ERROR::STALE;
-    } else if (mtime != InodeMtime(inode)) {
-        LOG(WARNING) << "open(" << ino << "): stale file handler"
-                     << ", cache(" << mtime << ") vs remote("
-                     << InodeMtime(inode) << ")";
-        return CURVEFS_ERROR::STALE;
-    }
-
-    openFiles_->Open(ino, inode);
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FileSystem::Release(Request req, Ino ino) {
-    openFiles_->Close(ino);
-    return CURVEFS_ERROR::OK;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/filesystem.h

@@ -1,167 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-08
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_
-
-#include <gtest/gtest_prod.h>
-
-#include <memory>
-#include <string>
-
-#include "curvefs/src/client/filesystem/error.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/package.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/lookup_cache.h"
-#include "curvefs/src/client/filesystem/dir_cache.h"
-#include "curvefs/src/client/filesystem/openfile.h"
-#include "curvefs/src/client/filesystem/attr_watcher.h"
-#include "curvefs/src/client/filesystem/rpc_client.h"
-#include "curvefs/src/client/filesystem/defer_sync.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curvefs::client::common::FileSystemOption;
-
-struct FileSystemMember {
-    FileSystemMember(std::shared_ptr<DeferSync> deferSync,
-                     std::shared_ptr<OpenFiles> openFiles,
-                     std::shared_ptr<AttrWatcher> attrWatcher)
-        : deferSync(deferSync),
-          openFiles(openFiles),
-          attrWatcher(attrWatcher) {}
-
-    std::shared_ptr<DeferSync> deferSync;
-    std::shared_ptr<OpenFiles> openFiles;
-    std::shared_ptr<AttrWatcher> attrWatcher;
-};
-
-class FileSystem {
- public:
-    FileSystem(struct FileSystemOption option, ExternalMember member);
-
-    void Run();
-
-    void Destory();
-
-    // fuse request
-    CURVEFS_ERROR Lookup(Request req,
-                         Ino parent,
-                         const std::string& name,
-                         EntryOut* entryOut);
-
-    CURVEFS_ERROR GetAttr(Request req, Ino ino, AttrOut* attrOut);
-
-    CURVEFS_ERROR OpenDir(Request req, Ino ino, FileInfo* fi);
-
-    CURVEFS_ERROR ReadDir(Request req,
-                          Ino ino,
-                          FileInfo* fi,
-                          std::shared_ptr<DirEntryList>* entries);
-
-    CURVEFS_ERROR ReleaseDir(Request req, Ino ino, FileInfo* fi);
-
-    CURVEFS_ERROR Open(Request req, Ino ino, FileInfo* fi);
-
-    CURVEFS_ERROR Release(Request req, Ino ino);
-
-    // fuse reply: we control all replies to vfs layer in same entrance.
-    void ReplyError(Request req, CURVEFS_ERROR code);
-
-    void ReplyEntry(Request req, EntryOut* entryOut);
-
-    void ReplyAttr(Request req, AttrOut* attrOut);
-
-    void ReplyReadlink(Request req, const std::string& link);
-
-    void ReplyOpen(Request req, FileInfo *fi);
-
-    void ReplyOpen(Request req, FileOut* fileOut);
-
-    void ReplyData(Request req,
-                   struct fuse_bufvec *bufv,
-                   enum fuse_buf_copy_flags flags);
-
-    void ReplyWrite(Request req, FileOut* fileOut);
-
-    void ReplyBuffer(Request req, const char *buf, size_t size);
-
-    void ReplyStatfs(Request req, const struct statvfs *stbuf);
-
-    void ReplyXattr(Request req, size_t size);
-
-    void ReplyCreate(Request req, EntryOut* entryOut, FileInfo* fi);
-
-    void AddDirEntry(Request req,
-                     DirBufferHead* buffer,
-                     DirEntry* dirEntry);
-
-    void AddDirEntryPlus(Request req,
-                         DirBufferHead* buffer,
-                         DirEntry* dirEntry);
-
-    // utility: file handler
-    std::shared_ptr<FileHandler> NewHandler();
-
-    std::shared_ptr<FileHandler> FindHandler(uint64_t fh);
-
-    void ReleaseHandler(uint64_t fh);
-
-    // utility: others
-    FileSystemMember BorrowMember();
-
- private:
-    FRIEND_TEST(FileSystemTest, Attr2Stat);
-    FRIEND_TEST(FileSystemTest, Entry2Param);
-    FRIEND_TEST(FileSystemTest, SetEntryTimeout);
-    FRIEND_TEST(FileSystemTest, SetAttrTimeout);
-
-    // utility: convert to system type.
-    void Attr2Stat(InodeAttr* attr, struct stat* stat);
-
-    void Entry2Param(EntryOut* entryOut, fuse_entry_param* e);
-
-    // utility: set entry/attribute timeout
-    void SetEntryTimeout(EntryOut* entryOut);
-
-    void SetAttrTimeout(AttrOut* attrOut);
-
- private:
-    FileSystemOption option_;
-    ExternalMember member;
-    std::shared_ptr<DeferSync> deferSync_;
-    std::shared_ptr<LookupCache> negative_;
-    std::shared_ptr<DirCache> dirCache_;
-    std::shared_ptr<OpenFiles> openFiles_;
-    std::shared_ptr<AttrWatcher> attrWatcher_;
-    std::shared_ptr<HandlerManager> handlerManager_;
-    std::shared_ptr<RPCClient> rpc_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_FILESYSTEM_H_

curvefs/src/client/filesystem/lookup_cache.cpp

@@ -1,104 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-31
- * Author: Jingli Chen (Wine93)
- */
-
-#include <glog/logging.h>
-
-#include <ctime>
-
-#include "absl/strings/str_format.h"
-#include "curvefs/src/client/filesystem/utils.h"
-#include "curvefs/src/client/filesystem/lookup_cache.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-#define RETURN_FALSE_IF_DISABLED() \
-    do {                           \
-        if (!enable_) {            \
-            return false;          \
-        }                          \
-    } while (0)
-
-
-LookupCache::LookupCache(LookupCacheOption option)
-    : enable_(option.negativeTimeoutSec > 0),
-      rwlock_(),
-      option_(option) {
-    lru_ = std::make_shared<LRUType>(option.lruSize);
-    if (enable_) {
-        LOG(INFO) << "Using lookup negative lru cache"
-                  << ", timeout = " << option.negativeTimeoutSec
-                  << ", capacity = " << option.lruSize;
-    }
-}
-
-std::string LookupCache::CacheKey(Ino parent, const std::string& name) {
-    return absl::StrFormat("%d:%s", parent, name);
-}
-
-bool LookupCache::Get(Ino parent, const std::string& name) {
-    RETURN_FALSE_IF_DISABLED();
-    ReadLockGuard lk(rwlock_);
-    CacheEntry entry;
-    auto key = CacheKey(parent, name);
-    bool yes = lru_->Get(key, &entry);
-    if (!yes) {
-        VLOG(1) << absl::StrFormat("Lookup cache not found: key(%d,%s)",
-                                   parent, name);
-        return false;
-    } else if (entry.uses < option_.minUses) {
-        return false;
-    } else if (entry.expireTime < Now()) {
-        return false;
-    }
-    return true;
-}
-
-bool LookupCache::Put(Ino parent, const std::string& name) {
-    RETURN_FALSE_IF_DISABLED();
-    WriteLockGuard lk(rwlock_);
-    CacheEntry entry;
-    auto key = CacheKey(parent, name);
-    bool yes = lru_->Get(key, &entry);
-    if (yes) {
-        entry.uses++;
-    } else {
-        entry.uses = 0;
-    }
-
-    entry.expireTime = Now() + TimeSpec(option_.negativeTimeoutSec, 0);
-    lru_->Put(key, entry);
-    return true;
-}
-
-bool LookupCache::Delete(Ino parent, const std::string& name) {
-    RETURN_FALSE_IF_DISABLED();
-    WriteLockGuard lk(rwlock_);
-    auto key = CacheKey(parent, name);
-    lru_->Remove(key);
-    return true;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/lookup_cache.h

@@ -1,77 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-31
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_
-
-#include <memory>
-#include <string>
-
-#include "src/common/lru_cache.h"
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::RWLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::common::LookupCacheOption;
-
-// memory cache for lookup result, now we only support cache negative result,
-// and other positive entry will be cached in kernel.
-class LookupCache {
- public:
-    struct CacheEntry {
-        uint32_t uses;
-        TimeSpec expireTime;
-    };
-
-    using LRUType = LRUCache<std::string, CacheEntry>;
-
- public:
-    explicit LookupCache(LookupCacheOption option);
-
-    bool Get(Ino parent, const std::string& name);
-
-    bool Put(Ino parent, const std::string& name);
-
-    bool Delete(Ino parent, const std::string& name);
-
- private:
-    std::string CacheKey(Ino parent, const std::string& name);
-
- private:
-    bool enable_;
-    RWLock rwlock_;
-    LookupCacheOption option_;
-    std::shared_ptr<LRUType> lru_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_LOOKUP_CACHE_H_

curvefs/src/client/filesystem/message_queue.h

@@ -1,118 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_
-
-#include <glog/logging.h>
-
-#include <string>
-#include <atomic>
-#include <thread>
-
-#include "src/common/concurrent/task_queue.h"
-#include "curvefs/src/common/threading.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::TaskQueue;
-using ::curvefs::common::SetThreadName;
-
-template <typename MessageT>
-class MessageQueue {
- public:
-    using MessageHandler = std::function<void(const MessageT& message)>;
-
- public:
-    MessageQueue(const std::string& name, size_t bufferSize)
-        : name_(name),
-          running_(false),
-          thread_(),
-          handler_(),
-          queue_(bufferSize) {}
-
-    void Start() {
-        if (running_.exchange(true)) {
-            return;
-        }
-
-        thread_ = std::thread(&MessageQueue::Consumer, this);
-        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
-                  << "consumer thread start success";
-    }
-
-    void Stop() {
-        if (!running_.exchange(false)) {
-            return;
-        }
-
-        auto wakeup = []() {};
-        queue_.Push(wakeup);
-
-        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
-                  << "consumer thread stoping...";
-
-        thread_.join();
-
-        LOG(INFO) << "MessageQueue [ " << name_ << " ] "
-                  << "consumer thread stopped";
-    }
-
-    void Publish(MessageT message) {
-        if (handler_ != nullptr) {
-            queue_.Push([this, message](){
-                this->handler_(message);
-            });
-        }
-    }
-
-    void Subscribe(MessageHandler handler) {
-        handler_ = handler;
-    }
-
- private:
-    void Consumer() {
-        SetThreadName(name_.c_str());
-        while (running_.load(std::memory_order_relaxed)) {
-            queue_.Pop()();
-        }
-
-        while (queue_.Size() > 0) {
-            queue_.Pop()();
-        }
-    }
-
- private:
-    std::string name_;
-    std::atomic<bool> running_;
-    std::thread thread_;
-    MessageHandler handler_;
-    TaskQueue queue_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_MESSAGE_QUEUE_H_

curvefs/src/client/filesystem/meta.cpp

@@ -1,129 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <sys/stat.h>
-
-#include "absl/strings/str_format.h"
-#include "curvefs/src/client/filesystem/meta.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-HandlerManager::HandlerManager()
-    : mutex_(),
-      dirBuffer_(std::make_shared<DirBuffer>()),
-      handlers_() {}
-
-HandlerManager::~HandlerManager() {
-    dirBuffer_->DirBufferFreeAll();
-}
-
-std::shared_ptr<FileHandler> HandlerManager::NewHandler() {
-    UniqueLock lk(mutex_);
-    auto handler = std::make_shared<FileHandler>();
-    handler->fh = dirBuffer_->DirBufferNew();
-    handler->buffer = dirBuffer_->DirBufferGet(handler->fh);
-    handler->padding = false;
-    handlers_.emplace(handler->fh, handler);
-    return handler;
-}
-
-std::shared_ptr<FileHandler> HandlerManager::FindHandler(uint64_t fh) {
-    UniqueLock lk(mutex_);
-    auto iter = handlers_.find(fh);
-    if (iter == handlers_.end()) {
-        return nullptr;
-    }
-    return iter->second;
-}
-
-void HandlerManager::ReleaseHandler(uint64_t fh) {
-    UniqueLock lk(mutex_);
-    dirBuffer_->DirBufferRelease(fh);
-    handlers_.erase(fh);
-}
-
-std::string StrMode(uint16_t mode) {
-    static std::map<uint16_t, char> type2char = {
-        { S_IFSOCK, 's' },
-        { S_IFLNK, 'l' },
-        { S_IFREG, '-' },
-        { S_IFBLK, 'b' },
-        { S_IFDIR, 'd' },
-        { S_IFCHR, 'c' },
-        { S_IFIFO, 'f' },
-        { 0, '?' },
-    };
-
-    std::string s("?rwxrwxrwx");
-    s[0] = type2char[mode & (S_IFMT & 0xffff)];
-    if (mode & S_ISUID) {
-        s[3] = 's';
-    }
-    if (mode & S_ISGID) {
-        s[6] = 's';
-    }
-    if (mode & S_ISVTX) {
-        s[9] = 't';
-    }
-
-    for (auto i = 0; i < 9; i++) {
-        if ((mode & (1 << i)) == 0) {
-            if ((s[9-i] == 's') || (s[9-i] == 't')) {
-                s[9-i] &= 0xDF;
-            } else {
-                s[9-i] = '-';
-            }
-        }
-    }
-    return s;
-}
-
-namespace {
-
-std::string Attr2Str(const InodeAttr& attr) {
-    if (!attr.IsInitialized()) {
-        return "";
-    }
-
-    std::string smode;
-    return absl::StrFormat(" (%d,[%s:0%06o,%d,%d,%d,%d,%d,%d,%d])",
-        attr.inodeid(), StrMode(attr.mode()).c_str(), attr.mode(), attr.nlink(),
-        attr.uid(), attr.gid(),
-        attr.atime(), attr.mtime(), attr.ctime(),
-        attr.length());
-}
-
-}  // namespace
-
-std::string StrEntry(EntryOut entryOut) {
-    return Attr2Str(entryOut.attr);
-}
-
-std::string StrAttr(AttrOut attrOut) {
-    return Attr2Str(attrOut.attr);
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/meta.h

@@ -1,178 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_
-
-#include <map>
-#include <memory>
-#include <string>
-#include <iostream>
-
-#include "src/common/concurrent/concurrent.h"
-#include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/fuse_common.h"
-#include "curvefs/src/client/dir_buffer.h"
-#include "curvefs/src/client/inode_wrapper.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::Mutex;
-using ::curve::common::RWLock;
-using ::curve::common::UniqueLock;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::metaserver::XAttr;
-using ::curvefs::metaserver::Dentry;
-using ::curvefs::metaserver::InodeAttr;
-using ::curvefs::metaserver::FsFileType;
-using ::curvefs::client::DirBufferHead;
-using ::curvefs::client::InodeWrapper;
-
-using Ino = fuse_ino_t;
-using Request = fuse_req_t;
-using FileInfo = struct fuse_file_info;
-
-struct EntryOut {
-    EntryOut() = default;
-
-    explicit EntryOut(InodeAttr attr) : attr(attr) {}
-
-    InodeAttr attr;
-    double entryTimeout;
-    double attrTimeout;
-};
-
-struct AttrOut {
-    AttrOut() = default;
-
-    explicit AttrOut(InodeAttr attr) : attr(attr) {}
-
-    InodeAttr attr;
-    double attrTimeout;
-};
-
-struct DirEntry {
-    DirEntry() = default;
-
-    DirEntry(Ino ino, const std::string& name, InodeAttr attr)
-        : ino(ino), name(name), attr(attr) {}
-
-    Ino ino;
-    std::string name;
-    InodeAttr attr;
-};
-
-struct FileOut {
-    FileOut() = default;
-
-    FileOut(FileInfo* fi, InodeAttr attr)
-        : fi(fi), attr(attr), nwritten(0) {}
-
-    FileOut(InodeAttr attr, size_t nwritten)
-        : fi(nullptr), attr(attr), nwritten(nwritten) {}
-
-    FileInfo* fi;
-    InodeAttr attr;
-    size_t nwritten;
-};
-
-struct TimeSpec {
-    TimeSpec() : seconds(0), nanoSeconds(0) {}
-
-    TimeSpec(uint64_t seconds, uint32_t nanoSeconds)
-        : seconds(seconds), nanoSeconds(nanoSeconds) {}
-
-    TimeSpec(const TimeSpec& time)
-        : seconds(time.seconds), nanoSeconds(time.nanoSeconds) {}
-
-    TimeSpec& operator=(const TimeSpec& time) = default;
-
-    TimeSpec operator+(const TimeSpec& time) const {
-        return TimeSpec(seconds + time.seconds, nanoSeconds + time.nanoSeconds);
-    }
-
-    uint64_t seconds;
-    uint32_t nanoSeconds;
-};
-
-struct FileHandler {
-    uint64_t fh;
-    DirBufferHead* buffer;
-    TimeSpec mtime;
-    bool padding;  // padding buffer
-};
-
-class HandlerManager {
- public:
-    HandlerManager();
-
-    ~HandlerManager();
-
-    std::shared_ptr<FileHandler> NewHandler();
-
-    std::shared_ptr<FileHandler> FindHandler(uint64_t id);
-
-    void ReleaseHandler(uint64_t id);
-
- private:
-    Mutex mutex_;
-    std::shared_ptr<DirBuffer> dirBuffer_;
-    std::map<uint64_t, std::shared_ptr<FileHandler>> handlers_;
-};
-
-inline bool operator==(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return (lhs.seconds == rhs.seconds) &&
-        (lhs.nanoSeconds == rhs.nanoSeconds);
-}
-
-inline bool operator!=(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return !(lhs == rhs);
-}
-
-inline bool operator<(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return (lhs.seconds < rhs.seconds) ||
-        (lhs.seconds == rhs.seconds && lhs.nanoSeconds < rhs.nanoSeconds);
-}
-
-inline bool operator>(const TimeSpec& lhs, const TimeSpec& rhs) {
-    return (lhs.seconds > rhs.seconds) ||
-        (lhs.seconds == rhs.seconds && lhs.nanoSeconds > rhs.nanoSeconds);
-}
-
-inline std::ostream &operator<<(std::ostream &os, const TimeSpec& time) {
-    return os << time.seconds << "." << time.nanoSeconds;
-}
-
-std::string StrMode(uint16_t mode);
-
-std::string StrEntry(EntryOut entryOut);
-
-std::string StrAttr(AttrOut attrOut);
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_META_H_

curvefs/src/client/filesystem/openfile.cpp

@@ -1,141 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#include "curvefs/src/client/filesystem/openfile.h"
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-OpenFiles::OpenFiles(OpenFilesOption option,
-                     std::shared_ptr<DeferSync> deferSync)
-    : rwlock_(),
-      option_(option),
-      deferSync_(deferSync),
-      files_(std::make_shared<LRUType>(option.lruSize)) {
-    LOG(INFO) << "Using openfile lru cache, capacity " << option.lruSize;
-}
-
-/*
- * Delete(...) does:
- *   1) publish to message queue which will flush file to server with async
- *   2) delete file from lru cache
- */
-void OpenFiles::Delete(Ino ino,
-                       const std::shared_ptr<OpenFile>& file,
-                       bool flush) {
-    if (flush) {
-        deferSync_->Push(file->inode);
-    }
-    files_->Remove(ino);
-
-    VLOG(1) << "Delete open file cache: ino = " << ino
-            << ", refs = " << file->refs
-            << ", mtime = " << InodeMtime(file->inode);
-}
-
-void OpenFiles::Evit(size_t size) {
-    Ino ino;
-    std::shared_ptr<OpenFile> file;
-    while (files_->Size() + size >= option_.lruSize) {
-        bool yes = files_->GetLast(&ino, &file);
-        if (!yes) {
-            break;
-        }
-        Delete(ino, file, true);
-    }
-}
-
-void OpenFiles::Open(Ino ino, std::shared_ptr<InodeWrapper> inode) {
-    WriteLockGuard lk(rwlock_);
-
-    Evit(1);
-
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (yes) {
-        file->refs++;
-        return;
-    }
-
-    file = std::make_shared<OpenFile>(inode);
-    file->refs++;
-    files_->Put(ino, file);
-
-    VLOG(1) << "Insert open file cache: ino = " << ino
-            << ", refs = " << file->refs
-            << ", mtime = " << InodeMtime(file->inode);
-}
-
-bool OpenFiles::IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
-    ReadLockGuard lk(rwlock_);
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (!yes) {
-        return false;
-    }
-    *inode = file->inode;
-    return true;
-}
-
-void OpenFiles::Close(Ino ino) {
-    WriteLockGuard lk(rwlock_);
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (!yes) {
-        return;
-    }
-
-    if (file->refs > 0) {
-        file->refs--;
-    }
-
-    if (file->refs == 0) {
-        Delete(ino, file, false);  // file already flushed before close
-    }
-}
-
-/*
- * CloseAll() does:
- *   flush all file to server and delete from LRU cache
- */
-void OpenFiles::CloseAll() {
-    WriteLockGuard lk(rwlock_);
-    Evit(option_.lruSize);
-}
-
-bool OpenFiles::GetFileAttr(Ino ino, InodeAttr* attr) {
-    ReadLockGuard lk(rwlock_);
-    std::shared_ptr<OpenFile> file;
-    bool yes = files_->Get(ino, &file);
-    if (!yes) {
-        return false;
-    }
-
-    file->inode->GetInodeAttr(attr);
-    return true;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/openfile.h

@@ -1,86 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-09
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_
-
-#include <memory>
-
-#include "src/common/lru_cache.h"
-#include "curvefs/src/client/inode_wrapper.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/defer_sync.h"
-#include "curvefs/src/client/filesystem/dir_cache.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curve::common::LRUCache;
-using ::curve::common::ReadLockGuard;
-using ::curve::common::WriteLockGuard;
-using ::curvefs::client::InodeWrapper;
-using ::curvefs::client::common::OpenFilesOption;
-
-struct OpenFile {
-    explicit OpenFile(std::shared_ptr<InodeWrapper> inode)
-        : inode(inode), refs(0) {}
-
-    std::shared_ptr<InodeWrapper> inode;
-    uint64_t refs;
-};
-
-class OpenFiles {
- public:
-    using LRUType = LRUCache<Ino, std::shared_ptr<OpenFile>>;
-
- public:
-    explicit OpenFiles(OpenFilesOption option,
-                       std::shared_ptr<DeferSync> deferSync);
-
-    void Open(Ino ino, std::shared_ptr<InodeWrapper> inode);
-
-    bool IsOpened(Ino ino, std::shared_ptr<InodeWrapper>* inode);
-
-    void Close(Ino ino);
-
-    void CloseAll();
-
-    bool GetFileAttr(Ino ino, InodeAttr* inode);
-
- private:
-    void Delete(Ino ino, const std::shared_ptr<OpenFile>& file, bool flush);
-
-    void Evit(size_t size);
-
- private:
-    RWLock rwlock_;
-    OpenFilesOption option_;
-    std::shared_ptr<DeferSync> deferSync_;
-    std::shared_ptr<LRUType> files_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_OPENFILE_H_

curvefs/src/client/filesystem/package.h

@@ -1,51 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_
-
-#include <memory>
-
-#include "curvefs/src/client/dentry_cache_manager.h"
-#include "curvefs/src/client/inode_cache_manager.h"
-#include "curvefs/src/client/xattr_manager.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-struct ExternalMember {  // external member depended by FileSystem
-    ExternalMember() = delete;
-    ExternalMember(std::shared_ptr<DentryCacheManager> dentryManager,
-                   std::shared_ptr<InodeCacheManager> inodeManager)
-        : dentryManager(dentryManager),
-          inodeManager(inodeManager) {}
-
-    std::shared_ptr<DentryCacheManager> dentryManager;
-    std::shared_ptr<InodeCacheManager> inodeManager;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_PACKAGE_H_

curvefs/src/client/filesystem/rpc_client.cpp

@@ -1,128 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#include <map>
-#include <set>
-#include <list>
-#include <string>
-#include <memory>
-#include <utility>
-
-#include "curvefs/src/client/filesystem/rpc_client.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-RPCClient::RPCClient(RPCOption option,
-                     ExternalMember member)
-    : option_(option),
-      inodeManager_(member.inodeManager),
-      dentryManager_(member.dentryManager) {}
-
-CURVEFS_ERROR RPCClient::GetAttr(Ino ino, InodeAttr* attr) {
-    CURVEFS_ERROR rc = inodeManager_->GetInodeAttr(ino, attr);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(getattr::GetInodeAttr) failed, retCode = " << rc
-                   << ", ino = " << ino;
-    }
-    return rc;
-}
-
-CURVEFS_ERROR RPCClient::Lookup(Ino parent,
-                                const std::string& name,
-                                EntryOut* entryOut) {
-    Dentry dentry;
-    CURVEFS_ERROR rc = dentryManager_->GetDentry(parent, name, &dentry);
-    if (rc != CURVEFS_ERROR::OK) {
-        if (rc != CURVEFS_ERROR::NOTEXIST) {
-            LOG(ERROR) << "rpc(lookup::GetDentry) failed, retCode = " << rc
-                       << ", parent = " << parent << ", name = " << name;
-        }
-        return rc;
-    }
-
-    Ino ino = dentry.inodeid();
-    rc = inodeManager_->GetInodeAttr(ino, &entryOut->attr);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(lookup::GetInodeAttr) failed, retCode = " << rc
-                   << ", ino = " << ino;
-    }
-    return rc;
-}
-
-CURVEFS_ERROR RPCClient::ReadDir(Ino ino,
-                                 std::shared_ptr<DirEntryList>* entries) {
-    uint32_t limit = option_.listDentryLimit;
-
-    std::list<Dentry> dentries;
-    CURVEFS_ERROR rc = dentryManager_->ListDentry(ino, &dentries, limit);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(readdir::ListDentry) failed, retCode = " << rc
-                   << ", ino = " << ino;
-        return rc;
-    }
-
-    std::set<uint64_t> inos;
-    std::map<uint64_t, InodeAttr> attrs;
-    std::for_each(dentries.begin(), dentries.end(), [&](Dentry& dentry){
-        inos.emplace(dentry.inodeid());
-    });
-    rc = inodeManager_->BatchGetInodeAttrAsync(ino, &inos, &attrs);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(readdir::BatchGetInodeAttrAsync) failed"
-                   << ", retCode = " << rc << ", ino = " << ino;
-        return rc;
-    }
-
-    DirEntry dirEntry;
-    for (const auto& dentry : dentries) {
-        Ino ino = dentry.inodeid();
-        auto iter = attrs.find(ino);
-        if (iter == attrs.end()) {
-            LOG(WARNING) << "rpc(readdir::BatchGetInodeAttrAsync) "
-                         << "missing attribute, ino = " << ino;
-            continue;
-        }
-
-        // NOTE: we can't use std::move() for attribute for hard link
-        // which will sharing inode attribute.
-        dirEntry.ino = ino;
-        dirEntry.name = std::move(dentry.name());
-        dirEntry.attr = iter->second;
-        (*entries)->Add(dirEntry);
-    }
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR RPCClient::Open(Ino ino, std::shared_ptr<InodeWrapper>* inode) {
-    CURVEFS_ERROR rc = inodeManager_->GetInode(ino, *inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "rpc(open/GetInode) failed" << ", retCode = " << rc
-                   << ", ino = " << ino;
-    }
-    return rc;
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/rpc_client.h

@@ -1,65 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-07
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_
-
-#include <memory>
-#include <string>
-
-#include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/package.h"
-#include "curvefs/src/client/filesystem/dir_cache.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-using ::curvefs::client::common::RPCOption;
-
-class RPCClient {
- public:
-    RPCClient(RPCOption option,
-              ExternalMember member);
-
-    CURVEFS_ERROR GetAttr(Ino ino, InodeAttr* attr);
-
-    CURVEFS_ERROR Lookup(Ino parent,
-                         const std::string& name,
-                         EntryOut* entryOut);
-
-    CURVEFS_ERROR ReadDir(Ino ino, std::shared_ptr<DirEntryList>* entries);
-
-    CURVEFS_ERROR Open(Ino ino, std::shared_ptr<InodeWrapper>* inode);
-
- private:
-    RPCOption option_;
-    std::shared_ptr<InodeCacheManager> inodeManager_;
-    std::shared_ptr<DentryCacheManager> dentryManager_;
-};
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_RPC_CLIENT_H_

curvefs/src/client/filesystem/utils.cpp

@@ -1,69 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#include <glog/logging.h>
-
-#include "curvefs/src/client/filesystem/utils.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-bool IsDir(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_DIRECTORY;
-}
-
-bool IsS3File(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_S3;
-}
-
-bool IsVolmeFile(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_FILE;
-}
-
-bool IsSymLink(const InodeAttr& attr) {
-    return attr.type() == FsFileType::TYPE_SYM_LINK;
-}
-
-TimeSpec AttrMtime(const InodeAttr& attr) {
-    return TimeSpec(attr.mtime(), attr.mtime_ns());
-}
-
-TimeSpec AttrCtime(const InodeAttr& attr) {
-    return TimeSpec(attr.ctime(), attr.ctime_ns());
-}
-
-TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode) {
-    InodeAttr attr;
-    inode->GetInodeAttr(&attr);
-    return AttrMtime(attr);
-}
-
-TimeSpec Now() {
-    struct timespec now;
-    clock_gettime(CLOCK_REALTIME, &now);
-    return TimeSpec(now.tv_sec, now.tv_nsec);
-}
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs

curvefs/src/client/filesystem/utils.h

@@ -1,59 +0,0 @@
-/*
- *  Copyright (c) 2023 NetEase Inc.
- *
- *  Licensed under the Apache License, Version 2.0 (the "License");
- *  you may not use this file except in compliance with the License.
- *  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- *  Unless required by applicable law or agreed to in writing, software
- *  distributed under the License is distributed on an "AS IS" BASIS,
- *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  See the License for the specific language governing permissions and
- *  limitations under the License.
- */
-
-/*
- * Project: Curve
- * Created Date: 2023-03-06
- * Author: Jingli Chen (Wine93)
- */
-
-#ifndef CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
-#define CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_
-
-#include <memory>
-
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/package.h"
-
-namespace curvefs {
-namespace client {
-namespace filesystem {
-
-// directory
-bool IsDir(const InodeAttr& attr);
-
-// file which data is stored in s3
-bool IsS3File(const InodeAttr& attr);
-
-// file which data is stored in volume
-bool IsVolmeFile(const InodeAttr& attr);
-
-// symbol link
-bool IsSymLink(const InodeAttr& attr);
-
-struct TimeSpec AttrMtime(const InodeAttr& attr);
-
-struct TimeSpec AttrCtime(const InodeAttr& attr);
-
-struct TimeSpec InodeMtime(const std::shared_ptr<InodeWrapper> inode);
-
-struct TimeSpec Now();
-
-}  // namespace filesystem
-}  // namespace client
-}  // namespace curvefs
-
-#endif  // CURVEFS_SRC_CLIENT_FILESYSTEM_UTILS_H_

curvefs/src/client/fuse_client.cpp

@@ -35,7 +35,7 @@
 
 #include "curvefs/proto/mds.pb.h"
 #include "curvefs/src/client/common/common.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/fuse_common.h"
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/inode_wrapper.h"
@@ -55,10 +55,7 @@ using ::curvefs::mds::topology::PartitionTxId;
 using ::curvefs::mds::FSStatusCode_Name;
 using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH;
 using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
-using ::curvefs::client::filesystem::ExternalMember;
-using ::curvefs::client::filesystem::DirEntry;
-using ::curvefs::client::filesystem::DirEntryList;
-using ::curvefs::client::filesystem::FileOut;
+using ::curvefs::client::common::FileHandle;
 
 #define RETURN_IF_UNSUCCESS(action)                                            \
     do {                                                                       \
@@ -73,21 +70,6 @@ namespace curvefs {
 namespace client {
 namespace common {
 DECLARE_bool(enableCto);
-DECLARE_uint64(fuseClientAvgWriteIops);
-DECLARE_uint64(fuseClientBurstWriteIops);
-DECLARE_uint64(fuseClientBurstWriteIopsSecs);
-
-DECLARE_uint64(fuseClientAvgWriteBytes);
-DECLARE_uint64(fuseClientBurstWriteBytes);
-DECLARE_uint64(fuseClientBurstWriteBytesSecs);
-
-DECLARE_uint64(fuseClientAvgReadIops);
-DECLARE_uint64(fuseClientBurstReadIops);
-DECLARE_uint64(fuseClientBurstReadIopsSecs);
-
-DECLARE_uint64(fuseClientAvgReadBytes);
-DECLARE_uint64(fuseClientBurstReadBytes);
-DECLARE_uint64(fuseClientBurstReadBytesSecs);
 }  // namespace common
 }  // namespace client
 }  // namespace curvefs
@@ -101,27 +83,6 @@ using rpcclient::Cli2ClientImpl;
 using rpcclient::MetaCache;
 using common::FLAGS_enableCto;
 
-using common::FLAGS_fuseClientAvgWriteIops;
-using common::FLAGS_fuseClientBurstWriteIops;
-using common::FLAGS_fuseClientBurstWriteIopsSecs;
-
-using common::FLAGS_fuseClientAvgWriteBytes;
-using common::FLAGS_fuseClientBurstWriteBytes;
-using common::FLAGS_fuseClientBurstWriteBytesSecs;
-
-using common::FLAGS_fuseClientAvgReadIops;
-using common::FLAGS_fuseClientBurstReadIops;
-using common::FLAGS_fuseClientBurstReadIopsSecs;
-
-using common::FLAGS_fuseClientAvgReadBytes;
-using common::FLAGS_fuseClientBurstReadBytes;
-using common::FLAGS_fuseClientBurstReadBytesSecs;
-
-static void on_throttle_timer(void *arg) {
-    FuseClient *fuseClient = reinterpret_cast<FuseClient *>(arg);
-    fuseClient->InitQosParam();
-}
-
 CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     option_ = option;
 
@@ -137,8 +98,7 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     auto channelManager = std::make_shared<ChannelManager<MetaserverID>>();
 
     leaseExecutor_ = absl::make_unique<LeaseExecutor>(option.leaseOpt,
-                                                      metaCache, mdsClient_,
-                                                      &enableSumInDir_);
+                                                      metaCache, mdsClient_);
 
     xattrManager_ = std::make_shared<XattrManager>(inodeManager_,
         dentryManager_, option_.listDentryLimit, option_.listDentryThreads);
@@ -157,11 +117,6 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     curve::client::ClientDummyServerInfo::GetInstance().SetPort(listenPort);
     curve::client::ClientDummyServerInfo::GetInstance().SetIP(localIp);
 
-    {
-        ExternalMember member(dentryManager_, inodeManager_);
-        fs_ = std::make_shared<FileSystem>(option_.fileSystemOption, member);
-    }
-
     MetaStatusCode ret2 =
         metaClient_->Init(option.excutorOpt, option.excutorInternalOpt,
                           metaCache, channelManager);
@@ -169,14 +124,18 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
         return CURVEFS_ERROR::INTERNAL;
     }
 
-    {  // init inode manager
-        auto member = fs_->BorrowMember();
-        CURVEFS_ERROR rc = inodeManager_->Init(option.refreshDataOption,
-                                               member.openFiles,
-                                               member.deferSync);
-        if (rc != CURVEFS_ERROR::OK) {
-            return rc;
-        }
+    CURVEFS_ERROR ret3 =
+        inodeManager_->Init(option.iCacheLruSize, option.enableICacheMetrics,
+                            option.flushPeriodSec, option.refreshDataOption,
+                            option.lruTimeOutSec);
+    if (ret3 != CURVEFS_ERROR::OK) {
+        return ret3;
+    }
+    ret3 =
+        dentryManager_->Init(option.dCacheLruSize, option.enableDCacheMetrics,
+            option.lruTimeOutSec);
+    if (ret3 != CURVEFS_ERROR::OK) {
+        return ret3;
     }
 
     if (warmupManager_ != nullptr) {
@@ -184,9 +143,7 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
         warmupManager_->SetFsInfo(fsInfo_);
     }
 
-    InitQosParam();
-
-    return CURVEFS_ERROR::OK;
+    return ret3;
 }
 
 void FuseClient::UnInit() {
@@ -196,14 +153,11 @@ void FuseClient::UnInit() {
 
     delete mdsBase_;
     mdsBase_ = nullptr;
-
-    while (bthread_timer_del(throttleTimer_) == 1) {
-        bthread_usleep(1000);
-    }
 }
 
 CURVEFS_ERROR FuseClient::Run() {
     if (isStop_.exchange(false)) {
+        inodeManager_->Run();
         return CURVEFS_ERROR::OK;
     }
     return CURVEFS_ERROR::INTERNAL;
@@ -211,6 +165,7 @@ CURVEFS_ERROR FuseClient::Run() {
 
 void FuseClient::Fini() {
     if (!isStop_.exchange(true)) {
+        inodeManager_->Stop();
         xattrManager_->Stop();
     }
 }
@@ -219,7 +174,6 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata,
                                      struct fuse_conn_info *conn) {
     (void)userdata;
     (void)conn;
-    fs_->Run();
     return CURVEFS_ERROR::OK;
 }
 
@@ -229,7 +183,7 @@ void FuseClient::FuseOpDestroy(void *userdata) {
     }
 
     FlushAll();
-    fs_->Destory();
+    dirBuf_->DirBufferFreeAll();
 
     // stop lease before umount fs, otherwise, lease request after umount fs
     // will add a mountpoint entry.
@@ -261,38 +215,91 @@ void FuseClient::FuseOpDestroy(void *userdata) {
               << " success!";
 }
 
-CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       EntryOut* entryOut) {
-    CURVEFS_ERROR rc = fs_->Lookup(req, parent, name, entryOut);
-    if (rc != CURVEFS_ERROR::OK && rc != CURVEFS_ERROR::NOTEXIST) {
-        LOG(ERROR) << "Lookup() failed, retCode = " << rc
-                   << ", parent = " << parent << ", name = " << name;
+void InodeAttr2ParamAttr(const InodeAttr &inodeAttr, struct stat *attr) {
+    attr->st_ino = inodeAttr.inodeid();
+    attr->st_mode = inodeAttr.mode();
+    attr->st_nlink = inodeAttr.nlink();
+    attr->st_uid = inodeAttr.uid();
+    attr->st_gid = inodeAttr.gid();
+    attr->st_size = inodeAttr.length();
+    attr->st_rdev = inodeAttr.rdev();
+    attr->st_atim.tv_sec = inodeAttr.atime();
+    attr->st_atim.tv_nsec = inodeAttr.atime_ns();
+    attr->st_mtim.tv_sec = inodeAttr.mtime();
+    attr->st_mtim.tv_nsec = inodeAttr.mtime_ns();
+    attr->st_ctim.tv_sec = inodeAttr.ctime();
+    attr->st_ctim.tv_nsec = inodeAttr.ctime_ns();
+    attr->st_blksize = kOptimalIOBlockSize;
+
+    switch (inodeAttr.type()) {
+        case metaserver::TYPE_S3:
+            attr->st_blocks = (inodeAttr.length() + 511) / 512;
+            break;
+        default:
+            attr->st_blocks = 0;
+            break;
     }
-    return rc;
 }
 
-CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
-                                          fuse_ino_t ino,
-                                          struct fuse_file_info* fi,
-                                          FileOut* fileOut) {
+void GetDentryParamFromInodeAttr(
+    const FuseClientOption &option,
+    const InodeAttr &inodeAttr,
+    fuse_entry_param *param) {
+    memset(param, 0, sizeof(fuse_entry_param));
+    param->ino = inodeAttr.inodeid();
+    param->generation = 0;
+    InodeAttr2ParamAttr(inodeAttr, &param->attr);
+    param->attr_timeout = option.attrTimeOut;
+    param->entry_timeout = option.entryTimeOut;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
+                                       const char *name, fuse_entry_param *e) {
+    (void)req;
+    VLOG(1) << "FuseOpLookup parent: " << parent
+            << ", name: " << name;
+    if (strlen(name) > option_.maxNameLength) {
+        return CURVEFS_ERROR::NAMETOOLONG;
+    }
+
+    Dentry dentry;
+    CURVEFS_ERROR ret = dentryManager_->GetDentry(parent, name, &dentry);
+    if (ret != CURVEFS_ERROR::OK) {
+        if (ret != CURVEFS_ERROR::NOTEXIST) {
+            LOG(WARNING) << "dentryManager_ get dentry fail, ret = " << ret
+                         << ", parent inodeid = " << parent
+                         << ", name = " << name;
+        }
+        return ret;
+    }
+
+    fuse_ino_t ino = dentry.inodeid();
+    InodeAttr attr;
+    ret = inodeManager_->GetInodeAttr(ino, &attr);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
+    }
+
+    GetDentryParamFromInodeAttr(option_, attr, e);
+    return ret;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
+                                     struct fuse_file_info *fi) {
+    (void)req;
+    VLOG(1) << "FuseOpOpen, ino: " << ino;
     std::shared_ptr<InodeWrapper> inodeWrapper;
-    // alredy opened
     CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
                    << ", inodeid = " << ino;
         return ret;
     }
-
-    fileOut->fi = fi;
-    inodeWrapper->GetInodeAttr(&fileOut->attr);
-
+    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
     if (fi->flags & O_TRUNC) {
         if (fi->flags & O_WRONLY || fi->flags & O_RDWR) {
-            ::curve::common::UniqueLock lgGuard =
-                inodeWrapper->GetUniqueLock();
             uint64_t length = inodeWrapper->GetLengthLocked();
             CURVEFS_ERROR tRet = Truncate(inodeWrapper.get(), 0);
             if (tRet != CURVEFS_ERROR::OK) {
@@ -311,7 +318,7 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
                 inodeWrapper->MarkDirty();
             }
 
-            if (enableSumInDir_.load() && length != 0) {
+            if (enableSumInDir_ && length != 0) {
                 // update parent summary info
                 const Inode *inode = inodeWrapper->GetInodeLocked();
                 XAttr xattr;
@@ -327,29 +334,18 @@ CURVEFS_ERROR FuseClient::HandleOpenFlags(fuse_req_t req,
                     }
                 }
             }
-            inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
         } else {
             return CURVEFS_ERROR::NOPERMISSION;
         }
     }
-    return CURVEFS_ERROR::OK;
-}
-
-CURVEFS_ERROR FuseClient::FuseOpOpen(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     struct fuse_file_info* fi,
-                                     FileOut* fileOut) {
-    CURVEFS_ERROR rc = fs_->Open(req, ino, fi);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "open(" << ino << ") failed, retCode = " << rc;
-        return rc;
+    if (FLAGS_enableCto) {
+        inodeManager_->AddOpenedInode(ino);
     }
-    return HandleOpenFlags(req, ino, fi, fileOut);
+    return ret;
 }
 
 CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
     fuse_ino_t parent, FsFileType type, NlinkChange nlink) {
-
     std::shared_ptr<InodeWrapper> parentInodeWrapper;
     auto ret = inodeManager_->GetInode(parent, parentInodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
@@ -365,27 +361,16 @@ CURVEFS_ERROR FuseClient::UpdateParentMCTimeAndNlink(
         if (FsFileType::TYPE_DIRECTORY == type) {
             parentInodeWrapper->UpdateNlinkLocked(nlink);
         }
-
-        if (option_.fileSystemOption.deferSyncOption.deferDirMtime) {
-            inodeManager_->ShipToFlush(parentInodeWrapper);
-        } else {
-            return parentInodeWrapper->SyncAttr();
-        }
     }
-
+    inodeManager_->ShipToFlush(parentInodeWrapper);
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::MakeNode(
-    fuse_req_t req,
-    fuse_ino_t parent,
-    const char* name,
-    mode_t mode,
-    FsFileType type,
-    dev_t rdev,
-    bool internal,
-    std::shared_ptr<InodeWrapper>& inodeWrapper) {
-    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+CURVEFS_ERROR FuseClient::MakeNode(fuse_req_t req, fuse_ino_t parent,
+                                   const char *name, mode_t mode,
+                                   FsFileType type, dev_t rdev, bool internal,
+                                   fuse_entry_param *e) {
+    if (strlen(name) > option_.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -416,6 +401,7 @@ CURVEFS_ERROR FuseClient::MakeNode(
     param.rdev = rdev;
     param.parent = parent;
 
+    std::shared_ptr<InodeWrapper> inodeWrapper;
     CURVEFS_ERROR ret = inodeManager_->CreateInode(param, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "inodeManager CreateInode fail, ret = " << ret
@@ -467,7 +453,7 @@ CURVEFS_ERROR FuseClient::MakeNode(
             << ", parent = " << parent << ", name = " << name
             << ", mode = " << mode;
 
-    if (enableSumInDir_.load()) {
+    if (enableSumInDir_) {
         // update parent summary info
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
@@ -486,26 +472,20 @@ CURVEFS_ERROR FuseClient::MakeNode(
         }
     }
 
+    InodeAttr attr;
+    inodeWrapper->GetInodeAttr(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req,
-                                      fuse_ino_t parent,
-                                      const char *name,
-                                      mode_t mode,
-                                      EntryOut* entryOut) {
+CURVEFS_ERROR FuseClient::FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
+                                      const char *name, mode_t mode,
+                                      fuse_entry_param *e) {
     VLOG(1) << "FuseOpMkDir, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
     bool internal = false;
-    std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR rc = MakeNode(req, parent, name, S_IFDIR | mode,
-                                FsFileType::TYPE_DIRECTORY, 0, internal, inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    inode->GetInodeAttr(&entryOut->attr);
-    return CURVEFS_ERROR::OK;
+    return MakeNode(req, parent, name, S_IFDIR | mode,
+                    FsFileType::TYPE_DIRECTORY, 0, internal, e);
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
@@ -546,7 +526,7 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent,
                    << ", parent = " << parent << ", name = " << name;
     }
 
-    if (enableSumInDir_.load()) {
+    if (enableSumInDir_) {
         // update parent summary info
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
@@ -564,6 +544,8 @@ CURVEFS_ERROR FuseClient::DeleteNode(uint64_t ino, fuse_ino_t parent,
                          << ", xattr = " << xattr.DebugString();
         }
     }
+
+    inodeManager_->ClearInodeCache(ino);
     return ret;
 }
 
@@ -576,13 +558,10 @@ std::string GetRecycleTimeDirName() {
     return now;
 }
 
-CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req,
-                                           uint64_t parent,
-                                           const char* name,
-                                           mode_t mode,
-                                           ManageInodeType manageType,
-                                           EntryOut* entryOut) {
-    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req, uint64_t parent,
+                    const char *name, mode_t mode, ManageInodeType manageType,
+                    fuse_entry_param *e) {
+    if (strlen(name) > option_.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -647,7 +626,7 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req,
             << ", parent = " << parent << ", name = " << name
             << ", mode = " << mode;
 
-    if (enableSumInDir_.load()) {
+    if (enableSumInDir_) {
         // update parent summary info
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
@@ -666,7 +645,9 @@ CURVEFS_ERROR FuseClient::CreateManageNode(fuse_req_t req,
         }
     }
 
-    inodeWrapper->GetInodeAttrLocked(&entryOut->attr);
+    InodeAttr attr;
+    inodeWrapper->GetInodeAttrLocked(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
@@ -680,10 +661,9 @@ CURVEFS_ERROR FuseClient::GetOrCreateRecycleDir(fuse_req_t req, Dentry *out) {
     } else if (ret == CURVEFS_ERROR::NOTEXIST) {
         LOG(INFO) << "recycle dir is not exist, create " << RECYCLENAME
                   << ", parentid = " << ROOTINODEID;
-        EntryOut entryOut;
+        fuse_entry_param param;
         ret = CreateManageNode(req, ROOTINODEID, RECYCLENAME,
-                               S_IFDIR | 0755, ManageInodeType::TYPE_RECYCLE,
-                               &entryOut);
+                       S_IFDIR | 0755, ManageInodeType::TYPE_RECYCLE, &param);
         if (ret != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "CreateManageNode failed, ret = " << ret
                        << ", inode = " << ROOTINODEID
@@ -720,17 +700,17 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
                    << ", name = " << recycleTimeDirName;
         return ret;
     } else if (ret == CURVEFS_ERROR::NOTEXIST) {
-        std::shared_ptr<InodeWrapper> inode;
+        fuse_entry_param param;
         bool internal = true;
         ret = MakeNode(req, RECYCLEINODEID, recycleTimeDirName.c_str(),
-              S_IFDIR | 0755, FsFileType::TYPE_DIRECTORY, 0, internal, inode);
+              S_IFDIR | 0755, FsFileType::TYPE_DIRECTORY, 0, internal, &param);
         if (ret != CURVEFS_ERROR::OK) {
             LOG(ERROR) << "MakeNode failed, ret = " << ret
                        << ", inode = " << RECYCLEINODEID
                        << ", name = " << recycleTimeDirName;
             return ret;
         }
-        recycleTimeDirIno = inode->GetInodeId();
+        recycleTimeDirIno = param.ino;
     } else {
         recycleTimeDirIno = dentry.inodeid();
     }
@@ -740,7 +720,7 @@ CURVEFS_ERROR FuseClient::MoveToRecycle(fuse_req_t req, fuse_ino_t ino,
 
     // 4. move inode to recycle time dir
     ret = FuseOpRename(req, parent, name,
-                       recycleTimeDirIno, newName.c_str(), 0);
+                                     recycleTimeDirIno, newName.c_str());
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "MoveToRecycle failed, ret = " << ret
                    << ", inodeid = " << ino
@@ -777,7 +757,7 @@ bool FuseClient::ShouldMoveToRecycle(fuse_ino_t parent) {
 
 CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
                                      const char *name, FsFileType type) {
-    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+    if (strlen(name) > option_.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -833,87 +813,146 @@ CURVEFS_ERROR FuseClient::RemoveNode(fuse_req_t req, fuse_ino_t parent,
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req,
-                                        fuse_ino_t ino,
-                                        struct fuse_file_info* fi) {
-    CURVEFS_ERROR rc = fs_->OpenDir(req, ino, fi);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "opendir() failed, retCode = " << rc
-                   << ", ino = " << ino;
+CURVEFS_ERROR FuseClient::FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
+                                        struct fuse_file_info *fi) {
+    (void)req;
+    VLOG(1) << "FuseOpOpenDir ino = " << ino;
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
     }
-    return rc;
+
+    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+
+    uint64_t dindex = dirBuf_->DirBufferNew();
+    fi->fh = dindex;
+    VLOG(1) << "FuseOpOpenDir, ino: " << ino << ", dindex: " << dindex;
+    return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpReadDir(fuse_req_t req,
-                                        fuse_ino_t ino,
-                                        size_t size,
-                                        off_t off,
-                                        struct fuse_file_info *fi,
-                                        char** bufferOut,
-                                        size_t* rSize,
-                                        bool plus) {
-    auto handler = fs_->FindHandler(fi->fh);
-    DirBufferHead* buffer = handler->buffer;
-    if (!handler->padding) {
-        auto entries = std::make_shared<DirEntryList>();
-        CURVEFS_ERROR rc = fs_->ReadDir(req, ino, fi, &entries);
-        if (rc != CURVEFS_ERROR::OK) {
-            LOG(ERROR) << "readdir() failed, retCode = " << rc
-                       << ", ino = " << ino << ", fh = " << fi->fh;
-            return rc;
-        }
+CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
+                                           struct fuse_file_info *fi) {
+    (void)req;
+    uint64_t dindex = fi->fh;
+    VLOG(1) << "FuseOpReleaseDir, ino: " << ino << ", dindex: " << dindex;
+    dirBuf_->DirBufferRelease(dindex);
 
-        entries->Iterate([&](DirEntry* dirEntry){
-            if (plus) {
-                fs_->AddDirEntryPlus(req, buffer, dirEntry);
-            } else {
-                fs_->AddDirEntry(req, buffer, dirEntry);
-            }
-        });
-        handler->padding = true;
-    }
-
-    if (off < buffer->size) {
-        *bufferOut = buffer->p + off;
-        *rSize = std::min(buffer->size - off, size);
-    } else {
-        *bufferOut = nullptr;
-        *rSize = 0;
-    }
+    // release inodeAttr cache
+    inodeManager_->ReleaseCache(ino);
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpReleaseDir(fuse_req_t req,
-                                           fuse_ino_t ino,
-                                           struct fuse_file_info* fi) {
-    CURVEFS_ERROR rc = fs_->ReleaseDir(req, ino, fi);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "releasedir() failed, retCode = " << rc
-                   << ", ino = " << ino;
+static void dirbuf_add(fuse_req_t req, struct DirBufferHead *b,
+                       const Dentry &dentry,
+                       const FuseClientOption &option,
+                       bool cacheDir,
+                       InodeAttr *attr = nullptr) {
+    struct stat stbuf;
+    struct fuse_entry_param param;
+    size_t oldsize = b->size;
+    if (!cacheDir) {
+        b->size += fuse_add_direntry(req, NULL, 0, dentry.name().c_str(),
+                                     NULL, 0);
+        b->p = static_cast<char *>(realloc(b->p, b->size));
+        memset(&stbuf, 0, sizeof(stbuf));
+        stbuf.st_ino = dentry.inodeid();
+        fuse_add_direntry(req, b->p + oldsize, b->size - oldsize,
+                          dentry.name().c_str(), &stbuf, b->size);
+    } else {
+        b->size += fuse_add_direntry_plus(req, NULL, 0, dentry.name().c_str(),
+                                          NULL, 0);
+        b->p = static_cast<char *>(realloc(b->p, b->size));
+        GetDentryParamFromInodeAttr(option, *attr, &param);
+        fuse_add_direntry_plus(req, b->p + oldsize, b->size - oldsize,
+                               dentry.name().c_str(), &param, b->size);
     }
-    return rc;
+}
+
+CURVEFS_ERROR FuseClient::FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
+                                            size_t size, off_t off,
+                                            struct fuse_file_info *fi,
+                                            char **buffer, size_t *rSize,
+                                            bool cacheDir) {
+    VLOG(1) << "FuseOpReadDirPlus ino: " << ino << ", size: " << size
+            << ", off = " << off << ", cacheDir = " << cacheDir;
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
+    }
+
+    uint64_t dindex = fi->fh;
+    DirBufferHead *bufHead = dirBuf_->DirBufferGet(dindex);
+    if (!bufHead->wasRead) {
+        std::list<Dentry> dentryList;
+        std::set<uint64_t> inodeIds;
+        std::map<uint64_t, InodeAttr> inodeAttrMap;
+        auto limit = option_.listDentryLimit;
+        ret = dentryManager_->ListDentry(ino, &dentryList, limit);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "dentryManager_ ListDentry fail, ret = " << ret
+                       << ", parent = " << ino;
+            return ret;
+        }
+
+        if (!cacheDir) {
+            for (const auto &dentry : dentryList) {
+                dirbuf_add(req, bufHead, dentry, option_, cacheDir);
+            }
+        } else {
+            for (const auto &dentry : dentryList) {
+                inodeIds.emplace(dentry.inodeid());
+            }
+            VLOG(3) << "batch get inode size = " << inodeIds.size();
+            ret = inodeManager_->BatchGetInodeAttrAsync(ino, &inodeIds,
+                                                        &inodeAttrMap);
+            if (ret != CURVEFS_ERROR::OK) {
+                LOG(ERROR) << "BatchGetInodeAttr failed when FuseOpReadDir"
+                           << ", parentId = " << ino;
+                return ret;
+            }
+
+            for (const auto &dentry : dentryList) {
+                auto iter = inodeAttrMap.find(dentry.inodeid());
+                if (iter != inodeAttrMap.end()) {
+                    dirbuf_add(req, bufHead, dentry, option_,
+                               cacheDir, &iter->second);
+                } else {
+                    LOG(WARNING) << "BatchGetInodeAttr missing some inodes,"
+                                 << " inodeId = " << dentry.inodeid();
+                }
+            }
+        }
+        bufHead->wasRead = true;
+    }
+
+    if (off < static_cast<ssize_t>(bufHead->size)) {
+        *buffer = bufHead->p + off;
+        *rSize = std::min(bufHead->size - off, size);
+    } else {
+        *buffer = nullptr;
+        *rSize = 0;
+    }
+    return ret;
 }
 
 CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
                                        const char *name, fuse_ino_t newparent,
-                                       const char *newname,
-                                       unsigned int flags) {
+                                       const char *newname) {
+    (void)req;
     VLOG(1) << "FuseOpRename from (" << parent << ", " << name << ") to ("
             << newparent << ", " << newname << ")";
-
-    // TODO(Wine93): the flag RENAME_EXCHANGE and RENAME_NOREPLACE
-    // is only used in linux interface renameat(), not required by posix,
-    // we can ignore it now
-    if (flags != 0) {
-        return CURVEFS_ERROR::INVALIDPARAM;
-    }
-
-    uint64_t maxNameLength = option_.fileSystemOption.maxNameLength;
-    if (strlen(name) > maxNameLength || strlen(newname) > maxNameLength) {
+    if (strlen(name) > option_.maxNameLength ||
+        strlen(newname) > option_.maxNameLength) {
         LOG(WARNING) << "FuseOpRename name too long, name = " << name
                      << ", name len = " << strlen(name) << ", new name = "
                      << newname << ", new name len = " << strlen(newname)
-                     << ", maxNameLength = " << maxNameLength;
+                     << ", maxNameLength = " << option_.maxNameLength;
         return CURVEFS_ERROR::NAMETOOLONG;
     }
 
@@ -944,7 +983,7 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
     renameOp.UpdateInodeCtime();
     renameOp.UpdateCache();
 
-    if (enableSumInDir_.load()) {
+    if (enableSumInDir_) {
         xattrManager_->UpdateParentXattrAfterRename(
             parent, newparent, newname, &renameOp);
     }
@@ -952,24 +991,30 @@ CURVEFS_ERROR FuseClient::FuseOpRename(fuse_req_t req, fuse_ino_t parent,
     return rc;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req,
-                                        fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi,
-                                        AttrOut* attrOut) {
-    CURVEFS_ERROR rc = fs_->GetAttr(req, ino, attrOut);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "getattr() fail, retCode = " << rc
-                   << ", ino = " << ino;
+                                        struct stat *attr) {
+    (void)req;
+    (void)fi;
+    VLOG(1) << "FuseOpGetAttr ino = " << ino;
+    InodeAttr inodeAttr;
+    CURVEFS_ERROR ret =
+        inodeManager_->GetInodeAttr(ino, &inodeAttr);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "inodeManager get inodeAttr fail, ret = " << ret
+                   << ", inodeid = " << ino;
+        return ret;
     }
-    return rc;
+    InodeAttr2ParamAttr(inodeAttr, attr);
+    return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
-                                        fuse_ino_t ino,
-                                        struct stat* attr,
-                                        int to_set,
-                                        struct fuse_file_info* fi,
-                                        struct AttrOut* attrOut) {
+CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
+                                        struct stat *attr, int to_set,
+                                        struct fuse_file_info *fi,
+                                        struct stat *attrOut) {
+    (void)req;
+    (void)fi;
     VLOG(1) << "FuseOpSetAttr to_set: " << to_set << ", ino: " << ino
             << ", attr: " << *attr;
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -1026,9 +1071,11 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
         if (ret != CURVEFS_ERROR::OK) {
             return ret;
         }
-        inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
+        InodeAttr inodeAttr;
+        inodeWrapper->GetInodeAttrLocked(&inodeAttr);
+        InodeAttr2ParamAttr(inodeAttr, attrOut);
 
-        if (enableSumInDir_.load() && changeSize != 0) {
+        if (enableSumInDir_ && changeSize != 0) {
             // update parent summary info
             const Inode* inode = inodeWrapper->GetInodeLocked();
             XAttr xattr;
@@ -1051,7 +1098,9 @@ CURVEFS_ERROR FuseClient::FuseOpSetAttr(fuse_req_t req,
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-    inodeWrapper->GetInodeAttrLocked(&attrOut->attr);
+    InodeAttr inodeAttr;
+    inodeWrapper->GetInodeAttrLocked(&inodeAttr);
+    InodeAttr2ParamAttr(inodeAttr, attrOut);
     return ret;
 }
 
@@ -1061,8 +1110,8 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
     (void)req;
     VLOG(9) << "FuseOpGetXattr, ino: " << ino
             << ", name: " << name << ", size = " << size;
-    if (option_.fileSystemOption.disableXattr) {
-        return CURVEFS_ERROR::NOSYS;
+    if (option_.disableXattr) {
+        return CURVEFS_ERROR::NOTSUPPORT;
     }
 
     InodeAttr inodeAttr;
@@ -1073,8 +1122,7 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
         return ret;
     }
 
-    ret = xattrManager_->GetXattr(name, value,
-                    &inodeAttr, enableSumInDir_.load());
+    ret = xattrManager_->GetXattr(name, value, &inodeAttr, enableSumInDir_);
     if (CURVEFS_ERROR::OK != ret) {
         LOG(ERROR) << "xattrManager get xattr failed, name = " << name;
         return ret;
@@ -1099,6 +1147,12 @@ CURVEFS_ERROR FuseClient::FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseClient::FuseOpSetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, const char* value,
                                          size_t size, int flags) {
+    (void)req;
+    (void)flags;
+    if (option_.disableXattr) {
+        return CURVEFS_ERROR::NOTSUPPORT;
+    }
+
     std::string strname(name);
     std::string strvalue(value, size);
     VLOG(1) << "FuseOpSetXattr ino: " << ino << ", name: " << name
@@ -1181,12 +1235,10 @@ CURVEFS_ERROR FuseClient::FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
     return CURVEFS_ERROR::OUT_OF_RANGE;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req,
-                                        const char* link,
-                                        fuse_ino_t parent,
-                                        const char* name,
-                                        EntryOut* entryOut) {
-    if (strlen(name) > option_.fileSystemOption.maxNameLength) {
+CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req, const char *link,
+                                        fuse_ino_t parent, const char *name,
+                                        fuse_entry_param *e) {
+    if (strlen(name) > option_.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     const struct fuse_ctx *ctx = fuse_req_ctx(req);
@@ -1240,7 +1292,7 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req,
         return ret;
     }
 
-    if (enableSumInDir_.load()) {
+    if (enableSumInDir_) {
         // update parent summary info
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
@@ -1255,17 +1307,18 @@ CURVEFS_ERROR FuseClient::FuseOpSymlink(fuse_req_t req,
         }
     }
 
-    inodeWrapper->GetInodeAttr(&entryOut->attr);
+    InodeAttr attr;
+    inodeWrapper->GetInodeAttr(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     fuse_ino_t newparent,
-                                     const char* newname,
+CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char *newname,
                                      FsFileType type,
-                                     EntryOut* entryOut) {
-    if (strlen(newname) > option_.fileSystemOption.maxNameLength) {
+                                     fuse_entry_param *e) {
+    (void)req;
+    if (strlen(newname) > option_.maxNameLength) {
         return CURVEFS_ERROR::NAMETOOLONG;
     }
     std::shared_ptr<InodeWrapper> inodeWrapper;
@@ -1310,7 +1363,7 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
         return ret;
     }
 
-    if (enableSumInDir_.load()) {
+    if (enableSumInDir_) {
         // update parent summary info
         XAttr xattr;
         xattr.mutable_xattrinfos()->insert({XATTRENTRIES, "1"});
@@ -1326,7 +1379,9 @@ CURVEFS_ERROR FuseClient::FuseOpLink(fuse_req_t req,
         }
     }
 
-    inodeWrapper->GetInodeAttr(&entryOut->attr);
+    InodeAttr attr;
+    inodeWrapper->GetInodeAttr(&attr);
+    GetDentryParamFromInodeAttr(option_, attr, e);
     return ret;
 }
 
@@ -1345,18 +1400,24 @@ CURVEFS_ERROR FuseClient::FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
     return CURVEFS_ERROR::OK;
 }
 
-CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req,
-                                        fuse_ino_t ino,
+CURVEFS_ERROR FuseClient::FuseOpRelease(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info *fi) {
-    CURVEFS_ERROR rc = fs_->Release(req, ino);
-    if (rc != CURVEFS_ERROR::OK) {
-        LOG(ERROR) << "release() failed, ino = " << ino;
+    (void)req;
+    (void)fi;
+    VLOG(1) << "FuseOpRelease, ino: " << ino;
+    if (FLAGS_enableCto) {
+        inodeManager_->RemoveOpenedInode(ino);
     }
-    return rc;
+    return CURVEFS_ERROR::OK;
 }
 
+void FuseClient::FlushInode() { inodeManager_->FlushInodeOnce(); }
+
+void FuseClient::FlushInodeAll() { inodeManager_->FlushAll(); }
+
 void FuseClient::FlushAll() {
     FlushData();
+    FlushInodeAll();
 }
 
 CURVEFS_ERROR
@@ -1385,15 +1446,14 @@ FuseClient::SetMountStatus(const struct MountOption *mountOption) {
     }
     inodeManager_->SetFsId(fsInfo_->fsid());
     dentryManager_->SetFsId(fsInfo_->fsid());
-    enableSumInDir_.store(fsInfo_->enablesumindir());
+    enableSumInDir_ = fsInfo_->enablesumindir() && !FLAGS_enableCto;
     if (fsInfo_->has_recycletimehour()) {
-        enableSumInDir_.store(enableSumInDir_.load() &&
-        (fsInfo_->recycletimehour() == 0));
+        enableSumInDir_ = enableSumInDir_ && (fsInfo_->recycletimehour() == 0);
     }
 
     LOG(INFO) << "Mount " << fsName << " on " << mountpoint_.ShortDebugString()
               << " success!"
-              << " enableSumInDir = " << enableSumInDir_.load();
+              << " enableSumInDir = " << enableSumInDir_;
 
     fsMetric_ = std::make_shared<FSMetric>(fsName);
 
@@ -1411,32 +1471,5 @@ FuseClient::SetMountStatus(const struct MountOption *mountOption) {
     return CURVEFS_ERROR::OK;
 }
 
-void FuseClient::InitQosParam() {
-    ReadWriteThrottleParams params;
-    params.iopsWrite = ThrottleParams(FLAGS_fuseClientAvgWriteIops,
-                                      FLAGS_fuseClientBurstWriteIops,
-                                      FLAGS_fuseClientBurstWriteIopsSecs);
-
-    params.bpsWrite = ThrottleParams(FLAGS_fuseClientAvgWriteBytes,
-                                     FLAGS_fuseClientBurstWriteBytes,
-                                     FLAGS_fuseClientBurstWriteBytesSecs);
-
-    params.iopsRead = ThrottleParams(FLAGS_fuseClientAvgReadIops,
-                                     FLAGS_fuseClientBurstReadIops,
-                                     FLAGS_fuseClientBurstReadIopsSecs);
-
-    params.bpsRead = ThrottleParams(FLAGS_fuseClientAvgReadBytes,
-                                    FLAGS_fuseClientBurstReadBytes,
-                                    FLAGS_fuseClientBurstReadBytesSecs);
-
-    throttle_.UpdateThrottleParams(params);
-
-    int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1),
-                                on_throttle_timer, this);
-    if (ret != 0) {
-        LOG(ERROR) << "Create fuse client throttle timer failed!";
-    }
-}
-
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/fuse_client.h

@@ -25,7 +25,6 @@
 
 #include <unistd.h>
 #include <sys/stat.h>
-#include <bthread/unstable.h>
 
 #include <map>
 #include <memory>
@@ -33,7 +32,6 @@
 #include <list>
 #include <utility>
 #include <vector>
-#include <atomic>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/mds.pb.h"
@@ -55,9 +53,6 @@
 #include "curvefs/src/client/lease/lease_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
-#include "src/common/throttle.h"
-#include "curvefs/src/client/filesystem/meta.h"
-#include "curvefs/src/client/filesystem/filesystem.h"
 
 #define DirectIOAlignment 512
 
@@ -72,7 +67,6 @@ using ::curvefs::client::metric::FSMetric;
 namespace curvefs {
 namespace client {
 
-
 namespace warmup {
 class WarmupManager;
 }
@@ -83,10 +77,6 @@ using rpcclient::MdsClient;
 using rpcclient::MdsClientImpl;
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
-using ::curvefs::client::filesystem::FileSystem;
-using ::curvefs::client::filesystem::EntryOut;
-using ::curvefs::client::filesystem::AttrOut;
-using ::curvefs::client::filesystem::FileOut;
 
 using curvefs::common::is_aligned;
 
@@ -101,6 +91,7 @@ class FuseClient {
         metaClient_(std::make_shared<MetaServerClientImpl>()),
         inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
         dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
+        dirBuf_(std::make_shared<DirBuffer>()),
         fsInfo_(nullptr),
         init_(false),
         enableSumInDir_(false),
@@ -119,6 +110,7 @@ class FuseClient {
             metaClient_(metaClient),
             inodeManager_(inodeManager),
             dentryManager_(dentryManager),
+            dirBuf_(std::make_shared<DirBuffer>()),
             fsInfo_(nullptr),
             init_(false),
             enableSumInDir_(false),
@@ -142,42 +134,31 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                       const char* buf, size_t size, off_t off,
                                       struct fuse_file_info* fi,
-                                      FileOut* fileOut) = 0;
+                                      size_t* wSize) = 0;
 
     virtual CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
                                      size_t size, off_t off,
                                      struct fuse_file_info* fi, char* buffer,
                                      size_t* rSize) = 0;
 
-    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       EntryOut* entryOut);
+    virtual CURVEFS_ERROR FuseOpLookup(fuse_req_t req, fuse_ino_t parent,
+                                       const char* name, fuse_entry_param* e);
 
-    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     struct fuse_file_info* fi,
-                                     FileOut* fileOut);
+    virtual CURVEFS_ERROR FuseOpOpen(fuse_req_t req, fuse_ino_t ino,
+                                     struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       mode_t mode,
+    virtual CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
+                                       const char* name, mode_t mode,
                                        struct fuse_file_info* fi,
-                                       EntryOut* entryOut) = 0;
+                                       fuse_entry_param* e) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
-                                      fuse_ino_t parent,
-                                      const char* name,
-                                      mode_t mode,
-                                      dev_t rdev,
-                                      EntryOut* entryOut) = 0;
+    virtual CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+                                      const char* name, mode_t mode, dev_t rdev,
+                                      fuse_entry_param* e) = 0;
 
-    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req,
-                                      fuse_ino_t parent,
-                                      const char* name,
-                                      mode_t mode,
-                                      EntryOut* entryOut);
+    virtual CURVEFS_ERROR FuseOpMkDir(fuse_req_t req, fuse_ino_t parent,
+                                      const char* name, mode_t mode,
+                                      fuse_entry_param* e);
 
     virtual CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                        const char* name) = 0;
@@ -185,38 +166,30 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpRmDir(fuse_req_t req, fuse_ino_t parent,
                                       const char* name);
 
-    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req,
-                                        fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpOpenDir(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req,
-                                           fuse_ino_t ino,
+    virtual CURVEFS_ERROR FuseOpReleaseDir(fuse_req_t req, fuse_ino_t ino,
                                            struct fuse_file_info* fi);
 
-    virtual CURVEFS_ERROR FuseOpReadDir(fuse_req_t req,
-                                        fuse_ino_t ino,
-                                        size_t size,
-                                        off_t off,
-                                        struct fuse_file_info* fi,
-                                        char** bufferOut,
-                                        size_t* rSize,
-                                        bool plus);
+    virtual CURVEFS_ERROR FuseOpReadDirPlus(fuse_req_t req, fuse_ino_t ino,
+                                            size_t size, off_t off,
+                                            struct fuse_file_info* fi,
+                                            char** buffer, size_t* rSize,
+                                            bool cacheDir);
 
-    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req,
-                                       fuse_ino_t parent,
-                                       const char* name,
-                                       fuse_ino_t newparent,
-                                       const char* newname,
-                                       unsigned int flags);
+    virtual CURVEFS_ERROR FuseOpRename(fuse_req_t req, fuse_ino_t parent,
+                                       const char* name, fuse_ino_t newparent,
+                                       const char* newname);
 
     virtual CURVEFS_ERROR FuseOpGetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct fuse_file_info* fi,
-                                        struct AttrOut* out);
+                                        struct stat* attr);
 
     virtual CURVEFS_ERROR FuseOpSetAttr(fuse_req_t req, fuse_ino_t ino,
                                         struct stat* attr, int to_set,
                                         struct fuse_file_info* fi,
-                                        struct AttrOut* out);
+                                        struct stat* attrOut);
 
     virtual CURVEFS_ERROR FuseOpGetXattr(fuse_req_t req, fuse_ino_t ino,
                                          const char* name, std::string* value,
@@ -229,17 +202,13 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpListXattr(fuse_req_t req, fuse_ino_t ino,
                             char *value, size_t size, size_t *realSize);
 
-    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req,
-                                        const char* link,
-                                        fuse_ino_t parent,
-                                        const char* name,
-                                        EntryOut* entryOut);
+    virtual CURVEFS_ERROR FuseOpSymlink(fuse_req_t req, const char* link,
+                                        fuse_ino_t parent, const char* name,
+                                        fuse_entry_param* e);
 
-    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req,
-                                     fuse_ino_t ino,
-                                     fuse_ino_t newparent,
-                                     const char* newname,
-                                     EntryOut* entryOut) = 0;
+    virtual CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char* newname,
+                                     fuse_entry_param* e) = 0;
 
     virtual CURVEFS_ERROR FuseOpReadLink(fuse_req_t req, fuse_ino_t ino,
                                          std::string* linkStr);
@@ -273,12 +242,10 @@ class FuseClient {
         stbuf->f_fsid = fsInfo_->fsid();
 
         stbuf->f_flag = 0;
-        stbuf->f_namemax = option_.fileSystemOption.maxNameLength;
+        stbuf->f_namemax = option_.maxNameLength;
         return CURVEFS_ERROR::OK;
     }
 
-    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
-
     void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
         fsInfo_ = fsInfo;
         init_ = true;
@@ -294,9 +261,9 @@ class FuseClient {
         return fsInfo_;
     }
 
-    std::shared_ptr<FileSystem> GetFileSystem() {
-        return fs_;
-    }
+    virtual void FlushInode();
+
+    virtual void FlushInodeAll();
 
     virtual void FlushAll();
 
@@ -329,29 +296,18 @@ class FuseClient {
 
     CURVEFS_ERROR SetMountStatus(const struct MountOption *mountOption);
 
-    void Add(bool isRead, size_t size) { throttle_.Add(isRead, size); }
-
-    void InitQosParam();
-
  protected:
-    CURVEFS_ERROR MakeNode(fuse_req_t req,
-                           fuse_ino_t parent,
-                           const char* name,
-                           mode_t mode,
-                           FsFileType type,
-                           dev_t rdev,
-                           bool internal,
-                           std::shared_ptr<InodeWrapper>& InodeWrapper);  // NOLINT
+    CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
+                           mode_t mode, FsFileType type, dev_t rdev,
+                           bool internal, fuse_entry_param* e);
 
     CURVEFS_ERROR RemoveNode(fuse_req_t req, fuse_ino_t parent,
                              const char* name, FsFileType type);
 
-    CURVEFS_ERROR CreateManageNode(fuse_req_t req,
-                                   uint64_t parent,
-                                   const char* name,
-                                   mode_t mode,
+    CURVEFS_ERROR CreateManageNode(fuse_req_t req, uint64_t parent,
+                                   const char *name, mode_t mode,
                                    ManageInodeType manageType,
-                                   EntryOut* entryOut);
+                                   fuse_entry_param *e);
 
     CURVEFS_ERROR GetOrCreateRecycleDir(fuse_req_t req, Dentry *out);
 
@@ -364,17 +320,10 @@ class FuseClient {
 
     bool ShouldMoveToRecycle(fuse_ino_t parent);
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
-                             fuse_ino_t ino,
-                             fuse_ino_t newparent,
-                             const char* newname,
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                             fuse_ino_t newparent, const char* newname,
                              FsFileType type,
-                             EntryOut* entryOut);
-
-    CURVEFS_ERROR HandleOpenFlags(fuse_req_t req,
-                                  fuse_ino_t ino,
-                                  struct fuse_file_info* fi,
-                                  FileOut* fileOut);
+                             fuse_entry_param* e);
 
     int SetHostPortInMountPoint(Mountpoint* out) {
         char hostname[kMaxHostNameLength];
@@ -389,8 +338,9 @@ class FuseClient {
         return 0;
     }
 
-
  private:
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inode, uint64_t length) = 0;
+
     virtual void FlushData() = 0;
 
     CURVEFS_ERROR UpdateParentMCTimeAndNlink(
@@ -401,8 +351,8 @@ class FuseClient {
         std::string newName(name);
         newName = std::to_string(parent) + "_" + std::to_string(ino)
                     + "_" + newName;
-        if (newName.length() > option_.fileSystemOption.maxNameLength) {
-            newName = newName.substr(0, option_.fileSystemOption.maxNameLength);
+        if (newName.length() > option_.maxNameLength) {
+            newName = newName.substr(0, option_.maxNameLength);
         }
 
         return newName;
@@ -426,6 +376,9 @@ class FuseClient {
 
     std::shared_ptr<LeaseExecutor> leaseExecutor_;
 
+    // dir buffer
+    std::shared_ptr<DirBuffer> dirBuf_;
+
     // filesystem info
     std::shared_ptr<FsInfo> fsInfo_;
 
@@ -435,7 +388,7 @@ class FuseClient {
     bool init_;
 
     // enable record summary info in dir inode xattr
-    std::atomic<bool> enableSumInDir_;
+    bool enableSumInDir_;
 
     std::shared_ptr<FSMetric> fsMetric_;
 
@@ -444,18 +397,12 @@ class FuseClient {
     // warmup manager
     std::shared_ptr<warmup::WarmupManager> warmupManager_;
 
-    std::shared_ptr<FileSystem> fs_;
-
  private:
     MDSBaseClient* mdsBase_;
 
     Atomic<bool> isStop_;
 
     curve::common::Mutex renameMutex_;
-
-    Throttle throttle_;
-
-    bthread_timer_t throttleTimer_;
 };
 
 }  // namespace client

curvefs/src/client/fuse_s3_client.cpp

@@ -67,20 +67,12 @@ CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
 
     auto s3Client = std::make_shared<S3ClientImpl>();
     s3Client->Init(opt.s3Opt.s3AdaptrOpt);
-
-    const uint64_t writeCacheMaxByte =
-        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
-    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
-        LOG(ERROR) << "writeCacheMaxByte is too small"
-                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
-                      ", writeCacheMaxByte = " << writeCacheMaxByte;
-        return CURVEFS_ERROR::CACHETOOSMALL;
-    }
-
     auto fsCacheManager = std::make_shared<FsCacheManager>(
         dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte, writeCacheMaxByte,
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheThreads, kvClientManager_);
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
+        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte,
+        opt.s3Opt.s3ClientAdaptorOpt.readCacheThreads,
+        kvClientManager_);
     if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
         DiskCacheType::Disable) {
         auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
@@ -155,8 +147,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
 CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
                                         const char *buf, size_t size, off_t off,
                                         struct fuse_file_info *fi,
-                                        FileOut* fileOut) {
-    size_t *wSize = &fileOut->nwritten;
+                                        size_t *wSize) {
+    (void)req;
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
@@ -218,8 +210,6 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
             }
         }
     }
-
-    inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
     return ret;
 }
 
@@ -282,56 +272,33 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino,
 CURVEFS_ERROR FuseS3Client::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                          const char *name, mode_t mode,
                                          struct fuse_file_info *fi,
-                                         EntryOut* entryOut) {
+                                         fuse_entry_param *e) {
     VLOG(1) << "FuseOpCreate, parent: " << parent << ", name: " << name
             << ", mode: " << mode;
-
-    std::shared_ptr<InodeWrapper> inode;
     CURVEFS_ERROR ret =
-        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, inode);
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, 0, false, e);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-
-    auto openFiles = fs_->BorrowMember().openFiles;
-    openFiles->Open(inode->GetInodeId(), inode);
-
-    inode->GetInodeAttr(&entryOut->attr);
-    return CURVEFS_ERROR::OK;
+    return FuseOpOpen(req, e->ino, fi);
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req,
-                                        fuse_ino_t parent,
-                                        const char* name,
-                                        mode_t mode,
-                                        dev_t rdev,
-                                        EntryOut* entryOut) {
+CURVEFS_ERROR FuseS3Client::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+                                        const char *name, mode_t mode,
+                                        dev_t rdev, fuse_entry_param *e) {
     VLOG(1) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-
-    std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
-                                FsFileType::TYPE_S3, rdev, false,
-                                inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    InodeAttr attr;
-    inode->GetInodeAttr(&attr);
-    *entryOut = EntryOut(attr);
-    return CURVEFS_ERROR::OK;
+    return MakeNode(req, parent, name, mode, FsFileType::TYPE_S3, rdev, false,
+                    e);
 }
 
-CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req,
-                                       fuse_ino_t ino,
-                                       fuse_ino_t newparent,
-                                       const char* newname,
-                                       EntryOut* entryOut) {
+CURVEFS_ERROR FuseS3Client::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char *newname,
+                                     fuse_entry_param *e) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_S3, entryOut);
+        req, ino, newparent, newname, FsFileType::TYPE_S3, e);
 }
 
 CURVEFS_ERROR FuseS3Client::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,

curvefs/src/client/fuse_s3_client.h

@@ -33,7 +33,6 @@
 #include "curvefs/src/client/fuse_client.h"
 #include "curvefs/src/client/s3/client_s3_cache_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
-#include "curvefs/src/volume/common.h"
 #include "src/common/s3_adapter.h"
 
 namespace curvefs {
@@ -41,7 +40,6 @@ namespace client {
 
 using curve::common::GetObjectAsyncContext;
 using curve::common::GetObjectAsyncCallBack;
-using curvefs::volume::kMiB;
 namespace warmup {
 class WarmupManager;
 class WarmupManagerS3Impl;
@@ -81,7 +79,7 @@ class FuseS3Client : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, FileOut* fileOut) override;
+        struct fuse_file_info *fi, size_t *wSize) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
         fuse_ino_t ino, size_t size, off_t off,
@@ -89,25 +87,17 @@ class FuseS3Client : public FuseClient {
         char *buffer,
         size_t *rSize) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
-                               fuse_ino_t parent,
-                               const char* name,
-                               mode_t mode,
-                               struct fuse_file_info *fi,
-                               EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, struct fuse_file_info *fi,
+        fuse_entry_param *e) override;
 
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
-                              fuse_ino_t parent,
-                              const char* name,
-                              mode_t mode,
-                              dev_t rdev,
-                              EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, dev_t rdev,
+        fuse_entry_param *e) override;
 
-    CURVEFS_ERROR FuseOpLink(fuse_req_t req,
-                             fuse_ino_t ino,
-                             fuse_ino_t newparent,
-                             const char *newname,
-                             EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+        fuse_ino_t newparent, const char *newname,
+        fuse_entry_param *e) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
         const char *name) override;
@@ -118,19 +108,17 @@ class FuseS3Client : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
         struct fuse_file_info *fi) override;
 
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
  private:
     bool InitKVCache(const KVClientManagerOpt &opt);
 
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
     void FlushData() override;
 
  private:
     // s3 adaptor
     std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
     std::shared_ptr<KVClientManager> kvClientManager_;
-
-    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };
 
 

curvefs/src/client/fuse_volume_client.cpp

@@ -32,7 +32,7 @@
 #include "absl/cleanup/cleanup.h"
 #include "absl/memory/memory.h"
 #include "curvefs/proto/mds.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/volume/default_volume_storage.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/volume/common.h"
@@ -111,9 +111,12 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpInit(void *userdata,
         volOpts_.allocatorOption.bitmapAllocatorOption.sizePerBit;
     option.allocatorOption.bitmapAllocatorOption.smallAllocProportion =
         volOpts_.allocatorOption.bitmapAllocatorOption.smallAllocProportion;
+    option.threshold = volOpts_.threshold;
+    option.releaseInterSec = volOpts_.releaseInterSec;
 
     spaceManager_ = absl::make_unique<SpaceManagerImpl>(option, mdsClient_,
                                                         blockDeviceClient_);
+    spaceManager_->Run();
 
     storage_ = absl::make_unique<DefaultVolumeStorage>(
         spaceManager_.get(), blockDeviceClient_.get(), inodeManager_.get());
@@ -133,11 +136,11 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
                                             size_t size,
                                             off_t off,
                                             struct fuse_file_info *fi,
-                                            FileOut* fileOut) {
+                                            size_t *wSize) {
+    (void)req;
     VLOG(9) << "write start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
-    size_t* wSize = &fileOut->nwritten;
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
               is_aligned(size, DirectIOAlignment))) {
@@ -149,7 +152,7 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
     butil::Timer timer;
     timer.start();
 
-    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf, fileOut);
+    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf);
     if (ret != CURVEFS_ERROR::OK) {
         if (fsMetric_) {
             fsMetric_->userWrite.eps.count << 1;
@@ -235,57 +238,34 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
 CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
                                              const char *name, mode_t mode,
                                              struct fuse_file_info *fi,
-                                             EntryOut* entryOut) {
+                                             fuse_entry_param *e) {
     VLOG(3) << "FuseOpCreate, parent: " << parent
               << ", name: " << name
               << ", mode: " << mode;
-
-    std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR ret = MakeNode(
-        req, parent, name, mode, FsFileType::TYPE_FILE, 0, false, inode);
+    CURVEFS_ERROR ret =
+        MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, false, e);
     if (ret != CURVEFS_ERROR::OK) {
         return ret;
     }
-
-    auto openFiles = fs_->BorrowMember().openFiles;
-    openFiles->Open(inode->GetInodeId(), inode);
-
-    inode->GetInodeAttr(&entryOut->attr);
-    return CURVEFS_ERROR::OK;
+    return FuseOpOpen(req, e->ino, fi);
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req,
-                                            fuse_ino_t parent,
-                                            const char* name,
-                                            mode_t mode,
-                                            dev_t rdev,
-                                            EntryOut* entryOut) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+                                            const char *name, mode_t mode,
+                                            dev_t rdev, fuse_entry_param *e) {
     VLOG(3) << "FuseOpMkNod, parent: " << parent << ", name: " << name
             << ", mode: " << mode << ", rdev: " << rdev;
-
-    std::shared_ptr<InodeWrapper> inode;
-    CURVEFS_ERROR rc = MakeNode(req, parent, name, mode,
-                                FsFileType::TYPE_FILE, rdev,
-                                false, inode);
-    if (rc != CURVEFS_ERROR::OK) {
-        return rc;
-    }
-
-    InodeAttr attr;
-    inode->GetInodeAttr(&attr);
-    *entryOut = EntryOut(attr);
-    return CURVEFS_ERROR::OK;
+    return MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, rdev,
+                    false, e);
 }
 
-CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req,
-                                           fuse_ino_t ino,
-                                           fuse_ino_t newparent,
-                                           const char* newname,
-                                           EntryOut* entryOut) {
+CURVEFS_ERROR FuseVolumeClient::FuseOpLink(fuse_req_t req, fuse_ino_t ino,
+                                     fuse_ino_t newparent, const char *newname,
+                                     fuse_entry_param *e) {
     VLOG(1) << "FuseOpLink, ino: " << ino << ", newparent: " << newparent
             << ", newname: " << newname;
     return FuseClient::FuseOpLink(
-        req, ino, newparent, newname, FsFileType::TYPE_FILE, entryOut);
+        req, ino, newparent, newname, FsFileType::TYPE_FILE, e);
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,

curvefs/src/client/fuse_volume_client.h

@@ -65,32 +65,24 @@ class FuseVolumeClient : public FuseClient {
 
     CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
         const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, FileOut* fileOut) override;
+        struct fuse_file_info *fi, size_t *wSize) override;
 
     CURVEFS_ERROR FuseOpRead(fuse_req_t req,
             fuse_ino_t ino, size_t size, off_t off,
             struct fuse_file_info *fi,
             char *buffer,
             size_t *rSize) override;
+    CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, struct fuse_file_info *fi,
+        fuse_entry_param *e) override;
 
-    CURVEFS_ERROR FuseOpCreate(fuse_req_t req,
-                               fuse_ino_t parent,
-                               const char* name,
-                               mode_t mode,
-                               struct fuse_file_info* fi,
-                               EntryOut* entryOut) override;
-
-    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req,
-                              fuse_ino_t parent,
-                              const char* name,
-                              mode_t mode,
-                              dev_t rdev,
-                              EntryOut* entryOut) override;
+    CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
+        const char *name, mode_t mode, dev_t rdev,
+        fuse_entry_param *e) override;
 
     CURVEFS_ERROR FuseOpLink(fuse_req_t req, fuse_ino_t ino,
-                             fuse_ino_t newparent,
-                             const char *newname,
-                             EntryOut* entryOut) override;
+        fuse_ino_t newparent, const char *newname,
+        fuse_entry_param *e) override;
 
     CURVEFS_ERROR FuseOpUnlink(fuse_req_t req, fuse_ino_t parent,
                                const char *name) override;
@@ -101,13 +93,13 @@ class FuseVolumeClient : public FuseClient {
     CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
                               struct fuse_file_info *fi) override;
 
-    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
-
     void SetSpaceManagerForTesting(SpaceManager *manager);
 
     void SetVolumeStorageForTesting(VolumeStorage *storage);
 
  private:
+    CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
+
     void FlushData() override;
 
  private:

curvefs/src/client/inode_cache_manager.cpp

@@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/inode_wrapper.h"
 
 using ::curvefs::metaserver::Inode;
@@ -46,11 +46,36 @@ DECLARE_bool(enableCto);
 namespace curvefs {
 namespace client {
 
-using ::curvefs::client::filesystem::ToFSError;
-
 using NameLockGuard = ::curve::common::GenericNameLockGuard<Mutex>;
 using curvefs::client::common::FLAGS_enableCto;
 
+class TrimICacheAsyncDone : public MetaServerClientDone {
+ public:
+    explicit TrimICacheAsyncDone(
+        const std::shared_ptr<InodeWrapper> &inodeWrapper,
+        const std::shared_ptr<InodeCacheManagerImpl> &inodeCacheManager)
+        : inodeWrapper_(inodeWrapper), inodeCacheManager_(inodeCacheManager) {}
+
+    void Run() override {
+        std::unique_ptr<TrimICacheAsyncDone> self_guard(this);
+        MetaStatusCode ret = GetStatusCode();
+        if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
+            LOG(ERROR) << "metaClient_ UpdateInode failed, "
+                       << "MetaStatusCode: " << ret
+                       << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
+                       << ", inodeid: " << inodeWrapper_->GetInodeId();
+            inodeWrapper_->MarkInodeError();
+        }
+        VLOG(9) << "Trim inode " << inodeWrapper_->GetInodeId()
+                << " async success.";
+        inodeCacheManager_->RemoveICache(inodeWrapper_);
+    };
+
+ private:
+    std::shared_ptr<InodeWrapper> inodeWrapper_;
+    std::shared_ptr<InodeCacheManagerImpl> inodeCacheManager_;
+};
+
 #define GET_INODE_REMOTE(FSID, INODEID, OUT, STREAMING)                        \
     MetaStatusCode ret = metaClient_->GetInode(FSID, INODEID, OUT, STREAMING); \
     if (ret != MetaStatusCode::OK) {                                           \
@@ -58,7 +83,17 @@ using curvefs::client::common::FLAGS_enableCto;
             << "metaClient_ GetInode failed, MetaStatusCode = " << ret         \
             << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)          \
             << ", inodeid = " << INODEID;                                      \
-        return ToFSError(ret);                            \
+        return MetaStatusCodeToCurvefsErrCode(ret);                            \
+    }
+
+#define PUT_INODE_CACHE(INODEID, INODEWRAPPER)                                 \
+    std::shared_ptr<InodeWrapper> eliminatedOne;                               \
+    bool eliminated = iCache_->Put(INODEID, INODEWRAPPER, &eliminatedOne);     \
+    if (eliminated) {                                                          \
+        VLOG(3) << "GetInode eliminate one inode, ino: "                       \
+                << eliminatedOne->GetInodeId()                                 \
+                << ", iCache does not evict inodes via put interface ";        \
+        assert(0);                                                             \
     }
 
 #define REFRESH_DATA_REMOTE(OUT, STREAMING)                                    \
@@ -71,8 +106,15 @@ CURVEFS_ERROR
 InodeCacheManagerImpl::GetInode(uint64_t inodeId,
                                 std::shared_ptr<InodeWrapper> &out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
-    bool yes = openFiles_->IsOpened(inodeId, &out);
-    if (yes) {
+    // get inode from cache
+    bool ok = iCache_->Get(inodeId, &out);
+    if (ok && NeedUseCache(inodeId, out, false)) {
+        curve::common::UniqueLock lgGuard = out->GetUniqueLock();
+        if (out->GetType() == FsFileType::TYPE_FILE) {
+            return CURVEFS_ERROR::OK;
+        }
+
+        REFRESH_DATA_REMOTE(out, out->NeedRefreshData());
         return CURVEFS_ERROR::OK;
     }
 
@@ -87,12 +129,24 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
     // refresh data
     REFRESH_DATA_REMOTE(out, streaming);
 
+    // put to cache
+    PUT_INODE_CACHE(inodeId, out);
+
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
                                                   InodeAttr *out) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
+    // 1. find in icache
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    bool ok = iCache_->Get(inodeId, &inodeWrapper);
+    if (ok && NeedUseCache(inodeId, inodeWrapper, true)) {
+        inodeWrapper->GetInodeAttr(out);
+        return CURVEFS_ERROR::OK;
+    }
+
+    // 2. get form metaserver
     std::set<uint64_t> inodeIds;
     std::list<InodeAttr> attrs;
     inodeIds.emplace(inodeId);
@@ -102,7 +156,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
         LOG(ERROR) << "metaClient BatchGetInodeAttr failed"
                    << ", inodeId = " << inodeId << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
 
     if (attrs.size() != 1) {
@@ -119,6 +173,21 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInodeAttr(uint64_t inodeId,
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
     std::set<uint64_t> *inodeIds,
     std::list<InodeAttr> *attrs) {
+    // get some inode attr in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
+            InodeAttr tmpAttr;
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace_back(std::move(tmpAttr));
+            iter = inodeIds->erase(iter);
+            continue;
+        }
+        ++iter;
+    }
+
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -130,7 +199,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return ToFSError(ret);
+    return MetaStatusCodeToCurvefsErrCode(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
@@ -138,6 +207,26 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
     std::set<uint64_t> *inodeIds,
     std::map<uint64_t, InodeAttr> *attrs) {
     NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
+    std::map<uint64_t, InodeAttr> cachedAttr;
+    bool cache  = iAttrCache_->Get(parentId, &cachedAttr);
+
+    // get some inode attr in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
+            InodeAttr tmpAttr;
+            inodeWrapper->GetInodeAttr(&tmpAttr);
+            attrs->emplace(*iter, std::move(tmpAttr));
+            iter = inodeIds->erase(iter);
+        } else if (cache && cachedAttr.find(*iter) != cachedAttr.end()) {
+            attrs->emplace(*iter, cachedAttr[*iter]);
+            iter = inodeIds->erase(iter);
+        } else {
+            ++iter;
+        }
+    }
 
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
@@ -149,12 +238,12 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
         return CURVEFS_ERROR::NOTEXIST;
     }
 
-    ::curve::common::Mutex mutex;
     std::shared_ptr<CountDownEvent> cond =
         std::make_shared<CountDownEvent>(inodeGroups.size());
     for (const auto& it : inodeGroups) {
         VLOG(3) << "BatchGetInodeAttrAsync Send " << it.size();
-        auto* done = new BatchGetInodeAttrAsyncDone(attrs, &mutex, cond);
+        auto* done = new BatchGetInodeAttrAsyncDone(shared_from_this(),
+                                                    cond, parentId);
         MetaStatusCode ret = metaClient_->BatchGetInodeAttrAsync(fsId_, it,
                                                                  done);
         if (MetaStatusCode::OK != ret) {
@@ -167,12 +256,30 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetInodeAttrAsync(
 
     // wait for all sudrequest finished
     cond->Wait();
+
+    bool ok  = iAttrCache_->Get(parentId, attrs);
+    if (!ok) {
+        LOG(WARNING) << "get attrs form iAttrCache_ failed.";
+    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
     std::set<uint64_t> *inodeIds,
     std::list<XAttr> *xattr) {
+    // get some inode in icache
+    for (auto iter = inodeIds->begin(); iter != inodeIds->end();) {
+        std::shared_ptr<InodeWrapper> inodeWrapper;
+        NameLockGuard lock(nameLock_, std::to_string(*iter));
+        bool ok = iCache_->Get(*iter, &inodeWrapper);
+        if (ok && NeedUseCache(*iter, inodeWrapper, true)) {
+            xattr->emplace_back(inodeWrapper->GetXattr());
+            iter = inodeIds->erase(iter);
+        } else {
+            ++iter;
+        }
+    }
+
     if (inodeIds->empty()) {
         return CURVEFS_ERROR::OK;
     }
@@ -183,7 +290,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::BatchGetXAttr(
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
     }
-    return ToFSError(ret);
+    return MetaStatusCodeToCurvefsErrCode(ret);
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
@@ -194,11 +301,23 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateInode(
     if (ret != MetaStatusCode::OK) {
         LOG(ERROR) << "metaClient_ CreateInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret);
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
+    uint64_t inodeid = inode.inodeid();
     out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
         s3ChunkInfoMetric_, option_.maxDataSize,
         option_.refreshDataIntervalSec);
+
+    std::shared_ptr<InodeWrapper> eliminatedOne;
+    bool eliminated = false;
+    {
+        NameLockGuard lock(nameLock_, std::to_string(inodeid));
+        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
+    }
+    if (eliminated) {
+        /* iCache does not evict inodes via put interface */
+        assert(0);
+    }
     return CURVEFS_ERROR::OK;
 }
 
@@ -211,29 +330,159 @@ CURVEFS_ERROR InodeCacheManagerImpl::CreateManageInode(
         LOG(ERROR) << "metaClient_ CreateManageInode failed, MetaStatusCode = "
                    << ret << ", MetaStatusCode_Name = "
                    << MetaStatusCode_Name(ret);
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
+    uint64_t inodeid = inode.inodeid();
     out = std::make_shared<InodeWrapper>(std::move(inode), metaClient_,
         s3ChunkInfoMetric_, option_.maxDataSize,
         option_.refreshDataIntervalSec);
+
+    std::shared_ptr<InodeWrapper> eliminatedOne;
+    bool eliminated = false;
+    {
+        NameLockGuard lock(nameLock_, std::to_string(inodeid));
+        eliminated = iCache_->Put(inodeid, out, &eliminatedOne);
+    }
+    if (eliminated) {
+        /* iCache does not evict inodes via put interface */
+        assert(0);
+    }
     return CURVEFS_ERROR::OK;
 }
 
 CURVEFS_ERROR InodeCacheManagerImpl::DeleteInode(uint64_t inodeId) {
     NameLockGuard lock(nameLock_, std::to_string(inodeId));
+    iCache_->Remove(inodeId);
     MetaStatusCode ret = metaClient_->DeleteInode(fsId_, inodeId);
     if (ret != MetaStatusCode::OK && ret != MetaStatusCode::NOT_FOUND) {
         LOG(ERROR) << "metaClient_ DeleteInode failed, MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeId = " << inodeId;
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
+
+    curve::common::LockGuard lg2(dirtyMapMutex_);
+    dirtyMap_.erase(inodeId);
     return CURVEFS_ERROR::OK;
 }
 
+void InodeCacheManagerImpl::AddInodeAttrs(
+    uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
+    iAttrCache_->Set(parentId, inodeAttrs);
+}
+
+void InodeCacheManagerImpl::ClearInodeCache(uint64_t inodeId) {
+    {
+        NameLockGuard lock(nameLock_, std::to_string(inodeId));
+        iCache_->Remove(inodeId);
+    }
+    curve::common::LockGuard lg2(dirtyMapMutex_);
+    dirtyMap_.erase(inodeId);
+}
+
 void InodeCacheManagerImpl::ShipToFlush(
-    const std::shared_ptr<InodeWrapper>& inode) {
-    deferSync_->Push(inode);
+    const std::shared_ptr<InodeWrapper> &inodeWrapper) {
+    curve::common::LockGuard lg(dirtyMapMutex_);
+    dirtyMap_.emplace(inodeWrapper->GetInodeId(), inodeWrapper);
+}
+
+void InodeCacheManagerImpl::FlushAll() {
+    while (!dirtyMap_.empty()) {
+        FlushInodeOnce();
+    }
+}
+
+void InodeCacheManagerImpl::FlushInodeOnce() {
+    std::map<uint64_t, std::shared_ptr<InodeWrapper>> temp_;
+    {
+        curve::common::LockGuard lg(dirtyMapMutex_);
+        temp_.swap(dirtyMap_);
+    }
+    for (auto it = temp_.begin(); it != temp_.end(); it++) {
+        curve::common::UniqueLock ulk = it->second->GetUniqueLock();
+        it->second->Async(nullptr, true);
+    }
+}
+
+void InodeCacheManagerImpl::ReleaseCache(uint64_t parentId) {
+    NameLockGuard lg(asyncNameLock_, std::to_string(parentId));
+    iAttrCache_->Release(parentId);
+}
+
+void InodeCacheManagerImpl::FlushInodeBackground() {
+    LOG(INFO) << "flush thread is start.";
+    while (!isStop_.load()) {
+        if (iCache_->Size() > maxCacheSize_) {
+            TrimIcache(iCache_->Size() - maxCacheSize_);
+        }
+        FlushInodeOnce();
+        sleeper_.wait_for(std::chrono::seconds(flushPeriodSec_));
+    }
+    LOG(INFO) << "flush thread is stop.";
+}
+
+namespace {
+// Wether an inode is dirty or not.
+// if |needLock| is true, we acquire the lock firstly and do the test,
+// otherwise, we assume the lock is already held.
+bool IsDirtyInode(InodeWrapper *ino, bool needLock) {
+    auto check = [ino]() {
+        return ino->IsDirty() || !ino->S3ChunkInfoEmptyNolock() ||
+               ino->GetMutableExtentCacheLocked()->HasDirtyExtents();
+    };
+
+    if (needLock) {
+        auto lk = ino->GetUniqueLock();
+        return check();
+    }
+
+    return check();
+}
+}  // namespace
+
+void InodeCacheManagerImpl::RemoveICache(
+    const std::shared_ptr<InodeWrapper> &inode) {
+    if (!IsDirtyInode(inode.get(), true)) {
+        uint64_t inodeId = inode->GetInodeId();
+        NameLockGuard lock(nameLock_, std::to_string(inodeId));
+        ::curve::common::UniqueLock lgGuard = inode->GetUniqueLock();
+        iCache_->Remove(inodeId);
+    }
+}
+
+void InodeCacheManagerImpl::TrimIcache(uint64_t trimSize) {
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    uint64_t inodeId;
+    VLOG(3) << "TrimIcache trimSize " << trimSize;
+    while (trimSize > 0) {
+        bool ok = iCache_->GetLast(&inodeId, &inodeWrapper);
+        if (ok) {
+            NameLockGuard lock(nameLock_, std::to_string(inodeId));
+            ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+            if (IsDirtyInode(inodeWrapper.get(), false)) {
+                VLOG(9) << "TrimIcache sync dirty inode " << inodeId;
+                dirtyMapMutex_.lock();
+                dirtyMap_.erase(inodeId);
+                dirtyMapMutex_.unlock();
+                auto *done =
+                    new TrimICacheAsyncDone(inodeWrapper, shared_from_this());
+                inodeWrapper->Async(done);
+            } else {
+                VLOG(9) << "TrimIcache remove inode " << inodeId
+                        << " from iCache";
+                iCache_->Remove(inodeId);
+            }
+            trimSize--;
+            // remove the attr of the inode in iattrcache
+            auto parents = inodeWrapper->GetParentLocked();
+            for (uint64_t parent : parents) {
+                iAttrCache_->Remove(parent, inodeId);
+            }
+        } else {
+            LOG(ERROR) << "icache size " << iCache_->Size();
+            assert(0);
+        }
+    }
 }
 
 CURVEFS_ERROR
@@ -270,5 +519,57 @@ InodeCacheManagerImpl::RefreshData(std::shared_ptr<InodeWrapper> &inode,
     return rc;
 }
 
+void InodeCacheManagerImpl::AddOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "AddOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    openedInodes_.emplace(inodeId);
+}
+
+void InodeCacheManagerImpl::RemoveOpenedInode(uint64_t inodeId) {
+    VLOG(1) << "RemoveOpenedInode inodeId: " <<  inodeId;
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    if (iter != openedInodes_.end()) {
+        openedInodes_.erase(iter);
+    }
+}
+
+bool InodeCacheManagerImpl::OpenInodeCached(uint64_t inodeId) {
+    curve::common::LockGuard lg(openInodesMutex_);
+    auto iter = openedInodes_.find(inodeId);
+    return iter != openedInodes_.end();
+}
+
+bool InodeCacheManagerImpl::NeedUseCache(uint64_t inodeId,
+    const std::shared_ptr<InodeWrapper> &inodeWrapper,
+    bool onlyAttr) {
+    auto lock = inodeWrapper->GetUniqueLock();
+    if (onlyAttr) {
+        if (inodeWrapper->IsDirty()) {
+            return true;
+        }
+    } else {
+        if (IsDirtyInode(inodeWrapper.get(), false)) {
+            return true;
+        }
+    }
+
+    if (((FLAGS_enableCto && OpenInodeCached(inodeId)) || !FLAGS_enableCto)
+        && !IsTimeOut(inodeWrapper)) {
+        return true;
+    }
+    return false;
+}
+
+bool InodeCacheManagerImpl::IsTimeOut(
+    const std::shared_ptr<InodeWrapper> &inodeWrapper) const {
+    uint32_t time = inodeWrapper->GetCachedTime();
+    if (cacheTimeOutSec_ > 0 &&
+        TimeUtility::GetTimeofDaySec() - time >= cacheTimeOutSec_) {
+        return true;
+    }
+    return false;
+}
+
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/inode_cache_manager.h

@@ -38,14 +38,12 @@
 
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "src/common/concurrent/name_lock.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/openfile.h"
-#include "curvefs/src/client/filesystem/defer_sync.h"
 
 using ::curve::common::LRUCache;
 using ::curve::common::CacheMetrics;
@@ -66,8 +64,57 @@ using rpcclient::BatchGetInodeAttrDone;
 using curve::common::CountDownEvent;
 using metric::S3ChunkInfoMetric;
 using common::RefreshDataOption;
-using ::curvefs::client::filesystem::OpenFiles;
-using ::curvefs::client::filesystem::DeferSync;
+
+class InodeAttrCache {
+ public:
+    InodeAttrCache() {}
+    ~InodeAttrCache() {}
+
+    bool Get(uint64_t parentId, std::map<uint64_t, InodeAttr> *imap) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            imap->insert(iter->second.begin(), iter->second.end());
+            return true;
+        }
+        return false;
+    }
+
+    void Set(uint64_t parentId, const RepeatedPtrField<InodeAttr>& inodeAttrs) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        VLOG(1) << "parentId = " << parentId
+                << ", iAttrCache set size = " << inodeAttrs.size();
+        auto& inner = iAttrCache_[parentId];
+        for (const auto &it : inodeAttrs) {
+            inner.emplace(it.inodeid(), it);
+        }
+    }
+
+    void Release(uint64_t parentId) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto size = iAttrCache_.size();
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            iAttrCache_.erase(iter);
+        }
+        VLOG(1) << "inodeId = " << parentId
+                << ", inode attr cache release, before = "
+                << size << ", after = " << iAttrCache_.size();
+    }
+
+    void Remove(uint64_t parentId, uint64_t inodeId) {
+        curve::common::LockGuard lg(iAttrCacheMutex_);
+        auto iter = iAttrCache_.find(parentId);
+        if (iter != iAttrCache_.end()) {
+            iter->second.erase(inodeId);
+        }
+    }
+
+ private:
+    // inodeAttr cache; <parentId, <inodeId, inodeAttr>>
+    std::map<uint64_t, std::map<uint64_t, InodeAttr>> iAttrCache_;
+    curve::common::Mutex iAttrCacheMutex_;
+};
 
 class InodeCacheManager {
  public:
@@ -79,9 +126,14 @@ class InodeCacheManager {
         fsId_ = fsId;
     }
 
-    virtual CURVEFS_ERROR Init(RefreshDataOption option,
-                               std::shared_ptr<OpenFiles> openFiles,
-                               std::shared_ptr<DeferSync> deferSync) = 0;
+    virtual CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+                               uint32_t flushPeriodSec,
+                               RefreshDataOption option,
+                               uint32_t cacheTimeOutSec) = 0;
+
+    virtual void Run() = 0;
+
+    virtual void Stop() = 0;
 
     virtual CURVEFS_ERROR
     GetInode(uint64_t inodeId,
@@ -109,9 +161,24 @@ class InodeCacheManager {
 
     virtual CURVEFS_ERROR DeleteInode(uint64_t inodeId) = 0;
 
+    virtual void AddInodeAttrs(uint64_t parentId,
+        const RepeatedPtrField<InodeAttr>& inodeAttrs) = 0;
+
+    virtual void ClearInodeCache(uint64_t inodeId) = 0;
+
     virtual void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) = 0;
 
+    virtual void FlushAll() = 0;
+
+    virtual void FlushInodeOnce() = 0;
+
+    virtual void ReleaseCache(uint64_t parentId) = 0;
+
+    virtual void AddOpenedInode(uint64_t inodeId) = 0;
+
+    virtual void RemoveOpenedInode(uint64_t inodeId) = 0;
+
  protected:
     uint32_t fsId_;
 };
@@ -120,22 +187,59 @@ class InodeCacheManagerImpl : public InodeCacheManager,
     public std::enable_shared_from_this<InodeCacheManagerImpl> {
  public:
     InodeCacheManagerImpl()
-      : metaClient_(std::make_shared<MetaServerClientImpl>()) {}
+      : metaClient_(std::make_shared<MetaServerClientImpl>()),
+        iCache_(nullptr),
+        iAttrCache_(nullptr),
+        isStop_(true),
+        cacheTimeOutSec_(0) {}
 
     explicit InodeCacheManagerImpl(
         const std::shared_ptr<MetaServerClient> &metaClient)
-      : metaClient_(metaClient) {}
+      : metaClient_(metaClient),
+        iCache_(nullptr),
+        iAttrCache_(nullptr),
+        cacheTimeOutSec_(0) {}
 
-    CURVEFS_ERROR Init(RefreshDataOption option,
-                       std::shared_ptr<OpenFiles> openFiles,
-                       std::shared_ptr<DeferSync> deferSync) override {
+    CURVEFS_ERROR Init(uint64_t cacheSize, bool enableCacheMetrics,
+                       uint32_t flushPeriodSec,
+                       RefreshDataOption option,
+                       uint32_t cacheTimeOutSec) override {
+        if (enableCacheMetrics) {
+            iCache_ = std::make_shared<
+                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0,
+                    std::make_shared<CacheMetrics>("icache"));
+        } else {
+            iCache_ = std::make_shared<
+                LRUCache<uint64_t, std::shared_ptr<InodeWrapper>>>(0);
+        }
+        maxCacheSize_ = cacheSize;
         option_ = option;
+        flushPeriodSec_ = flushPeriodSec;
+        cacheTimeOutSec_ = cacheTimeOutSec;
+        iAttrCache_ = std::make_shared<InodeAttrCache>();
         s3ChunkInfoMetric_ = std::make_shared<S3ChunkInfoMetric>();
-        openFiles_ =  openFiles;
-        deferSync_ = deferSync;
         return CURVEFS_ERROR::OK;
     }
 
+    void Run() {
+        isStop_.exchange(false);
+        flushThread_ =
+            Thread(&InodeCacheManagerImpl::FlushInodeBackground, this);
+        LOG(INFO) << "Start inodeManager flush thread ok.";
+    }
+
+    void Stop() {
+        isStop_.exchange(true);
+        LOG(INFO) << "stop inodeManager flush thread ...";
+        sleeper_.interrupt();
+        flushThread_.join();
+    }
+
+    bool IsDirtyMapExist(uint64_t inodeId) {
+        curve::common::LockGuard lg(dirtyMapMutex_);
+        return dirtyMap_.count(inodeId) > 0;
+    }
+
     CURVEFS_ERROR GetInode(uint64_t inodeId,
                            std::shared_ptr<InodeWrapper> &out) override;
 
@@ -159,37 +263,78 @@ class InodeCacheManagerImpl : public InodeCacheManager,
 
     CURVEFS_ERROR DeleteInode(uint64_t inodeId) override;
 
+    void AddInodeAttrs(uint64_t parentId,
+        const RepeatedPtrField<InodeAttr>& inodeAttrs) override;
+
+    void ClearInodeCache(uint64_t inodeId) override;
+
     void ShipToFlush(
         const std::shared_ptr<InodeWrapper> &inodeWrapper) override;
 
+    void FlushAll() override;
+
+    void FlushInodeOnce() override;
+
+    void ReleaseCache(uint64_t parentId) override;
+
+    void RemoveICache(const std::shared_ptr<InodeWrapper> &inode);
+
+    void AddOpenedInode(uint64_t inodeId) override;
+
+    void RemoveOpenedInode(uint64_t inodeId) override;
+
+    bool NeedUseCache(uint64_t inodeId,
+        const std::shared_ptr<InodeWrapper> &inodeWrapper,
+        bool onlyAttr);
+
+    bool IsTimeOut(const std::shared_ptr<InodeWrapper> &inodeWrapper) const;
+
  private:
+    virtual void FlushInodeBackground();
+    void TrimIcache(uint64_t trimSize);
     CURVEFS_ERROR RefreshData(std::shared_ptr<InodeWrapper> &inode,  // NOLINT
                               bool streaming = true);
+    bool OpenInodeCached(uint64_t inodeId);
 
  private:
     std::shared_ptr<MetaServerClient> metaClient_;
+    std::shared_ptr<LRUCache<uint64_t,
+        std::shared_ptr<InodeWrapper>>> iCache_;
     std::shared_ptr<S3ChunkInfoMetric> s3ChunkInfoMetric_;
 
-    std::shared_ptr<OpenFiles> openFiles_;
+    std::shared_ptr<InodeAttrCache> iAttrCache_;
 
-    std::shared_ptr<DeferSync> deferSync_;
+    // dirty map, key is inodeid
+    std::map<uint64_t, std::shared_ptr<InodeWrapper>> dirtyMap_;
+    curve::common::Mutex dirtyMapMutex_;
+
+    // record opened inode
+    std::multiset<uint64_t> openedInodes_;
+    curve::common::Mutex openInodesMutex_;
 
     curve::common::GenericNameLock<Mutex> nameLock_;
 
     curve::common::GenericNameLock<Mutex> asyncNameLock_;
 
+    uint64_t maxCacheSize_;
     RefreshDataOption option_;
+    uint32_t flushPeriodSec_;
+    Thread flushThread_;
+    InterruptibleSleeper sleeper_;
+    Atomic<bool> isStop_;
+    // cache timeout seconds, 0 means never timeout
+    uint32_t cacheTimeOutSec_;
 };
 
 class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
  public:
-    BatchGetInodeAttrAsyncDone(std::map<uint64_t, InodeAttr>* attrs,
-                               ::curve::common::Mutex* mutex,
-                               std::shared_ptr<CountDownEvent> cond):
-        attrs_(attrs),
-        mutex_(mutex),
-        cond_(cond) {}
-
+    BatchGetInodeAttrAsyncDone(
+        const std::shared_ptr<InodeCacheManager> &inodeCacheManager,
+        std::shared_ptr<CountDownEvent> cond,
+        uint64_t parentId):
+        inodeCacheManager_(inodeCacheManager),
+        cond_(cond),
+        parentId_(parentId) {}
     ~BatchGetInodeAttrAsyncDone() {}
 
     void Run() override {
@@ -197,25 +342,22 @@ class BatchGetInodeAttrAsyncDone : public BatchGetInodeAttrDone {
         MetaStatusCode ret = GetStatusCode();
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "BatchGetInodeAttrAsync failed, "
+                       << "parentId = " << parentId_
                        << ", MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret);
         } else {
             auto inodeAttrs = GetInodeAttrs();
             VLOG(3) << "BatchGetInodeAttrAsyncDone update inodeAttrCache"
                     << " size = " << inodeAttrs.size();
-
-            curve::common::LockGuard lk(*mutex_);
-            for (const auto& attr : inodeAttrs) {
-                attrs_->emplace(attr.inodeid(), attr);
-            }
+            inodeCacheManager_->AddInodeAttrs(parentId_, inodeAttrs);
         }
         cond_->Signal();
     };
 
  private:
-    ::curve::common::Mutex* mutex_;
-    std::map<uint64_t, InodeAttr>* attrs_;
+    std::shared_ptr<InodeCacheManager> inodeCacheManager_;
     std::shared_ptr<CountDownEvent> cond_;
+    uint64_t parentId_;
 };
 
 }  // namespace client

curvefs/src/client/inode_wrapper.cpp

@@ -33,6 +33,7 @@
 
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/async_request_closure.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/src/client/xattr_manager.h"
@@ -46,7 +47,6 @@ namespace client {
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::DataIndices;
-using ::curvefs::client::filesystem::ToFSError;
 
 bvar::Adder<int64_t> g_alive_inode_count{"alive_inode_count"};
 
@@ -193,7 +193,7 @@ CURVEFS_ERROR InodeWrapper::SyncAttr(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
 
         dirty_ = false;
@@ -213,7 +213,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3ChunkInfo(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
         ClearS3ChunkInfoAdd();
     }
@@ -294,7 +294,7 @@ CURVEFS_ERROR InodeWrapper::RefreshS3ChunkInfo() {
                    << "MetaStatusCode: " << ret
                    << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                    << ", inodeid: " << inode_.inodeid();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
     auto before = s3ChunkInfoSize_;
     inode_.mutable_s3chunkinfomap()->swap(s3ChunkInfoMap);
@@ -328,7 +328,7 @@ CURVEFS_ERROR InodeWrapper::Link(uint64_t parent) {
                    <<", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
 
     dirty_ = false;
@@ -396,7 +396,7 @@ CURVEFS_ERROR InodeWrapper::UnLink(uint64_t parent) {
                        << ", MetaStatusCode = " << ret
                        << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                        << ", inodeid = " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
         dirty_ = false;
         dirtyAttr_.Clear();
@@ -427,7 +427,7 @@ CURVEFS_ERROR InodeWrapper::UpdateParent(
                    << ", MetaStatusCode = " << ret
                    << ", MetaStatusCode_Name = " << MetaStatusCode_Name(ret)
                    << ", inodeid = " << inode_.inodeid();
-        return ToFSError(ret);
+        return MetaStatusCodeToCurvefsErrCode(ret);
     }
     dirty_ = false;
     dirtyAttr_.Clear();
@@ -514,7 +514,7 @@ CURVEFS_ERROR InodeWrapper::SyncS3(bool internal) {
                        << "MetaStatusCode: " << ret
                        << ", MetaStatusCode_Name: " << MetaStatusCode_Name(ret)
                        << ", inodeid: " << inode_.inodeid();
-            return ToFSError(ret);
+            return MetaStatusCodeToCurvefsErrCode(ret);
         }
         ClearS3ChunkInfoAdd();
         dirty_ = false;
@@ -583,7 +583,7 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
-    VolumeExtentList extents;
+    VolumeExtentSliceList extents;
     auto st = metaClient_->GetVolumeExtent(inode_.fsid(), inode_.inodeid(),
                                            true, &extents);
     VLOG(9) << "RefreshVolumeExtent, ino: " << inode_.inodeid()
@@ -597,7 +597,7 @@ CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
         LOG(ERROR) << "GetVolumeExtent failed, inodeid: " << inode_.inodeid();
     }
 
-    return ToFSError(st);
+    return MetaStatusCodeToCurvefsErrCode(st);
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshNlink() {

curvefs/src/client/inode_wrapper.h

@@ -34,12 +34,12 @@
 
 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "src/common/concurrent/concurrent.h"
 #include "curvefs/src/client/volume/extent_cache.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/timeutility.h"
-#include "curvefs/src/client/filesystem/error.h"
 
 using ::curvefs::metaserver::Inode;
 using ::curvefs::metaserver::S3ChunkInfoList;
@@ -52,14 +52,16 @@ constexpr int kAccessTime = 1 << 0;
 constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;
 
-using ::curvefs::metaserver::VolumeExtentList;
-using ::curvefs::client::filesystem::CURVEFS_ERROR;
+using ::curvefs::metaserver::VolumeExtentSliceList;
 
 enum class InodeStatus {
     kNormal = 0,
     kError = -1,
 };
 
+// TODO(xuchaojie) : get from conf maybe?
+const uint32_t kOptimalIOBlockSize = 0x10000u;
+
 using rpcclient::MetaServerClient;
 using rpcclient::MetaServerClientImpl;
 using rpcclient::MetaServerClientDone;

curvefs/src/client/lease/lease_excutor.cpp

@@ -41,7 +41,7 @@ LeaseExecutor::~LeaseExecutor() {
 bool LeaseExecutor::Start() {
     if (opt_.leaseTimeUs <= 0 || opt_.refreshTimesPerLease <= 0) {
         LOG(ERROR) << "LeaseExecutor start fail. Invalid param in leaseopt, "
-                      "leaseTimeUs = "
+                      "leasTimesUs = "
                    << opt_.leaseTimeUs
                    << ", refreshTimePerLease = " << opt_.refreshTimesPerLease;
         return false;
@@ -78,8 +78,7 @@ bool LeaseExecutor::RefreshLease() {
     // refresh from mds
     std::vector<PartitionTxId> latestTxIdList;
     FSStatusCode ret = mdsCli_->RefreshSession(txIds, &latestTxIdList,
-                                               fsName_, mountpoint_,
-                                               enableSumInDir_);
+                                               fsName_, mountpoint_);
     if (ret != FSStatusCode::OK) {
         LOG(ERROR) << "LeaseExecutor refresh session fail, ret = " << ret
                    << ", errorName = " << FSStatusCode_Name(ret);

curvefs/src/client/lease/lease_excutor.h

@@ -25,7 +25,6 @@
 
 #include <memory>
 #include <string>
-#include <atomic>
 
 #include "curvefs/src/client/rpcclient/metacache.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
@@ -45,10 +44,8 @@ namespace client {
 class LeaseExecutor : public LeaseExecutorBase {
  public:
     LeaseExecutor(const LeaseOpt &opt, std::shared_ptr<MetaCache> metaCache,
-                  std::shared_ptr<MdsClient> mdsCli,
-                  std::atomic<bool>* enableSumInDir)
-         : opt_(opt), metaCache_(metaCache), mdsCli_(mdsCli),
-         enableSumInDir_(enableSumInDir) {}
+                  std::shared_ptr<MdsClient> mdsCli)
+        : opt_(opt), metaCache_(metaCache), mdsCli_(mdsCli) {}
 
     ~LeaseExecutor();
 
@@ -76,7 +73,6 @@ class LeaseExecutor : public LeaseExecutorBase {
     std::unique_ptr<RefreshSessionTask> task_;
     std::string fsName_;
     Mountpoint mountpoint_;
-    std::atomic<bool>* enableSumInDir_;
 };
 
 }  // namespace client

curvefs/src/client/main.cpp

@@ -51,11 +51,13 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     release : FuseOpRelease,
     fsync : FuseOpFsync,
     opendir : FuseOpOpenDir,
-    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    readdir : 0,
-    #else
+    // TODO(wuhongsong): readdirplus is problematic,
+    // resulting in inconsistent metadata
+    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
+    // readdir : 0,
+    // #else
     readdir : FuseOpReadDir,
-    #endif
+    // #endif
     releasedir : FuseOpReleaseDir,
     fsyncdir : 0,
     statfs : FuseOpStatFs,
@@ -79,11 +81,11 @@ static const struct fuse_lowlevel_ops curve_ll_oper = {
     flock : 0,
     fallocate : 0,
     #endif
-    #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
-    readdirplus : FuseOpReadDirPlus,
-    #else
+    // TODO(wuhongsong): The current implementation is problematic,
+    // resulting in inconsistent metadata
+    // #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 0)
     readdirplus : 0,
-    #endif
+    // #endif
     #if FUSE_VERSION >= FUSE_MAKE_VERSION(3, 4)
     copy_file_range : 0,
     #endif
@@ -220,8 +222,8 @@ int main(int argc, char *argv[]) {
 
     fuse_daemonize(opts.foreground);
 
-    if (InitLog(mOpts.conf, argv[0]) < 0) {
-        printf("Init log failed, confpath = %s\n", mOpts.conf);
+    if (InitGlog(mOpts.conf, argv[0]) < 0) {
+        printf("Init glog failed, confpath = %s\n", mOpts.conf);
     }
 
     ret = InitFuseClient(&mOpts);

curvefs/src/client/metric/client_metric.h

@@ -92,13 +92,12 @@ struct MetaServerClientMetric {
     // volume extent
     InterfaceMetric updateVolumeExtent;
     InterfaceMetric getVolumeExtent;
+    InterfaceMetric updateDeallocatableBlockGroup;
 
     MetaServerClientMetric()
-        : getDentry(prefix, "getDentry"),
-          listDentry(prefix, "listDentry"),
+        : getDentry(prefix, "getDentry"), listDentry(prefix, "listDentry"),
           createDentry(prefix, "createDentry"),
-          deleteDentry(prefix, "deleteDentry"),
-          getInode(prefix, "getInode"),
+          deleteDentry(prefix, "deleteDentry"), getInode(prefix, "getInode"),
           batchGetInodeAttr(prefix, "batchGetInodeAttr"),
           batchGetXattr(prefix, "batchGetXattr"),
           createInode(prefix, "createInode"),
@@ -107,7 +106,9 @@ struct MetaServerClientMetric {
           appendS3ChunkInfo(prefix, "appendS3ChunkInfo"),
           prepareRenameTx(prefix, "prepareRenameTx"),
           updateVolumeExtent(prefix, "updateVolumeExtent"),
-          getVolumeExtent(prefix, "getVolumeExtent") {}
+          getVolumeExtent(prefix, "getVolumeExtent"),
+          updateDeallocatableBlockGroup(prefix,
+                                        "updateDeallocatableBlockGroup") {}
 };
 
 struct InflightGuard {

curvefs/src/client/rpcclient/mds_client.cpp

@@ -501,8 +501,7 @@ FSStatusCode
 MdsClientImpl::RefreshSession(const std::vector<PartitionTxId> &txIds,
                               std::vector<PartitionTxId> *latestTxIdList,
                               const std::string& fsName,
-                              const Mountpoint& mountpoint,
-                              std::atomic<bool>* enableSumInDir) {
+                              const Mountpoint& mountpoint) {
     auto task = RPCTask {
         (void)addrindex;
         (void)rpctimeoutMS;
@@ -533,11 +532,6 @@ MdsClientImpl::RefreshSession(const std::vector<PartitionTxId> &txIds,
             LOG(INFO) << "RefreshSession need update partition txid list: "
                       << response.DebugString();
         }
-        if (enableSumInDir->load() && !response.enablesumindir()) {
-            enableSumInDir->store(response.enablesumindir());
-            LOG(INFO) << "update enableSumInDir to "
-                      << response.enablesumindir();
-        }
 
         return ret;
     };

curvefs/src/client/rpcclient/mds_client.h

@@ -26,7 +26,6 @@
 #include <memory>
 #include <string>
 #include <vector>
-#include <atomic>
 
 #include "curvefs/proto/mds.pb.h"
 #include "curvefs/proto/topology.pb.h"
@@ -116,8 +115,7 @@ class MdsClient {
     RefreshSession(const std::vector<PartitionTxId> &txIds,
                    std::vector<PartitionTxId> *latestTxIdList,
                    const std::string& fsName,
-                   const Mountpoint& mountpoint,
-                   std::atomic<bool>* enableSumInDir) = 0;
+                   const Mountpoint& mountpoint) = 0;
 
     virtual FSStatusCode GetLatestTxId(uint32_t fsId,
                                        std::vector<PartitionTxId>* txIds) = 0;
@@ -204,8 +202,7 @@ class MdsClientImpl : public MdsClient {
     FSStatusCode RefreshSession(const std::vector<PartitionTxId> &txIds,
                                 std::vector<PartitionTxId> *latestTxIdList,
                                 const std::string& fsName,
-                                const Mountpoint& mountpoint,
-                                std::atomic<bool>* enableSumInDir) override;
+                                const Mountpoint& mountpoint) override;
 
     FSStatusCode GetLatestTxId(uint32_t fsId,
                                std::vector<PartitionTxId>* txIds) override;

curvefs/src/client/rpcclient/metaserver_client.cpp

@@ -65,6 +65,7 @@ using BatchGetXAttrExcutor = TaskExecutor;
 using GetOrModifyS3ChunkInfoExcutor = TaskExecutor;
 using UpdateVolumeExtentExecutor = TaskExecutor;
 using GetVolumeExtentExecutor = TaskExecutor;
+using UpdateDeallocatableBlockGroupExcutor = TaskExecutor;
 
 using ::curvefs::common::LatencyUpdater;
 using ::curvefs::common::StreamConnection;
@@ -1465,7 +1466,7 @@ void UpdateVolumeExtentRpcDone::Run() {
     } while (0)
 
 void MetaServerClientImpl::AsyncUpdateVolumeExtent(
-    uint32_t fsId, uint64_t inodeId, const VolumeExtentList &extents,
+    uint32_t fsId, uint64_t inodeId, const VolumeExtentSliceList &extents,
     MetaServerClientDone *done) {
     auto task = AsyncRPCTask {
         (void)txId;
@@ -1473,7 +1474,7 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
         metric_.updateVolumeExtent.qps.count << 1;
         metaserver::UpdateVolumeExtentRequest request;
         SET_COMMON_FIELDS;
-        request.set_allocated_extents(new VolumeExtentList{extents});
+        request.set_allocated_extents(new VolumeExtentSliceList{extents});
 
         auto *rpcDone =
             new UpdateVolumeExtentRpcDone(taskExecutorDone, &metric_);
@@ -1493,7 +1494,8 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
 namespace {
 
 struct ParseVolumeExtentCallBack {
-    explicit ParseVolumeExtentCallBack(VolumeExtentList *ext) : extents(ext) {}
+    explicit ParseVolumeExtentCallBack(VolumeExtentSliceList *ext)
+        : extents(ext) {}
 
     bool operator()(butil::IOBuf *data) const {
         metaserver::VolumeExtentSlice slice;
@@ -1506,7 +1508,7 @@ struct ParseVolumeExtentCallBack {
         return true;
     }
 
-    VolumeExtentList *extents;
+    VolumeExtentSliceList *extents;
 };
 
 }  // namespace
@@ -1514,7 +1516,7 @@ struct ParseVolumeExtentCallBack {
 MetaStatusCode
 MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
                                       bool streaming,
-                                      VolumeExtentList *extents) {
+                                      VolumeExtentSliceList *extents) {
     auto task = RPCTask {
         (void)txId;
         (void)applyIndex;
@@ -1596,9 +1598,66 @@ MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
     return ConvertToMetaStatusCode(executor.DoRPCTask());
 }
 
-MetaStatusCode MetaServerClientImpl::GetInodeAttr(uint32_t fsId,
-                                                  uint64_t inodeid,
-                                                  InodeAttr *attr) {
+MetaStatusCode MetaServerClientImpl::UpdateDeallocatableBlockGroup(
+    uint32_t fsId, uint64_t inodeId, DeallocatableBlockGroupMap *statistic) {
+    auto task = RPCTask {
+        metric_.updateDeallocatableBlockGroup.qps.count << 1;
+        LatencyUpdater updater(&metric_.updateDeallocatableBlockGroup.latency);
+
+        metaserver::UpdateDeallocatableBlockGroupRequest request;
+        metaserver::UpdateDeallocatableBlockGroupResponse response;
+        request.set_poolid(poolID);
+        request.set_copysetid(copysetID);
+        request.set_partitionid(partitionID);
+        request.set_fsid(fsId);
+        auto *upadte = request.mutable_update();
+        for (auto &it : *statistic) {
+            upadte->Add()->CopyFrom(std::move(it.second));
+        }
+
+        curvefs::metaserver::MetaServerService_Stub stub(channel);
+        stub.UpdateDeallocatableBlockGroup(cntl, &request, &response, nullptr);
+
+        if (cntl->Failed()) {
+            metric_.updateDeallocatableBlockGroup.eps.count << 1;
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup failed"
+                         << ", errorCode = " << cntl->ErrorCode()
+                         << ", errorText = " << cntl->ErrorText()
+                         << ", logId = " << cntl->log_id();
+            return -cntl->ErrorCode();
+        }
+
+        auto rc = response.statuscode();
+        if (rc != MetaStatusCode::OK) {
+            metric_.updateDeallocatableBlockGroup.eps.count << 1;
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup: retCode = " << rc
+                         << ", message = " << MetaStatusCode_Name(rc);
+        } else if (response.has_appliedindex()) {
+            metaCache_->UpdateApplyIndex(CopysetGroupID(poolID, copysetID),
+                                         response.appliedindex());
+        } else {
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup OK"
+                         << ", but applyIndex not set in response:"
+                         << response.DebugString();
+            return -1;
+        }
+
+        VLOG(6) << "UpdateDeallocatableBlockGroup done, request: "
+                << request.DebugString()
+                << "response: " << response.DebugString();
+        return rc;
+    };
+
+    auto taskCtx = std::make_shared<TaskContext>(
+        MetaServerOpType::UpdateDeallocatableBlockGroup, task, fsId, inodeId);
+
+    UpdateDeallocatableBlockGroupExcutor excutor(
+        opt_, metaCache_, channelManager_, std::move(taskCtx));
+    return ConvertToMetaStatusCode(excutor.DoRPCTask());
+}
+
+MetaStatusCode MetaServerClientImpl::GetInodeAttr(
+    uint32_t fsId, uint64_t inodeid, InodeAttr *attr) {
     std::set<uint64_t> inodeIds;
     inodeIds.insert(inodeid);
     std::list<InodeAttr> attrs;

curvefs/src/client/rpcclient/metaserver_client.h

@@ -30,6 +30,7 @@
 #include <set>
 #include <unordered_map>
 #include <utility>
+#include <map>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/metaserver.pb.h"
@@ -49,21 +50,24 @@ using ::curvefs::metaserver::InodeAttr;
 using ::curvefs::metaserver::XAttr;
 using ::curvefs::metaserver::MetaStatusCode;
 using ::curvefs::metaserver::S3ChunkInfoList;
+using ::curvefs::metaserver::DeallocatableBlockGroup;
 using ::curvefs::common::StreamStatus;
 using ::curvefs::common::StreamClient;
-using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 using ::curvefs::metaserver::Time;
 
+using DeallocatableBlockGroupMap = std::map<uint64_t, DeallocatableBlockGroup>;
+using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+
 namespace curvefs {
 namespace client {
 namespace rpcclient {
 
 using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtentSliceList;
 
 struct DataIndices {
     absl::optional<S3ChunkInfoMap> s3ChunkInfoMap;
-    absl::optional<VolumeExtentList> volumeExtents;
+    absl::optional<VolumeExtentSliceList> volumeExtents;
 };
 
 class MetaServerClient {
@@ -161,13 +165,17 @@ class MetaServerClient {
 
     virtual void AsyncUpdateVolumeExtent(uint32_t fsId,
                                          uint64_t inodeId,
-                                         const VolumeExtentList &extents,
+                                         const VolumeExtentSliceList &extents,
                                          MetaServerClientDone *done) = 0;
 
     virtual MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                            uint64_t inodeId,
                                            bool streaming,
-                                           VolumeExtentList *extents) = 0;
+                                           VolumeExtentSliceList *extents) = 0;
+
+    virtual MetaStatusCode
+    UpdateDeallocatableBlockGroup(uint32_t fsId, uint64_t inodeId,
+                                  DeallocatableBlockGroupMap *statistic) = 0;
 };
 
 class MetaServerClientImpl : public MetaServerClient {
@@ -265,13 +273,17 @@ class MetaServerClientImpl : public MetaServerClient {
 
     void AsyncUpdateVolumeExtent(uint32_t fsId,
                                  uint64_t inodeId,
-                                 const VolumeExtentList &extents,
+                                 const VolumeExtentSliceList &extents,
                                  MetaServerClientDone *done) override;
 
     MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                    uint64_t inodeId,
                                    bool streaming,
-                                   VolumeExtentList *extents) override;
+                                   VolumeExtentSliceList *extents) override;
+
+    MetaStatusCode UpdateDeallocatableBlockGroup(
+        uint32_t fsId, uint64_t inodeId,
+        DeallocatableBlockGroupMap *statistic) override;
 
  private:
     MetaStatusCode UpdateInode(const UpdateInodeRequest &request,

curvefs/src/client/s3/client_s3_adaptor.cpp

@@ -42,6 +42,7 @@ S3ClientAdaptorImpl::Init(
     std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
     std::shared_ptr<KVClientManager> kvClientManager,
     bool startBackGround) {
+    pendingReq_ = 0;
     blockSize_ = option.blockSize;
     chunkSize_ = option.chunkSize;
     pageSize_ = option.pageSize;
@@ -51,6 +52,7 @@ S3ClientAdaptorImpl::Init(
                    << blockSize_;
         return CURVEFS_ERROR::INVALIDPARAM;
     }
+    fuseMaxSize_ = option.fuseMaxSize;
     prefetchBlocks_ = option.prefetchBlocks;
     prefetchExecQueueNum_ = option.prefetchExecQueueNum;
     diskCacheType_ = option.diskCacheOpt.diskCacheType;
@@ -111,25 +113,30 @@ int S3ClientAdaptorImpl::Write(uint64_t inodeId, uint64_t offset,
     VLOG(6) << "write start offset:" << offset << ", len:" << length
             << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
     uint64_t start = butil::cpuwide_time_us();
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindOrCreateFileCacheManager(fsId_, inodeId);
     {
-        std::lock_guard<std::mutex> lockGuard(ioMtx_);
+        std::lock_guard<std::mutex> lockguard(ioMtx_);
+        pendingReq_.fetch_add(1, std::memory_order_seq_cst);
+        VLOG(6) << "pendingReq_ is: " << pendingReq_;
+        uint64_t pendingReq = pendingReq_.load(std::memory_order_seq_cst);
         fsCacheManager_->DataCacheByteInc(length);
         uint64_t size = fsCacheManager_->GetDataCacheSize();
-        const uint64_t maxSize = fsCacheManager_->GetDataCacheMaxSize();
-        if (size >= maxSize) {
+        uint64_t maxSize = fsCacheManager_->GetDataCacheMaxSize();
+        if ((size + pendingReq * fuseMaxSize_) >= maxSize) {
             VLOG(6) << "write cache is full, wait flush. size: " << size
-                    << ", maxSize: " << maxSize;
+                    << ", maxSize:" << maxSize;
             // offer to do flush
             waitInterval_.StopWait();
             fsCacheManager_->WaitFlush();
         }
     }
-    const uint64_t memCacheRatio = fsCacheManager_->MemCacheRatio();
+    uint64_t memCacheRatio = fsCacheManager_->MemCacheRatio();
     int64_t exceedRatio = memCacheRatio - memCacheNearfullRatio_;
     if (exceedRatio > 0) {
         // offer to do flush
         waitInterval_.StopWait();
-        // upload to s3 directly or cache disk full
+        // upload to s3 derectly or cache disk full
         bool needSleep =
             (DisableDiskCache() || IsReadCache()) ||
             (IsReadWriteCache() && diskCacheManagerImpl_->IsDiskCacheFull());
@@ -140,15 +147,15 @@ int S3ClientAdaptorImpl::Write(uint64_t inodeId, uint64_t offset,
                     << memCacheRatio << ", exponent is: " << exponent;
         }
     }
-    FileCacheManagerPtr fileCacheManager =
-        fsCacheManager_->FindOrCreateFileCacheManager(fsId_, inodeId);
     int ret = fileCacheManager->Write(offset, length, buf);
+    pendingReq_.fetch_sub(1, std::memory_order_seq_cst);
     fsCacheManager_->DataCacheByteDec(length);
-    if (s3Metric_ != nullptr) {
+    if (s3Metric_.get() != nullptr) {
         CollectMetrics(&s3Metric_->adaptorWrite, ret, start);
         s3Metric_->writeSize.set_value(length);
     }
-    VLOG(6) << "write end inodeId: " << inodeId << ", ret: " << ret;
+    VLOG(6) << "write end inodeId:" << inodeId << ",ret:" << ret
+            << ", pendingReq_ is: " << pendingReq_;
     return ret;
 }
 

curvefs/src/client/s3/client_s3_adaptor.h

@@ -33,7 +33,7 @@
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/s3/client_s3.h"
@@ -249,6 +249,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
     std::shared_ptr<S3Client> client_;
     uint64_t blockSize_;
     uint64_t chunkSize_;
+    uint32_t fuseMaxSize_;
     uint32_t prefetchBlocks_;
     uint32_t prefetchExecQueueNum_;
     std::string allocateServerEps_;
@@ -269,6 +270,7 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
     std::shared_ptr<InodeCacheManager> inodeManager_;
     std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl_;
     DiskCacheType diskCacheType_;
+    std::atomic<uint64_t> pendingReq_;
     std::shared_ptr<MdsClient> mdsClient_;
     uint32_t fsId_;
     std::string fsName_;

curvefs/src/client/s3/client_s3_cache_manager.h

@@ -34,7 +34,7 @@
 #include <set>
 
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/s3/client_s3.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/concurrent/task_thread_pool.h"

curvefs/src/client/volume/default_volume_storage.cpp

@@ -30,7 +30,7 @@
 #include <vector>
 
 #include "absl/meta/type_traits.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/volume/extent_cache.h"
@@ -74,8 +74,7 @@ std::ostream& operator<<(std::ostream& os, const std::vector<IOPart>& iov) {
 CURVEFS_ERROR DefaultVolumeStorage::Write(uint64_t ino,
                                           off_t offset,
                                           size_t len,
-                                          const char* data,
-                                          FileOut* fileOut) {
+                                          const char* data) {
     std::shared_ptr<InodeWrapper> inodeWrapper;
     LatencyUpdater updater(&metric_.writeLatency);
     auto ret = inodeCacheManager_->GetInode(ino, inodeWrapper);
@@ -117,7 +116,6 @@ CURVEFS_ERROR DefaultVolumeStorage::Write(uint64_t ino,
         }
 
         inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
-        inodeWrapper->GetInodeAttrLocked(&fileOut->attr);
     }
 
     inodeCacheManager_->ShipToFlush(inodeWrapper);

curvefs/src/client/volume/default_volume_storage.h

@@ -31,16 +31,12 @@
 #include "curvefs/src/client/volume/volume_storage.h"
 #include "curvefs/src/volume/block_device_client.h"
 #include "curvefs/src/volume/space_manager.h"
-#include "curvefs/src/client/filesystem/error.h"
-#include "curvefs/src/client/filesystem/meta.h"
 
 namespace curvefs {
 namespace client {
 
 using ::curvefs::volume::BlockDeviceClient;
 using ::curvefs::volume::SpaceManager;
-using ::curvefs::client::filesystem::CURVEFS_ERROR;
-using ::curvefs::client::filesystem::FileOut;
 
 class InodeCacheManager;
 
@@ -70,8 +66,7 @@ class DefaultVolumeStorage final : public VolumeStorage {
     CURVEFS_ERROR Write(uint64_t ino,
                         off_t offset,
                         size_t len,
-                        const char* data,
-                        FileOut* fileOut) override;
+                        const char* data) override;
 
     CURVEFS_ERROR Flush(uint64_t ino) override;
 

curvefs/src/client/volume/extent_cache.cpp

@@ -228,7 +228,7 @@ void ExtentCache::SetOption(const ExtentCacheOption& option) {
     LOG(INFO) << "ExtentCacheOption: [" << option_ << "]";
 }
 
-void ExtentCache::Build(const VolumeExtentList &extents) {
+void ExtentCache::Build(const VolumeExtentSliceList &extents) {
     WriteLockGuard lk(lock_);
     slices_.clear();
     dirties_.clear();
@@ -238,8 +238,8 @@ void ExtentCache::Build(const VolumeExtentList &extents) {
     }
 }
 
-VolumeExtentList ExtentCache::GetDirtyExtents() {
-    VolumeExtentList result;
+VolumeExtentSliceList ExtentCache::GetDirtyExtents() {
+    VolumeExtentSliceList result;
     WriteLockGuard lk(lock_);
     for (const auto* slice : dirties_) {
         *result.add_slices() = slice->ToVolumeExtentSlice();

curvefs/src/client/volume/extent_cache.h

@@ -43,7 +43,7 @@ namespace client {
 using ::curvefs::volume::ReadPart;
 using ::curvefs::volume::WritePart;
 using ::curvefs::metaserver::VolumeExtentSlice;
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtentSliceList;
 
 struct ExtentCacheOption {
     // preallocation size if offset ~ length is not allocated
@@ -61,7 +61,7 @@ class ExtentCache {
 
     static void SetOption(const ExtentCacheOption& option);
 
-    void Build(const VolumeExtentList& extents);
+    void Build(const VolumeExtentSliceList& extents);
 
     void DivideForWrite(uint64_t offset,
                         uint64_t len,
@@ -81,7 +81,7 @@ class ExtentCache {
 
     bool HasDirtyExtents() const;
 
-    VolumeExtentList GetDirtyExtents();
+    VolumeExtentSliceList GetDirtyExtents();
 
     std::unordered_map<uint64_t, std::map<uint64_t, PExtent>>
     GetExtentsForTesting() const;

curvefs/src/client/volume/volume_storage.h

@@ -28,15 +28,11 @@
 #include <cstddef>
 #include <cstdint>
 
-#include "curvefs/src/client/filesystem/error.h"
-#include "curvefs/src/client/filesystem/meta.h"
+#include "curvefs/src/client/error_code.h"
 
 namespace curvefs {
 namespace client {
 
-using ::curvefs::client::filesystem::CURVEFS_ERROR;
-using ::curvefs::client::filesystem::FileOut;
-
 class VolumeStorage {
  public:
     virtual ~VolumeStorage() = default;
@@ -49,8 +45,7 @@ class VolumeStorage {
     virtual CURVEFS_ERROR Write(uint64_t ino,
                                 off_t offset,
                                 size_t len,
-                                const char* data,
-                                FileOut* fileOut) = 0;
+                                const char* data) = 0;
 
     virtual CURVEFS_ERROR Flush(uint64_t ino) = 0;
 

curvefs/src/client/warmup/warmup_manager.cpp

@@ -473,7 +473,7 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
             }
             if (context->retCode == 0) {
                 VLOG(9) << "Get Object success: " << context->key;
-                PutObjectToCache(key, context);
+                PutObjectToCache(key, context->key, context->buf, context->len);
                 CollectMetrics(&warmupS3Metric_.warmupS3Cached, context->len,
                                start);
                 warmupS3Metric_.warmupS3CacheSize << context->len;
@@ -481,6 +481,7 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
                     VLOG(6) << "pendingReq is over";
                     cond.Signal();
                 }
+                delete[] context->buf;
                 return;
             }
             warmupS3Metric_.warmupS3Cached.eps.count << 1;
@@ -677,8 +678,9 @@ void WarmupManagerS3Impl::AddFetchS3objectsTask(fuse_ino_t key,
     }
 }
 
-void WarmupManagerS3Impl::PutObjectToCache(
-    fuse_ino_t key, const std::shared_ptr<GetObjectAsyncContext> &context) {
+void WarmupManagerS3Impl::PutObjectToCache(fuse_ino_t key,
+                                           const std::string &filename,
+                                           const char *data, uint64_t len) {
     ReadLockGuard lock(inode2ProgressMutex_);
     auto iter = FindWarmupProgressByKeyLocked(key);
     if (iter == inode2Progress_.end()) {
@@ -690,21 +692,17 @@ void WarmupManagerS3Impl::PutObjectToCache(
     iter->second.FinishedPlusOne();
     switch (iter->second.GetStorageType()) {
     case curvefs::client::common::WarmupStorageType::kWarmupStorageTypeDisk:
-        ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(
-            context->key, context->buf, context->len);
+        ret = s3Adaptor_->GetDiskCacheManager()->WriteReadDirect(filename, data,
+                                                                 len);
         if (ret < 0) {
             LOG_EVERY_SECOND(INFO)
-                << "write read directly failed, key: " << context->key;
+                << "write read directly failed, key: " << filename;
         }
-        delete[] context->buf;
         break;
     case curvefs::client::common::WarmupStorageType::kWarmupStorageTypeKvClient:
         if (kvClientManager_ != nullptr) {
-            kvClientManager_->Set(std::make_shared<SetKVCacheTask>(
-                context->key, context->buf, context->len,
-                [context](const std::shared_ptr<SetKVCacheTask> &) {
-                    delete[] context->buf;
-                }));
+            kvClientManager_->Set(
+                std::make_shared<SetKVCacheTask>(filename, data, len));
         }
         break;
     default:

curvefs/src/client/warmup/warmup_manager.h

@@ -413,9 +413,8 @@ class WarmupManagerS3Impl : public WarmupManager {
 
     void AddFetchS3objectsTask(fuse_ino_t key, std::function<void()> task);
 
-    void
-    PutObjectToCache(fuse_ino_t key,
-                     const std::shared_ptr<GetObjectAsyncContext> &context);
+    void PutObjectToCache(fuse_ino_t key, const std::string &filename,
+                          const char *data, uint64_t len);
 
  protected:
     std::deque<WarmupFilelist> warmupFilelistDeque_;

curvefs/src/client/xattr_manager.cpp

@@ -78,14 +78,20 @@ CURVEFS_ERROR XattrManager::CalOneLayerSumInfo(InodeAttr *attr) {
             summaryInfo.entries++;
             summaryInfo.fbytes += it.length();
         }
-        (*attr->mutable_xattr())[XATTRFILES] =
-            std::to_string(summaryInfo.files);
-        (*attr->mutable_xattr())[XATTRSUBDIRS] =
-            std::to_string(summaryInfo.subdirs);
-        (*attr->mutable_xattr())[XATTRENTRIES] =
-            std::to_string(summaryInfo.entries);
-        (*attr->mutable_xattr())[XATTRFBYTES] =
-            std::to_string(summaryInfo.fbytes + attr->length());
+        if (!(AddUllStringToFirst(
+                &(attr->mutable_xattr()->find(XATTRFILES)->second),
+                summaryInfo.files, true) &&
+            AddUllStringToFirst(
+                &(attr->mutable_xattr()->find(XATTRSUBDIRS)->second),
+                summaryInfo.subdirs, true) &&
+            AddUllStringToFirst(
+                &(attr->mutable_xattr()->find(XATTRENTRIES)->second),
+                summaryInfo.entries, true) &&
+            AddUllStringToFirst(
+                &(attr->mutable_xattr()->find(XATTRFBYTES)->second),
+                summaryInfo.fbytes + attr->length(), true))) {
+            ret = CURVEFS_ERROR::INTERNAL;
+        }
     }
     return ret;
 }

curvefs/src/client/xattr_manager.h

@@ -36,7 +36,7 @@
 
 #include "curvefs/src/common/define.h"
 #include "curvefs/proto/metaserver.pb.h"
-#include "curvefs/src/client/filesystem/error.h"
+#include "curvefs/src/client/error_code.h"
 #include "curvefs/src/client/client_operator.h"
 #include "src/common/interruptible_sleeper.h"
 

curvefs/src/mds/fs_info_wrapper.cpp

@@ -113,10 +113,6 @@ void FsInfoWrapper::AddMountPoint(const Mountpoint& mp) {
     *p = mp;
 
     fsInfo_.set_mountnum(fsInfo_.mountnum() + 1);
-
-    if (fsInfo_.enablesumindir() && fsInfo_.mountnum() > 1) {
-        fsInfo_.set_enablesumindir(false);
-    }
 }
 
 FSStatusCode FsInfoWrapper::DeleteMountPoint(const Mountpoint& mp) {

curvefs/src/mds/fs_manager.cpp

@@ -843,16 +843,6 @@ void FsManager::RefreshSession(const RefreshSessionRequest* request,
 
     // update this client's alive time
     UpdateClientAliveTime(request->mountpoint(), request->fsname());
-    FsInfoWrapper wrapper;
-    FSStatusCode ret = fsStorage_->Get(request->fsname(), &wrapper);
-    if (ret != FSStatusCode::OK) {
-        LOG(WARNING) << "GetFsInfo fail, get fs fail, fsName = "
-                    << request->fsname()
-                    << ", errCode = " << FSStatusCode_Name(ret);
-        return;
-    }
-
-    response->set_enablesumindir(wrapper.ProtoFsInfo().enablesumindir());
 }
 
 FSStatusCode FsManager::ReloadMountedFsVolumeSpace() {

curvefs/src/mds/fs_storage.cpp

@@ -359,15 +359,6 @@ bool PersisKVStorage::LoadAllFs() {
 
         idToName_.emplace(fsInfo.fsid(), fsInfo.fsname());
 
-        // For compatibility when upgrading, the new field 'optional
-        // objectPrefix` in message `S3Info` needs to be set to the default
-        // value
-        if (fsInfo.fstype() == FSType::TYPE_S3) {
-            if (!fsInfo.detail().s3info().has_objectprefix()) {
-                fsInfo.mutable_detail()->mutable_s3info()->set_objectprefix(0);
-            }
-        }
-
         fs_.emplace(fsInfo.fsname(), std::move(fsInfo));
     }
 

curvefs/src/mds/heartbeat/BUILD

@@ -27,6 +27,7 @@ cc_library(
         "//curvefs/proto:curvefs_topology_cc_proto",
         "//curvefs/proto:metaserver_cc_proto",
         "//curvefs/src/mds/topology:curvefs_topology",
+        "//curvefs/src/mds/space:curvefs_mds_space",
         "//external:brpc",
         "//external:gflags",
         "//external:glog",

curvefs/src/mds/heartbeat/heartbeat_manager.cpp

@@ -42,8 +42,9 @@ namespace mds {
 namespace heartbeat {
 HeartbeatManager::HeartbeatManager(
     const HeartbeatOption &option, const std::shared_ptr<Topology> &topology,
-    const std::shared_ptr<Coordinator> &coordinator)
-    : topology_(topology) {
+    const std::shared_ptr<Coordinator> &coordinator,
+    const std::shared_ptr<SpaceManager> &spaceManager)
+    : topology_(topology), spaceManager_(spaceManager) {
     healthyChecker_ =
         std::make_shared<MetaserverHealthyChecker>(option, topology);
 
@@ -86,24 +87,6 @@ void HeartbeatManager::Stop() {
     }
 }
 
-void HeartbeatManager::MetaServerHealthyChecker() {
-    while (sleeper_.wait_for(
-        std::chrono::milliseconds(metaserverHealthyCheckerRunInter_))) {
-        healthyChecker_->CheckHeartBeatInterval();
-    }
-}
-
-void HeartbeatManager::UpdateMetaServerSpace(
-    const MetaServerHeartbeatRequest &request) {
-    MetaServerSpace space(request.spacestatus());
-    TopoStatusCode ret =
-        topology_->UpdateMetaServerSpace(space, request.metaserverid());
-    if (ret != TopoStatusCode::TOPO_OK) {
-        LOG(ERROR) << "heartbeat UpdateMetaServerSpace fail, ret = "
-                   << TopoStatusCode_Name(ret);
-    }
-}
-
 void HeartbeatManager::MetaServerHeartbeat(
     const MetaServerHeartbeatRequest &request,
     MetaServerHeartbeatResponse *response) {
@@ -127,6 +110,42 @@ void HeartbeatManager::MetaServerHeartbeat(
     UpdateMetaServerSpace(request);
 
     // dealing with copysets included in the heartbeat request
+    Coordinate(request, response);
+
+    // update deallocatable block group info
+    UpdateDeallocatableBlockGroup(request, response);
+}
+
+void HeartbeatManager::MetaServerHealthyChecker() {
+    while (sleeper_.wait_for(
+        std::chrono::milliseconds(metaserverHealthyCheckerRunInter_))) {
+        healthyChecker_->CheckHeartBeatInterval();
+
+        auto metaservers =
+            topology_->GetMetaServerInCluster([](const MetaServer &ms) {
+                return ms.GetOnlineState() == OnlineState::ONLINE;
+            });
+
+        uint32_t currentOnlineMetaServerNum =
+            onlineMetaServerNum_.load(std::memory_order_acquire);
+        onlineMetaServerNum_.compare_exchange_strong(currentOnlineMetaServerNum,
+                                                     metaservers.size());
+    }
+}
+
+void HeartbeatManager::UpdateMetaServerSpace(
+    const MetaServerHeartbeatRequest &request) {
+    MetaServerSpace space(request.spacestatus());
+    TopoStatusCode ret =
+        topology_->UpdateMetaServerSpace(space, request.metaserverid());
+    if (ret != TopoStatusCode::TOPO_OK) {
+        LOG(ERROR) << "heartbeat UpdateMetaServerSpace fail, ret = "
+                   << TopoStatusCode_Name(ret);
+    }
+}
+
+void HeartbeatManager::Coordinate(const MetaServerHeartbeatRequest &request,
+                                  MetaServerHeartbeatResponse *response) {
     for (auto &value : request.copysetinfos()) {
         // convert copysetInfo from heartbeat format to topology format
         ::curvefs::mds::topology::CopySetInfo reportCopySetInfo;
@@ -168,6 +187,34 @@ void HeartbeatManager::MetaServerHeartbeat(
     }
 }
 
+void HeartbeatManager::UpdateDeallocatableBlockGroup(
+    const MetaServerHeartbeatRequest &request,
+    MetaServerHeartbeatResponse *response) {
+    uint32_t metaserverId = request.metaserverid();
+
+    for (auto &info : request.blockgroupstatinfos()) {
+        auto volumeSpace = spaceManager_->GetVolumeSpace(info.fsid());
+        if (volumeSpace == nullptr) {
+            LOG(ERROR) << "HeartbeatManager fsid=" << info.fsid()
+                       << " do not have volumeSpace manager";
+            response->set_statuscode(HeartbeatStatusCode::hbMetaServerFSUnkown);
+            return;
+        }
+
+        uint64_t issued = 0;
+        bool hasissued = volumeSpace->UpdateDeallocatableBlockGroup(
+            metaserverId, onlineMetaServerNum_, info.deallocatableblockgroups(),
+            info.blockgroupdeallocatestatus(), &issued);
+        if (hasissued) {
+            response->mutable_issuedblockgroups()->insert(
+                {info.fsid(), issued});
+            LOG(INFO) << "HeartbeatManager issue metaserverid=" << metaserverId
+                      << " blockgroup, fsid=" << info.fsid()
+                      << ", issued=" << issued;
+        }
+    }
+}
+
 HeartbeatStatusCode HeartbeatManager::CheckRequest(
     const MetaServerHeartbeatRequest &request) {
     MetaServer metaServer;

curvefs/src/mds/heartbeat/heartbeat_manager.h

@@ -36,6 +36,7 @@
 #include "curvefs/src/mds/heartbeat/topo_updater.h"
 #include "curvefs/src/mds/schedule/coordinator.h"
 #include "curvefs/src/mds/topology/topology.h"
+#include "curvefs/src/mds/space/manager.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 
@@ -43,6 +44,7 @@ using ::curvefs::mds::topology::PoolIdType;
 using ::curvefs::mds::topology::CopySetIdType;
 using ::curvefs::mds::topology::Topology;
 using ::curvefs::mds::schedule::Coordinator;
+using ::curvefs::mds::space::SpaceManager;
 
 using ::curve::common::Thread;
 using ::curve::common::Atomic;
@@ -65,7 +67,8 @@ class HeartbeatManager {
  public:
     HeartbeatManager(const HeartbeatOption &option,
                      const std::shared_ptr<Topology> &topology,
-                     const std::shared_ptr<Coordinator> &coordinator);
+                     const std::shared_ptr<Coordinator> &coordinator,
+                     const std::shared_ptr<SpaceManager> &spaceManager);
 
     ~HeartbeatManager() { Stop(); }
 
@@ -98,6 +101,7 @@ class HeartbeatManager {
                              MetaServerHeartbeatResponse *response);
 
  private:
+    FRIEND_TEST(TestHeartbeatManager, TEST_UpdateDeallocatableBlockGroup);
     /**
      * @brief Background thread for heartbeat timeout inspection
      */
@@ -140,10 +144,18 @@ class HeartbeatManager {
 
     void UpdateMetaServerSpace(const MetaServerHeartbeatRequest &request);
 
+    void Coordinate(const MetaServerHeartbeatRequest &request,
+                    MetaServerHeartbeatResponse *response);
+
+    void
+    UpdateDeallocatableBlockGroup(const MetaServerHeartbeatRequest &request,
+                                  MetaServerHeartbeatResponse *response);
+
  private:
     // Dependencies of heartbeat
     std::shared_ptr<Topology> topology_;
     std::shared_ptr<Coordinator> coordinator_;
+    std::shared_ptr<SpaceManager> spaceManager_;
 
     // healthyChecker_ health checker running in background thread
     std::shared_ptr<MetaserverHealthyChecker> healthyChecker_;
@@ -156,6 +168,8 @@ class HeartbeatManager {
     // Manage metaserverHealthyChecker threads
     Thread backEndThread_;
 
+    std::atomic<uint32_t> onlineMetaServerNum_;
+
     Atomic<bool> isStop_;
     InterruptibleSleeper sleeper_;
     int metaserverHealthyCheckerRunInter_;

curvefs/src/mds/mds.cpp

@@ -66,8 +66,8 @@ void MDS::InitOptions(std::shared_ptr<Configuration> conf) {
     conf_ = std::move(conf);
     conf_->GetValueFatalIfFail("mds.listen.addr", &options_.mdsListenAddr);
     conf_->GetValueFatalIfFail("mds.dummy.port", &options_.dummyPort);
-    conf_->GetValueFatalIfFail("mds.server.idleTimeoutSec",
-                               &options_.idleTimeoutSec);
+    conf_->GetValueFatalIfFail("mds.space.calIntervalSec",
+                               &options_.mdsSpaceCalIntervalSec);
 
     InitMetaServerOption(&options_.metaserverOptions);
     InitTopologyOption(&options_.topologyOptions);
@@ -171,8 +171,8 @@ void MDS::Init() {
     space::MdsProxyManager::SetProxyOptions(options_.bsMdsProxyOptions);
 
     fsStorage_ = std::make_shared<PersisKVStorage>(etcdClient_);
-    spaceManager_ =
-        std::make_shared<SpaceManagerImpl>(etcdClient_, fsStorage_);
+    spaceManager_ = std::make_shared<SpaceManagerImpl>(
+        etcdClient_, fsStorage_, options_.mdsSpaceCalIntervalSec);
     metaserverClient_ =
         std::make_shared<MetaserverClient>(options_.metaserverOptions);
     auto dlock = std::make_shared<DLock>(options_.dLockOptions, etcdClient_);
@@ -277,7 +277,6 @@ void MDS::Run() {
 
     // start rpc server
     brpc::ServerOptions option;
-    option.idle_timeout_sec = options_.idleTimeoutSec;
     LOG_IF(FATAL, server.Start(options_.mdsListenAddr.c_str(), &option) != 0)
         << "start brpc server error";
     running_ = true;
@@ -420,7 +419,7 @@ void MDS::InitHeartbeatManager() {
 
     heartbeatOption.mdsStartTime = steady_clock::now();
     heartbeatManager_ = std::make_shared<HeartbeatManager>(
-        heartbeatOption, topology_, coordinator_);
+        heartbeatOption, topology_, coordinator_, spaceManager_);
     heartbeatManager_->Init();
 }
 

curvefs/src/mds/mds.h

@@ -85,11 +85,11 @@ using ::curvefs::mds::space::MdsProxyOptions;
 struct MDSOptions {
     int dummyPort;
     std::string mdsListenAddr;
-    int idleTimeoutSec;
-
     MetaserverOptions metaserverOptions;
     // TODO(add EtcdConf): add etcd configure
 
+    uint64_t mdsSpaceCalIntervalSec;
+
     TopologyOption topologyOptions;
     HeartbeatOption heartbeatOption;
     ScheduleOption scheduleOption;

curvefs/src/mds/space/BUILD

@@ -27,6 +27,8 @@ cc_library(
     deps = [
         "//curvefs/proto:mds_cc_proto",
         "//curvefs/proto:space_cc_proto",
+        "//curvefs/proto:metaserver_cc_proto",
+        "//curvefs/proto:curvefs_heartbeat_cc_proto",
         "//curvefs/src/mds:curvefs_mds_fs_storage",
         "//curvefs/src/mds/codec:fs_mds_codec",
         "//external:brpc",