fix bugs

1. fuse_volume_client just flush inode but do not flush data in Flush
2. Optimize blockgroup allocation logic:
	- fix the available group will always decrease in the case of insufficient allocation on the mds side
	- the blockgroup assigned by mds to the client is allowed to be smaller than the count required by the client
	- mds does not allocate blockgroups exceeding a certain water level
3. fix the issue where the cache of metaserver_client could only correspond to a single file system
4. Optimize log: add some vlog to print more details

Signed-off-by: ilixiaocui <ilixiaocui@163.com>

curvefs/conf/client.conf

@@ -58,10 +58,6 @@ executorOpt.maxRetryTimesBeforeConsiderSuspend=20
 # batch limit of get inode attr and xattr
 executorOpt.batchInodeAttrLimit=10000
 
-#### spaceserver
-spaceServer.spaceAddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
-spaceServer.rpcTimeoutMs=1000
-
 #### bdev
 # curve client's config file
 bdev.confPath=/etc/curve/client.conf
@@ -162,6 +158,15 @@ volume.bitmapAllocator.smallAllocProportion=0.2
 # number of block groups that allocated once
 volume.blockGroup.allocateOnce=4
 
+## spaceserver
+# the space used by the blockgroup exceeds this percentage and can
+# be returned to mds [0.8-1]
+volume.space.useThreshold=0.95
+
+# the background thread calculates the time interval for returning
+# the blockgroup to mds
+volume.space.releaseInterSec=300
+
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false

curvefs/conf/curvebs_client.conf

@@ -0,0 +1,191 @@
+#
+################### mds一侧配置信息 ##################
+#
+
+# mds的地址信息，对于mds集群，地址以逗号隔开
+mds.listen.addr=127.0.0.1:6666
+
+# 初始化阶段向mds注册开关，默认为开
+mds.registerToMDS=true
+
+# 与mds通信的rpc超时时间
+mds.rpcTimeoutMS=500
+
+# 与mds通信rpc最大的超时时间, 指数退避的超时间不能超过这个值
+mds.maxRPCTimeoutMS=2000
+
+# 与mds通信重试总时间
+mds.maxRetryMS=8000
+
+# 在当前mds上连续重试次数超过该限制就切换, 这个失败次数包含超时重试次数
+mds.maxFailedTimesBeforeChangeMDS=2
+
+# 与MDS一侧保持一个lease时间内多少次续约
+mds.refreshTimesPerLease=4
+
+# mds RPC接口每次重试之前需要先睡眠一段时间
+mds.rpcRetryIntervalUS=100000
+
+# The normal retry times for trigger wait strategy
+mds.normalRetryTimesBeforeTriggerWait=3
+
+# Max retry time for IO-Path request
+mds.maxRetryMsInIOPath=86400000
+
+# Sleep interval for wait
+mds.waitSleepMs=10000
+
+#
+################# metacache配置信息 ################
+#
+
+# 获取leader的rpc超时时间
+metacache.getLeaderTimeOutMS=500
+
+# 获取leader的重试次数
+metacache.getLeaderRetry=5
+
+# 获取leader接口每次重试之前需要先睡眠一段时间
+metacache.rpcRetryIntervalUS=100000
+
+#
+############### 调度层的配置信息 #############
+#
+
+# 调度层队列大小，每个文件对应一个队列
+# 调度队列的深度会影响client端整体吞吐，这个队列存放的是异步IO任务。。
+schedule.queueCapacity=1000000
+
+# 队列的执行线程数量
+# 执行线程所要做的事情就是将IO取出，然后发到网络就返回取下一个网络任务。一个任务从
+# 队列取出到发送完rpc请求大概在(20us-100us)，20us是正常情况下不需要获取leader的时候
+# 如果在发送的时候需要获取leader，时间会在100us左右，一个线程的吞吐在10w-50w
+# 性能已经满足需求
+schedule.threadpoolSize=2
+
+# 为隔离qemu侧线程引入的任务队列，因为qemu一侧只有一个IO线程
+# 当qemu一侧调用aio接口的时候直接将调用push到任务队列就返回，
+# 这样libcurve不占用qemu的线程，不阻塞其异步调用
+isolation.taskQueueCapacity=1000000
+
+# 隔离qemu线程的任务队列线程池大小, 默认值为1个线程
+isolation.taskThreadPoolSize=1
+
+
+#
+################ 与chunkserver通信相关配置 #############
+#
+# 读写接口失败的OP之间重试睡眠
+chunkserver.opRetryIntervalUS=100000
+
+# 失败的OP重试次数
+chunkserver.opMaxRetry=2500000
+
+# 与chunkserver通信的rpc超时时间
+chunkserver.rpcTimeoutMS=1000
+
+# 开启基于appliedindex的读，用于性能优化
+chunkserver.enableAppliedIndexRead=1
+
+# 重试请求之间睡眠最长时间
+# 因为当网络拥塞的时候或者chunkserver出现过载的时候，需要增加睡眠时间
+# 这个时间最大为maxRetrySleepIntervalUs
+chunkserver.maxRetrySleepIntervalUS=8000000
+
+# 重试请求的超时rpc时间最大值，超时时间会遵循指数退避策略
+# 因为当网络拥塞的时候出现超时，需要增加RPC超时时间
+# 这个时间最大为maxTimeoutMS
+chunkserver.maxRPCTimeoutMS=8000
+
+# 同一个chunkserver连续超时上限次数
+# 如果超过这个值，就会进行健康检查，健康检查失败后，会标记为unstable
+chunkserver.maxStableTimeoutTimes=10
+# chunkserver上rpc连续超时后，健康检查请求的超时间
+chunkserver.checkHealthTimeoutMs=100
+# 同一个server上unstable的chunkserver数量超过这个值之后
+# 所有的chunkserver都会标记为unstable
+chunkserver.serverStableThreshold=3
+
+# 当底层chunkserver压力大时，可能也会触发unstable
+# 由于copyset leader may change，会导致请求超时时间设置为默认值，从而导致IO hang
+# 真正宕机的情况下，请求重试一定次数后会处理完成
+# 如果一直重试，则不是宕机情况，这时候超时时间还是要进入指数退避逻辑
+# 当一个请求重试次数超过这个值时，其超时时间一定进入指数退避
+chunkserver.minRetryTimesForceTimeoutBackoff=5
+
+# 当一个rpc重试超过次数maxRetryTimesBeforeConsiderSuspend的时候
+# 记为悬挂IO，metric会报警
+chunkserver.maxRetryTimesBeforeConsiderSuspend=20
+
+#
+################# 文件级别配置项 #############
+#
+# libcurve底层rpc调度允许最大的未返回rpc数量，每个文件的inflight RPC独立
+global.fileMaxInFlightRPCNum=128
+
+# 文件IO下发到底层chunkserver最大的分片KB
+global.fileIOSplitMaxSizeKB=64
+
+#
+################# log相关配置 ###############
+#
+# enable logging or not
+global.logging.enable=True
+#
+# log等级 INFO=0/WARNING=1/ERROR=2/FATAL=3
+global.logLevel=0
+# 设置log的路径
+global.logPath=/data/log/curve/  # __CURVEADM_TEMPLATE__ /curvebs/client/logs __CURVEADM_TEMPLATE__
+# 单元测试情况下
+# logpath=./runlog/
+
+#
+################# 读源卷相关配置 ###############
+#
+# 读取源卷时打开的fd超时关闭时间300s
+closefd.timeout=300
+# 读取源卷时打开的fd后台线程每600s扫描一遍fdMap，关闭超时fd
+closefd.timeInterval=600
+
+#
+############### metric 配置信息 #############
+#
+global.metricDummyServerStartPort=9000
+
+# 是否关闭健康检查: true/关闭 false/不关闭
+global.turnOffHealthCheck=true
+
+#
+### throttle config
+#
+throttle.enable=false
+
+##### discard configurations #####
+# enable/disable discard
+discard.enable=true
+# discard granularity
+discard.granularity=4096
+# discard cleanup task delay times in millisecond
+discard.taskDelayMs=60000
+
+##### alignment #####
+# default alignment
+global.alignment.commonVolume=512
+# alignment for clone volume
+# default is 4096, because lazy clone chunk bitmap granularity is 4096
+global.alignment.cloneVolume=4096
+
+##### chunkserver client option #####
+# chunkserver client rpc timeout time
+csClientOpt.rpcTimeoutMs=500
+# chunkserver client rpc max try
+csClientOpt.rpcMaxTry=86400000
+# chunkserver client rpc retry interval
+csClientOpt.rpcIntervalUs=100000
+# chunkserver client rpc max timeout time
+csClientOpt.rpcMaxTimeoutMs=8000
+
+##### chunkserver broadcaster option #####
+# broad cast max machine num
+csBroadCasterOpt.broadCastMaxNum=200
+

curvefs/conf/mds.conf

@@ -159,3 +159,6 @@ bs.mds.maxFailedTimesBeforeChangeMDS=2
 bs.mds.normalRetryTimesBeforeTriggerWait=3
 # sleep interval in ms for wait
 bs.mds.waitSleepMs=1000
+
+#### Options for volume space deallcatable setting
+mds.space.calIntervalSec=60

curvefs/conf/metaserver.conf

@@ -307,3 +307,9 @@ metaCacheOpt.metacacheGetLeaderRetry=3
 metaCacheOpt.metacacheRPCRetryIntervalUS=100000
 # RPC timeout of get leader
 metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000
+
+#### volume deallocate
+volume.deallocate.enable=true
+volume.deallocate.workerNum=5
+volume.deallocate.batchClean=10
+volume.sdk.confPath=/etc/curvefs/curvebs_client.conf

curvefs/proto/BUILD

@@ -113,6 +113,7 @@ proto_library(
     name = "curvefs_heartbeat_proto",
     srcs = ["heartbeat.proto"],
     deps = [":curvefs_common_proto",
+            ":metaserver_proto",
             "//proto:heartbeat_proto"],
 )
 

curvefs/proto/common.proto

@@ -31,6 +31,13 @@ enum BitmapLocation {
     AtEnd = 2;
 }
 
+message EmptyMsg {}
+
+message BlockGroupID {
+    required uint64 fsId = 1;
+    required uint64 offset = 2;
+}
+
 // When creating fs, `volumeSize` and `extendAlignment` are fetched from the bs cluster
 message Volume {
     optional uint64 volumeSize = 1;

curvefs/proto/heartbeat.proto

@@ -16,6 +16,7 @@
 
 syntax = "proto2";
 import "curvefs/proto/common.proto";
+import "curvefs/proto/metaserver.proto";
 import "proto/heartbeat.proto";
 package curvefs.mds.heartbeat;
 option cc_generic_services = true;
@@ -27,6 +28,17 @@ option go_package = "curvefs/proto/heartbeat";
 //     required uint64 diskUsedByte = 2;  // the disk this copyset used
 // };
 
+enum BlockGroupDeallcateStatusCode {
+    BGDP_PROCESSING = 0;
+    BGDP_DONE = 1;
+}
+
+message BlockGroupStatInfo {
+    required uint32 fsId = 1;
+    repeated metaserver.DeallocatableBlockGroup deallocatableBlockGroups = 2;
+    map<uint64, BlockGroupDeallcateStatusCode> blockGroupDeallocateStatus = 3;
+}
+
 message CopySetInfo {
     required uint32 poolId = 1;
     required uint32 copysetId = 2;
@@ -81,6 +93,7 @@ message MetaServerHeartbeatRequest {
     required uint32 leaderCount = 7;
     required uint32 copysetCount = 8;
     required MetaServerSpaceStatus spaceStatus = 9;
+    repeated BlockGroupStatInfo blockGroupStatInfos = 10;
 };
 
 message CopySetConf {
@@ -107,11 +120,14 @@ enum HeartbeatStatusCode {
     hbMetaServerIpPortNotMatch = 2;
     hbMetaServerTokenNotMatch = 3;
     hbAnalyseCopysetError = 4;
+    hbMetaServerFSUnkown = 5;
 }
 
 message MetaServerHeartbeatResponse {
     required HeartbeatStatusCode statusCode = 1;
     repeated CopySetConf needUpdateCopysets = 2;
+    // key is fsid, value is blockgroup offset
+    map<uint64, uint64> issuedBlockGroups = 3;
 };
 
 service HeartbeatService {

curvefs/proto/mds.proto

@@ -60,6 +60,7 @@ enum FSStatusCode {
     INSERT_MANAGE_INODE_FAIL = 35;
     DELETE_DENTRY_FAIL = 36;
     UPDATE_FS_FAIL = 37;
+    SPACE_RELEASE_FAIL = 38;
 }
 
 // fs interface

curvefs/proto/metaserver.proto

@@ -167,6 +167,46 @@ enum FsFileType {
     TYPE_S3 = 4;
 };
 
+message DeallocatableBlockGroup {
+    required uint64 blockGroupOffset = 1;
+    optional uint64 deallocatableSize = 2;
+    repeated uint64 inodeIdlist = 3;
+    repeated uint64 inodeIdUnderDeallocate = 4;
+
+    oneof type {
+        IncreaseDeallocatableBlockGroup increase = 5;
+        DecreaseDeallocatableBlockGroup decrease = 6;
+        MarkDeallocatableBlockGroup mark = 7;
+    }
+}
+
+message IncreaseDeallocatableBlockGroup {
+    required uint64 increaseDeallocatableSize = 1;
+    repeated uint64 inodeIdlistAdd = 2;
+}
+
+message DecreaseDeallocatableBlockGroup {
+    required uint64 decreaseDeallocatableSize = 1;
+    repeated uint64 inodedDeallocated = 2;
+}
+
+message MarkDeallocatableBlockGroup {
+    repeated uint64 inodeIdUnderDeallocate = 2;
+}
+
+message UpdateDeallocatableBlockGroupRequest {
+    required uint32 poolId = 1;
+    required uint32 copysetId = 2;
+    required uint32 partitionId = 3;
+    required uint64 fsId = 4;
+    repeated DeallocatableBlockGroup update = 5;
+}
+
+message UpdateDeallocatableBlockGroupResponse {
+    required MetaStatusCode statusCode = 1; 
+    optional uint64 appliedIndex = 2;
+}
+
 message VolumeExtent {
     required uint64 fsOffset = 1;
     required uint64 volumeOffset = 2;
@@ -179,7 +219,7 @@ message VolumeExtentSlice {
     repeated VolumeExtent extents = 2;
 }
 
-message VolumeExtentList {
+message VolumeExtentSliceList {
     repeated VolumeExtentSlice slices = 1;
 }
 
@@ -317,7 +357,7 @@ message UpdateInodeRequest {
     map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
-    optional VolumeExtentList volumeExtents = 23;
+    optional VolumeExtentSliceList volumeExtents = 23;
 }
 
 message UpdateInodeResponse {
@@ -464,7 +504,7 @@ message GetVolumeExtentRequest {
 message GetVolumeExtentResponse {
     required MetaStatusCode statusCode = 1;
     optional uint64 appliedIndex = 2;
-    optional VolumeExtentList slices = 3;
+    optional VolumeExtentSliceList slices = 3;
 }
 
 message UpdateVolumeExtentRequest {
@@ -473,7 +513,7 @@ message UpdateVolumeExtentRequest {
     required uint32 partitionId = 3;
     required uint32 fsId = 4;
     required uint64 inodeId = 5;
-    required VolumeExtentList extents = 6;
+    required VolumeExtentSliceList extents = 6;
 }
 
 message UpdateVolumeExtentResponse {
@@ -508,4 +548,7 @@ service MetaServerService {
     // volume extent interface
     rpc GetVolumeExtent(GetVolumeExtentRequest) returns (GetVolumeExtentResponse);
     rpc UpdateVolumeExtent(UpdateVolumeExtentRequest) returns (UpdateVolumeExtentResponse);
+
+    // block group with deallocatable inode list interface
+    rpc UpdateDeallocatableBlockGroup(UpdateDeallocatableBlockGroupRequest) returns (UpdateDeallocatableBlockGroupResponse);
 }

curvefs/proto/space.proto

@@ -34,6 +34,7 @@ enum SpaceErrCode {
     SpaceErrParam = 10;
     SpaceErrNotSupport = 11;
     SpaceErrExtendVolumeError = 12;
+    SpaceErrRelease = 13;
 }
 
 message BlockGroup {
@@ -48,6 +49,10 @@ message BlockGroup {
     required common.BitmapLocation bitmaplocation = 4;
     // owner, who owns this block group
     optional string owner = 5;
+    // deallocating, metaserver who is deallocate this block group
+    repeated uint32 deallocating = 6;
+    // deallocated, metaserver who finish deallocate this block group
+    repeated uint32 deallocated = 7;
 }
 
 message AllocateBlockGroupRequest {

curvefs/src/client/BUILD

@@ -45,6 +45,11 @@ cc_library(
             "volume/*.h",
             "warmup/*.h",
             "warmup/*.cpp",
+            "cache/*.h",
+            "cache/*.cpp",
+            "cache/diskcache/*.h",
+            "cache/diskcache/*.cpp",
+
         ],
         exclude = ["main.cpp"],
     ),

curvefs/src/client/async_request_closure.cpp

@@ -69,13 +69,20 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
 void UpdateVolumeExtentClosure::Run() {
     auto st = GetStatusCode();
     if (!IsOK(st)) {
-        LOG(ERROR) << "UpdateVolumeExtent failed, error: "
-                   << MetaStatusCode_Name(st)
-                   << ", inodeid: " << inode_->GetInodeId();
-        inode_->MarkInodeError();
+        if (inode_ != nullptr) {
+            inode_->MarkInodeError();
+            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
+                       << MetaStatusCode_Name(st)
+                       << ", inodeid: " << inode_->GetInodeId();
+        } else {
+            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
+                       << MetaStatusCode_Name(st);
+        }
     }
 
-    inode_->syncingVolumeExtentsMtx_.unlock();
+    if (inode_ != nullptr) {
+        inode_->syncingVolumeExtentsMtx_.unlock();
+    }
 
     if (sync_) {
         std::lock_guard<bthread::Mutex> lk(mtx_);

curvefs/src/client/cache/diskcache/disk_cache_base.cpp

@@ -28,7 +28,7 @@
 #include <dirent.h>
 #include <functional>
 
-#include "curvefs/src/client/s3/disk_cache_base.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_base.h"
 
 namespace curvefs {
 

curvefs/src/client/cache/diskcache/disk_cache_base.h

@@ -20,8 +20,8 @@
  * Author: wuhongsong
  */
 
-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_BASE_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_BASE_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_BASE_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_BASE_H_
 
 #include <glog/logging.h>
 
@@ -71,4 +71,4 @@ class DiskCacheBase {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_BASE_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_BASE_H_

curvefs/src/client/cache/diskcache/disk_cache_manager.cpp

@@ -19,6 +19,9 @@
  * Created Date: 21-08-13
  * Author: hzwuhongsong
  */
+
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager.h"
+
 #include <sys/vfs.h>
 #include <errno.h>
 #include <string>
@@ -27,7 +30,6 @@
 #include <list>
 
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
-#include "curvefs/src/client/s3/disk_cache_manager.h"
 #include "curvefs/src/common/s3util.h"
 
 namespace curvefs {
@@ -230,7 +232,12 @@ int DiskCacheManager::WriteDiskFile(const std::string fileName, const char *buf,
                                     uint64_t length, bool force) {
     // write throttle
     diskCacheThrottle_.Add(false, length);
+    uint64_t start = butil::cpuwide_time_us();
     int ret = cacheWrite_->WriteDiskFile(fileName, buf, length, force);
+    if ((ret >= 0) &&
+      metric_.get() != nullptr) {
+        CollectMetrics(&metric_->adaptorWriteDiskCache, length, start);
+    }
     if (ret > 0)
         AddDiskUsedBytes(ret);
     return ret;
@@ -244,14 +251,26 @@ int DiskCacheManager::ReadDiskFile(const std::string name, char *buf,
                                    uint64_t offset, uint64_t length) {
     // read throttle
     diskCacheThrottle_.Add(true, length);
-    return cacheRead_->ReadDiskFile(name, buf, offset, length);
+    uint64_t start = butil::cpuwide_time_us();
+    int ret = cacheRead_->ReadDiskFile(
+      name, buf, offset, length);
+    if ((ret >= 0) &&
+      metric_.get() != nullptr) {
+        CollectMetrics(&metric_->adaptorWriteDiskCache, length, start);
+    }
+    return ret;
 }
 
 int DiskCacheManager::WriteReadDirect(const std::string fileName,
                                       const char *buf, uint64_t length) {
     // write hrottle
     diskCacheThrottle_.Add(false, length);
+    uint64_t start = butil::cpuwide_time_us();
     int ret = cacheRead_->WriteDiskFile(fileName, buf, length);
+    if ((ret >= 0) &&
+      metric_.get() != nullptr) {
+        CollectMetrics(&metric_->adaptorWriteDiskCache, length, start);
+    }
     if (ret > 0)
         AddDiskUsedBytes(ret);
     return ret;

curvefs/src/client/cache/diskcache/disk_cache_manager.h

@@ -19,8 +19,9 @@
  * Created Date: 21-08-13
  * Author: wuhongsong
  */
-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_
+
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_H_
 
 #include <bthread/mutex.h>
 
@@ -38,8 +39,8 @@
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/common/utils.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/s3/disk_cache_write.h"
-#include "curvefs/src/client/s3/disk_cache_read.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_write.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_read.h"
 #include "curvefs/src/client/common/config.h"
 namespace curvefs {
 namespace client {
@@ -59,7 +60,7 @@ class DiskCacheManager {
                      std::shared_ptr<DiskCacheWrite> cacheWrite,
                      std::shared_ptr<DiskCacheRead> cacheRead);
     DiskCacheManager() {}
-    virtual ~DiskCacheManager() { TrimStop(); }
+    virtual ~DiskCacheManager() { TrimStop();}
 
     virtual int Init(std::shared_ptr<S3Client> client,
         const S3ClientAdaptorOption option);
@@ -76,33 +77,48 @@ class DiskCacheManager {
     void AddCache(const std::string &name);
 
     int CreateDir();
+
     std::string GetCacheReadFullDir();
+
     std::string GetCacheWriteFullDir();
 
     int WriteDiskFile(const std::string fileName, const char *buf,
                       uint64_t length, bool force = true);
-    void AsyncUploadEnqueue(const std::string objName);
+
     virtual int WriteReadDirect(const std::string fileName, const char *buf,
                                 uint64_t length);
+
     int ReadDiskFile(const std::string name, char *buf, uint64_t offset,
                      uint64_t length);
+
     int LinkWriteToRead(const std::string fileName,
                         const std::string fullWriteDir,
                         const std::string fullReadDir);
+
+    void AsyncUploadEnqueue(const std::string objName);
+
     int UploadAllCacheWriteFile();
+
     int UploadWriteCacheByInode(const std::string &inode);
+
     int ClearReadCache(const std::list<std::string> &files);
+
     /**
      * @brief get use ratio of cache disk
      * @return the use ratio
      */
     int64_t SetDiskFsUsedRatio();
+
     virtual bool IsDiskCacheFull();
+
     bool IsDiskCacheSafe();
+
     /**
      * @brief: start trim thread.
      */
+
     int TrimRun();
+
     /**
      * @brief: stop trim thread.
      */
@@ -129,6 +145,7 @@ class DiskCacheManager {
                 << ", now is: " << usedBytes_.load();
         return;
     }
+
     /**
      * @brief dec the used bytes of disk cache.
      * can not dec disk used bytes after file have been loaded,
@@ -143,12 +160,15 @@ class DiskCacheManager {
                 << ", now is: " << usedBytes_.load();
         return;
     }
+
     void SetDiskInitUsedBytes();
+
     uint64_t GetDiskUsedbytes() {
         return usedBytes_.load();
     }
 
     void InitQosParam();
+
     /**
      * @brief trim cache func.
      */
@@ -164,6 +184,14 @@ class DiskCacheManager {
      */
     bool IsCacheClean();
 
+    void CollectMetrics(InterfaceMetric *interface,
+      int count, uint64_t start) {
+        interface->bps.count << count;
+        interface->qps.count << 1;
+        interface->latency << (butil::cpuwide_time_us() - start);
+    }
+
+ private:
     curve::common::Thread backEndThread_;
     curve::common::Atomic<bool> isRunning_;
     curve::common::InterruptibleSleeper sleeper_;
@@ -202,4 +230,4 @@ class DiskCacheManager {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_H_

curvefs/src/client/cache/diskcache/disk_cache_manager_impl.cpp

@@ -27,7 +27,7 @@
 #include <memory>
 
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
-#include "curvefs/src/client/s3/disk_cache_manager_impl.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h"
 
 namespace curvefs {
 
@@ -70,8 +70,6 @@ void DiskCacheManagerImpl::Enqueue(
     taskPool_.Enqueue(task);
 }
 
-
-
 int DiskCacheManagerImpl::WriteReadDirectClosure(
     std::shared_ptr<PutObjectAsyncContext> context) {
         VLOG(9) << "WriteReadClosure start, name: " << context->key;

curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h

@@ -19,8 +19,8 @@
  * Created Date: 21-08-13
  * Author: wuhongsong
  */
-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_IMPL_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_IMPL_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_IMPL_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_IMPL_H_
 
 #include <bthread/mutex.h>
 
@@ -35,9 +35,9 @@
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/s3/disk_cache_write.h"
-#include "curvefs/src/client/s3/disk_cache_read.h"
-#include "curvefs/src/client/s3/disk_cache_manager.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_write.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_read.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager.h"
 
 namespace curvefs {
 namespace client {
@@ -46,6 +46,7 @@ using curvefs::common::PosixWrapper;
 using curvefs::client::common::DiskCacheType;
 using curve::common::TaskThreadPool;
 
+// disk cache opt
 struct DiskCacheOption {
     DiskCacheType diskCacheType;
     uint64_t trimCheckIntervalSec;
@@ -72,13 +73,15 @@ class DiskCacheManagerImpl {
         std::shared_ptr<S3Client> client);
     DiskCacheManagerImpl() {}
     virtual ~DiskCacheManagerImpl() {}
+
     /**
      * @brief init DiskCacheManagerImpl
      * @param[in] client S3Client
      * @param[in] option config option
      * @return success: 0, fail : < 0
-     */
-    int Init(const S3ClientAdaptorOption option);
+    */
+    virtual int Init(const S3ClientAdaptorOption option);
+
     /**
      * @brief Write obj
      * @param[in] name obj name
@@ -87,12 +90,14 @@ class DiskCacheManagerImpl {
      * @return success: write length, fail : < 0
      */
     int Write(const std::string name, const char *buf, uint64_t length);
+
     /**
      * @brief whether obj is cached in cached disk
      * @param[in] name obj name
      * @return cached: true, not cached : < 0
      */
     bool IsCached(const std::string name);
+
     /**
      * @brief read obj
      * @param[in] name obj name
@@ -109,9 +114,18 @@ class DiskCacheManagerImpl {
      */
     int UmountDiskCache();
 
+    /// @brief disk cache is full or not
+    /// @return true: full, false: not full
     bool IsDiskCacheFull();
+
+    /// @brief write to cache read disk
+    /// @param fileName file name
+    /// @param buf data need to write
+    /// @param length data length need to write
+    /// @return data length write to cache read
     int WriteReadDirect(const std::string fileName, const char *buf,
                         uint64_t length);
+
     void InitMetrics(std::string fsName);
 
     virtual int UploadWriteCacheByInode(const std::string &inode);
@@ -124,14 +138,17 @@ class DiskCacheManagerImpl {
  private:
     int WriteDiskFile(const std::string name, const char *buf, uint64_t length);
 
-    std::shared_ptr<DiskCacheManager> diskCacheManager_;
-
-    bool forceFlush_;
-    std::shared_ptr<S3Client> client_;
-
     int WriteClosure(std::shared_ptr<PutObjectAsyncContext> context);
 
     int WriteReadDirectClosure(std::shared_ptr<PutObjectAsyncContext> context);
+
+ private:
+    // disk cache manager
+    std::shared_ptr<DiskCacheManager> diskCacheManager_;
+    // use fdatasync or not
+    bool forceFlush_;
+    // s3 client interface(put/get object from s3)
+    std::shared_ptr<S3Client> client_;
     // threads for disk cache
     uint32_t threads_;
     TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
@@ -141,4 +158,4 @@ class DiskCacheManagerImpl {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_MANAGER_IMPL_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_MANAGER_IMPL_H_

curvefs/src/client/cache/diskcache/disk_cache_read.cpp

@@ -29,7 +29,7 @@
 #include <memory>
 #include <utility>
 
-#include "curvefs/src/client/s3/disk_cache_read.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_read.h"
 #include "curvefs/src/common/s3util.h"
 
 namespace curvefs {

curvefs/src/client/cache/diskcache/disk_cache_read.h

@@ -19,8 +19,8 @@
  * Created Date: 21-08-13
  * Author: wuhongsong
  */
-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_READ_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_READ_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_READ_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_READ_H_
 
 #include <list>
 #include <string>
@@ -32,7 +32,7 @@
 #include "src/common/interruptible_sleeper.h"
 #include "src/common/lru_cache.h"
 #include "curvefs/src/common/wrap_posix.h"
-#include "curvefs/src/client/s3/disk_cache_base.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_base.h"
 
 namespace curvefs {
 namespace client {
@@ -73,4 +73,4 @@ class DiskCacheRead : public DiskCacheBase {
 
 }  // namespace client
 }  // namespace curvefs
-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_READ_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_READ_H_

curvefs/src/client/cache/diskcache/disk_cache_write.cpp

@@ -27,7 +27,8 @@
 #include <dirent.h>
 
 #include <vector>
-#include "curvefs/src/client/s3/disk_cache_write.h"
+
+#include "curvefs/src/client/cache/diskcache/disk_cache_write.h"
 #include "curvefs/src/common/s3util.h"
 
 namespace curvefs {

curvefs/src/client/cache/diskcache/disk_cache_write.h

@@ -19,8 +19,8 @@
  * Created Date: 21-08-13
  * Author: wuhongsong
  */
-#ifndef CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_
-#define CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_WRITE_H_
+#define CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_WRITE_H_
 
 #include <sys/stat.h>
 #include <bthread/condition_variable.h>
@@ -39,9 +39,8 @@
 #include "curvefs/src/common/wrap_posix.h"
 #include "curvefs/src/common/utils.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/s3/disk_cache_read.h"
 #include "curvefs/src/client/common/config.h"
-#include "curvefs/src/client/s3/disk_cache_base.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_base.h"
 
 namespace curvefs {
 namespace client {
@@ -167,4 +166,4 @@ class DiskCacheWrite : public DiskCacheBase {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_S3_DISK_CACHE_WRITE_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_DISKCACHE_DISK_CACHE_WRITE_H_

curvefs/src/client/cache/fuse_client_cache_manager.h

@@ -19,8 +19,8 @@
  * Created Date: 21-8-18
  * Author: huyao
  */
-#ifndef CURVEFS_SRC_CLIENT_S3_CLIENT_S3_CACHE_MANAGER_H_
-#define CURVEFS_SRC_CLIENT_S3_CLIENT_S3_CACHE_MANAGER_H_
+#ifndef CURVEFS_SRC_CLIENT_CACHE_FUSE_CLIENT_CACHE_MANAGER_H_
+#define CURVEFS_SRC_CLIENT_CACHE_FUSE_CLIENT_CACHE_MANAGER_H_
 
 #include <algorithm>
 #include <cstring>
@@ -48,19 +48,16 @@ using curve::common::WriteLockGuard;
 namespace curvefs {
 namespace client {
 
-class S3ClientAdaptorImpl;
+class StorageAdaptor;
 class ChunkCacheManager;
 class FileCacheManager;
 class FsCacheManager;
 class DataCache;
-class S3ReadRequest;
 using FileCacheManagerPtr = std::shared_ptr<FileCacheManager>;
 using ChunkCacheManagerPtr = std::shared_ptr<ChunkCacheManager>;
 using DataCachePtr = std::shared_ptr<DataCache>;
 using WeakDataCachePtr = std::weak_ptr<DataCache>;
-using curve::common::GetObjectAsyncCallBack;
-using curve::common::PutObjectAsyncCallBack;
-using curve::common::S3Adapter;
+using curve::common::TaskThreadPool;
 using curvefs::metaserver::Inode;
 using curvefs::metaserver::S3ChunkInfo;
 using curvefs::metaserver::S3ChunkInfoList;
@@ -87,34 +84,22 @@ struct ReadRequest {
     }
 };
 
-struct S3ReadRequest {
-    uint64_t chunkId;
-    uint64_t offset;  // file offset
-    uint64_t len;
-    uint64_t objectOffset;  // s3 object's begin in the block
-    uint64_t readOffset;    // read buf offset
-    uint64_t fsId;
+struct UperFlushRequest {
     uint64_t inodeId;
-    uint64_t compaction;
-
-    std::string DebugString() const {
-        std::ostringstream os;
-        os << "S3ReadRequest ( chunkId = " << chunkId << ", offset = " << offset
-           << ", len = " << len << ", objectOffset = " << objectOffset
-           << ", readOffset = " << readOffset << ", fsId = " << fsId
-           << ", inodeId = " << inodeId << ", compaction = " << compaction
-           << " )";
-        return os.str();
-    }
+    const char *buf;
+    uint64_t length;
+    uint64_t offset;   // offset at inode
+    uint64_t chunkId;
+    uint64_t chunkPos;
+    bool sync;
 };
 
-inline std::string
-S3ReadRequestVecDebugString(const std::vector<S3ReadRequest> &reqs) {
-    std::ostringstream os;
-    for_each(reqs.begin(), reqs.end(),
-             [&](const S3ReadRequest &req) { os << req.DebugString() << " "; });
-    return os.str();
-}
+struct  UperReadRequest {
+    std::vector<ReadRequest> requests;
+    uint64_t inodeId;
+    std::shared_ptr<InodeWrapper> inodeWapper;
+    char *buf;
+};
 
 struct ObjectChunkInfo {
     S3ChunkInfo s3ChunkInfo;
@@ -134,7 +119,7 @@ enum DataCacheStatus {
 
 class DataCache : public std::enable_shared_from_this<DataCache> {
  public:
-    DataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
+    DataCache(StorageAdaptor *s3ClientAdaptor,
               ChunkCacheManagerPtr chunkCacheManager, uint64_t chunkPos,
               uint64_t len, const char *data,
               std::shared_ptr<KVClientManager> kvClientManager);
@@ -209,21 +194,10 @@ class DataCache : public std::enable_shared_from_this<DataCache> {
                              const char *data);
     void AddDataBefore(uint64_t len, const char *data);
 
-    CURVEFS_ERROR PrepareFlushTasks(
-        uint64_t inodeId, char *data,
-        std::vector<std::shared_ptr<PutObjectAsyncContext>> *s3Tasks,
-        std::vector<std::shared_ptr<SetKVCacheTask>> *kvCacheTasks,
-        uint64_t *chunkId, uint64_t *writeOffset);
-
-    void FlushTaskExecute(
-        CachePolicy cachePolicy,
-        const std::vector<std::shared_ptr<PutObjectAsyncContext>> &s3Tasks,
-        const std::vector<std::shared_ptr<SetKVCacheTask>> &kvCacheTasks);
-
     CachePolicy GetCachePolicy(bool toS3);
 
  private:
-    S3ClientAdaptorImpl *s3ClientAdaptor_;
+    StorageAdaptor *storageAdaptor_;
     ChunkCacheManagerPtr chunkCacheManager_;
     uint64_t chunkPos_;  // useful chunkPos
     uint64_t len_;  // useful len
@@ -238,33 +212,19 @@ class DataCache : public std::enable_shared_from_this<DataCache> {
     std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
-class S3ReadResponse {
- public:
-    explicit S3ReadResponse(char *data, uint64_t length)
-        : data_(data), len_(length) {}
-
-    char *GetDataBuf() { return data_; }
-
-    uint64_t GetBufLen() { return len_; }
-
- private:
-    char *data_;
-    uint64_t len_;
-};
-
 class ChunkCacheManager
     : public std::enable_shared_from_this<ChunkCacheManager> {
  public:
-    ChunkCacheManager(uint64_t index, S3ClientAdaptorImpl *s3ClientAdaptor,
+    ChunkCacheManager(uint64_t index, StorageAdaptor *s3ClientAdaptor,
                       std::shared_ptr<KVClientManager> kvClientManager)
-        : index_(index), s3ClientAdaptor_(s3ClientAdaptor),
+        : index_(index), storageAdaptor_(s3ClientAdaptor),
           flushingDataCache_(nullptr),
           kvClientManager_(std::move(kvClientManager)) {}
     virtual ~ChunkCacheManager() = default;
-    void ReadChunk(uint64_t index, uint64_t chunkPos, uint64_t readLen,
+    virtual void ReadChunk(uint64_t index, uint64_t chunkPos, uint64_t readLen,
                    char *dataBuf, uint64_t dataBufOffset,
                    std::vector<ReadRequest> *requests);
-    virtual void WriteNewDataCache(S3ClientAdaptorImpl *s3ClientAdaptor,
+    virtual void WriteNewDataCache(StorageAdaptor *s3ClientAdaptor,
                                    uint32_t chunkPos, uint32_t len,
                                    const char *data);
     virtual void AddReadDataCache(DataCachePtr dataCache);
@@ -320,7 +280,7 @@ class ChunkCacheManager
         dataRCacheMap_;  // first is pos in chunk
 
     RWLock rwLockRead_;  //  for read cache
-    S3ClientAdaptorImpl *s3ClientAdaptor_;
+    StorageAdaptor *storageAdaptor_;
     curve::common::Mutex flushMtx_;
     DataCachePtr flushingDataCache_;
     curve::common::Mutex flushingDataCacheMtx_;
@@ -331,10 +291,10 @@ class ChunkCacheManager
 class FileCacheManager {
  public:
     FileCacheManager(uint32_t fsid, uint64_t inode,
-                     S3ClientAdaptorImpl *s3ClientAdaptor,
-                     std::shared_ptr<KVClientManager> kvClientManager,
-                     std::shared_ptr<TaskThreadPool<>> threadPool)
-        : fsId_(fsid), inode_(inode), s3ClientAdaptor_(s3ClientAdaptor),
+      StorageAdaptor *s3ClientAdaptor,
+      std::shared_ptr<KVClientManager> kvClientManager,
+      std::shared_ptr<TaskThreadPool<>> threadPool)
+        : fsId_(fsid), inode_(inode), storageAdaptor_(s3ClientAdaptor),
           kvClientManager_(std::move(kvClientManager)),
           readTaskPool_(threadPool) {}
     FileCacheManager() = default;
@@ -353,6 +313,8 @@ class FileCacheManager {
     virtual int Read(uint64_t inodeId, uint64_t offset, uint64_t length,
                      char *dataBuf);
 
+    // for test
+
     bool IsEmpty() { return chunkCacheMap_.empty(); }
 
     uint64_t GetInodeId() const { return inode_; }
@@ -369,24 +331,6 @@ class FileCacheManager {
  private:
     void WriteChunk(uint64_t index, uint64_t chunkPos, uint64_t writeLen,
                     const char *dataBuf);
-    void GenerateS3Request(ReadRequest request,
-                           const S3ChunkInfoList &s3ChunkInfoList,
-                           char *dataBuf, std::vector<S3ReadRequest> *requests,
-                           uint64_t fsId, uint64_t inodeId);
-
-    void PrefetchS3Objs(
-        const std::vector<std::pair<std::string, uint64_t>> &prefetchObjs);
-
-    void HandleReadRequest(const ReadRequest &request,
-                           const S3ChunkInfo &s3ChunkInfo,
-                           std::vector<ReadRequest> *addReadRequests,
-                           std::vector<uint64_t> *deletingReq,
-                           std::vector<S3ReadRequest> *requests, char *dataBuf,
-                           uint64_t fsId, uint64_t inodeId);
-
-    int HandleReadRequest(const std::vector<S3ReadRequest> &requests,
-                          std::vector<S3ReadResponse> *responses,
-                          uint64_t fileLen);
 
     // GetChunkLoc: get chunk info according to offset
     void GetChunkLoc(uint64_t offset, uint64_t *index, uint64_t *chunkPos,
@@ -401,12 +345,7 @@ class FileCacheManager {
                           uint64_t *actualReadLen,
                           std::vector<ReadRequest> *memCacheMissRequest);
 
-    // miss read from memory read/write cache, need read from
-    // kv(localdisk/remote cache/s3)
-    int GenerateKVRequest(const std::shared_ptr<InodeWrapper> &inodeWrapper,
-                          const std::vector<ReadRequest> &readRequest,
-                          char *dataBuf, std::vector<S3ReadRequest> *kvRequest);
-
+/*  whs
     enum class ReadStatus {
         OK = 0,
         S3_READ_FAIL = -1,
@@ -421,39 +360,7 @@ class FileCacheManager {
         }
         return st;
     }
-
-    // read kv request, need
-    ReadStatus ReadKVRequest(const std::vector<S3ReadRequest> &kvRequests,
-                             char *dataBuf, uint64_t fileLen);
-
-    // thread function for ReadKVRequest
-    void ProcessKVRequest(const S3ReadRequest &req, char *dataBuf,
-                          uint64_t fileLen,
-                          std::once_flag &cancelFlag,     // NOLINT
-                          std::atomic<bool> &isCanceled,  // NOLINT
-                          std::atomic<int> &retCode);     // NOLINT
-
-    // read kv request from local disk cache
-    bool ReadKVRequestFromLocalCache(const std::string &name, char *databuf,
-                                     uint64_t offset, uint64_t len);
-
-    // read kv request from remote cache like memcached
-    bool ReadKVRequestFromRemoteCache(const std::string &name, char *databuf,
-                                      uint64_t offset, uint64_t length);
-
-    // read kv request from s3
-    bool ReadKVRequestFromS3(const std::string &name, char *databuf,
-                             uint64_t offset, uint64_t length, int *ret);
-
-    // read retry policy when read from s3 occur not exist error
-    int HandleReadS3NotExist(uint32_t retry,
-                             const std::shared_ptr<InodeWrapper> &inodeWrapper);
-
-    // prefetch for block
-    void PrefetchForBlock(const S3ReadRequest &req, uint64_t fileLen,
-                         uint64_t blockSize, uint64_t chunkSize,
-                         uint64_t startBlockIndex);
-
+*/
  private:
     friend class AsyncPrefetchCallback;
 
@@ -462,9 +369,7 @@ class FileCacheManager {
     std::map<uint64_t, ChunkCacheManagerPtr> chunkCacheMap_;  // first is index
     RWLock rwLock_;
     curve::common::Mutex mtx_;
-    S3ClientAdaptorImpl *s3ClientAdaptor_;
-    curve::common::Mutex downloadMtx_;
-    std::set<std::string> downloadingObj_;
+    StorageAdaptor *storageAdaptor_;
 
     std::shared_ptr<KVClientManager> kvClientManager_;
     std::shared_ptr<TaskThreadPool<>> readTaskPool_;
@@ -472,19 +377,27 @@ class FileCacheManager {
 
 class FsCacheManager {
  public:
-    FsCacheManager(S3ClientAdaptorImpl *s3ClientAdaptor,
+    FsCacheManager(StorageAdaptor *s3ClientAdaptor,
                    uint64_t readCacheMaxByte, uint64_t writeCacheMaxByte,
                    uint32_t readCacheThreads,
                    std::shared_ptr<KVClientManager> kvClientManager)
         : lruByte_(0), wDataCacheNum_(0), wDataCacheByte_(0),
           readCacheMaxByte_(readCacheMaxByte),
           writeCacheMaxByte_(writeCacheMaxByte),
-          s3ClientAdaptor_(s3ClientAdaptor), isWaiting_(false),
+          storageAdaptor_(s3ClientAdaptor), isWaiting_(false),
           kvClientManager_(std::move(kvClientManager)) {
-        readTaskPool_->Start(readCacheThreads);
+            LOG(INFO) << "whs read task pool start. " << readCacheThreads;
+            readTaskPool_->Start(readCacheThreads);
+            LOG(INFO) << "whs read task pool start end";
     }
+
     FsCacheManager() = default;
-    virtual ~FsCacheManager() { readTaskPool_->Stop(); }
+    virtual ~FsCacheManager() {
+        LOG(INFO) << "whs read task pool stop";
+        readTaskPool_->Stop();
+        LOG(INFO) << "whs read task pool stop end";
+    }
+
     virtual FileCacheManagerPtr FindFileCacheManager(uint64_t inodeId);
     virtual FileCacheManagerPtr FindOrCreateFileCacheManager(uint64_t fsId,
                                                      uint64_t inodeId);
@@ -547,6 +460,11 @@ class FsCacheManager {
         assert(ret.second);
         (void)ret;
     }
+
+    std::shared_ptr<TaskThreadPool<>> GetReadTaskPool() {
+        return readTaskPool_;
+    }
+
     void DataCacheNumInc();
     void DataCacheNumFetchSub(uint64_t v);
     void DataCacheByteInc(uint64_t v);
@@ -585,7 +503,7 @@ class FsCacheManager {
     std::atomic<uint64_t> wDataCacheByte_;
     uint64_t readCacheMaxByte_;
     uint64_t writeCacheMaxByte_;
-    S3ClientAdaptorImpl *s3ClientAdaptor_;
+    StorageAdaptor *storageAdaptor_;
     bool isWaiting_;
     std::mutex mutex_;
     std::condition_variable cond_;
@@ -601,4 +519,4 @@ class FsCacheManager {
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_S3_CLIENT_S3_CACHE_MANAGER_H_
+#endif  // CURVEFS_SRC_CLIENT_CACHE_FUSE_CLIENT_CACHE_MANAGER_H_

curvefs/src/client/client_storage_adaptor.cpp

@@ -0,0 +1,301 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*
+ * Project: curve
+ * Created Date: Thur March 14 2023
+ * Author: wuhongsong
+ */
+
+#include "curvefs/src/client/client_storage_adaptor.h"
+
+#include <brpc/channel.h>
+#include <brpc/controller.h>
+#include <algorithm>
+#include <list>
+
+#include "absl/memory/memory.h"
+
+
+#include "curvefs/src/common/s3util.h"
+
+namespace curvefs {
+namespace client {
+
+CURVEFS_ERROR
+StorageAdaptor::Init(const FuseClientOption &fuseOption,
+    std::shared_ptr<InodeCacheManager> inodeManager,
+    std::shared_ptr<MdsClient> mdsClient,
+    std::shared_ptr<FsCacheManager> fsCacheManager,
+    std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+    std::shared_ptr<KVClientManager> kvClientManager,
+    std::shared_ptr<FsInfo> fsInfo) {
+        const S3ClientAdaptorOption option =
+          fuseOption.s3Opt.s3ClientAdaptorOpt;
+        LOG(INFO) << "StorageAdaptor Init. block size:" << blockSize_
+            << ", chunk size: " << chunkSize_
+            << ", intervalSec: " << option.intervalSec
+            << ", flushIntervalSec: " << option.flushIntervalSec
+            << ", writeCacheMaxByte: " << option.writeCacheMaxByte
+            << ", readCacheMaxByte: " << option.readCacheMaxByte
+            << ", nearfullRatio: " << option.nearfullRatio
+            << ", baseSleepUs: " << option.baseSleepUs;
+        pageSize_ = option.pageSize;
+        if (chunkSize_ % blockSize_ != 0) {
+            LOG(ERROR) << "chunkSize:" << chunkSize_
+                    << " is not integral multiple for the blockSize:"
+                    << blockSize_;
+            return CURVEFS_ERROR::INVALIDPARAM;
+        }
+        diskCacheType_ = option.diskCacheOpt.diskCacheType;
+        memCacheNearfullRatio_ = option.nearfullRatio;
+        throttleBaseSleepUs_ = option.baseSleepUs;
+        flushIntervalSec_ = option.flushIntervalSec;
+        chunkFlushThreads_ = option.chunkFlushThreads;
+        inodeManager_ = inodeManager;
+        mdsClient_ = mdsClient;
+        fsCacheManager_ = fsCacheManager;
+        waitInterval_.Init(option.intervalSec * 1000);
+        fsInfo_ = fsInfo;
+
+        if (nullptr != diskCacheManagerImpl) {
+            diskCacheManagerImpl_ = diskCacheManagerImpl;
+            if (diskCacheManagerImpl_->Init(option) < 0) {
+                LOG(ERROR) << "Init disk cache failed";
+                return CURVEFS_ERROR::INTERNAL;
+            }
+        }
+
+        if (enableBgFlush_) {        // start background flush thread
+            toStop_.store(false, std::memory_order_release);
+            bgFlushThread_ = Thread(&StorageAdaptor::BackGroundFlush, this);
+        }
+
+        // start chunk flush threads
+        taskPool_.Start(chunkFlushThreads_);
+        LOG(INFO) << "storage Adaptor init success.";
+        return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR StorageAdaptor::FuseOpInit(void *userdata,
+  struct fuse_conn_info *conn) {
+    LOG(INFO) << " storage adaptor fuse init start.";
+    SetFsId(fsInfo_->fsid());
+    InitMetrics(fsInfo_->fsname());
+    return CURVEFS_ERROR::OK;
+}
+
+int StorageAdaptor::Stop() {
+    LOG(INFO) << " stop storage adaptor start.";
+    waitInterval_.StopWait();
+    toStop_.store(true, std::memory_order_release);
+    FsSyncSignal();
+    if (bgFlushThread_.joinable()) {
+        bgFlushThread_.join();
+    }
+    if (HasDiskCache()) {
+        diskCacheManagerImpl_->UmountDiskCache();
+    }
+    taskPool_.Stop();
+    LOG(INFO) << " stop storage adaptor success.";
+    return 0;
+}
+
+int StorageAdaptor::Write(uint64_t inodeId, uint64_t offset,
+  uint64_t length, const char *buf) {
+    VLOG(6) << "write start offset:" << offset << ", len:" << length
+            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
+    uint64_t start = butil::cpuwide_time_us();
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindOrCreateFileCacheManager(fsId_, inodeId);
+    {
+        std::lock_guard<std::mutex> lockguard(ioMtx_);
+        fsCacheManager_->DataCacheByteInc(length);
+        uint64_t size = fsCacheManager_->GetDataCacheSize();
+        uint64_t maxSize = fsCacheManager_->GetDataCacheMaxSize();
+        if (size >= maxSize) {
+            VLOG(6) << "write cache is full, wait flush. size: " << size
+                    << ", maxSize:" << maxSize;
+            // offer to do flush
+            waitInterval_.StopWait();
+            fsCacheManager_->WaitFlush();
+        }
+    }
+    uint64_t memCacheRatio = fsCacheManager_->MemCacheRatio();
+    int64_t exceedRatio = memCacheRatio - memCacheNearfullRatio_;
+    if (exceedRatio > 0) {
+        // offer to do flush
+        waitInterval_.StopWait();
+        // upload to s3 derectly or cache disk full
+        bool needSleep =
+            (DisableDiskCache() || IsReadCache()) ||
+            (IsReadWriteCache() && diskCacheManagerImpl_->IsDiskCacheFull());
+        if (needSleep) {
+            uint32_t exponent = pow(2, (exceedRatio) / 10);
+            bthread_usleep(throttleBaseSleepUs_ * exceedRatio * exponent);
+            VLOG(6) << "write cache nearfull and use ratio is: "
+                    << memCacheRatio << ", exponent is: " << exponent;
+        }
+    }
+    int ret = fileCacheManager->Write(offset, length, buf);
+    fsCacheManager_->DataCacheByteDec(length);
+    if (nullptr != GetMetric()) {
+        CollectMetrics(&ioMetric_->adaptorWrite, ret, start);
+    }
+    VLOG(6) << "write end inodeId:" << inodeId << ",ret:" << ret;
+    return ret;
+}
+
+int StorageAdaptor::Read(uint64_t inodeId, uint64_t offset,
+  uint64_t length, char *buf) {
+    VLOG(6) << "read start offset:" << offset << ", len:" << length
+            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
+    uint64_t start = butil::cpuwide_time_us();
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindOrCreateFileCacheManager(fsId_, inodeId);
+
+    int ret = fileCacheManager->Read(inodeId, offset, length, buf);
+    VLOG(6) << "read end inodeId:" << inodeId << ",ret:" << ret;
+    if (ret < 0) {
+        return ret;
+    }
+    if (nullptr != GetMetric()) {
+        CollectMetrics(&ioMetric_->adaptorRead, ret, start);
+    }
+    VLOG(6) << "read end offset:" << offset << ", len:" << length
+            << ", fsId:" << fsId_ << ", inodeId:" << inodeId;
+    return ret;
+}
+
+void StorageAdaptor::BackGroundFlush() {
+    while (!toStop_.load(std::memory_order_acquire)) {
+        {
+            std::unique_lock<std::mutex> lck(mtx_);
+            if (fsCacheManager_->GetDataCacheNum() == 0) {
+                VLOG(3) << "BackGroundFlush has no write cache, so wait";
+                cond_.wait(lck);
+            }
+        }
+        uint64_t start = butil::cpuwide_time_us();
+        if (fsCacheManager_->MemCacheRatio() > memCacheNearfullRatio_) {
+            VLOG(3) << "BackGroundFlush radically, write cache num is: "
+                    << fsCacheManager_->GetDataCacheNum()
+                    << "cache ratio is: " << fsCacheManager_->MemCacheRatio();
+            fsCacheManager_->FsSync(true);
+
+        } else {
+            waitInterval_.WaitForNextExcution();
+            VLOG(6) << "BackGroundFlush, write cache num is:"
+                    << fsCacheManager_->GetDataCacheNum()
+                    << "cache ratio is: " << fsCacheManager_->MemCacheRatio();
+            fsCacheManager_->FsSync(false);
+            VLOG(6) << "background fssync end";
+        }
+        if (nullptr != GetMetric()) {
+            CollectMetrics(&ioMetric_->adaptorBgFlush, 1, start);
+        }
+    }
+    return;
+}
+
+
+CURVEFS_ERROR StorageAdaptor::Flush(uint64_t inodeId) {
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindFileCacheManager(inodeId);
+    if (nullptr == fileCacheManager) {
+        return CURVEFS_ERROR::OK;
+    }
+    VLOG(6) << "Flush data of inodeId:" << inodeId;
+    return fileCacheManager->Flush(true, false);
+}
+
+CURVEFS_ERROR StorageAdaptor::FlushAllCache(uint64_t inodeId) {
+    VLOG(6) << "FlushAllCache, inodeId:" << inodeId;
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindFileCacheManager(inodeId);
+    if (!fileCacheManager) {
+        return CURVEFS_ERROR::OK;
+    }
+
+    // force flush data in memory to s3
+    VLOG(6) << "FlushAllCache, flush memory data of inodeId:" << inodeId;
+    CURVEFS_ERROR ret = fileCacheManager->Flush(true, false);
+    if (ret != CURVEFS_ERROR::OK) {
+        return ret;
+    }
+
+    // force flush data in diskcache to s3
+    if (HasDiskCache()) {
+        VLOG(6) << "FlushAllCache, wait inodeId:" << inodeId
+                << "related chunk upload to s3";
+        if (ClearDiskCache(inodeId) < 0) {
+            return CURVEFS_ERROR::INTERNAL;
+        }
+    }
+
+    return ret;
+}
+
+CURVEFS_ERROR StorageAdaptor::FsSync() {
+    return fsCacheManager_->FsSync(true);
+}
+
+int StorageAdaptor::ClearDiskCache(int64_t inodeId) {
+    // flush disk cache. read cache do not need clean
+    int ret =
+        diskCacheManagerImpl_->UploadWriteCacheByInode(std::to_string(inodeId));
+    LOG_IF(ERROR, ret < 0) << "FlushAllCache, inode:" << inodeId
+                           << ", upload write cache fail";
+    return ret;
+}
+
+void StorageAdaptor::ReleaseCache(uint64_t inodeId) {
+    FileCacheManagerPtr fileCacheManager =
+        fsCacheManager_->FindFileCacheManager(inodeId);
+    if (!fileCacheManager) {
+        return;
+    }
+    VLOG(9) << "ReleaseCache inode:" << inodeId;
+    fileCacheManager->ReleaseCache();
+    fsCacheManager_->ReleaseFileCacheManager(inodeId);
+    return;
+}
+
+void StorageAdaptor::Enqueue(
+  std::shared_ptr<FlushChunkCacheContext> context) {
+    auto task = [this, context]() {
+        this->FlushChunkClosure(context);
+    };
+    taskPool_.Enqueue(task);
+}
+
+int StorageAdaptor::FlushChunkClosure(
+  std::shared_ptr<FlushChunkCacheContext> context) {
+    VLOG(9) << "FlushChunkCacheClosure start: " << context->inode;
+    CURVEFS_ERROR ret = context->chunkCacheManptr->Flush(
+      context->inode, context->force);
+    // set the returned value
+    // it is need in FlushChunkCacheCallBack
+    context->retCode = ret;
+    context->cb(context);
+    VLOG(9) << "FlushChunkCacheClosure end: " << context->inode;
+    return 0;
+}
+
+}  // namespace client
+}  // namespace curvefs
+

curvefs/src/client/client_storage_adaptor.h

@@ -0,0 +1,332 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*
+ * Project: curve
+ * Created Date: Thur March 14 2023
+ * Author: wuhongsong
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_CLIENT_STORAGE_ADAPTOR_H_
+#define CURVEFS_SRC_CLIENT_CLIENT_STORAGE_ADAPTOR_H_
+
+#include <memory>
+#include <string>
+
+#include "curvefs/proto/common.pb.h"
+#include "curvefs/proto/mds.pb.h"
+#include "curvefs/proto/metaserver.pb.h"
+
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h"
+#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
+#include "curvefs/src/client/common/common.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/common/define.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/metric/client_metric.h"
+#include "curvefs/src/client/rpcclient/mds_client.h"
+#include "curvefs/src/client/s3/client_s3.h"
+
+using ::curve::common::Thread;
+using ::curve::common::TaskThreadPool;
+using curvefs::client::common::FuseClientOption;
+using curvefs::client::common::DiskCacheType;
+using curvefs::client::metric::IoMetric;
+using curvefs::client::rpcclient::MdsClient;
+using curvefs::metaserver::Inode;
+
+namespace curvefs {
+namespace client {
+
+class DiskCacheManagerImpl;
+class FlushChunkCacheContext;
+class ChunkCacheManager;
+
+// callback function for FlushChunkCache
+using FlushChunkCacheCallBack = std::function<
+  void(const std::shared_ptr<FlushChunkCacheContext>&)>;
+
+
+/// @brief flush chunk cache context
+/// @param inode inode id
+/// @param chunkCacheManptr chunk cache manager
+/// @param force force flush memory cache
+/// @param cb callback function
+/// @param retCode error code
+struct FlushChunkCacheContext {
+    uint64_t inode;
+    ChunkCacheManagerPtr chunkCacheManptr;
+    bool force;
+    FlushChunkCacheCallBack cb;
+    CURVEFS_ERROR retCode;
+};
+
+// the base class of the underlying storage adaptation layer
+class StorageAdaptor {
+ public:
+    StorageAdaptor() {}
+    virtual ~StorageAdaptor() {
+        Stop();
+    }
+
+    /// @brief init storage adaptor
+    /// @param option fuse client option
+    /// @param inodeManager inode cache manager
+    /// @param mdsClient mds client
+    /// @param fsCacheManager fscache manager
+    /// @param diskCacheManagerImpl disk cache manager
+    /// @param kvClientManager kv client manager
+    /// @param fsInfo file system information
+    /// @return error code
+    virtual CURVEFS_ERROR
+    Init(const FuseClientOption &option,
+        std::shared_ptr<InodeCacheManager> inodeManager,
+        std::shared_ptr<MdsClient> mdsClient,
+        std::shared_ptr<FsCacheManager> fsCacheManager,
+        std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+        std::shared_ptr<KVClientManager> kvClientManager,
+        std::shared_ptr<FsInfo> fsInfo);
+
+    virtual CURVEFS_ERROR FuseOpInit(void *userdata,
+        struct fuse_conn_info *conn);
+
+    virtual int Stop();
+
+    /// @brief flush data cache to backend storage(s3 or volume)
+    /// @param req flush request
+    /// @param writeOffset length that has been written to backend storage
+    /// @return error code
+    virtual CURVEFS_ERROR FlushDataCache(const UperFlushRequest& req,
+      uint64_t* writeOffset) = 0;
+
+    /// @brief read data from backend storage(s3 or volume)
+    /// @param request read request
+    /// @return error code
+    virtual CURVEFS_ERROR ReadFromLowlevel(UperReadRequest request) = 0;
+
+    /// @brief wrire data to memory cache
+    /// @param inodeId file inode id
+    /// @param offset offset of data in file
+    /// @param length length of data
+    /// @param buf data need to write
+    /// @return length of data that has been written to memory cache
+    virtual int Write(uint64_t inodeId, uint64_t offset,
+      uint64_t length, const char *buf);
+
+    /// @brief read data from memory cache
+    /// @param inodeId file inode id
+    /// @param offset offset of data in file
+    /// @param length length need to read
+    /// @param buf data buffer
+    /// @return length of data that has been read from memory cache
+    virtual int Read(uint64_t inodeId, uint64_t offset,
+      uint64_t length, char *buf);
+
+    virtual CURVEFS_ERROR Truncate(InodeWrapper* inodeWrapper,
+      uint64_t size) = 0;
+
+    /// @brief allocate chunk id from mds
+    /// @param fsId file system id
+    /// @param idNum chunks number need to allocate
+    /// @param chunkId chunk id that has been allocated
+    /// @return error code
+    FSStatusCode AllocChunkId(uint32_t fsId,
+      uint32_t idNum, uint64_t *chunkId) {
+        return mdsClient_->AllocS3ChunkId(fsId, idNum, chunkId);
+    }
+
+    /// @brief release all cache of file
+    /// @param inodeId file inode id
+    void ReleaseCache(uint64_t inodeId);
+
+    virtual CURVEFS_ERROR Flush(uint64_t inodeId);
+    virtual CURVEFS_ERROR FlushAllCache(uint64_t inodeId);
+    virtual CURVEFS_ERROR FsSync();
+
+    /// @brief enqueue flush chunk cache context
+    void Enqueue(std::shared_ptr<FlushChunkCacheContext> context);
+
+    void FsSyncSignal() {
+        std::lock_guard<std::mutex> lk(mtx_);
+        VLOG(3) << "fs sync signal";
+        cond_.notify_one();
+    }
+
+    void FsSyncSignalAndDataCacheInc() {
+        std::lock_guard<std::mutex> lk(mtx_);
+        fsCacheManager_->DataCacheNumInc();
+        VLOG(3) << "fs sync signal";
+        cond_.notify_one();
+    }
+
+/*** get and set element ***/
+
+    void SetFsId(uint32_t fsId) {
+        fsId_ = fsId;
+    }
+
+    uint64_t GetBlockSize() {
+        return blockSize_;
+    }
+
+    uint64_t GetChunkSize() {
+        return chunkSize_;
+    }
+
+    void SetBlockSize(const uint64_t& blockSize) {
+        blockSize_ = blockSize;
+    }
+
+    void SetChunkSize(const uint64_t& chunkSize) {
+        chunkSize_ = chunkSize;
+    }
+
+    std::shared_ptr<FsCacheManager> GetFsCacheManager() {
+        return fsCacheManager_;
+    }
+
+    uint32_t GetFlushInterval() { return flushIntervalSec_; }
+
+    uint32_t GetDiskCacheType() {
+        return diskCacheType_;
+    }
+
+    bool DisableDiskCache() {
+        return diskCacheType_ == DiskCacheType::Disable;
+    }
+
+    bool HasDiskCache() {
+        return diskCacheType_ != DiskCacheType::Disable;
+    }
+
+    bool IsReadCache() {
+        return diskCacheType_ == DiskCacheType::OnlyRead;
+    }
+
+    bool IsReadWriteCache() {
+        return diskCacheType_ == DiskCacheType::ReadWrite;
+    }
+    virtual std::shared_ptr<InodeCacheManager> GetInodeCacheManager() {
+        return inodeManager_;
+    }
+
+    std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() {
+        return diskCacheManagerImpl_;
+    }
+
+    uint32_t GetFsId() {
+        return fsId_;
+    }
+
+    uint32_t GetPageSize() {
+        return pageSize_;
+    }
+
+    void DisableBgFlush() {
+        enableBgFlush_ = false;
+    }
+
+    std::shared_ptr<IoMetric> GetMetric() {
+      return ioMetric_;
+    }
+
+    void CollectMetrics(InterfaceMetric *interface,
+      int count, uint64_t start) {
+        interface->bps.count << count;
+        interface->qps.count << 1;
+        interface->latency << (butil::cpuwide_time_us() - start);
+    }
+
+    void SetDiskCache(DiskCacheType type) {
+       diskCacheType_ = type;
+    }
+
+    CachePolicy GetCachePolicy(bool sync) {
+      const bool mayCache =
+        HasDiskCache() && !GetDiskCacheManager()->IsDiskCacheFull() && !sync;
+        if (IsReadCache() && mayCache) {
+            return CachePolicy::RCache;
+        } else if (IsReadWriteCache() && mayCache) {
+            return CachePolicy::WRCache;
+        } else {
+            return CachePolicy::NCache;
+        }
+    }
+
+    std::shared_ptr<MdsClient> GetMdsClient() {
+        return mdsClient_;
+    }
+
+    std::string GetMountOwner() {
+        return mountOwner_;
+    }
+
+    void SetMountOwner(const std::string& mountOwner) {
+        mountOwner_ = mountOwner;
+    }
+
+ private:
+    int FlushChunkClosure(std::shared_ptr<FlushChunkCacheContext> context);
+    int ClearDiskCache(int64_t inodeId);
+    void BackGroundFlush();
+
+    void InitMetrics(const std::string &fsName) {
+        fsName_ = fsName;
+        ioMetric_ = std::make_shared<IoMetric>(fsName);
+        // init disk cache metrics(needed)
+        if (HasDiskCache()) {
+        diskCacheManagerImpl_->InitMetrics(fsName);
+        }
+    }
+
+ protected:
+  std::shared_ptr<FsCacheManager> fsCacheManager_;
+  std::shared_ptr<MdsClient> mdsClient_;
+  std::shared_ptr<FsInfo> fsInfo_;
+
+ private:
+    uint64_t blockSize_;
+    uint64_t chunkSize_;
+    uint32_t pageSize_;
+    uint32_t flushIntervalSec_;
+    uint32_t chunkFlushThreads_;
+    uint32_t memCacheNearfullRatio_;
+    uint32_t throttleBaseSleepUs_;
+    Thread bgFlushThread_;
+    std::atomic<bool> toStop_;
+    std::mutex mtx_;
+    std::mutex ioMtx_;
+    std::condition_variable cond_;
+    curve::common::WaitInterval waitInterval_;
+    std::shared_ptr<InodeCacheManager> inodeManager_;
+    std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl_;
+    DiskCacheType diskCacheType_;
+    uint32_t fsId_;
+    std::string fsName_;
+    std::string mountOwner_;
+    std::shared_ptr<IoMetric> ioMetric_;
+    bool enableBgFlush_ = true;
+
+    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
+        taskPool_;
+};
+
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_CLIENT_STORAGE_ADAPTOR_H_

curvefs/src/client/common/common.h

@@ -60,6 +60,7 @@ enum class MetaServerOpType {
     GetVolumeExtent,
     UpdateVolumeExtent,
     CreateManageInode,
+    UpdateDeallocatableBlockGroup,
 };
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);

curvefs/src/client/common/config.cpp

@@ -204,6 +204,7 @@ void InitDiskCacheOption(Configuration *conf,
                               &diskCacheOption->avgReadFileIops);
 }
 
+// TODO(@hzwuhongsong)
 void InitS3Option(Configuration *conf, S3Option *s3Opt) {
     conf->GetValueFatalIfFail("s3.fakeS3", &FLAGS_useFakeS3);
     conf->GetValueFatalIfFail("s3.pageSize",
@@ -244,6 +245,10 @@ void InitVolumeOption(Configuration *conf, VolumeOption *volumeOpt) {
     conf->GetValueFatalIfFail("volume.fsBlockSize", &volumeOpt->fsBlockSize);
     conf->GetValueFatalIfFail("volume.allocator.type",
                               &volumeOpt->allocatorOption.type);
+    conf->GetValueFatalIfFail("volume.space.useThreshold",
+                              &volumeOpt->threshold);
+    conf->GetValueFatalIfFail("volume.space.releaseInterSec",
+                              &volumeOpt->releaseInterSec);
 
     conf->GetValueFatalIfFail(
         "volume.blockGroup.allocateOnce",
@@ -396,6 +401,9 @@ void SetFuseClientS3Option(FuseClientOption *clientOption,
     clientOption->s3Opt.s3AdaptrOpt.ak = fsS3Opt.ak;
     clientOption->s3Opt.s3AdaptrOpt.sk = fsS3Opt.sk;
     clientOption->s3Opt.s3AdaptrOpt.bucketName = fsS3Opt.bucketName;
+    VLOG(1) << "fuse s3 option, chunk size is: " << fsS3Opt.chunkSize
+            << "block size is: " << fsS3Opt.blockSize
+            << "s3 adress is: " << fsS3Opt.s3Address;
 }
 
 void S3Info2FsS3Option(const curvefs::common::S3Info& s3,

curvefs/src/client/common/config.h

@@ -73,11 +73,6 @@ struct LeaseOpt {
     uint32_t leaseTimeUs = 20000000;
 };
 
-struct SpaceAllocServerOption {
-    std::string spaceaddr;
-    uint64_t rpcTimeoutMs;
-};
-
 struct KVClientManagerOpt {
     int setThreadPooln = 4;
     int getThreadPooln = 4;
@@ -164,6 +159,9 @@ struct VolumeOption {
     uint64_t volBlockSize;
     uint64_t fsBlockSize;
     VolumeAllocatorOption allocatorOption;
+
+    double threshold{1.0};
+    uint64_t releaseInterSec{300};
 };
 
 struct ExtentManagerOption {
@@ -179,7 +177,6 @@ struct FuseClientOption {
     MetaCacheOpt metaCacheOpt;
     ExcutorOpt excutorOpt;
     ExcutorOpt excutorInternalOpt;
-    SpaceAllocServerOption spaceOpt;
     BlockDeviceClientOptions bdevOpt;
     S3Option s3Opt;
     ExtentManagerOption extentManagerOpt;

curvefs/src/client/curve_fuse_op.cpp

@@ -35,8 +35,8 @@
 #include "src/common/configuration.h"
 #include "src/common/gflags_helper.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
-#include "curvefs/src/client/fuse_volume_client.h"
-#include "curvefs/src/client/fuse_s3_client.h"
+#include "curvefs/src/client/volume/fuse_volume_client.h"
+#include "curvefs/src/client/s3/fuse_s3_client.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/rpcclient/base_client.h"
 #include "curvefs/src/client/metric/client_metric.h"
@@ -164,7 +164,6 @@ int InitFuseClient(const struct MountOption *mountOption) {
     } else if (fsInfo->fstype() == FSType::TYPE_VOLUME) {
        fsTypeMds = "volume";
     }
-
     if (fsTypeMds != fsTypeStr) {
         LOG(ERROR) << "The parameter fstype is inconsistent with mds!";
         return -1;

curvefs/src/client/fuse_client.cpp

@@ -39,6 +39,7 @@
 #include "curvefs/src/client/fuse_common.h"
 #include "curvefs/src/client/client_operator.h"
 #include "curvefs/src/client/inode_wrapper.h"
+#include "curvefs/src/client/kvclient/memcache_client.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
 #include "curvefs/src/client/xattr_manager.h"
 #include "curvefs/src/common/define.h"
@@ -49,13 +50,9 @@
 
 #define PORT_LIMIT 65535
 
-using ::curvefs::common::S3Info;
-using ::curvefs::common::Volume;
-using ::curvefs::mds::topology::PartitionTxId;
 using ::curvefs::mds::FSStatusCode_Name;
 using ::curvefs::client::common::MAX_XATTR_NAME_LENGTH;
 using ::curvefs::client::common::MAX_XATTR_VALUE_LENGTH;
-using ::curvefs::client::common::FileHandle;
 
 #define RETURN_IF_UNSUCCESS(action)                                            \
     do {                                                                       \
@@ -114,16 +111,25 @@ using common::FLAGS_fuseClientAvgReadBytes;
 using common::FLAGS_fuseClientBurstReadBytes;
 using common::FLAGS_fuseClientBurstReadBytesSecs;
 
-static void on_throttle_timer(void *arg) {
-    FuseClient *fuseClient = reinterpret_cast<FuseClient *>(arg);
-    fuseClient->InitQosParam();
-}
+// static void on_throttle_timer(void *arg) {
+//     FuseClient *fuseClient = reinterpret_cast<FuseClient *>(arg);
+//     fuseClient->InitQosParam();
+// }
 
 CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
+    LOG(INFO) << "fuse client init start.";
     option_ = option;
+    const uint64_t writeCacheMaxByte =
+      option_.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
+    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
+        LOG(ERROR) << "writeCacheMaxByte is too small"
+                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
+                      ", writeCacheMaxByte = " << writeCacheMaxByte;
+        return CURVEFS_ERROR::CACHETOOSMALL;
+    }
 
     mdsBase_ = new MDSBaseClient();
-    FSStatusCode ret = mdsClient_->Init(option.mdsOpt, mdsBase_);
+    auto ret = mdsClient_->Init(option.mdsOpt, mdsBase_);
     if (ret != FSStatusCode::OK) {
         return CURVEFS_ERROR::INTERNAL;
     }
@@ -154,14 +160,14 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     curve::client::ClientDummyServerInfo::GetInstance().SetPort(listenPort);
     curve::client::ClientDummyServerInfo::GetInstance().SetIP(localIp);
 
-    MetaStatusCode ret2 =
+    auto ret2 =
         metaClient_->Init(option.excutorOpt, option.excutorInternalOpt,
                           metaCache, channelManager);
     if (ret2 != MetaStatusCode::OK) {
         return CURVEFS_ERROR::INTERNAL;
     }
 
-    CURVEFS_ERROR ret3 =
+    auto ret3 =
         inodeManager_->Init(option.iCacheLruSize, option.enableICacheMetrics,
                             option.flushPeriodSec, option.refreshDataOption,
                             option.lruTimeOutSec);
@@ -181,7 +187,7 @@ CURVEFS_ERROR FuseClient::Init(const FuseClientOption &option) {
     }
 
     InitQosParam();
-
+    LOG(INFO) << "fuse client init success.";
     return ret3;
 }
 
@@ -193,9 +199,10 @@ void FuseClient::UnInit() {
     delete mdsBase_;
     mdsBase_ = nullptr;
 
-    while (bthread_timer_del(throttleTimer_) == 1) {
-        bthread_usleep(1000);
-    }
+    // while (bthread_timer_del(throttleTimer_) == 1) {
+    //     bthread_usleep(1000);
+    // }
+    // s3Adaptor_->Stop();
 }
 
 CURVEFS_ERROR FuseClient::Run() {
@@ -273,8 +280,8 @@ void InodeAttr2ParamAttr(const InodeAttr &inodeAttr, struct stat *attr) {
     attr->st_ctim.tv_sec = inodeAttr.ctime();
     attr->st_ctim.tv_nsec = inodeAttr.ctime_ns();
     attr->st_blksize = kOptimalIOBlockSize;
-
     switch (inodeAttr.type()) {
+        // TODO(@hzwuhongsong): type_volume?
         case metaserver::TYPE_S3:
             attr->st_blocks = (inodeAttr.length() + 511) / 512;
             break;
@@ -1536,11 +1543,11 @@ void FuseClient::InitQosParam() {
 
     throttle_.UpdateThrottleParams(params);
 
-    int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1),
-                                on_throttle_timer, this);
-    if (ret != 0) {
-        LOG(ERROR) << "Create fuse client throttle timer failed!";
-    }
+    // int ret = bthread_timer_add(&throttleTimer_, butil::seconds_from_now(1),
+    //                             on_throttle_timer, this);
+    // if (ret != 0) {
+    //     LOG(ERROR) << "Create fuse client throttle timer failed!";
+    // }
 }
 
 }  // namespace client

curvefs/src/client/fuse_client.h

@@ -45,6 +45,7 @@
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
+#include "curvefs/src/client/volume/client_volume_adaptor.h"
 #include "curvefs/src/common/fast_align.h"
 #include "curvefs/src/client/metric/client_metric.h"
 #include "src/common/concurrent/concurrent.h"
@@ -66,6 +67,7 @@ using ::curvefs::common::FSType;
 using ::curvefs::metaserver::DentryFlag;
 using ::curvefs::metaserver::ManageInodeType;
 using ::curvefs::client::metric::FSMetric;
+using curvefs::volume::kMiB;
 
 namespace curvefs {
 namespace client {
@@ -90,17 +92,17 @@ using mds::Mountpoint;
 class FuseClient {
  public:
     FuseClient()
-      : mdsClient_(std::make_shared<MdsClientImpl>()),
-        metaClient_(std::make_shared<MetaServerClientImpl>()),
-        inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
-        dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
-        dirBuf_(std::make_shared<DirBuffer>()),
-        fsInfo_(nullptr),
-        init_(false),
-        enableSumInDir_(false),
-        warmupManager_(nullptr),
-        mdsBase_(nullptr),
-        isStop_(true) {}
+    : mdsClient_(std::make_shared<MdsClientImpl>()),
+      metaClient_(std::make_shared<MetaServerClientImpl>()),
+      inodeManager_(std::make_shared<InodeCacheManagerImpl>(metaClient_)),
+      dentryManager_(std::make_shared<DentryCacheManagerImpl>(metaClient_)),
+      dirBuf_(std::make_shared<DirBuffer>()),
+      fsInfo_(nullptr),
+      init_(false),
+      enableSumInDir_(false),
+      warmupManager_(nullptr),
+      mdsBase_(nullptr),
+      isStop_(true) {}
 
     virtual ~FuseClient() {}
 
@@ -129,6 +131,8 @@ class FuseClient {
 
     virtual void Fini();
 
+/*** fuse op***/
+
     virtual CURVEFS_ERROR FuseOpInit(
         void* userdata, struct fuse_conn_info* conn);
 
@@ -222,13 +226,9 @@ class FuseClient {
     virtual CURVEFS_ERROR FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
                                       int datasync,
                                       struct fuse_file_info* fi) = 0;
+
     virtual CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
-                                      struct fuse_file_info *fi) {
-        (void)req;
-        (void)ino;
-        (void)fi;
-        return CURVEFS_ERROR::OK;
-    }
+                                      struct fuse_file_info *fi) = 0;
 
     virtual CURVEFS_ERROR FuseOpStatFs(fuse_req_t req, fuse_ino_t ino,
                                        struct statvfs* stbuf) {
@@ -249,6 +249,16 @@ class FuseClient {
         return CURVEFS_ERROR::OK;
     }
 
+/*** flush inode ***/
+
+    virtual void FlushInode();
+
+    virtual void FlushInodeAll();
+
+    virtual void FlushAll();
+
+/*** get or set some elements***/
+
     void SetFsInfo(const std::shared_ptr<FsInfo>& fsInfo) {
         fsInfo_ = fsInfo;
         init_ = true;
@@ -264,12 +274,6 @@ class FuseClient {
         return fsInfo_;
     }
 
-    virtual void FlushInode();
-
-    virtual void FlushInodeAll();
-
-    virtual void FlushAll();
-
     // for unit test
     void SetEnableSumInDir(bool enable) {
         enableSumInDir_ = enable;
@@ -302,6 +306,9 @@ class FuseClient {
     void Add(bool isRead, size_t size) { throttle_.Add(isRead, size); }
 
     void InitQosParam();
+    Mountpoint& GetMountPoint() {
+        return mountpoint_;
+    }
 
  protected:
     CURVEFS_ERROR MakeNode(fuse_req_t req, fuse_ino_t parent, const char* name,
@@ -414,6 +421,8 @@ class FuseClient {
     Throttle throttle_;
 
     bthread_timer_t throttleTimer_;
+
+    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };
 
 }  // namespace client

curvefs/src/client/inode_cache_manager.cpp

@@ -113,7 +113,8 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
         if (out->GetType() == FsFileType::TYPE_FILE) {
             return CURVEFS_ERROR::OK;
         }
-
+        VLOG(9) << "get inode: " << inodeId
+                << " from icache ok, need refresh data";
         REFRESH_DATA_REMOTE(out, out->NeedRefreshData());
         return CURVEFS_ERROR::OK;
     }
@@ -127,6 +128,7 @@ InodeCacheManagerImpl::GetInode(uint64_t inodeId,
         option_.refreshDataIntervalSec);
 
     // refresh data
+    VLOG(9) << "get inode: " << inodeId << " from icache fail, get from remote";
     REFRESH_DATA_REMOTE(out, streaming);
 
     // put to cache
@@ -544,6 +546,7 @@ bool InodeCacheManagerImpl::NeedUseCache(uint64_t inodeId,
     const std::shared_ptr<InodeWrapper> &inodeWrapper,
     bool onlyAttr) {
     auto lock = inodeWrapper->GetUniqueLock();
+
     if (onlyAttr) {
         if (inodeWrapper->IsDirty()) {
             return true;
@@ -558,6 +561,8 @@ bool InodeCacheManagerImpl::NeedUseCache(uint64_t inodeId,
         && !IsTimeOut(inodeWrapper)) {
         return true;
     }
+
+    VLOG(9) << "inodeId " << inodeId << " is not need use cache";
     return false;
 }
 

curvefs/src/client/inode_wrapper.cpp

@@ -251,6 +251,8 @@ void InodeWrapper::FlushS3ChunkInfoAsync() {
 CURVEFS_ERROR InodeWrapper::FlushVolumeExtent() {
     std::lock_guard<::curve::common::Mutex> guard(syncingVolumeExtentsMtx_);
     if (!extentCache_.HasDirtyExtents()) {
+        VLOG(9) << "FlushVolumeExtent, ino: " << inode_.inodeid()
+                << ", no dirty extents";
         return CURVEFS_ERROR::OK;
     }
 
@@ -476,12 +478,18 @@ void InodeWrapper::Async(MetaServerClientDone *done, bool internal) {
 
 void InodeWrapper::AsyncFlushAttrAndExtents(MetaServerClientDone *done,
                                             bool /*internal*/) {
+    VLOG(9) << "async inode: " << inode_.ShortDebugString()
+            << ", is dirty: " << dirty_
+            << ", has dirty extents: " << extentCache_.HasDirtyExtents();
     if (dirty_ || extentCache_.HasDirtyExtents()) {
         LockSyncingInode();
         syncingVolumeExtentsMtx_.lock();
         DataIndices indices;
         if (extentCache_.HasDirtyExtents()) {
             indices.volumeExtents = extentCache_.GetDirtyExtents();
+            VLOG(9) << "aync inode: " << inode_.ShortDebugString()
+                    << ", volume extents: "
+                    << indices.volumeExtents->ShortDebugString();
         }
 
         metaClient_->UpdateInodeWithOutNlinkAsync(
@@ -583,7 +591,7 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
-    VolumeExtentList extents;
+    VolumeExtentSliceList extents;
     auto st = metaClient_->GetVolumeExtent(inode_.fsid(), inode_.inodeid(),
                                            true, &extents);
     VLOG(9) << "RefreshVolumeExtent, ino: " << inode_.inodeid()

curvefs/src/client/inode_wrapper.h

@@ -52,7 +52,7 @@ constexpr int kAccessTime = 1 << 0;
 constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;
 
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtentSliceList;
 
 enum class InodeStatus {
     kNormal = 0,

curvefs/src/client/kvclient/kvclient_manager.cpp

@@ -41,6 +41,7 @@ namespace client {
 
 bool KVClientManager::Init(const KVClientManagerOpt &config,
                            const std::shared_ptr<KVClient> &kvclient) {
+    LOG(INFO) << "KVClientManager init.";
     client_ = kvclient;
     return threadPool_.Start(config.setThreadPooln) == 0;
 }

curvefs/src/client/kvclient/kvclient_manager.h

@@ -57,6 +57,7 @@ struct SetKVCacheTask {
     std::string key;
     const char *value;
     uint64_t length;
+    uint64_t startTime;
     SetKVCacheDone done;
     SetKVCacheTask() = default;
     SetKVCacheTask(
@@ -71,6 +72,7 @@ struct GetKVCacheTask {
     uint64_t offset;
     uint64_t length;
     bool res;
+    uint64_t startTime;
     GetKVCacheDone done;
     GetKVCacheTask(const std::string &k, char *v, uint64_t off, uint64_t len)
         : key(k), value(v), offset(off), length(len), res(false) {

curvefs/src/client/metric/client_metric.cpp

@@ -31,7 +31,7 @@ const std::string MetaServerClientMetric::prefix = "curvefs_metaserver_client";
 const std::string ClientOpMetric::prefix = "curvefs_client";  // NOLINT
 const std::string S3MultiManagerMetric::prefix = "curvefs_client_manager";  // NOLINT
 const std::string FSMetric::prefix = "curvefs_client";  // NOLINT
-const std::string S3Metric::prefix = "curvefs_s3";  // NOLINT
+const std::string IoMetric::prefix = "curvefs_s3";  // NOLINT
 const std::string DiskCacheMetric::prefix = "curvefs_disk_cache";  // NOLINT
 const std::string KVClientMetric::prefix = "curvefs_kvclient";  // NOLINT
 const std::string S3ChunkInfoMetric::prefix = "inode_s3_chunk_info";  // NOLINT

curvefs/src/client/metric/client_metric.h

@@ -92,13 +92,12 @@ struct MetaServerClientMetric {
     // volume extent
     InterfaceMetric updateVolumeExtent;
     InterfaceMetric getVolumeExtent;
+    InterfaceMetric updateDeallocatableBlockGroup;
 
     MetaServerClientMetric()
-        : getDentry(prefix, "getDentry"),
-          listDentry(prefix, "listDentry"),
+        : getDentry(prefix, "getDentry"), listDentry(prefix, "listDentry"),
           createDentry(prefix, "createDentry"),
-          deleteDentry(prefix, "deleteDentry"),
-          getInode(prefix, "getInode"),
+          deleteDentry(prefix, "deleteDentry"), getInode(prefix, "getInode"),
           batchGetInodeAttr(prefix, "batchGetInodeAttr"),
           batchGetXattr(prefix, "batchGetXattr"),
           createInode(prefix, "createInode"),
@@ -107,7 +106,9 @@ struct MetaServerClientMetric {
           appendS3ChunkInfo(prefix, "appendS3ChunkInfo"),
           prepareRenameTx(prefix, "prepareRenameTx"),
           updateVolumeExtent(prefix, "updateVolumeExtent"),
-          getVolumeExtent(prefix, "getVolumeExtent") {}
+          getVolumeExtent(prefix, "getVolumeExtent"),
+          updateDeallocatableBlockGroup(prefix,
+                                        "updateDeallocatableBlockGroup") {}
 };
 
 struct InflightGuard {
@@ -227,27 +228,35 @@ struct FSMetric {
           userReadIoSize(prefix, fsName + "_userReadIoSize", 0) {}
 };
 
-struct S3Metric {
-    static const std::string prefix;
 
+struct IoMetric {
+    static const std::string prefix;
     std::string fsName;
     InterfaceMetric adaptorWrite;
     InterfaceMetric adaptorRead;
+    InterfaceMetric adaptorBgFlush;
+    InterfaceMetric adaptorFlushBackend;
     InterfaceMetric adaptorWriteS3;
-    InterfaceMetric adaptorWriteDiskCache;
     InterfaceMetric adaptorReadS3;
+    InterfaceMetric adaptorWriteKvCache;
+    InterfaceMetric adaptorReadKvCache;
+    InterfaceMetric adaptorReadBackend;
     InterfaceMetric adaptorReadDiskCache;
     bvar::Status<uint32_t> readSize;
     bvar::Status<uint32_t> writeSize;
 
-    explicit S3Metric(const std::string &name = "")
+    explicit IoMetric(const std::string &name = "")
         : fsName(!name.empty() ? name
                                : prefix + curve::common::ToHexString(this)),
           adaptorWrite(prefix, fsName + "_adaptor_write"),
           adaptorRead(prefix, fsName + "_adaptor_read"),
+          adaptorBgFlush(prefix, fsName + "_adaptor_bg_flush"),
+          adaptorFlushBackend(prefix, fsName + "_adaptor_flush_backend"),
+          adaptorReadBackend(prefix, fsName + "_adaptor_read_backend"),
           adaptorWriteS3(prefix, fsName + "_adaptor_write_s3"),
-          adaptorWriteDiskCache(prefix, fsName + "_adaptor_write_disk_cache"),
           adaptorReadS3(prefix, fsName + "_adaptor_read_s3"),
+          adaptorWriteKvCache(prefix, fsName + "_adaptor_write_kv_cache"),
+          adaptorReadKvCache(prefix, fsName + "_adaptor_read_kv_cache"),
           adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache"),
           readSize(prefix, fsName + "_adaptor_read_size", 0),
           writeSize(prefix, fsName + "_adaptor_write_size", 0) {}
@@ -257,12 +266,16 @@ struct DiskCacheMetric {
     static const std::string prefix;
 
     std::string fsName;
+    InterfaceMetric adaptorWriteDiskCache;
+    InterfaceMetric adaptorReadDiskCache;
     InterfaceMetric writeS3;
     bvar::Status<uint64_t> diskUsedBytes;
 
     explicit DiskCacheMetric(const std::string &name = "")
         : fsName(!name.empty() ? name
                                : prefix + curve::common::ToHexString(this)),
+          adaptorWriteDiskCache(prefix, fsName + "_adaptor_write_disk_cache"),
+          adaptorReadDiskCache(prefix, fsName + "_adaptor_read_disk_cache"),
           writeS3(prefix, fsName + "_write_s3"),
           diskUsedBytes(prefix, fsName + "_diskcache_usedbytes", 0) {}
 };

curvefs/src/client/rpcclient/base_client.cpp

@@ -180,7 +180,6 @@ void MDSBaseClient::AllocateVolumeBlockGroup(
     request.set_fsid(fsId);
     request.set_count(count);
     request.set_owner(owner);
-
     SpaceService_Stub stub(channel);
     stub.AllocateBlockGroup(cntl, &request, response, nullptr);
 }

curvefs/src/client/rpcclient/mds_client.cpp

@@ -710,14 +710,17 @@ SpaceErrCode MdsClientImpl::AllocateVolumeBlockGroup(
         AllocateBlockGroupResponse response;
         mdsbasecli_->AllocateVolumeBlockGroup(fsId, count, owner, &response,
                                               cntl, channel);
+        VLOG(9) << "AllocateVolumeBlockGroup, response: "
+                    << response.DebugString();
 
         CHECK_RPC_AND_RETRY_IF_ERROR("AllocateVolumeBlockGroup");
 
         auto status = response.status();
-        if (status != SpaceErrCode::SpaceOk) {
-            LOG(WARNING) << "Allocate volume block group failed, err: "
-                         << SpaceErrCode_Name(status);
-        } else if (response.blockgroups_size() == 0) {
+        LOG_IF(WARNING, status != SpaceErrCode::SpaceOk)
+            << "Allocate volume block group failed, err: "
+            << SpaceErrCode_Name(status);
+
+        if (response.blockgroups_size() == 0) {
             LOG(WARNING) << "Allocate volume block group failed, no block "
                             "group allcoated";
             return SpaceErrCode::SpaceErrNoSpace;

curvefs/src/client/rpcclient/metaserver_client.cpp

@@ -65,6 +65,7 @@ using BatchGetXAttrExcutor = TaskExecutor;
 using GetOrModifyS3ChunkInfoExcutor = TaskExecutor;
 using UpdateVolumeExtentExecutor = TaskExecutor;
 using GetVolumeExtentExecutor = TaskExecutor;
+using UpdateDeallocatableBlockGroupExcutor = TaskExecutor;
 
 using ::curvefs::common::LatencyUpdater;
 using ::curvefs::common::StreamConnection;
@@ -970,6 +971,7 @@ void MetaServerClientImpl::UpdateInodeAsync(const UpdateInodeRequest &request,
         req.set_poolid(poolID);
         req.set_copysetid(copysetID);
         req.set_partitionid(partitionID);
+        VLOG(9) << "update inode async req: " << req.ShortDebugString();
 
         auto *rpcDone = new UpdateInodeRpcDone(taskExecutorDone, &metric_);
         curvefs::metaserver::MetaServerService_Stub stub(channel);
@@ -1355,8 +1357,8 @@ MetaStatusCode MetaServerClientImpl::CreateManageInode(const InodeParam &param,
 
     auto taskCtx = std::make_shared<TaskContext>(
         MetaServerOpType::CreateManageInode, task, param.fsId, 0);
-    CreateInodeExcutor excutor(opt_, metaCache_, channelManager_,
-                               std::move(taskCtx));
+    CreateManagerInodeExcutor excutor(opt_, metaCache_, channelManager_,
+                                      std::move(taskCtx));
     return ConvertToMetaStatusCode(excutor.DoRPCTask());
 }
 
@@ -1465,7 +1467,7 @@ void UpdateVolumeExtentRpcDone::Run() {
     } while (0)
 
 void MetaServerClientImpl::AsyncUpdateVolumeExtent(
-    uint32_t fsId, uint64_t inodeId, const VolumeExtentList &extents,
+    uint32_t fsId, uint64_t inodeId, const VolumeExtentSliceList &extents,
     MetaServerClientDone *done) {
     auto task = AsyncRPCTask {
         (void)txId;
@@ -1473,7 +1475,7 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
         metric_.updateVolumeExtent.qps.count << 1;
         metaserver::UpdateVolumeExtentRequest request;
         SET_COMMON_FIELDS;
-        request.set_allocated_extents(new VolumeExtentList{extents});
+        request.set_allocated_extents(new VolumeExtentSliceList{extents});
 
         auto *rpcDone =
             new UpdateVolumeExtentRpcDone(taskExecutorDone, &metric_);
@@ -1493,7 +1495,8 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
 namespace {
 
 struct ParseVolumeExtentCallBack {
-    explicit ParseVolumeExtentCallBack(VolumeExtentList *ext) : extents(ext) {}
+    explicit ParseVolumeExtentCallBack(VolumeExtentSliceList *ext)
+        : extents(ext) {}
 
     bool operator()(butil::IOBuf *data) const {
         metaserver::VolumeExtentSlice slice;
@@ -1506,7 +1509,7 @@ struct ParseVolumeExtentCallBack {
         return true;
     }
 
-    VolumeExtentList *extents;
+    VolumeExtentSliceList *extents;
 };
 
 }  // namespace
@@ -1514,7 +1517,7 @@ struct ParseVolumeExtentCallBack {
 MetaStatusCode
 MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
                                       bool streaming,
-                                      VolumeExtentList *extents) {
+                                      VolumeExtentSliceList *extents) {
     auto task = RPCTask {
         (void)txId;
         (void)applyIndex;
@@ -1596,9 +1599,66 @@ MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
     return ConvertToMetaStatusCode(executor.DoRPCTask());
 }
 
-MetaStatusCode MetaServerClientImpl::GetInodeAttr(uint32_t fsId,
-                                                  uint64_t inodeid,
-                                                  InodeAttr *attr) {
+MetaStatusCode MetaServerClientImpl::UpdateDeallocatableBlockGroup(
+    uint32_t fsId, uint64_t inodeId, DeallocatableBlockGroupMap *statistic) {
+    auto task = RPCTask {
+        metric_.updateDeallocatableBlockGroup.qps.count << 1;
+        LatencyUpdater updater(&metric_.updateDeallocatableBlockGroup.latency);
+
+        metaserver::UpdateDeallocatableBlockGroupRequest request;
+        metaserver::UpdateDeallocatableBlockGroupResponse response;
+        request.set_poolid(poolID);
+        request.set_copysetid(copysetID);
+        request.set_partitionid(partitionID);
+        request.set_fsid(fsId);
+        auto *upadte = request.mutable_update();
+        for (auto &it : *statistic) {
+            upadte->Add()->CopyFrom(std::move(it.second));
+        }
+
+        curvefs::metaserver::MetaServerService_Stub stub(channel);
+        stub.UpdateDeallocatableBlockGroup(cntl, &request, &response, nullptr);
+
+        if (cntl->Failed()) {
+            metric_.updateDeallocatableBlockGroup.eps.count << 1;
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup failed"
+                         << ", errorCode = " << cntl->ErrorCode()
+                         << ", errorText = " << cntl->ErrorText()
+                         << ", logId = " << cntl->log_id();
+            return -cntl->ErrorCode();
+        }
+
+        auto rc = response.statuscode();
+        if (rc != MetaStatusCode::OK) {
+            metric_.updateDeallocatableBlockGroup.eps.count << 1;
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup: retCode = " << rc
+                         << ", message = " << MetaStatusCode_Name(rc);
+        } else if (response.has_appliedindex()) {
+            metaCache_->UpdateApplyIndex(CopysetGroupID(poolID, copysetID),
+                                         response.appliedindex());
+        } else {
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup OK"
+                         << ", but applyIndex not set in response:"
+                         << response.DebugString();
+            return -1;
+        }
+
+        VLOG(6) << "UpdateDeallocatableBlockGroup done, request: "
+                << request.DebugString()
+                << "response: " << response.DebugString();
+        return rc;
+    };
+
+    auto taskCtx = std::make_shared<TaskContext>(
+        MetaServerOpType::UpdateDeallocatableBlockGroup, task, fsId, inodeId);
+
+    UpdateDeallocatableBlockGroupExcutor excutor(
+        opt_, metaCache_, channelManager_, std::move(taskCtx));
+    return ConvertToMetaStatusCode(excutor.DoRPCTask());
+}
+
+MetaStatusCode MetaServerClientImpl::GetInodeAttr(
+    uint32_t fsId, uint64_t inodeid, InodeAttr *attr) {
     std::set<uint64_t> inodeIds;
     inodeIds.insert(inodeid);
     std::list<InodeAttr> attrs;

curvefs/src/client/rpcclient/metaserver_client.h

@@ -30,6 +30,7 @@
 #include <set>
 #include <unordered_map>
 #include <utility>
+#include <map>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/metaserver.pb.h"
@@ -49,21 +50,24 @@ using ::curvefs::metaserver::InodeAttr;
 using ::curvefs::metaserver::XAttr;
 using ::curvefs::metaserver::MetaStatusCode;
 using ::curvefs::metaserver::S3ChunkInfoList;
+using ::curvefs::metaserver::DeallocatableBlockGroup;
 using ::curvefs::common::StreamStatus;
 using ::curvefs::common::StreamClient;
-using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 using ::curvefs::metaserver::Time;
 
+using DeallocatableBlockGroupMap = std::map<uint64_t, DeallocatableBlockGroup>;
+using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+
 namespace curvefs {
 namespace client {
 namespace rpcclient {
 
 using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtentSliceList;
 
 struct DataIndices {
     absl::optional<S3ChunkInfoMap> s3ChunkInfoMap;
-    absl::optional<VolumeExtentList> volumeExtents;
+    absl::optional<VolumeExtentSliceList> volumeExtents;
 };
 
 class MetaServerClient {
@@ -161,13 +165,17 @@ class MetaServerClient {
 
     virtual void AsyncUpdateVolumeExtent(uint32_t fsId,
                                          uint64_t inodeId,
-                                         const VolumeExtentList &extents,
+                                         const VolumeExtentSliceList &extents,
                                          MetaServerClientDone *done) = 0;
 
     virtual MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                            uint64_t inodeId,
                                            bool streaming,
-                                           VolumeExtentList *extents) = 0;
+                                           VolumeExtentSliceList *extents) = 0;
+
+    virtual MetaStatusCode
+    UpdateDeallocatableBlockGroup(uint32_t fsId, uint64_t inodeId,
+                                  DeallocatableBlockGroupMap *statistic) = 0;
 };
 
 class MetaServerClientImpl : public MetaServerClient {
@@ -265,13 +273,17 @@ class MetaServerClientImpl : public MetaServerClient {
 
     void AsyncUpdateVolumeExtent(uint32_t fsId,
                                  uint64_t inodeId,
-                                 const VolumeExtentList &extents,
+                                 const VolumeExtentSliceList &extents,
                                  MetaServerClientDone *done) override;
 
     MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                    uint64_t inodeId,
                                    bool streaming,
-                                   VolumeExtentList *extents) override;
+                                   VolumeExtentSliceList *extents) override;
+
+    MetaStatusCode UpdateDeallocatableBlockGroup(
+        uint32_t fsId, uint64_t inodeId,
+        DeallocatableBlockGroupMap *statistic) override;
 
  private:
     MetaStatusCode UpdateInode(const UpdateInodeRequest &request,

curvefs/src/client/rpcclient/task_excutor.cpp

@@ -27,8 +27,10 @@
 
 #include "curvefs/src/client/rpcclient/task_excutor.h"
 #include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/src/common/define.h"
 
 using ::curvefs::metaserver::MetaStatusCode;
+using ::curvefs::RECYCLEINODEID;
 
 namespace curvefs {
 namespace client {
@@ -339,6 +341,17 @@ bool CreateInodeExcutor::GetTarget() {
     return true;
 }
 
+bool CreateManagerInodeExcutor::GetTarget() {
+    if (!metaCache_->GetTarget(task_->fsID, RECYCLEINODEID, &task_->target,
+                               &task_->applyIndex)) {
+        LOG(ERROR) << "CreateManagerInodeExcutor select target for task fail, "
+                   << task_->TaskContextStr()
+                   << ", recycleInodeId = " << RECYCLEINODEID;
+        return false;
+    }
+    return true;
+}
+
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/rpcclient/task_excutor.h

@@ -275,6 +275,18 @@ class CreateInodeExcutor : public TaskExecutor {
     bool GetTarget() override;
 };
 
+class CreateManagerInodeExcutor : public TaskExecutor {
+ public:
+    explicit CreateManagerInodeExcutor(
+        const ExcutorOpt &opt, const std::shared_ptr<MetaCache> &metaCache,
+        const std::shared_ptr<ChannelManager<MetaserverID>> &channelManager,
+        const std::shared_ptr<TaskContext> &task)
+        : TaskExecutor(opt, metaCache, channelManager, task) {}
+
+ protected:
+    bool GetTarget() override;
+};
+
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs

curvefs/src/client/s3/client_s3_adaptor.h

@@ -19,12 +19,15 @@
  * Created Date: 21-5-31
  * Author: huyao
  */
+
 #ifndef CURVEFS_SRC_CLIENT_S3_CLIENT_S3_ADAPTOR_H_
 #define CURVEFS_SRC_CLIENT_S3_CLIENT_S3_ADAPTOR_H_
 
 #include <bthread/execution_queue.h>
 
 #include <memory>
+#include <utility>
+#include <set>
 #include <string>
 #include <vector>
 
@@ -33,204 +36,216 @@
 #include "curvefs/proto/metaserver.pb.h"
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/client_storage_adaptor.h"
+#include "curvefs/src/client/kvclient/kvclient_manager.h"
+#include "curvefs/src/client/kvclient/kvclient.h"
 #include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/inode_cache_manager.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/client/s3/client_s3.h"
-#include "curvefs/src/client/s3/client_s3_cache_manager.h"
-#include "curvefs/src/client/s3/disk_cache_manager_impl.h"
+#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
+#include "curvefs/src/client/cache/diskcache/disk_cache_manager_impl.h"
 #include "src/common/wait_interval.h"
+
 namespace curvefs {
 namespace client {
 
-using ::curve::common::Thread;
-using ::curve::common::TaskThreadPool;
+using curve::common::GetObjectAsyncCallBack;
+using curve::common::PutObjectAsyncCallBack;
+using curve::common::S3Adapter;
 using curvefs::client::common::S3ClientAdaptorOption;
-using curvefs::client::common::DiskCacheType;
-using curvefs::metaserver::Inode;
-using curvefs::metaserver::S3ChunkInfo;
-using curvefs::metaserver::S3ChunkInfoList;
-using rpcclient::MdsClient;
-using curvefs::client::metric::S3Metric;
+using curvefs::client::metric::IoMetric;
 
-class DiskCacheManagerImpl;
-class FlushChunkCacheContext;
-class ChunkCacheManager;
+/// @brief s3 read request
+/// @param chunkId chunk id
+/// @param offset file offset
+/// @param len read length
+/// @param objectOffset first offset in the block
+/// @param readOffset read buf offset
+/// @param fsId file system id
+/// @param inodeId inode id
+/// @param compaction compaction flag
+struct S3ReadRequest {
+    uint64_t chunkId;
+    uint64_t offset;
+    uint64_t len;
+    uint64_t objectOffset;
+    uint64_t readOffset;
+    uint64_t fsId;
+    uint64_t inodeId;
+    uint64_t compaction;
 
-class S3ClientAdaptor {
- public:
-    S3ClientAdaptor() {}
-    virtual ~S3ClientAdaptor() {}
-    /**
-     * @brief Initailize s3 client
-     * @param[in] options the options for s3 client
-     */
-    virtual CURVEFS_ERROR
-    Init(const S3ClientAdaptorOption &option, std::shared_ptr<S3Client> client,
-         std::shared_ptr<InodeCacheManager> inodeManager,
-         std::shared_ptr<MdsClient> mdsClient,
-         std::shared_ptr<FsCacheManager> fsCacheManager,
-         std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
-         std::shared_ptr<KVClientManager> kvClientManager,
-         bool startBackGround = false) = 0;
-    /**
-     * @brief write data to s3
-     * @param[in] options the options for s3 client
-     */
-    virtual int Write(uint64_t inodeId, uint64_t offset, uint64_t length,
-                      const char *buf) = 0;
-    virtual int Read(uint64_t inodeId, uint64_t offset, uint64_t length,
-                     char *buf) = 0;
-    virtual CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper,
-                                   uint64_t size) = 0;
-    virtual void ReleaseCache(uint64_t inodeId) = 0;
-    virtual CURVEFS_ERROR Flush(uint64_t inodeId) = 0;
-    virtual CURVEFS_ERROR FlushAllCache(uint64_t inodeId) = 0;
-    virtual CURVEFS_ERROR FsSync() = 0;
-    virtual int Stop() = 0;
-    virtual FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum,
-                                        uint64_t *chunkId) = 0;
-    virtual void SetFsId(uint32_t fsId) = 0;
-    virtual void InitMetrics(const std::string &fsName) = 0;
-    virtual void CollectMetrics(InterfaceMetric *interface, int count,
-                                uint64_t start) = 0;
-    virtual std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() = 0;
-    virtual std::shared_ptr<S3Client> GetS3Client() = 0;
-    virtual uint64_t GetBlockSize() = 0;
-    virtual uint64_t GetChunkSize() = 0;
-    virtual uint32_t GetObjectPrefix() = 0;
-    virtual bool HasDiskCache() = 0;
+    std::string DebugString() const {
+        std::ostringstream os;
+        os << "S3ReadRequest ( chunkId = " << chunkId << ", offset = " << offset
+           << ", len = " << len << ", objectOffset = " << objectOffset
+           << ", readOffset = " << readOffset << ", fsId = " << fsId
+           << ", inodeId = " << inodeId << ", compaction = " << compaction
+           << " )";
+        return os.str();
+    }
 };
 
-using FlushChunkCacheCallBack = std::function<
-  void(const std::shared_ptr<FlushChunkCacheContext>&)>;
-
-struct FlushChunkCacheContext {
-    uint64_t inode;
-    ChunkCacheManagerPtr chunkCacheManptr;
-    bool force;
-    FlushChunkCacheCallBack cb;
-    CURVEFS_ERROR retCode;
-};
+inline std::string
+S3ReadRequestVecDebugString(const std::vector<S3ReadRequest> &reqs) {
+    std::ostringstream os;
+    for_each(reqs.begin(), reqs.end(),
+             [&](const S3ReadRequest &req) { os << req.DebugString() << " "; });
+    return os.str();
+}
 
 // client use s3 internal interface
-class S3ClientAdaptorImpl : public S3ClientAdaptor {
+class S3ClientAdaptorImpl : public StorageAdaptor {
  public:
-    S3ClientAdaptorImpl() {}
+    S3ClientAdaptorImpl() : StorageAdaptor() {}
+
+    // for unittest
+    explicit S3ClientAdaptorImpl(std::shared_ptr<
+      S3Client> client) : StorageAdaptor() {
+        client_ = client;
+    }
+
     virtual ~S3ClientAdaptorImpl() {
         LOG(INFO) << "delete S3ClientAdaptorImpl";
     }
-    /**
-     * @brief Initailize s3 client
-     * @param[in] options the options for s3 client
-     */
-    CURVEFS_ERROR
-    Init(const S3ClientAdaptorOption &option, std::shared_ptr<S3Client> client,
+
+    /// @brief init s3 storage adaptor
+    /// @param option fuse client option
+    /// @param inodeManager inode cache manager
+    /// @param mdsClient mds client
+    /// @param fsCacheManager fscache manager
+    /// @param diskCacheManagerImpl disk cache manager
+    /// @param kvClientManager kv client manager
+    /// @param fsInfo file system information
+    /// @return error code
+    CURVEFS_ERROR Init(const FuseClientOption &option,
          std::shared_ptr<InodeCacheManager> inodeManager,
          std::shared_ptr<MdsClient> mdsClient,
          std::shared_ptr<FsCacheManager> fsCacheManager,
          std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
          std::shared_ptr<KVClientManager> kvClientManager,
-         bool startBackGround = false);
-    /**
-     * @brief write data to s3
-     * @param[in] options the options for s3 client
-     */
-    int Write(uint64_t inodeId, uint64_t offset, uint64_t length,
-              const char *buf);
-    int Read(uint64_t inodeId, uint64_t offset, uint64_t length, char *buf);
+         std::shared_ptr<FsInfo> fsInfo) override;
+
+    int Stop() override;
+
+    /// @brief read data from s3 storage
+    /// @param request read request
+    /// @return error code
+    CURVEFS_ERROR FlushDataCache(const UperFlushRequest& req,
+      uint64_t* writeOffset) override;
+
+    /// @brief read data from s3 storage
+    /// @param request read request
+    /// @return error code
+    CURVEFS_ERROR ReadFromLowlevel(UperReadRequest request) override;
+
     CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper, uint64_t size);
-    void ReleaseCache(uint64_t inodeId);
-    CURVEFS_ERROR Flush(uint64_t inodeId);
-    CURVEFS_ERROR FlushAllCache(uint64_t inodeId);
-    CURVEFS_ERROR FsSync();
-    int Stop();
-    uint64_t GetBlockSize() {
-        return blockSize_;
-    }
-    uint64_t GetChunkSize() {
-        return chunkSize_;
-    }
+
     uint32_t GetObjectPrefix() {
         return objectPrefix_;
     }
 
-    std::shared_ptr<FsCacheManager> GetFsCacheManager() {
-        return fsCacheManager_;
-    }
-    uint32_t GetFlushInterval() { return flushIntervalSec_; }
     std::shared_ptr<S3Client> GetS3Client() { return client_; }
-    uint32_t GetPrefetchBlocks() {
-        return prefetchBlocks_;
-    }
-    uint32_t GetDiskCacheType() {
-        return diskCacheType_;
-    }
-    bool DisableDiskCache() {
-        return diskCacheType_ == DiskCacheType::Disable;
-    }
-    bool HasDiskCache() {
-        return diskCacheType_ != DiskCacheType::Disable;
-    }
-    bool IsReadCache() {
-        return diskCacheType_ == DiskCacheType::OnlyRead;
-    }
-    bool IsReadWriteCache() {
-        return diskCacheType_ == DiskCacheType::ReadWrite;
-    }
-    std::shared_ptr<InodeCacheManager> GetInodeCacheManager() {
-        return inodeManager_;
-    }
-    std::shared_ptr<DiskCacheManagerImpl> GetDiskCacheManager() {
-        return diskCacheManagerImpl_;
-    }
-    FSStatusCode AllocS3ChunkId(uint32_t fsId, uint32_t idNum,
-                                uint64_t *chunkId);
-    void FsSyncSignal() {
-        std::lock_guard<std::mutex> lk(mtx_);
-        VLOG(3) << "fs sync signal";
-        cond_.notify_one();
-    }
-    void FsSyncSignalAndDataCacheInc() {
-        std::lock_guard<std::mutex> lk(mtx_);
-        fsCacheManager_->DataCacheNumInc();
-        VLOG(3) << "fs sync signal";
-        cond_.notify_one();
-    }
-    void SetFsId(uint32_t fsId) {
-        fsId_ = fsId;
-    }
-    uint32_t GetFsId() {
-        return fsId_;
-    }
-    uint32_t GetPageSize() {
-        return pageSize_;
-    }
-    void InitMetrics(const std::string &fsName);
-    void CollectMetrics(InterfaceMetric *interface, int count, uint64_t start);
-    void SetDiskCache(DiskCacheType type) {
-       diskCacheType_ = type;
+
+    CURVEFS_ERROR FuseOpInit(void *userdata,
+      struct fuse_conn_info *conn) override {
+        StorageAdaptor::FuseOpInit(userdata, conn);
+        return CURVEFS_ERROR::OK;
     }
 
-    uint32_t GetMaxReadRetryIntervalMs() const {
-        return maxReadRetryIntervalMs_;
+ private:
+     enum class ReadStatus {
+        OK = 0,
+        S3_READ_FAIL = -1,
+        S3_NOT_EXIST = -2,
+    };
+
+    ReadStatus toReadStatus(const int retCode) {
+        ReadStatus st = ReadStatus::OK;
+        if (retCode < 0) {
+            st = (retCode == -2) ? ReadStatus::S3_NOT_EXIST
+                                 : ReadStatus::S3_READ_FAIL;
+        }
+        return st;
+    }
+
+    S3ClientAdaptorImpl::ReadStatus ReadKVRequest(
+      const std::vector<S3ReadRequest> &kvRequests,
+      char *dataBuf, uint64_t fileLen);
+
+    CURVEFS_ERROR PrepareFlushTasks(const UperFlushRequest& req,
+      std::vector<std::shared_ptr<PutObjectAsyncContext>> *s3Tasks,
+      std::vector<std::shared_ptr<SetKVCacheTask>> *kvCacheTasks,
+      uint64_t* writeOffset);
+
+    void FlushTaskExecute(CachePolicy cachePoily,
+      const std::vector<std::shared_ptr<PutObjectAsyncContext>> &s3Tasks,
+      const std::vector<std::shared_ptr<SetKVCacheTask>> &kvCacheTasks);
+
+    void PrefetchS3Objs(uint64_t inodeId,
+      const std::vector<std::pair<std::string, uint64_t>> &prefetchObjs);
+
+    void HandleReadRequest(
+      const ReadRequest &request, const S3ChunkInfo &s3ChunkInfo,
+      std::vector<ReadRequest> *addReadRequests,
+      std::vector<uint64_t> *deletingReq, std::vector<S3ReadRequest> *requests,
+      char *dataBuf, uint64_t fsId, uint64_t inodeId);
+
+    void GenerateS3Request(ReadRequest request,
+      const S3ChunkInfoList &s3ChunkInfoList,
+      char *dataBuf,
+      std::vector<S3ReadRequest> *requests,
+      uint64_t fsId,
+      uint64_t inodeId);
+
+    // miss read from memory read/write cache, need read from
+    // kv(localdisk/remote cache/s3)
+    int GenerateKVReuqest(const std::shared_ptr<InodeWrapper> &inodeWrapper,
+                          const std::vector<ReadRequest> &readRequest,
+                          char *dataBuf, std::vector<S3ReadRequest> *kvRequest);
+
+    int HandleReadS3NotExist(uint32_t retry,
+      const std::shared_ptr<InodeWrapper> &inodeWrapper);
+
+    bool ReadKVRequestFromS3(const std::string &name,
+      char *databuf, uint64_t offset, uint64_t length, int *ret);
+
+    bool ReadKVRequestFromRemoteCache(const std::string &name,
+      char *databuf, uint64_t offset, uint64_t length);
+
+    bool ReadKVRequestFromLocalCache(const std::string &name, char *databuf,
+      uint64_t offset, uint64_t len);
+
+    // thread function for ReadKVRequest
+    void ProcessKVRequest(const S3ReadRequest &req, char *dataBuf,
+        uint64_t fileLen,
+        std::once_flag &cancelFlag,     // NOLINT
+        std::atomic<bool> &isCanceled,  // NOLINT
+        std::atomic<int> &retCode);     // NOLINT
+
+    void PrefetchForBlock(const S3ReadRequest &req, uint64_t fileLen,
+      uint64_t blockSize, uint64_t chunkSize, uint64_t startBlockIndex);
+
+    void GetChunkLoc(uint64_t offset, uint64_t *index,
+      uint64_t *chunkPos, uint64_t *chunkSize);
+
+    void GetBlockLoc(uint64_t offset, uint64_t *chunkIndex, uint64_t *chunkPos,
+      uint64_t *blockIndex, uint64_t *blockPos);
+
+    uint32_t GetPrefetchBlocks() {
+        return prefetchBlocks_;
     }
 
     uint32_t GetReadRetryIntervalMs() const {
         return readRetryIntervalMs_;
     }
 
- private:
-    void BackGroundFlush();
-
     using AsyncDownloadTask = std::function<void()>;
-
     static int ExecAsyncDownloadTask(void* meta, bthread::TaskIterator<AsyncDownloadTask>& iter);  // NOLINT
 
-    int ClearDiskCache(int64_t inodeId);
-
  public:
-    void PushAsyncTask(const AsyncDownloadTask& task) {
+  void PushAsyncTask(const AsyncDownloadTask& task) {
         static thread_local unsigned int seed = time(nullptr);
 
         int idx = rand_r(&seed) % downloadTaskQueues_.size();
@@ -241,47 +256,62 @@ class S3ClientAdaptorImpl : public S3ClientAdaptor {
             task();
         }
     }
-    std::shared_ptr<S3Metric> s3Metric_;
-
-    void Enqueue(std::shared_ptr<FlushChunkCacheContext> context);
 
  private:
-    std::shared_ptr<S3Client> client_;
-    uint64_t blockSize_;
-    uint64_t chunkSize_;
-    uint32_t prefetchBlocks_;
-    uint32_t prefetchExecQueueNum_;
-    std::string allocateServerEps_;
-    uint32_t flushIntervalSec_;
-    uint32_t chunkFlushThreads_;
-    uint32_t memCacheNearfullRatio_;
-    uint32_t throttleBaseSleepUs_;
-    uint32_t maxReadRetryIntervalMs_;
-    uint32_t readRetryIntervalMs_;
-    uint32_t objectPrefix_;
-    Thread bgFlushThread_;
-    std::atomic<bool> toStop_;
-    std::mutex mtx_;
-    std::mutex ioMtx_;
-    std::condition_variable cond_;
-    curve::common::WaitInterval waitInterval_;
-    std::shared_ptr<FsCacheManager> fsCacheManager_;
-    std::shared_ptr<InodeCacheManager> inodeManager_;
-    std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl_;
-    DiskCacheType diskCacheType_;
-    std::shared_ptr<MdsClient> mdsClient_;
-    uint32_t fsId_;
-    std::string fsName_;
+  class AsyncPrefetchCallback {
+   public:
+      AsyncPrefetchCallback(uint64_t inode, S3ClientAdaptorImpl *s3Client)
+          : inode_(inode), s3Client_(s3Client) {}
+
+      void operator()(const S3Adapter *,
+                      const std::shared_ptr<GetObjectAsyncContext> &context) {
+          std::unique_ptr<char[]> guard(context->buf);
+
+          if (context->retCode != 0) {
+              LOG(WARNING) << "prefetch failed, key: " << context->key;
+              return;
+          }
+
+          int ret = s3Client_->GetDiskCacheManager()->WriteReadDirect(
+              context->key, context->buf, context->actualLen);
+          if (ret < 0) {
+              LOG_EVERY_SECOND(INFO) <<
+                "prefetch failed, write read directly failed, key: "
+                << context->key;
+          }
+          {
+            curve::common::LockGuard lg(s3Client_->downloadMtx_);
+            s3Client_->downloadingObj_.erase(context->key);
+          }
+          VLOG(9) << "prefetch end, objectname is: " << context->key
+                  << ", len is: " << context->len
+                  << ", actual len is: " <<  context->actualLen;
+      }
+
+   private:
+      const uint64_t inode_;
+      S3ClientAdaptorImpl *s3Client_;
+  };
+
+ protected:
+  curve::common::Mutex downloadMtx_;
+  std::set<std::string> downloadingObj_;
+
+ private:
     std::vector<bthread::ExecutionQueueId<AsyncDownloadTask>>
       downloadTaskQueues_;
-    uint32_t pageSize_;
-
-    int FlushChunkClosure(std::shared_ptr<FlushChunkCacheContext> context);
-
-    TaskThreadPool<bthread::Mutex, bthread::ConditionVariable>
-        taskPool_;
-
-    std::shared_ptr<KVClientManager> kvClientManager_ = nullptr;
+    // prefetch blocks nums
+    uint32_t prefetchBlocks_;
+    // prefetch thread nums
+    uint32_t prefetchExecQueueNum_;
+    // read and max retry times when read s3 failed
+    uint32_t readRetryIntervalMs_;
+    uint32_t objectPrefix_;
+    uint32_t maxReadRetryIntervalMs_;
+    // s3 client manager(put or get object form s3)
+    std::shared_ptr<S3Client> client_;
+    // kv client manager
+    std::shared_ptr<KVClientManager> kvClientManager_;
 };
 
 }  // namespace client

curvefs/src/client/s3/fuse_s3_client.cpp

@@ -24,7 +24,7 @@
 #include <memory>
 #include <vector>
 
-#include "curvefs/src/client/fuse_s3_client.h"
+#include "curvefs/src/client/s3/fuse_s3_client.h"
 #include "curvefs/src/client/kvclient/memcache_client.h"
 
 namespace curvefs {
@@ -47,63 +47,30 @@ using curvefs::mds::topology::MemcacheClusterInfo;
 using curvefs::mds::topology::MemcacheServerInfo;
 
 CURVEFS_ERROR FuseS3Client::Init(const FuseClientOption &option) {
-    FuseClientOption opt(option);
-
-    CURVEFS_ERROR ret = FuseClient::Init(opt);
+    LOG(INFO) << "fuse init start.";
+    auto ret = FuseClient::Init(option);
     if (ret != CURVEFS_ERROR::OK) {
+        LOG(INFO) << "fuse init failed, " << ret;
         return ret;
     }
 
     // init kvcache
     if (FLAGS_supportKVcache && !InitKVCache(option.kvClientManagerOpt)) {
+        LOG(INFO) << "init kv cache failed.";
         return CURVEFS_ERROR::INTERNAL;
     }
 
-    // set fs S3Option
-    const auto& s3Info = fsInfo_->detail().s3info();
-    ::curve::common::S3InfoOption fsS3Option;
-    ::curvefs::client::common::S3Info2FsS3Option(s3Info, &fsS3Option);
-    SetFuseClientS3Option(&opt, fsS3Option);
-
-    auto s3Client = std::make_shared<S3ClientImpl>();
-    s3Client->Init(opt.s3Opt.s3AdaptrOpt);
-
-    const uint64_t writeCacheMaxByte =
-        opt.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte;
-    if (writeCacheMaxByte < MIN_WRITE_CACHE_SIZE) {
-        LOG(ERROR) << "writeCacheMaxByte is too small"
-                   << ", at least " << MIN_WRITE_CACHE_SIZE << " (8MB)"
-                      ", writeCacheMaxByte = " << writeCacheMaxByte;
-        return CURVEFS_ERROR::CACHETOOSMALL;
+    // init storage adaptor
+    ret = InitStorageAdaptor(option);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(INFO) << "init storage adaptor failed, " << ret;
+        return ret;
     }
 
-    auto fsCacheManager = std::make_shared<FsCacheManager>(
-        dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte, writeCacheMaxByte,
-        opt.s3Opt.s3ClientAdaptorOpt.readCacheThreads, kvClientManager_);
-    if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
-        DiskCacheType::Disable) {
-        auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
-        s3DiskCacheClient->Init(opt.s3Opt.s3AdaptrOpt);
-        auto wrapper = std::make_shared<PosixWrapper>();
-        auto diskCacheRead = std::make_shared<DiskCacheRead>();
-        auto diskCacheWrite = std::make_shared<DiskCacheWrite>();
-        auto diskCacheManager = std::make_shared<DiskCacheManager>(
-            wrapper, diskCacheWrite, diskCacheRead);
-        auto diskCacheManagerImpl = std::make_shared<DiskCacheManagerImpl>(
-            diskCacheManager, s3DiskCacheClient);
-        ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
-                               inodeManager_, mdsClient_, fsCacheManager,
-                               diskCacheManagerImpl, kvClientManager_, true);
-    } else {
-        ret = s3Adaptor_->Init(opt.s3Opt.s3ClientAdaptorOpt, s3Client,
-                               inodeManager_, mdsClient_, fsCacheManager,
-                               nullptr, kvClientManager_, true);
-    }
+    LOG(INFO) << "fuse init success.";
     return ret;
 }
 
-
 bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
      // get kvcache cluster
     MemcacheClusterInfo kvcachecluster;
@@ -136,6 +103,50 @@ bool FuseS3Client::InitKVCache(const KVClientManagerOpt &opt) {
     return true;
 }
 
+CURVEFS_ERROR FuseS3Client::InitStorageAdaptor(const FuseClientOption &option) {
+    auto opt(option);
+    auto ret = CURVEFS_ERROR::OK;
+
+    // set fs S3Option
+    const auto& s3Info = fsInfo_->detail().s3info();
+    ::curve::common::S3InfoOption fsS3Option;
+    ::curvefs::client::common::S3Info2FsS3Option(s3Info, &fsS3Option);
+    // update s3 info: s3 adress, bucket, etc
+    SetFuseClientS3Option(&opt, fsS3Option);
+
+    auto s3Client = std::make_shared<S3ClientImpl>();
+    s3Client->Init(opt.s3Opt.s3AdaptrOpt);
+
+    auto fsCacheManager = std::make_shared<FsCacheManager>(
+        dynamic_cast<S3ClientAdaptorImpl *>(s3Adaptor_.get()),
+        option.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
+        option.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte,
+        option.s3Opt.s3ClientAdaptorOpt.readCacheThreads,
+        kvClientManager_);
+
+    if (opt.s3Opt.s3ClientAdaptorOpt.diskCacheOpt.diskCacheType !=
+        DiskCacheType::Disable) {
+        LOG(INFO) << "has disk cache.";
+        auto s3DiskCacheClient = std::make_shared<S3ClientImpl>();
+        s3DiskCacheClient->Init(opt.s3Opt.s3AdaptrOpt);
+        auto wrapper = std::make_shared<PosixWrapper>();
+        auto diskCacheRead = std::make_shared<DiskCacheRead>();
+        auto diskCacheWrite = std::make_shared<DiskCacheWrite>();
+        auto diskCacheManager = std::make_shared<DiskCacheManager>(
+            wrapper, diskCacheWrite, diskCacheRead);
+        auto diskCacheManagerImpl = std::make_shared<DiskCacheManagerImpl>(
+            diskCacheManager, s3DiskCacheClient);
+
+        ret = s3Adaptor_->Init(opt, inodeManager_, mdsClient_, fsCacheManager,
+                               diskCacheManagerImpl, kvClientManager_, fsInfo_);
+    } else {
+        ret = s3Adaptor_->Init(opt, inodeManager_, mdsClient_, fsCacheManager,
+                               nullptr, kvClientManager_, fsInfo_);
+    }
+    LOG(INFO) << "init storage adaptor success.";
+    return ret;
+}
+
 void FuseS3Client::UnInit() {
     FuseClient::UnInit();
     s3Adaptor_->Stop();
@@ -144,10 +155,15 @@ void FuseS3Client::UnInit() {
 
 CURVEFS_ERROR FuseS3Client::FuseOpInit(void *userdata,
                                        struct fuse_conn_info *conn) {
-    CURVEFS_ERROR ret = FuseClient::FuseOpInit(userdata, conn);
-    if (init_) {
-        s3Adaptor_->SetFsId(fsInfo_->fsid());
-        s3Adaptor_->InitMetrics(fsInfo_->fsname());
+    auto ret = FuseClient::FuseOpInit(userdata, conn);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "fuse op init failed.";
+        return ret;
+    }
+    ret = s3Adaptor_->FuseOpInit(userdata, conn);
+    if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "fuse op init failed.";
+        return ret;
     }
     return ret;
 }

curvefs/src/client/s3/fuse_s3_client.h

@@ -21,8 +21,8 @@
  * Author: xuchaojie
  */
 
-#ifndef CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
-#define CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
+#ifndef CURVEFS_SRC_CLIENT_S3_FUSE_S3_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_S3_FUSE_S3_CLIENT_H_
 
 #include <memory>
 #include <string>
@@ -31,7 +31,7 @@
 #include <utility>
 
 #include "curvefs/src/client/fuse_client.h"
-#include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
 #include "curvefs/src/client/warmup/warmup_manager.h"
 #include "curvefs/src/volume/common.h"
 #include "src/common/s3_adapter.h"
@@ -47,11 +47,13 @@ class WarmupManager;
 class WarmupManagerS3Impl;
 }  // namespace warmup
 
-
+// s3 client
 class FuseS3Client : public FuseClient {
  public:
     FuseS3Client()
-        : FuseClient(), s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()) {
+        : FuseClient(),
+          s3Adaptor_(std::make_shared<S3ClientAdaptorImpl>()),
+          kvClientManager_(nullptr) {
         auto readFunc = [this](fuse_req_t req, fuse_ino_t ino, size_t size,
                                off_t off, struct fuse_file_info *fi,
                                char *buffer, size_t *rSize) {
@@ -66,16 +68,19 @@ class FuseS3Client : public FuseClient {
                  const std::shared_ptr<MetaServerClient> &metaClient,
                  const std::shared_ptr<InodeCacheManager> &inodeManager,
                  const std::shared_ptr<DentryCacheManager> &dentryManager,
-                 const std::shared_ptr<S3ClientAdaptor> &s3Adaptor,
+                 const std::shared_ptr<StorageAdaptor> &s3Adaptor,
                  const std::shared_ptr<warmup::WarmupManager> &warmupManager)
         : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
                      warmupManager),
-          s3Adaptor_(s3Adaptor) {}
+          s3Adaptor_(s3Adaptor),
+          kvClientManager_(nullptr) {}
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
 
     void UnInit() override;
 
+/*** fuse op ***/
+
     CURVEFS_ERROR FuseOpInit(
         void *userdata, struct fuse_conn_info *conn) override;
 
@@ -112,21 +117,20 @@ class FuseS3Client : public FuseClient {
 
  private:
     bool InitKVCache(const KVClientManagerOpt &opt);
-
+    CURVEFS_ERROR InitStorageAdaptor(const FuseClientOption &option);
     CURVEFS_ERROR Truncate(InodeWrapper *inode, uint64_t length) override;
 
     void FlushData() override;
 
  private:
-    // s3 adaptor
-    std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
+    // s3 storage adaptor
+    std::shared_ptr<StorageAdaptor> s3Adaptor_;
+    // kv client manager
     std::shared_ptr<KVClientManager> kvClientManager_;
-
-    static constexpr auto MIN_WRITE_CACHE_SIZE = 8 * kMiB;
 };
 
 
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_FUSE_S3_CLIENT_H_
+#endif  // CURVEFS_SRC_CLIENT_S3_FUSE_S3_CLIENT_H_

curvefs/src/client/volume/client_volume_adaptor.cpp

@@ -0,0 +1,182 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+
+/*
+ * Project: curve
+ * Created Date: Thur March 14 2023
+ * Author: wuhongsong
+ */
+
+#include <utility>
+
+#include "curvefs/src/client/volume/client_volume_adaptor.h"
+
+#include "curvefs/src/client/rpcclient/mds_client.h"
+
+namespace curvefs {
+namespace client {
+
+#define VOLUME_BLOCK_SIZE 4194304
+#define VOLUME_CHUNK_SIZE 67108864
+
+CURVEFS_ERROR VolumeClientAdaptorImpl::Init(const FuseClientOption &option,
+         std::shared_ptr<InodeCacheManager> inodeManager,
+         std::shared_ptr<MdsClient> mdsClient,
+         std::shared_ptr<FsCacheManager> fsCacheManager,
+         std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+         std::shared_ptr<KVClientManager> kvClientManager,
+         std::shared_ptr<FsInfo> fsInfo) {
+    LOG(INFO) << "volume adaptor init start.";
+    volOpts_ = option.volumeOpt;
+    SetBlockSize(VOLUME_BLOCK_SIZE);
+    SetChunkSize(VOLUME_CHUNK_SIZE);
+    auto ret = StorageAdaptor::Init(option, inodeManager,
+      mdsClient, fsCacheManager, diskCacheManagerImpl, kvClientManager, fsInfo);
+    if (ret != CURVEFS_ERROR::OK) {
+        return ret;
+    }
+    BlockDeviceClientOptions opts;
+    opts.configPath = option.bdevOpt.configPath;
+    auto ret2 = blockDeviceClient_->Init(opts);
+    if (!ret2) {
+        LOG(ERROR) << "Init block device client failed, " << ret2;
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    LOG(INFO) << "volume adaptor init sucess.";
+    return ret;
+}
+
+CURVEFS_ERROR VolumeClientAdaptorImpl::FuseOpInit(void *userdata,
+  struct fuse_conn_info *conn) {
+    LOG(INFO) << "volume fuse op init start!";
+    const auto &vol = fsInfo_->detail().volume();
+    const auto &volName = vol.volumename();
+    const auto &user = vol.user();
+    auto ret = blockDeviceClient_->Open(volName, user);
+    if (!ret) {
+        LOG(ERROR) << "BlockDeviceClientImpl open failed, ret = " << ret
+                   << ", volName = " << volName << ", user = " << user;
+        return CURVEFS_ERROR::INTERNAL;
+    }
+
+    SpaceManagerOption option;
+    option.blockGroupManagerOption.fsId = fsInfo_->fsid();
+
+    option.blockGroupManagerOption.owner = GetMountOwner();
+
+    option.blockGroupManagerOption.blockGroupAllocateOnce =
+        volOpts_.allocatorOption.blockGroupOption.allocateOnce;
+    option.blockGroupManagerOption.blockGroupSize =
+        fsInfo_->detail().volume().blockgroupsize();
+    option.blockGroupManagerOption.blockSize =
+        fsInfo_->detail().volume().blocksize();
+
+    option.allocatorOption.type = volOpts_.allocatorOption.type;
+    option.allocatorOption.bitmapAllocatorOption.sizePerBit =
+        volOpts_.allocatorOption.bitmapAllocatorOption.sizePerBit;
+    option.allocatorOption.bitmapAllocatorOption.smallAllocProportion =
+        volOpts_.allocatorOption.bitmapAllocatorOption.smallAllocProportion;
+
+    option.threshold = volOpts_.threshold;
+    option.releaseInterSec = volOpts_.releaseInterSec;
+
+    spaceManager_ = absl::make_unique<SpaceManagerImpl>(
+        option, mdsClient_, blockDeviceClient_);
+    spaceManager_->Run();
+
+    storage_ = absl::make_unique<DefaultVolumeStorage>(spaceManager_.get(),
+      blockDeviceClient_.get(), GetInodeCacheManager().get());
+
+    ExtentCacheOption extentOpt;
+    extentOpt.blockSize = vol.blocksize();
+    extentOpt.sliceSize = vol.slicesize();
+
+    ExtentCache::SetOption(extentOpt);
+    LOG(INFO) << "volume fuse op init sucess.";
+    return CURVEFS_ERROR::OK;
+}
+
+int VolumeClientAdaptorImpl::Stop() {
+    LOG(INFO) << "volume adaptor stop...";
+    StorageAdaptor::Stop();
+    if (nullptr != storage_) {
+        storage_->Shutdown();
+    }
+    if (nullptr != spaceManager_) {
+        spaceManager_->Shutdown();
+    }
+    blockDeviceClient_->UnInit();
+    LOG(INFO) << "volume adaptor stop sucess.";
+    return 0;
+}
+
+CURVEFS_ERROR VolumeClientAdaptorImpl::FlushDataCache(
+  const UperFlushRequest& req, uint64_t* writeOffset) {
+    uint64_t inodeId = req.inodeId;
+    uint64_t inodeOffset = req.offset;
+    uint64_t len = req.length;
+    const char* data = req.buf;
+    VLOG(9) << "volume flush dataCache, inode: "
+       << inodeId<< ", offset: " << inodeOffset
+       << ", length: " << len;
+
+    CURVEFS_ERROR ret = storage_->Write(inodeId, inodeOffset, len, data);
+    if (ret != CURVEFS_ERROR::OK) {
+        VLOG(0) << "volume flush dataCache err, inode: "
+            << inodeId<< ", offset: " << inodeOffset
+            << ", length: " << len;
+        return ret;
+    }
+
+    VLOG(9) << "volume flush dataCache end, inode: "
+       << inodeId<< ", offset: " << inodeOffset
+       << ", length: " << len;
+// 这个正确么？
+    *writeOffset = len;
+    return CURVEFS_ERROR::OK;
+}
+
+CURVEFS_ERROR VolumeClientAdaptorImpl::ReadFromLowlevel(
+  UperReadRequest uperRequest) {
+    uint64_t inodeId = uperRequest.inodeId;
+    VLOG(9) << "read lowlevel start, inodeId is: " << inodeId;
+    std::vector<ReadRequest> requests;
+    char *buf = uperRequest.buf;;
+    requests = std::move(uperRequest.requests);
+    uint64_t chunkSize = GetChunkSize();
+    CURVEFS_ERROR ret;
+    for (auto req : requests) {
+        VLOG(9) << "read from storage " << req.DebugString();
+        uint64_t len = req.len;
+        uint64_t readOffset = chunkSize * req.index + req.chunkPos;
+        ret = storage_->Read(
+          inodeId, readOffset, len, buf + req.bufOffset);
+        if (ret != CURVEFS_ERROR::OK) {
+            VLOG(0) << "volume flush dataCache err, inode: "
+                << inodeId << ", offset: " << readOffset
+                << ", length: " << len;
+            return ret;
+        }
+    }
+
+    VLOG(9) << "read lowlevel end, inodeId is: "<< inodeId;
+    return ret;
+}
+
+}  // namespace client
+}  // namespace curvefs

curvefs/src/client/volume/client_volume_adaptor.h

@@ -0,0 +1,145 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Created Date: Thur March 14 2023
+ * Author: wuhongsong
+ */
+
+#ifndef CURVEFS_SRC_CLIENT_VOLUME_CLIENT_VOLUME_ADAPTOR_H_
+#define CURVEFS_SRC_CLIENT_VOLUME_CLIENT_VOLUME_ADAPTOR_H_
+
+#include <bthread/execution_queue.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "curvefs/proto/common.pb.h"
+#include "curvefs/proto/mds.pb.h"
+#include "curvefs/proto/metaserver.pb.h"
+
+#include "src/common/wait_interval.h"
+#include "curvefs/src/client/common/common.h"
+#include "curvefs/src/client/common/config.h"
+#include "curvefs/src/client/client_storage_adaptor.h"
+#include "curvefs/src/client/error_code.h"
+#include "curvefs/src/client/inode_cache_manager.h"
+#include "curvefs/src/client/rpcclient/mds_client.h"
+#include "curvefs/src/client/s3/client_s3.h"
+
+#include "curvefs/src/client/volume/default_volume_storage.h"
+#include "curvefs/src/client/volume/volume_storage.h"
+#include "curvefs/src/volume/block_device_client.h"
+#include "curvefs/src/volume/space_manager.h"
+
+namespace curvefs {
+namespace client {
+
+using common::VolumeOption;
+using ::curvefs::volume::BlockDeviceClient;
+using ::curvefs::volume::BlockDeviceClientImpl;
+using ::curvefs::volume::BlockDeviceClientOptions;
+using ::curvefs::volume::SpaceManager;
+using ::curvefs::volume::SpaceManagerImpl;
+using ::curvefs::volume::SpaceManagerOption;
+
+// client use volume internal interface
+class VolumeClientAdaptorImpl : public StorageAdaptor {
+ public:
+    VolumeClientAdaptorImpl() : StorageAdaptor(),
+      blockDeviceClient_(std::make_shared<BlockDeviceClientImpl>()) {}
+
+    explicit VolumeClientAdaptorImpl(const std::shared_ptr<
+      BlockDeviceClient> &blockDeviceClient) : StorageAdaptor(),
+      blockDeviceClient_(blockDeviceClient) {}
+
+    virtual ~VolumeClientAdaptorImpl() {
+        LOG(INFO) << "delete VolumeClientAdaptorImpl";
+    }
+
+ public:
+    /// @brief init volume storage adaptor
+    /// @param option fuse client option
+    /// @param inodeManager inode cache manager
+    /// @param mdsClient mds client
+    /// @param fsCacheManager fscache manager
+    /// @param diskCacheManagerImpl disk cache manager
+    /// @param kvClientManager kv client manager
+    /// @param fsInfo file system information
+    /// @return error code
+    CURVEFS_ERROR Init(const FuseClientOption &option,
+      std::shared_ptr<InodeCacheManager> inodeManager,
+      std::shared_ptr<MdsClient> mdsClient,
+      std::shared_ptr<FsCacheManager> fsCacheManager,
+      std::shared_ptr<DiskCacheManagerImpl> diskCacheManagerImpl,
+      std::shared_ptr<KVClientManager> kvClientManager,
+      std::shared_ptr<FsInfo> fsInfo) override;
+
+    /// @brief fuse op init
+    /// @param userdata fuse user data
+    /// @param conn fuse connect info
+    /// @param fsid  fs id
+    /// @param fsname fs name
+    /// @return error code
+    CURVEFS_ERROR FuseOpInit(void *userdata,
+      struct fuse_conn_info *conn) override;
+
+    int Stop() override;
+
+    /// @brief read data from volume storage
+    /// @param request read request
+    /// @return error code
+    CURVEFS_ERROR FlushDataCache(const UperFlushRequest& req,
+      uint64_t* writeOffset) override;
+
+    /// @brief read data from volume storage
+    /// @param request read request
+    /// @return error code
+    CURVEFS_ERROR ReadFromLowlevel(UperReadRequest request) override;
+
+    // TODO(@hzwuhongsong)
+    virtual CURVEFS_ERROR Truncate(InodeWrapper *inodeWrapper,
+      uint64_t size) { return CURVEFS_ERROR::OK;}
+
+    /// @brief get volume storage
+    /// @return volume storage
+    std::shared_ptr<VolumeStorage> getUnderStorage() {
+        return storage_;
+    }
+
+    /// @brief get space manager
+    /// @return space manager
+    std::shared_ptr<SpaceManager> getSpaceManager() {
+        return spaceManager_;
+    }
+
+ private:
+    // block device client(write/read data from volume)
+    std::shared_ptr<BlockDeviceClient> blockDeviceClient_;
+    // volume space manager
+    std::shared_ptr<SpaceManager> spaceManager_;
+    // volume storage adaptor
+    std::shared_ptr<VolumeStorage> storage_;
+    // volume option
+    VolumeOption volOpts_;
+};
+
+}  // namespace client
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_CLIENT_VOLUME_CLIENT_VOLUME_ADAPTOR_H_

curvefs/src/client/volume/default_volume_storage.cpp

@@ -186,6 +186,8 @@ CURVEFS_ERROR DefaultVolumeStorage::Flush(uint64_t ino) {
         return CURVEFS_ERROR::OK;
     }
 
+    VLOG(9) << "volume storage flush: " << ino;
+
     LatencyUpdater updater(&metric_.flushLatency);
     std::shared_ptr<InodeWrapper> inodeWrapper;
     auto ret = inodeCacheManager_->GetInode(ino, inodeWrapper);

curvefs/src/client/volume/default_volume_storage.h

@@ -73,9 +73,13 @@ class DefaultVolumeStorage final : public VolumeStorage {
     bool Shutdown() override;
 
  private:
+    // volume space manager
     SpaceManager* spaceManager_;
+    // block device client(write/read data from volume)
     BlockDeviceClient* blockDeviceClient_;
+    // inode cache manager
     InodeCacheManager* inodeCacheManager_;
+    // metric from volume storage
     VolumeStorageMetric metric_;
 };
 

curvefs/src/client/volume/extent_cache.cpp

@@ -90,6 +90,10 @@ void ExtentCache::Merge(uint64_t loffset, const PExtent& pExt) {
 
     slice->second.Merge(loffset, pExt);
     dirties_.insert(&slice->second);
+    VLOG(9) << "merge extent, loffset: " << loffset
+            << ", physical offset: " << pExt.pOffset << ", len: " << pExt.len
+            << ", written: " << !pExt.UnWritten
+            << ", slice: " << slice->second.ToVolumeExtentSlice().DebugString();
 }
 
 void ExtentCache::DivideForWrite(uint64_t offset,
@@ -162,12 +166,24 @@ void ExtentCache::MarkWritten(uint64_t offset, uint64_t len) {
     auto cur = align_down(offset, option_.blockSize);
     const auto end = align_up(offset + len, option_.blockSize);
 
+    VLOG(9) << "mark written for offset: " << offset << ", len: " << len
+            << ", cur: " << cur << ", end: " << end;
+
     while (cur < end) {
         const auto length =
             std::min(end - cur, option_.sliceSize - (cur & ~option_.sliceSize));
         auto slice = slices_.find(align_down(cur, option_.sliceSize));
         assert(slice != slices_.end());
+        VLOG(9) << "mark written for offset: " << offset << ", len: " << len
+                << ", cur: " << cur << ", end: " << end
+                << ", before mark written slice: "
+                << slice->second.ToVolumeExtentSlice().DebugString();
         auto changed = slice->second.MarkWritten(cur, length);
+        VLOG(9) << "mark written for offset: " << offset << ", len: " << len
+                << ", cur: " << cur << ", end: " << end
+                << ", after mark written slice changed: " << changed
+                << ", slice: "
+                << slice->second.ToVolumeExtentSlice().DebugString();
         cur += length;
         if (changed) {
             dirties_.insert((&slice->second));
@@ -194,6 +210,8 @@ void ExtentCache::DivideForRead(uint64_t offset,
                                 std::vector<ReadPart>* holes) {
     LatencyUpdater updater(&g_read_divide_latency);
     ReadLockGuard lk(lock_);
+    VLOG(9) << "extent cache divide for read offset: " << offset
+            << ", length: " << len;
 
     const auto end = offset + len;
     char* datap = data;
@@ -205,8 +223,15 @@ void ExtentCache::DivideForRead(uint64_t offset,
         auto slice = slices_.find(align_down(offset, option_.sliceSize));
         if (slice != slices_.end()) {
             slice->second.DivideForRead(offset, length, datap, reads, holes);
+            VLOG(9) << "extent cache find slice for read offset: " << offset
+                    << ", length: " << len << ", slice: "
+                    << slice->second.ToVolumeExtentSlice().DebugString()
+                    << ", slices size: " << slices_.size();
         } else {
             holes->emplace_back(offset, length, datap);
+            VLOG(9) << "extent cache not find slice for read offset: " << offset
+                    << ", length: " << len
+                    << ", slices size: " << slices_.size();
         }
 
         datap += length;
@@ -228,7 +253,7 @@ void ExtentCache::SetOption(const ExtentCacheOption& option) {
     LOG(INFO) << "ExtentCacheOption: [" << option_ << "]";
 }
 
-void ExtentCache::Build(const VolumeExtentList &extents) {
+void ExtentCache::Build(const VolumeExtentSliceList &extents) {
     WriteLockGuard lk(lock_);
     slices_.clear();
     dirties_.clear();
@@ -238,14 +263,15 @@ void ExtentCache::Build(const VolumeExtentList &extents) {
     }
 }
 
-VolumeExtentList ExtentCache::GetDirtyExtents() {
-    VolumeExtentList result;
+VolumeExtentSliceList ExtentCache::GetDirtyExtents() {
+    VolumeExtentSliceList result;
     WriteLockGuard lk(lock_);
     for (const auto* slice : dirties_) {
         *result.add_slices() = slice->ToVolumeExtentSlice();
     }
 
     dirties_.clear();
+    VLOG(9) << "extent cache get and clear dirty extents";
     return result;
 }
 

curvefs/src/client/volume/extent_cache.h

@@ -43,7 +43,7 @@ namespace client {
 using ::curvefs::volume::ReadPart;
 using ::curvefs::volume::WritePart;
 using ::curvefs::metaserver::VolumeExtentSlice;
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtentSliceList;
 
 struct ExtentCacheOption {
     // preallocation size if offset ~ length is not allocated
@@ -61,7 +61,7 @@ class ExtentCache {
 
     static void SetOption(const ExtentCacheOption& option);
 
-    void Build(const VolumeExtentList& extents);
+    void Build(const VolumeExtentSliceList& extents);
 
     void DivideForWrite(uint64_t offset,
                         uint64_t len,
@@ -81,7 +81,7 @@ class ExtentCache {
 
     bool HasDirtyExtents() const;
 
-    VolumeExtentList GetDirtyExtents();
+    VolumeExtentSliceList GetDirtyExtents();
 
     std::unordered_map<uint64_t, std::map<uint64_t, PExtent>>
     GetExtentsForTesting() const;

curvefs/src/client/volume/fuse_volume_client.cpp

@@ -21,7 +21,7 @@
  * Author: xuchaojie
  */
 
-#include "curvefs/src/client/fuse_volume_client.h"
+#include "curvefs/src/client/volume/fuse_volume_client.h"
 
 #include <butil/time.h>
 #include <bvar/bvar.h>
@@ -41,38 +41,38 @@
 namespace curvefs {
 namespace client {
 
+namespace common {
+DECLARE_bool(enableCto);
+}  // namespace common
+
 using ::curvefs::volume::SpaceManagerImpl;
 using ::curvefs::volume::SpaceManagerOption;
 using ::curvefs::volume::BlockDeviceClientOptions;
 using ::curvefs::volume::BlockDeviceClientImpl;
+using ::curvefs::client::common::FLAGS_enableCto;
 
 CURVEFS_ERROR FuseVolumeClient::Init(const FuseClientOption &option) {
-    volOpts_ = option.volumeOpt;
-
     CURVEFS_ERROR ret = FuseClient::Init(option);
-
     if (ret != CURVEFS_ERROR::OK) {
+        LOG(ERROR) << "Init failed: " << ret;
         return ret;
     }
 
-    BlockDeviceClientOptions opts;
-    opts.configPath = option.bdevOpt.configPath;
-
-    bool ret2 = blockDeviceClient_->Init(opts);
-
-    if (!ret2) {
-        LOG(ERROR) << "Init block device client failed";
-        return CURVEFS_ERROR::INTERNAL;
-    }
+    auto fsCacheManager = std::make_shared<FsCacheManager>(
+        dynamic_cast<StorageAdaptor*>(storageAdaptor_.get()),
+        option.s3Opt.s3ClientAdaptorOpt.readCacheMaxByte,
+        option.s3Opt.s3ClientAdaptorOpt.writeCacheMaxByte,
+        option.s3Opt.s3ClientAdaptorOpt.readCacheThreads,
+        nullptr);  // bs no need cache cluster
+    ret = storageAdaptor_->Init(option,
+        inodeManager_, mdsClient_, fsCacheManager,
+        nullptr, nullptr, fsInfo_);  // no need cache cluster and diskcache
 
     return ret;
 }
 
 void FuseVolumeClient::UnInit() {
-    storage_->Shutdown();
-    spaceManager_->Shutdown();
-    blockDeviceClient_->UnInit();
-
+    storageAdaptor_->Stop();
     FuseClient::UnInit();
 }
 
@@ -83,47 +83,13 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpInit(void *userdata,
         LOG(ERROR) << "fuse op init failed, error: " << ret;
         return ret;
     }
-
-    const auto &vol = fsInfo_->detail().volume();
-    const auto &volName = vol.volumename();
-    const auto &user = vol.user();
-    auto ret2 = blockDeviceClient_->Open(volName, user);
-    if (!ret2) {
-        LOG(ERROR) << "BlockDeviceClientImpl open failed, ret = " << ret
-                   << ", volName = " << volName << ", user = " << user;
-        return CURVEFS_ERROR::INTERNAL;
-    }
-
-    SpaceManagerOption option;
-    option.blockGroupManagerOption.fsId = fsInfo_->fsid();
-    option.blockGroupManagerOption.owner = mountpoint_.hostname() + ":" +
-                                           std::to_string(mountpoint_.port()) +
-                                           ":" + mountpoint_.path();
-    option.blockGroupManagerOption.blockGroupAllocateOnce =
-        volOpts_.allocatorOption.blockGroupOption.allocateOnce;
-    option.blockGroupManagerOption.blockGroupSize =
-        fsInfo_->detail().volume().blockgroupsize();
-    option.blockGroupManagerOption.blockSize =
-        fsInfo_->detail().volume().blocksize();
-
-    option.allocatorOption.type = volOpts_.allocatorOption.type;
-    option.allocatorOption.bitmapAllocatorOption.sizePerBit =
-        volOpts_.allocatorOption.bitmapAllocatorOption.sizePerBit;
-    option.allocatorOption.bitmapAllocatorOption.smallAllocProportion =
-        volOpts_.allocatorOption.bitmapAllocatorOption.smallAllocProportion;
-
-    spaceManager_ = absl::make_unique<SpaceManagerImpl>(option, mdsClient_,
-                                                        blockDeviceClient_);
-
-    storage_ = absl::make_unique<DefaultVolumeStorage>(
-        spaceManager_.get(), blockDeviceClient_.get(), inodeManager_.get());
-
-    ExtentCacheOption extentOpt;
-    extentOpt.blockSize = vol.blocksize();
-    extentOpt.sliceSize = vol.slicesize();
-
-    ExtentCache::SetOption(extentOpt);
-
+    Mountpoint mountPoint = GetMountPoint();
+    std::string mountOwner = mountPoint.hostname() + ":" +
+        std::to_string(mountPoint.port()) +
+        ":" + mountPoint.path();
+    storageAdaptor_->SetMountOwner(mountOwner);
+    storageAdaptor_->FuseOpInit(userdata, conn);
+    LOG(INFO) << "fuse op Init success: " << mountOwner;
     return CURVEFS_ERROR::OK;
 }
 
@@ -140,47 +106,65 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req,
 
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
-              is_aligned(size, DirectIOAlignment))) {
-            fsMetric_->userWrite.eps.count << 1;
+              is_aligned(size, DirectIOAlignment)))
             return CURVEFS_ERROR::INVALIDPARAM;
-        }
+    }
+    uint64_t start = butil::cpuwide_time_us();
+    int wRet = storageAdaptor_->Write(ino, off, size, buf);
+    if (wRet < 0) {
+        LOG(ERROR) << "storageAdaptor_ write failed, ret = " << wRet;
+        return CURVEFS_ERROR::INTERNAL;
     }
 
-    butil::Timer timer;
-    timer.start();
+    if (fsMetric_.get() != nullptr) {
+        fsMetric_->userWrite.bps.count << wRet;
+        fsMetric_->userWrite.qps.count << 1;
+        uint64_t duration = butil::cpuwide_time_us() - start;
+        fsMetric_->userWrite.latency << duration;
+        fsMetric_->userWriteIoSize.set_value(wRet);
+    }
 
-    CURVEFS_ERROR ret = storage_->Write(ino, off, size, buf);
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
-        if (fsMetric_) {
-            fsMetric_->userWrite.eps.count << 1;
-        }
-        LOG(ERROR) << "write error, ino: " << ino << ", offset: " << off
-                   << ", len: " << size
-                   << ", error: " << ret;
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << ino;
         return ret;
     }
 
-    *wSize = size;
+    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+
+    *wSize = wRet;
+    size_t changeSize = 0;
+    // update file len
+    if (inodeWrapper->GetLengthLocked() < off + *wSize) {
+        changeSize = off + *wSize - inodeWrapper->GetLengthLocked();
+        inodeWrapper->SetLengthLocked(off + *wSize);
+    }
+
+    inodeWrapper->UpdateTimestampLocked(kModifyTime | kChangeTime);
+
+    inodeManager_->ShipToFlush(inodeWrapper);
 
-    // NOTE: O_DIRECT/O_SYNC/O_DSYNC have simillar semantic, but not exactly the
-    // same, see `man 2 open` for more details
     if (fi->flags & O_DIRECT || fi->flags & O_SYNC || fi->flags & O_DSYNC) {
         // Todo: do some cache flush later
     }
 
-    timer.stop();
-
-    if (fsMetric_) {
-        fsMetric_->userWrite.bps.count << size;
-        fsMetric_->userWrite.qps.count << 1;
-        fsMetric_->userWrite.latency << timer.u_elapsed();
-        fsMetric_->userWriteIoSize.set_value(size);
+    if (enableSumInDir_ && changeSize != 0) {
+        const Inode* inode = inodeWrapper->GetInodeLocked();
+        XAttr xattr;
+        xattr.mutable_xattrinfos()->insert({XATTRFBYTES,
+            std::to_string(changeSize)});
+        for (const auto &it : inode->parent()) {
+            auto tret = xattrManager_->UpdateParentInodeXattr(it, xattr, true);
+            if (tret != CURVEFS_ERROR::OK) {
+                LOG(ERROR) << "UpdateParentInodeXattr failed,"
+                           << " inodeId = " << it
+                           << ", xattr = " << xattr.DebugString();
+            }
+        }
     }
-
-    VLOG(9) << "write end, ino: " << ino << ", offset: " << off
-            << ", length: " << size << ", written: " << *wSize;
-
-    return CURVEFS_ERROR::OK;
+    return ret;
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
@@ -191,45 +175,57 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req,
                                            char *buffer,
                                            size_t *rSize) {
     (void)req;
-    VLOG(3) << "read start, ino: " << ino << ", offset: " << off
+    VLOG(9) << "read start, ino: " << ino << ", offset: " << off
             << ", length: " << size;
 
     // check align
     if (fi->flags & O_DIRECT) {
         if (!(is_aligned(off, DirectIOAlignment) &&
-              is_aligned(size, DirectIOAlignment))) {
-            fsMetric_->userRead.eps.count << 1;
-
+              is_aligned(size, DirectIOAlignment)))
             return CURVEFS_ERROR::INVALIDPARAM;
-        }
     }
 
-    butil::Timer timer;
-    timer.start();
-
-    CURVEFS_ERROR ret = storage_->Read(ino, off, size, buffer);
+    uint64_t start = butil::cpuwide_time_us();
+    std::shared_ptr<InodeWrapper> inodeWrapper;
+    CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper);
     if (ret != CURVEFS_ERROR::OK) {
-        if (fsMetric_) {
-            fsMetric_->userRead.eps.count << 1;
-        }
-        LOG(ERROR) << "read error, ino: " << ino << ", offset: " << off
-                   << ", len: " << size << ", error: " << ret;
+        LOG(ERROR) << "inodeManager get inode fail, ret = " << ret
+                   << ", inodeid = " << ino;
         return ret;
     }
+    uint64_t fileSize = inodeWrapper->GetLength();
 
-    if (fsMetric_) {
-        fsMetric_->userRead.bps.count << size;
-        fsMetric_->userRead.qps.count << 1;
-        fsMetric_->userRead.latency << timer.u_elapsed();
-        fsMetric_->userReadIoSize.set_value(size);
+    size_t len = 0;
+    if (fileSize <= off) {
+        *rSize = 0;
+        return CURVEFS_ERROR::OK;
+    } else if (fileSize < off + size) {
+        len = fileSize - off;
+    } else {
+        len = size;
     }
 
-    *rSize = size;
+    int rRet = storageAdaptor_->Read(ino, off, len, buffer);
+    if (rRet < 0) {
+        LOG(ERROR) << "storageAdaptor_ read failed, ret = " << rRet;
+        return CURVEFS_ERROR::INTERNAL;
+    }
+    *rSize = rRet;
 
-    VLOG(3) << "read end, ino: " << ino << ", offset: " << off
-            << ", length: " << size << ", rsize: " << *rSize;
+    if (fsMetric_.get() != nullptr) {
+        fsMetric_->userRead.bps.count << rRet;
+        fsMetric_->userRead.qps.count << 1;
+        uint64_t duration = butil::cpuwide_time_us() - start;
+        fsMetric_->userRead.latency << duration;
+        fsMetric_->userReadIoSize.set_value(rRet);
+    }
 
-    return CURVEFS_ERROR::OK;
+    ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock();
+    inodeWrapper->UpdateTimestampLocked(kAccessTime);
+    inodeManager_->ShipToFlush(inodeWrapper);
+
+    VLOG(9) << "read end, read size = " << *rSize;
+    return ret;
 }
 
 CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
@@ -277,8 +273,9 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino,
     (void)req;
     (void)fi;
     VLOG(3) << "FuseOpFsync start, ino: " << ino << ", datasync: " << datasync;
-
-    CURVEFS_ERROR ret = storage_->Flush(ino);
+    CURVEFS_ERROR ret = CURVEFS_ERROR::OK;
+    ret = dynamic_cast<VolumeClientAdaptorImpl*>(
+      storageAdaptor_.get())->getUnderStorage()->Flush(ino);
     if (ret != CURVEFS_ERROR::OK) {
         LOG(ERROR) << "Storage flush ino: " << ino << " failed, error: " << ret;
         return ret;
@@ -314,7 +311,20 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpFlush(fuse_req_t req, fuse_ino_t ino,
     (void)fi;
     VLOG(9) << "FuseOpFlush, ino: " << ino;
 
-    CURVEFS_ERROR ret = storage_->Flush(ino);
+    if (FLAGS_enableCto) {
+        auto ret = storageAdaptor_->FlushAllCache(ino);
+        if (ret != CURVEFS_ERROR::OK) {
+            LOG(ERROR) << "FuseOpFlush, flush all cache fail, ret = " << ret
+                       << ", ino: " << ino;
+            return ret;
+        }
+        VLOG(3) << "FuseOpFlush, ino: " << ino << " flush to volume ok";
+    }
+
+    CURVEFS_ERROR ret =
+        dynamic_cast<VolumeClientAdaptorImpl *>(storageAdaptor_.get())
+            ->getUnderStorage()
+            ->Flush(ino);
     LOG_IF(ERROR, ret != CURVEFS_ERROR::OK)
         << "Flush error, ino: " << ino << ", error: " << ret;
 
@@ -326,11 +336,13 @@ void FuseVolumeClient::FlushData() {
 }
 
 void FuseVolumeClient::SetSpaceManagerForTesting(SpaceManager *manager) {
-    spaceManager_.reset(manager);
+    dynamic_cast< VolumeClientAdaptorImpl *>(
+      storageAdaptor_.get())->getSpaceManager().reset(manager);
 }
 
 void FuseVolumeClient::SetVolumeStorageForTesting(VolumeStorage *storage) {
-    storage_.reset(storage);
+    dynamic_cast< VolumeClientAdaptorImpl *>(
+      storageAdaptor_.get())->getUnderStorage().reset(storage);
 }
 
 }  // namespace client

curvefs/src/client/volume/fuse_volume_client.h

@@ -21,13 +21,13 @@
  * Author: xuchaojie
  */
 
-#ifndef CURVEFS_SRC_CLIENT_FUSE_VOLUME_CLIENT_H_
-#define CURVEFS_SRC_CLIENT_FUSE_VOLUME_CLIENT_H_
+#ifndef CURVEFS_SRC_CLIENT_VOLUME_FUSE_VOLUME_CLIENT_H_
+#define CURVEFS_SRC_CLIENT_VOLUME_FUSE_VOLUME_CLIENT_H_
 
 #include <memory>
 
 #include "curvefs/src/client/fuse_client.h"
-#include "curvefs/src/client/volume/volume_storage.h"
+#include "curvefs/src/client/volume/client_volume_adaptor.h"
 #include "curvefs/src/volume/block_device_client.h"
 #include "curvefs/src/volume/space_manager.h"
 
@@ -35,15 +35,18 @@ namespace curvefs {
 namespace client {
 
 using common::VolumeOption;
+using mds::Mountpoint;
 using ::curvefs::volume::BlockDeviceClient;
 using ::curvefs::volume::BlockDeviceClientImpl;
 using ::curvefs::volume::SpaceManager;
 
+// volume client
 class FuseVolumeClient : public FuseClient {
  public:
     FuseVolumeClient()
-        : FuseClient(),
-          blockDeviceClient_(std::make_shared<BlockDeviceClientImpl>()) {}
+        : FuseClient() {
+        storageAdaptor_ = std::make_shared<VolumeClientAdaptorImpl>();
+    }
 
     // for UNIT_TEST
     FuseVolumeClient(
@@ -51,31 +54,36 @@ class FuseVolumeClient : public FuseClient {
         const std::shared_ptr<MetaServerClient> &metaClient,
         const std::shared_ptr<InodeCacheManager> &inodeManager,
         const std::shared_ptr<DentryCacheManager> &dentryManager,
-        const std::shared_ptr<BlockDeviceClient> &blockDeviceClient)
+        const std::shared_ptr<BlockDeviceClient> &blockDeviceClient,
+        const std::shared_ptr<VolumeClientAdaptorImpl> &storageAdaptor)
         : FuseClient(mdsClient, metaClient, inodeManager, dentryManager,
-                     nullptr),
-          blockDeviceClient_(blockDeviceClient) {}
+                     nullptr) {
+            storageAdaptor_ = storageAdaptor;
+        }
 
     CURVEFS_ERROR Init(const FuseClientOption &option) override;
 
     void UnInit() override;
 
+/*** fuse op ***/
+
     CURVEFS_ERROR FuseOpInit(
         void *userdata, struct fuse_conn_info *conn) override;
 
-    CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
-        const char *buf, size_t size, off_t off,
-        struct fuse_file_info *fi, size_t *wSize) override;
-
-    CURVEFS_ERROR FuseOpRead(fuse_req_t req,
-            fuse_ino_t ino, size_t size, off_t off,
-            struct fuse_file_info *fi,
-            char *buffer,
-            size_t *rSize) override;
     CURVEFS_ERROR FuseOpCreate(fuse_req_t req, fuse_ino_t parent,
         const char *name, mode_t mode, struct fuse_file_info *fi,
         fuse_entry_param *e) override;
 
+    CURVEFS_ERROR FuseOpRead(fuse_req_t req, fuse_ino_t ino,
+        size_t size, off_t off,
+        struct fuse_file_info *fi, char *buffer,
+        size_t *rSize) override;
+
+    CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino,
+        const char *buf, size_t size, off_t off,
+        struct fuse_file_info *fi,
+        size_t *wSize) override;
+
     CURVEFS_ERROR FuseOpMkNod(fuse_req_t req, fuse_ino_t parent,
         const char *name, mode_t mode, dev_t rdev,
         fuse_entry_param *e) override;
@@ -103,14 +111,10 @@ class FuseVolumeClient : public FuseClient {
     void FlushData() override;
 
  private:
-    std::shared_ptr<BlockDeviceClient> blockDeviceClient_;
-    std::unique_ptr<SpaceManager> spaceManager_;
-    std::unique_ptr<VolumeStorage> storage_;
-
-    VolumeOption volOpts_;
+    std::shared_ptr<StorageAdaptor> storageAdaptor_;
 };
 
 }  // namespace client
 }  // namespace curvefs
 
-#endif  // CURVEFS_SRC_CLIENT_FUSE_VOLUME_CLIENT_H_
+#endif  // CURVEFS_SRC_CLIENT_VOLUME_FUSE_VOLUME_CLIENT_H_

curvefs/src/client/warmup/warmup_manager.cpp

@@ -35,7 +35,7 @@
 #include "curvefs/src/client/common/common.h"
 #include "curvefs/src/client/inode_wrapper.h"
 #include "curvefs/src/client/kvclient/kvclient_manager.h"
-#include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
 #include "curvefs/src/common/s3util.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/string_util.h"
@@ -364,7 +364,8 @@ void WarmupManagerS3Impl::TravelChunk(fuse_ino_t ino,
                                       ObjectListType *prefetchObjs) {
     uint64_t blockSize = s3Adaptor_->GetBlockSize();
     uint64_t chunkSize = s3Adaptor_->GetChunkSize();
-    uint32_t objectPrefix = s3Adaptor_->GetObjectPrefix();
+    uint32_t objectPrefix = dynamic_cast<S3ClientAdaptorImpl *>(
+      s3Adaptor_.get())->GetObjectPrefix();
     uint64_t offset, len, chunkid, compaction;
     for (const auto &chunkinfo : chunkInfo.s3chunks()) {
         auto fsId = fsInfo_->fsid();
@@ -498,7 +499,9 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
 
             LOG(WARNING) << "Get Object failed, key: " << context->key
                          << ", offset: " << context->offset;
-            s3Adaptor_->GetS3Client()->DownloadAsync(context);
+
+            dynamic_cast<S3ClientAdaptorImpl *>(
+              s3Adaptor_.get())->GetS3Client()->DownloadAsync(context);
         };
 
     pendingReq.fetch_add(prefetchObjs.size(), std::memory_order_seq_cst);
@@ -529,7 +532,9 @@ void WarmupManagerS3Impl::WarmUpAllObjs(
             context->len = readLen;
             context->cb = cb;
             context->retry = 0;
-            s3Adaptor_->GetS3Client()->DownloadAsync(context);
+
+            dynamic_cast<S3ClientAdaptorImpl *>(
+              s3Adaptor_.get())->GetS3Client()->DownloadAsync(context);
         }
         if (pendingReq.load())
             cond.Wait();

curvefs/src/client/warmup/warmup_manager.h

@@ -46,7 +46,7 @@
 #include "curvefs/src/client/kvclient/kvclient_manager.h"
 #include "curvefs/src/client/rpcclient/metaserver_client.h"
 #include "curvefs/src/client/s3/client_s3_adaptor.h"
-#include "curvefs/src/client/s3/client_s3_cache_manager.h"
+#include "curvefs/src/client/cache/fuse_client_cache_manager.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/concurrent/rw_lock.h"
 #include "curvefs/src/common/task_thread_pool.h"
@@ -291,7 +291,7 @@ class WarmupManagerS3Impl : public WarmupManager {
         std::shared_ptr<InodeCacheManager> inodeManager,
         std::shared_ptr<DentryCacheManager> dentryManager,
         std::shared_ptr<FsInfo> fsInfo, FuseOpReadFunctionType readFunc,
-        std::shared_ptr<S3ClientAdaptor> s3Adaptor,
+        std::shared_ptr<StorageAdaptor> s3Adaptor,
         std::shared_ptr<KVClientManager> kvClientManager)
         : WarmupManager(std::move(metaClient), std::move(inodeManager),
                         std::move(dentryManager), std::move(fsInfo),
@@ -433,7 +433,7 @@ class WarmupManagerS3Impl : public WarmupManager {
     mutable RWLock warmupInodesDequeMutex_;
 
     // s3 adaptor
-    std::shared_ptr<S3ClientAdaptor> s3Adaptor_;
+    std::shared_ptr<StorageAdaptor> s3Adaptor_;
 
     // TODO(chengyi01): limit thread nums
     std::unordered_map<fuse_ino_t, std::unique_ptr<ThreadPool>>

curvefs/src/mds/fs_info_wrapper.cpp

@@ -1,4 +1,3 @@
-
 /*
  *  Copyright (c) 2021 NetEase Inc.
  *
@@ -50,8 +49,13 @@ FsInfoWrapper::FsInfoWrapper(const ::curvefs::mds::CreateFsRequest* request,
     fsInfo.set_enablesumindir(request->enablesumindir());
     fsInfo.set_txsequence(0);
     fsInfo.set_txowner("");
+    // TODO(@lixiaocui1): Currently, curveadm does not support setting recycling
+    // when creating a file system. It is expected to be supported in
+    // curveadm 3.0
     if (request->has_recycletimehour()) {
         fsInfo.set_recycletimehour(request->recycletimehour());
+    } else {
+        fsInfo.set_recycletimehour(1);
     }
 
     const auto& detail = request->fsdetail();

curvefs/src/mds/fs_manager.cpp

@@ -682,25 +682,34 @@ FSStatusCode FsManager::UmountFs(const std::string& fsName,
                      << ", errCode = " << FSStatusCode_Name(ret);
         return ret;
     }
+    VLOG(3) << "FsManager delete mount point success, fsName = " << fsName
+            << ", mountpoint = " << mountpoint.ShortDebugString();
 
     std::string mountpath;
     MountPoint2Str(mountpoint, &mountpath);
     DeleteClientAliveTime(mountpath);
 
     // 3. if no mount point exist, uninit space
-    if (wrapper.GetFsType() == FSType::TYPE_VOLUME &&
-        wrapper.IsMountPointEmpty()) {
-        auto ret = spaceManager_->RemoveVolume(wrapper.GetFsId());
-        if (ret != space::SpaceOk) {
-            LOG(ERROR) << "UmountFs fail, uninit space fail, fsName = "
+    if (wrapper.GetFsType() == FSType::TYPE_VOLUME) {
+        auto volumeSpace = spaceManager_->GetVolumeSpace(wrapper.GetFsId());
+        if (volumeSpace == nullptr) {
+            LOG(ERROR) << "handle fs mount point timeout fail, get volume "
+                          "space fail, fsName = "
                        << fsName
-                       << ", mountpoint = " << mountpoint.ShortDebugString()
-                       << ", errCode = " << space::SpaceErrCode_Name(ret);
+                       << ", mountpoint = " << mountpoint.ShortDebugString();
             return UNINIT_SPACE_ERROR;
         }
 
-        LOG(INFO) << "Remove volume space success, fsName = " << fsName
-                  << ", fsId = " << wrapper.GetFsId();
+        auto ret = volumeSpace->ReleaseBlockGroups(mountpath);
+        if (ret != space::SpaceOk) {
+            LOG(ERROR)
+                << "handle fs mount point timeout fail,release block groups "
+                   "fail, fsName = "
+                << fsName << ", mountpoint = " << mountpoint.ShortDebugString();
+            return SPACE_RELEASE_FAIL;
+        }
+
+        VLOG(3) << "FsManager release block group for " << mountpath << " ok";
     }
 
     // 4. update fs info
@@ -865,9 +874,7 @@ FSStatusCode FsManager::ReloadMountedFsVolumeSpace() {
             continue;
         }
 
-        if (!fs.MountPoints().empty()) {
-            reloader.Add(fs.ProtoFsInfo());
-        }
+        reloader.Add(fs.ProtoFsInfo());
     }
 
     auto err = reloader.Wait();

curvefs/src/mds/heartbeat/BUILD

@@ -27,6 +27,7 @@ cc_library(
         "//curvefs/proto:curvefs_topology_cc_proto",
         "//curvefs/proto:metaserver_cc_proto",
         "//curvefs/src/mds/topology:curvefs_topology",
+        "//curvefs/src/mds/space:curvefs_mds_space",
         "//external:brpc",
         "//external:gflags",
         "//external:glog",

curvefs/src/mds/heartbeat/heartbeat_manager.cpp

@@ -42,8 +42,9 @@ namespace mds {
 namespace heartbeat {
 HeartbeatManager::HeartbeatManager(
     const HeartbeatOption &option, const std::shared_ptr<Topology> &topology,
-    const std::shared_ptr<Coordinator> &coordinator)
-    : topology_(topology) {
+    const std::shared_ptr<Coordinator> &coordinator,
+    const std::shared_ptr<SpaceManager> &spaceManager)
+    : topology_(topology), spaceManager_(spaceManager) {
     healthyChecker_ =
         std::make_shared<MetaserverHealthyChecker>(option, topology);
 
@@ -86,24 +87,6 @@ void HeartbeatManager::Stop() {
     }
 }
 
-void HeartbeatManager::MetaServerHealthyChecker() {
-    while (sleeper_.wait_for(
-        std::chrono::milliseconds(metaserverHealthyCheckerRunInter_))) {
-        healthyChecker_->CheckHeartBeatInterval();
-    }
-}
-
-void HeartbeatManager::UpdateMetaServerSpace(
-    const MetaServerHeartbeatRequest &request) {
-    MetaServerSpace space(request.spacestatus());
-    TopoStatusCode ret =
-        topology_->UpdateMetaServerSpace(space, request.metaserverid());
-    if (ret != TopoStatusCode::TOPO_OK) {
-        LOG(ERROR) << "heartbeat UpdateMetaServerSpace fail, ret = "
-                   << TopoStatusCode_Name(ret);
-    }
-}
-
 void HeartbeatManager::MetaServerHeartbeat(
     const MetaServerHeartbeatRequest &request,
     MetaServerHeartbeatResponse *response) {
@@ -127,6 +110,42 @@ void HeartbeatManager::MetaServerHeartbeat(
     UpdateMetaServerSpace(request);
 
     // dealing with copysets included in the heartbeat request
+    Coordinate(request, response);
+
+    // update deallocatable block group info
+    UpdateDeallocatableBlockGroup(request, response);
+}
+
+void HeartbeatManager::MetaServerHealthyChecker() {
+    while (sleeper_.wait_for(
+        std::chrono::milliseconds(metaserverHealthyCheckerRunInter_))) {
+        healthyChecker_->CheckHeartBeatInterval();
+
+        auto metaservers =
+            topology_->GetMetaServerInCluster([](const MetaServer &ms) {
+                return ms.GetOnlineState() == OnlineState::ONLINE;
+            });
+
+        uint32_t currentOnlineMetaServerNum =
+            onlineMetaServerNum_.load(std::memory_order_acquire);
+        onlineMetaServerNum_.compare_exchange_strong(currentOnlineMetaServerNum,
+                                                     metaservers.size());
+    }
+}
+
+void HeartbeatManager::UpdateMetaServerSpace(
+    const MetaServerHeartbeatRequest &request) {
+    MetaServerSpace space(request.spacestatus());
+    TopoStatusCode ret =
+        topology_->UpdateMetaServerSpace(space, request.metaserverid());
+    if (ret != TopoStatusCode::TOPO_OK) {
+        LOG(ERROR) << "heartbeat UpdateMetaServerSpace fail, ret = "
+                   << TopoStatusCode_Name(ret);
+    }
+}
+
+void HeartbeatManager::Coordinate(const MetaServerHeartbeatRequest &request,
+                                  MetaServerHeartbeatResponse *response) {
     for (auto &value : request.copysetinfos()) {
         // convert copysetInfo from heartbeat format to topology format
         ::curvefs::mds::topology::CopySetInfo reportCopySetInfo;
@@ -168,6 +187,40 @@ void HeartbeatManager::MetaServerHeartbeat(
     }
 }
 
+void HeartbeatManager::UpdateDeallocatableBlockGroup(
+    const MetaServerHeartbeatRequest &request,
+    MetaServerHeartbeatResponse *response) {
+    uint32_t metaserverId = request.metaserverid();
+    VLOG(6) << "HeartbeatManager get block group stat info from metaserver:"
+            << request.metaserverid()
+            << ", size:" << request.blockgroupstatinfos_size();
+
+    for (auto &info : request.blockgroupstatinfos()) {
+        VLOG(9) << "HeartbeatManager handle request from metaserver:"
+                << request.metaserverid() << ", fsid:" << info.fsid()
+                << ", block stat info:" << info.DebugString();
+        auto volumeSpace = spaceManager_->GetVolumeSpace(info.fsid());
+        if (volumeSpace == nullptr) {
+            LOG(ERROR) << "HeartbeatManager fsid=" << info.fsid()
+                       << " do not have volumeSpace manager";
+            response->set_statuscode(HeartbeatStatusCode::hbMetaServerFSUnkown);
+            return;
+        }
+
+        uint64_t issued = 0;
+        bool hasissued = volumeSpace->UpdateDeallocatableBlockGroup(
+            metaserverId, onlineMetaServerNum_, info.deallocatableblockgroups(),
+            info.blockgroupdeallocatestatus(), &issued);
+        if (hasissued) {
+            response->mutable_issuedblockgroups()->insert(
+                {info.fsid(), issued});
+            LOG(INFO) << "HeartbeatManager issue metaserverid=" << metaserverId
+                      << " blockgroup, fsid=" << info.fsid()
+                      << ", issued=" << issued;
+        }
+    }
+}
+
 HeartbeatStatusCode HeartbeatManager::CheckRequest(
     const MetaServerHeartbeatRequest &request) {
     MetaServer metaServer;
@@ -203,6 +256,8 @@ HeartbeatStatusCode HeartbeatManager::CheckRequest(
                    << metaServer.GetToken();
         return HeartbeatStatusCode::hbMetaServerTokenNotMatch;
     }
+
+    VLOG(6) << "HeartbeatManager get request:" << request.DebugString();
     return HeartbeatStatusCode::hbOK;
 }
 

curvefs/src/mds/heartbeat/heartbeat_manager.h

@@ -36,6 +36,7 @@
 #include "curvefs/src/mds/heartbeat/topo_updater.h"
 #include "curvefs/src/mds/schedule/coordinator.h"
 #include "curvefs/src/mds/topology/topology.h"
+#include "curvefs/src/mds/space/manager.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 
@@ -43,6 +44,7 @@ using ::curvefs::mds::topology::PoolIdType;
 using ::curvefs::mds::topology::CopySetIdType;
 using ::curvefs::mds::topology::Topology;
 using ::curvefs::mds::schedule::Coordinator;
+using ::curvefs::mds::space::SpaceManager;
 
 using ::curve::common::Thread;
 using ::curve::common::Atomic;
@@ -65,7 +67,8 @@ class HeartbeatManager {
  public:
     HeartbeatManager(const HeartbeatOption &option,
                      const std::shared_ptr<Topology> &topology,
-                     const std::shared_ptr<Coordinator> &coordinator);
+                     const std::shared_ptr<Coordinator> &coordinator,
+                     const std::shared_ptr<SpaceManager> &spaceManager);
 
     ~HeartbeatManager() { Stop(); }
 
@@ -98,6 +101,7 @@ class HeartbeatManager {
                              MetaServerHeartbeatResponse *response);
 
  private:
+    FRIEND_TEST(TestHeartbeatManager, TEST_UpdateDeallocatableBlockGroup);
     /**
      * @brief Background thread for heartbeat timeout inspection
      */
@@ -140,10 +144,18 @@ class HeartbeatManager {
 
     void UpdateMetaServerSpace(const MetaServerHeartbeatRequest &request);
 
+    void Coordinate(const MetaServerHeartbeatRequest &request,
+                    MetaServerHeartbeatResponse *response);
+
+    void
+    UpdateDeallocatableBlockGroup(const MetaServerHeartbeatRequest &request,
+                                  MetaServerHeartbeatResponse *response);
+
  private:
     // Dependencies of heartbeat
     std::shared_ptr<Topology> topology_;
     std::shared_ptr<Coordinator> coordinator_;
+    std::shared_ptr<SpaceManager> spaceManager_;
 
     // healthyChecker_ health checker running in background thread
     std::shared_ptr<MetaserverHealthyChecker> healthyChecker_;
@@ -156,6 +168,8 @@ class HeartbeatManager {
     // Manage metaserverHealthyChecker threads
     Thread backEndThread_;
 
+    std::atomic<uint32_t> onlineMetaServerNum_;
+
     Atomic<bool> isStop_;
     InterruptibleSleeper sleeper_;
     int metaserverHealthyCheckerRunInter_;

curvefs/src/mds/mds.cpp

@@ -66,6 +66,8 @@ void MDS::InitOptions(std::shared_ptr<Configuration> conf) {
     conf_ = std::move(conf);
     conf_->GetValueFatalIfFail("mds.listen.addr", &options_.mdsListenAddr);
     conf_->GetValueFatalIfFail("mds.dummy.port", &options_.dummyPort);
+    conf_->GetValueFatalIfFail("mds.space.calIntervalSec",
+                               &options_.mdsSpaceCalIntervalSec);
 
     InitMetaServerOption(&options_.metaserverOptions);
     InitTopologyOption(&options_.topologyOptions);
@@ -169,8 +171,8 @@ void MDS::Init() {
     space::MdsProxyManager::SetProxyOptions(options_.bsMdsProxyOptions);
 
     fsStorage_ = std::make_shared<PersisKVStorage>(etcdClient_);
-    spaceManager_ =
-        std::make_shared<SpaceManagerImpl>(etcdClient_, fsStorage_);
+    spaceManager_ = std::make_shared<SpaceManagerImpl>(
+        etcdClient_, fsStorage_, options_.mdsSpaceCalIntervalSec);
     metaserverClient_ =
         std::make_shared<MetaserverClient>(options_.metaserverOptions);
     auto dlock = std::make_shared<DLock>(options_.dLockOptions, etcdClient_);
@@ -417,7 +419,7 @@ void MDS::InitHeartbeatManager() {
 
     heartbeatOption.mdsStartTime = steady_clock::now();
     heartbeatManager_ = std::make_shared<HeartbeatManager>(
-        heartbeatOption, topology_, coordinator_);
+        heartbeatOption, topology_, coordinator_, spaceManager_);
     heartbeatManager_->Init();
 }
 

curvefs/src/mds/mds.h

@@ -88,6 +88,8 @@ struct MDSOptions {
     MetaserverOptions metaserverOptions;
     // TODO(add EtcdConf): add etcd configure
 
+    uint64_t mdsSpaceCalIntervalSec;
+
     TopologyOption topologyOptions;
     HeartbeatOption heartbeatOption;
     ScheduleOption scheduleOption;

curvefs/src/mds/space/BUILD

@@ -27,6 +27,8 @@ cc_library(
     deps = [
         "//curvefs/proto:mds_cc_proto",
         "//curvefs/proto:space_cc_proto",
+        "//curvefs/proto:metaserver_cc_proto",
+        "//curvefs/proto:curvefs_heartbeat_cc_proto",
         "//curvefs/src/mds:curvefs_mds_fs_storage",
         "//curvefs/src/mds/codec:fs_mds_codec",
         "//external:brpc",

curvefs/src/mds/space/manager.cpp

@@ -55,12 +55,15 @@ SpaceErrCode SpaceManagerImpl::AddVolume(const FsInfo& fsInfo) {
     {
         ReadLockGuard lk(rwlock_);
         if (volumes_.count(fsInfo.fsid()) != 0) {
-            return SpaceErrCode::SpaceErrExist;
+            LOG(WARNING) << "Volume space already exists, fsId: "
+                         << fsInfo.fsid();
+            return SpaceOk;
         }
     }
 
-    auto space = VolumeSpace::Create(fsInfo.fsid(), fsInfo.detail().volume(),
-                                     storage_.get(), fsStorage_.get());
+    auto space =
+        VolumeSpace::Create(fsInfo.fsid(), fsInfo.detail().volume(),
+                            storage_.get(), fsStorage_.get(), calcIntervalSec_);
 
     if (!space) {
         LOG(ERROR) << "Create volume space failed, fsId: " << fsInfo.fsid();

curvefs/src/mds/space/manager.h

@@ -53,10 +53,10 @@ class SpaceManager {
 class SpaceManagerImpl final : public SpaceManager {
  public:
     SpaceManagerImpl(
-        const std::shared_ptr<curve::kvstorage::KVStorageClient>& kvstore,
-        std::shared_ptr<FsStorage> fsStorage)
+        const std::shared_ptr<curve::kvstorage::KVStorageClient> &kvstore,
+        std::shared_ptr<FsStorage> fsStorage, uint64_t calcIntervalSec)
         : storage_(new BlockGroupStorageImpl(kvstore)),
-          fsStorage_(std::move(fsStorage)) {}
+          fsStorage_(std::move(fsStorage)), calcIntervalSec_(calcIntervalSec) {}
 
     SpaceManagerImpl(const SpaceManagerImpl&) = delete;
     SpaceManagerImpl& operator=(const SpaceManagerImpl&) = delete;
@@ -80,6 +80,8 @@ class SpaceManagerImpl final : public SpaceManager {
     curve::common::GenericNameLock<Mutex> namelock_;
 
     std::shared_ptr<FsStorage> fsStorage_;
+
+    uint64_t calcIntervalSec_;
 };
 
 }  // namespace space

curvefs/src/mds/space/service.cpp

@@ -51,16 +51,19 @@ void SpaceServiceImpl::AllocateBlockGroup(
     std::vector<BlockGroup> groups;
     auto err =
         space->AllocateBlockGroups(request->count(), request->owner(), &groups);
-    if (err != SpaceOk) {
-        LOG(ERROR) << "Allocate block groups failed, err: "
-                   << SpaceErrCode_Name(err);
-    } else {
-        for (auto& group : groups) {
-            response->add_blockgroups()->Swap(&group);
-        }
+    response->set_status(err);
+
+    LOG_IF(WARNING, err != SpaceOk)
+        << "Allocate block groups failed, err: " << SpaceErrCode_Name(err)
+        << ", allocated size: " << groups.size()
+        << ", request: " << request->ShortDebugString();
+    if (err == SpaceErrEncode) {
+        return;
     }
 
-    response->set_status(err);
+    for (auto &group : groups) {
+        response->add_blockgroups()->Swap(&group);
+    }
 }
 
 void SpaceServiceImpl::AcquireBlockGroup(

curvefs/src/mds/space/volume_space.cpp

@@ -65,9 +65,10 @@ BlockGroup BuildBlockGroupFromClean(uint64_t offset,
 }  // namespace
 
 std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
-                                                 const Volume& volume,
-                                                 BlockGroupStorage* storage,
-                                                 FsStorage* fsStorage) {
+                                                 const Volume &volume,
+                                                 BlockGroupStorage *storage,
+                                                 FsStorage *fsStorage,
+                                                 uint64_t calcIntervalSec) {
     if (!volume.has_volumesize()) {
         LOG(ERROR) << "Volume info doesn't have size";
         return nullptr;
@@ -94,7 +95,7 @@ std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
     // for allocated groups, client will send heartbeat to update usage
     uint64_t availableSize = 0;
     std::set<uint64_t> usedGroupOffsets;
-    for (auto& group : groups) {
+    for (auto &group : groups) {
         usedGroupOffsets.insert(group.offset());
         // availableSize += group.available();
 
@@ -102,11 +103,23 @@ std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
         assert(offset % blockGroupSize == 0);
         assert(group.size() == blockGroupSize);
         assert(group.bitmaplocation() == location);
+        assert((group.has_owner() && group.deallocating_size()) == 0);
         if (group.has_owner()) {
+            VLOG(6) << "VolumeSpace init for fsid=" << fsId
+                    << ", blockgroup=" << group.DebugString()
+                    << " to allocatedGroups_";
             space->allocatedGroups_.emplace(offset, std::move(group));
+        } else if (group.deallocating_size() || group.deallocated_size()) {
+            VLOG(6) << "VolumeSpace init for fsid=" << fsId
+                    << ", blockgroup=" << group.DebugString()
+                    << " to deallocatingGroups_";
+            space->deallocatingGroups_.emplace(offset, std::move(group));
         } else {
+            group.set_available(group.size());
+            VLOG(6) << "VolumeSpace init for fsid=" << fsId
+                    << ", blockgroup=" << group.DebugString()
+                    << " to availableGroups_";
             space->availableGroups_.emplace(offset, std::move(group));
-            availableSize += group.available();
         }
     }
 
@@ -123,6 +136,9 @@ std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
 
     space->cleanGroups_ = std::move(cleanGroupOffsets);
 
+    space->calcIntervalSec_ = calcIntervalSec;
+    space->metaserverNum_ = 0;
+
     LOG(INFO) << "Init volume space success, fsId: " << fsId
               << ", size: " << volumeSize << ", available: " << availableSize
               << ", block size: " << blockSize
@@ -130,8 +146,10 @@ std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
               << ", total groups: " << volumeSize / blockGroupSize
               << ", allocated groups: " << space->allocatedGroups_.size()
               << ", available groups: " << space->availableGroups_.size()
+              << ", deallocating groups: " << space->deallocatingGroups_.size()
               << ", clean groups: " << space->cleanGroups_.size();
 
+    space->Run();
     return space;
 }
 
@@ -144,30 +162,38 @@ VolumeSpace::VolumeSpace(uint32_t fsId,
       storage_(storage),
       fsStorage_(fsStorage) {}
 
+
 SpaceErrCode VolumeSpace::AllocateBlockGroups(
     uint32_t count,
     const std::string& owner,
     std::vector<BlockGroup>* blockGroups) {
     LockGuard lk(mtx_);
     auto err = AllocateBlockGroupsInternal(count, owner, blockGroups);
-    if (err != SpaceOk) {
-        LOG(WARNING) << "Allocate block groups failed, fsId: " << fsId_
-                     << ", err: " << SpaceErrCode_Name(err);
-        return err;
+    if (blockGroups->size() < count) {
+        LOG(WARNING) << "Allocate block groups not enough, fsId: " << fsId_
+                     << ", err: " << SpaceErrCode_Name(err)
+                     << ", need count: " << count
+                     << ", allocated count: " << blockGroups->size();
     }
 
-    for (auto& group : *blockGroups) {
-        allocatedGroups_.emplace(group.offset(), group);
+    if (blockGroups->size() > 0) {
+        for (auto &group : *blockGroups) {
+            allocatedGroups_.emplace(group.offset(), group);
+            VLOG(9) << "VolumeSpace fsid=" << fsId_
+                    << ", allocate blockgroup=" << group.DebugString()
+                    << " to owner:" << owner;
+        }
+
+        // TODO(@wu-hanqing): if persist fail, we should rollback
+        err = PersistBlockGroups(*blockGroups);
+        if (err != SpaceOk) {
+            LOG(WARNING) << "Mark group allocated failed, fsId: " << fsId_
+                         << ", err: " << SpaceErrCode_Name(err);
+            return err;
+        }
     }
 
-    err = PersistBlockGroups(*blockGroups);
-    if (err != SpaceOk) {
-        LOG(WARNING) << "Mark group allocated failed, fsId: " << fsId_
-                     << ", err: " << SpaceErrCode_Name(err);
-        return err;
-    }
-
-    return SpaceOk;
+    return err;
 }
 
 SpaceErrCode VolumeSpace::AllocateBlockGroupsInternal(
@@ -177,6 +203,8 @@ SpaceErrCode VolumeSpace::AllocateBlockGroupsInternal(
     bool extend = false;
     uint32_t allocated = 0;
 
+    VLOG(9) << "owner " << owner << " need allocate " << count
+            << " block groups";
     while (allocated < count) {
         allocated += AllocateFromCleanGroups(count, owner, blockGroups);
         if (allocated >= count) {
@@ -199,6 +227,8 @@ SpaceErrCode VolumeSpace::AllocateBlockGroupsInternal(
                 return SpaceErrNoSpace;
             }
         } else {
+            VLOG(9) << "only allocate " << count << " block groups to owner "
+                    << owner;
             break;
         }
     }
@@ -227,10 +257,27 @@ uint32_t VolumeSpace::AllocateFromAvailableGroups(
     uint32_t count,
     const std::string& owner,
     std::vector<BlockGroup>* groups) {
+    VLOG(9) << "VolumeSpace fsid=" << fsId_
+            << ", allocate from available groups, count: " << count
+            << ", owner: " << owner
+            << ", available size:" << availableGroups_.size();
     uint32_t allocated = 0;
     auto it = availableGroups_.begin();
     while (allocated < count && it != availableGroups_.end()) {
         assert(!it->second.has_owner());
+
+        float usePer = 1.0 - static_cast<float>(it->second.available()) /
+                                 static_cast<float>(it->second.size());
+        if (usePer > 0.95) {
+            LOG(WARNING) << "VolumeSpace fsid=" << fsId_
+                         << " available group=" << it->second.DebugString()
+                         << " has no available space";
+            it++;
+            continue;
+        }
+        VLOG(9) << "VolumeSpace fsid=" << fsId_
+                << ", allocate blockgroup=" << it->second.DebugString()
+                << " to owner:" << owner;
         ++allocated;
         it->second.set_owner(owner);
         groups->push_back(std::move(it->second));
@@ -244,6 +291,7 @@ SpaceErrCode VolumeSpace::AcquireBlockGroup(uint64_t blockGroupOffset,
                                             const std::string& owner,
                                             BlockGroup* group) {
     LockGuard lk(mtx_);
+
     auto err = AcquireBlockGroupInternal(blockGroupOffset, owner, group);
     if (err != SpaceOk) {
         LOG(WARNING) << "Acquire block group failed, fsId: " << fsId_
@@ -267,6 +315,22 @@ SpaceErrCode VolumeSpace::AcquireBlockGroup(uint64_t blockGroupOffset,
 SpaceErrCode VolumeSpace::AcquireBlockGroupInternal(uint64_t blockGroupOffset,
                                                     const std::string& owner,
                                                     BlockGroup* group) {
+    if (owner.empty()) {
+        // find in deallocating
+        auto it = deallocatingGroups_.find(blockGroupOffset);
+        if (it != deallocatingGroups_.end()) {
+            *group = it->second;
+            VLOG(6) << "VolumeSpace fsid=" << fsId_
+                    << ", recieve acquire blockgroup=" << group->DebugString()
+                    << " request from metaserver, current block group is under "
+                       "deallocating";
+        } else {
+            return SpaceErrNotFound;
+        }
+        return SpaceOk;
+    }
+
+
     // find in availables
     {
         auto it = availableGroups_.find(blockGroupOffset);
@@ -317,13 +381,17 @@ SpaceErrCode VolumeSpace::ReleaseBlockGroups(
     LockGuard lk(mtx_);
 
     for (auto& group : blockGroups) {
+        VLOG(3) << "VolumeSpace fsid=" << fsId_
+                << ", need release block group:" << group.DebugString();
         auto it = allocatedGroups_.find(group.offset());
         if (it != allocatedGroups_.end()) {
             if (it->second.owner() != group.owner()) {
                 LOG(WARNING)
                     << "Owner is not identical, block group may "
                        "assign to others, fsId: "
-                    << fsId_ << ", block group offset: " << group.offset();
+                    << fsId_ << ", block group offset: " << group.offset()
+                    << ", record owner: " << it->second.owner()
+                    << ", report owner: " << group.owner();
                 return SpaceErrConflict;
             }
 
@@ -337,6 +405,9 @@ SpaceErrCode VolumeSpace::ReleaseBlockGroups(
                 }
 
                 cleanGroups_.insert(group.offset());
+                VLOG(6) << "VolumeSpace fsid=" << fsId_
+                        << " return block group to cleanGroups:"
+                        << group.DebugString();
             } else {
                 auto copy = group;
                 copy.clear_owner();
@@ -348,14 +419,55 @@ SpaceErrCode VolumeSpace::ReleaseBlockGroups(
                         << ", err: " << SpaceErrCode_Name(err);
                     return err;
                 }
-
+                VLOG(6) << "VolumeSpace return block group for fsid=" << fsId_
+                        << " to availableGroups:" << group.DebugString();
                 availableGroups_.emplace(group.offset(), std::move(copy));
             }
 
             allocatedGroups_.erase(group.offset());
+            VLOG(6) << "VolumeSpace fsid=" << fsId_
+                    << " erase block group from allocatedGroups:"
+                    << group.DebugString();
+            continue;
+        }
+        LOG(WARNING) << "VolumeSpace fsid=" << fsId_
+                     << " could not get release block gorup:"
+                     << group.DebugString() << " in allocatedGroups_";
+        // and if it's not allocated, this request must be a retry request
+    }
+
+    return SpaceOk;
+}
+
+SpaceErrCode VolumeSpace::ReleaseBlockGroups(const std::string &owner) {
+    LockGuard lk(mtx_);
+
+    LOG(INFO) << "Release all block groups for " << owner << ", fsid=" << fsId_;
+    auto iter = allocatedGroups_.begin();
+    while (iter != allocatedGroups_.end()) {
+        auto &group = iter->second;
+        if (group.owner() != owner) {
+            VLOG(9) << "VolumeSpace fsid=" << fsId_ << " expect owner:" << owner
+                    << ", current block group:" << group.DebugString();
+            iter++;
+            continue;
         }
 
-        // and if it's not allocated, this request must be a retry request
+        auto copy = group;
+        copy.clear_owner();
+        auto err = PersistBlockGroup(copy);
+        if (err != SpaceOk) {
+            LOG(WARNING) << "Persist block group failed, fsId=" << fsId_
+                         << ", block group offset: " << group.offset()
+                         << ", err: " << SpaceErrCode_Name(err);
+            return err;
+        }
+
+        VLOG(3) << "VolumeSpace fsid=" << fsId_
+                << " return to available block group:" << group.DebugString();
+
+        availableGroups_.emplace(group.offset(), std::move(copy));
+        iter = allocatedGroups_.erase(iter);
     }
 
     return SpaceOk;
@@ -447,6 +559,98 @@ void VolumeSpace::AddCleanGroups(uint64_t origin, uint64_t extended) {
     }
 }
 
+void VolumeSpace::CalBlockGroupAvailableForDeAllocate() {
+    LockGuard lk(mtx_);
+    LockGuard statlk(statmtx_);
+    // check whether deallocatingGroups_ need move to availableGroups_
+    auto iter = deallocatingGroups_.begin();
+    while (iter != deallocatingGroups_.end()) {
+        if (iter->second.deallocating_size()) {
+            VLOG(6) << "VolumeSpace skip cal, fsId=" << fsId_
+                    << ", block group offset=" << iter->first
+                    << " is under deallocating";
+
+            ++iter;
+            continue;
+        }
+
+        assert(iter->second.deallocated_size() > 0);
+
+        LOG(INFO)
+            << "VolumeSpace move deallocatingGroups_ to availableGroups_, "
+               "fsId="
+            << fsId_ << ", block group offset=" << iter->first
+            << ", available size=" << availableGroups_.size();
+
+        iter->second.clear_deallocated();
+        auto err = PersistBlockGroup(iter->second);
+        if (err != SpaceOk) {
+            LOG(ERROR) << "VolumeSpace put block group failed, fsId=" << fsId_
+                       << ", block group offset=" << iter->first
+                       << ", err=" << SpaceErrCode_Name(err);
+            continue;
+        }
+
+        iter->second.set_available(iter->second.size());
+        availableGroups_.emplace(iter->first, std::move(iter->second));
+        VLOG(9) << "VolumeSpace move deallocatingGroups_ to availableGroups_, "
+                   "fsId="
+                << fsId_ << ", block group offset=" << iter->first
+                << ", available size=" << availableGroups_.size();
+        iter = deallocatingGroups_.erase(iter);
+        metric_.dealloc << 1;
+    }
+
+    // check whether the cal conditions are met
+    if (!waitDeallocateGroups_.empty() || !deallocatingGroups_.empty() ||
+        availableGroups_.empty() || summary_.empty()) {
+        VLOG(3) << "VolumeSpace wait for cal, "
+                     "waitDeallocateGroups_ size="
+                  << waitDeallocateGroups_.size()
+                  << ",deallocatingGroups_ size=" << deallocatingGroups_.size()
+                  << ",availableGroups_ size=" << availableGroups_.size()
+                  << ", allocatedGroups_ size=" << allocatedGroups_.size()
+                  << ", cleanGroups_ size=" << cleanGroups_.size()
+                  << ",summary_ size=" << summary_.size()
+                  << ", fsid=" << fsId_;
+        return;
+    }
+
+    // get the keys shared by availableGroups_ and summary_
+    std::vector<std::pair<uint64_t, uint64_t>> commonKeys;
+    for (const auto &item : summary_) {
+        if (availableGroups_.count(item.first)) {
+            commonKeys.push_back(item);
+        }
+    }
+
+    // sort
+    std::sort(commonKeys.begin(), commonKeys.end(),
+              [](const std::pair<uint64_t, uint64_t> &a,
+                 const std::pair<uint64_t, uint64_t> &b) {
+                  return a.second > b.second;
+              });
+
+    srand(time(nullptr));
+    uint64_t size = (commonKeys.size() <= 1 ? 1 : commonKeys.size() / 2);
+    uint64_t selectKey = commonKeys[rand() % size].first;
+    LOG(INFO) << "VolumeSpace cal blockgroup=" << selectKey << ",fsid=" << fsId_
+              << " wait for deallocate";
+
+    // move key from availableGroups_ to waitDeallocateGroups_
+    BlockGroup selectGroup;
+    auto it = availableGroups_.find(selectKey);
+    if (it != availableGroups_.end()) {
+        assert(!it->second.has_owner());
+        selectGroup = std::move(it->second);
+        selectGroup.clear_owner();
+        availableGroups_.erase(it);
+    }
+    assert(waitDeallocateGroups_.count(selectKey) == 0);
+    waitDeallocateGroups_.emplace(selectKey, std::move(selectGroup));
+    metric_.waitingDealloc << 1;
+}
+
 SpaceErrCode VolumeSpace::ExtendVolume() {
     if (!volume_.autoextend()) {
         LOG(WARNING) << "Auto extend is not supported, fsId: " << fsId_
@@ -459,34 +663,264 @@ SpaceErrCode VolumeSpace::ExtendVolume() {
         ExtendedSize(origin, volume_.extendfactor(), volume_.extendalignment());
 
     LOG(INFO) << "Going to extend volume size from " << volume_.volumesize()
-              << " to " << extended;
+              << " to " << extended << ", fsid=" << fsId_;
 
-    auto* proxy = MdsProxyManager::GetInstance().GetOrCreateProxy(
+    auto *proxy = MdsProxyManager::GetInstance().GetOrCreateProxy(
         {volume_.cluster().begin(), volume_.cluster().end()});
     if (proxy == nullptr) {
-        LOG(WARNING) << "Fail to get or create proxy";
+        LOG(WARNING) << "Fail to get or create proxy, fsid=" << fsId_;
         return SpaceErrUnknown;
     }
 
     auto ret = proxy->ExtendVolume(volume_, extended);
     if (!ret) {
-        LOG(WARNING) << "Fail to extend volume";
+        LOG(WARNING) << "Fail to extend volume, fsid=" << fsId_;
         return SpaceErrExtendVolumeError;
     }
 
     if (!UpdateFsInfo(origin, extended)) {
-        LOG(WARNING) << "Fail to update fs info";
+        LOG(WARNING) << "Fail to update fs info, fsid=" << fsId_;
         return SpaceErrStorage;
     }
 
     volume_.set_volumesize(extended);
     AddCleanGroups(origin, extended);
 
-    LOG(INFO) << "Extended volume size from " << origin << " to " << extended;
+    LOG(INFO) << "Extended volume size from " << origin << " to " << extended
+              << ", fsid=" << fsId_;
 
     return SpaceOk;
 }
 
+void VolumeSpace::Run() {
+    calThread_ = std::thread([&] {
+        while (sleeper_.wait_for(std::chrono::seconds(calcIntervalSec_))) {
+            CalBlockGroupAvailableForDeAllocate();
+        }
+    });
+    LOG(INFO) << "VolumeSpace start background, fsid=" << fsId_;
+}
+
+void VolumeSpace::Stop() {
+    LOG(INFO) << "VolumeSpace stopping, fsid=" << fsId_;
+
+    sleeper_.interrupt();
+    if (calThread_.joinable()) {
+        calThread_.join();
+    }
+
+    LOG(INFO) << "VolumeSpace stopped, fsid=" << fsId_;
+}
+
+bool VolumeSpace::UpdateDeallocatableBlockGroup(
+    uint32_t metaserverId, uint32_t metaserverNum,
+    const DeallocatableBlockGroupVec &groups,
+    const BlockGroupDeallcateStatusMap &stats, uint64_t *issue) {
+    uint32_t current = metaserverNum_.load(std::memory_order_acquire);
+    metaserverNum_.compare_exchange_strong(current, metaserverNum,
+                                           std::memory_order_acq_rel);
+
+    UpdateBlockGroupDeallocatableSpace(metaserverId, groups);
+
+    UpdateDeallocatingBlockGroup(metaserverId, stats);
+
+    return SelectBlockGroupForDeAllocate(metaserverId, issue);
+}
+
+void VolumeSpace::UpdateBlockGroupDeallocatableSpace(
+    uint32_t metaserverId, const DeallocatableBlockGroupVec &groups) {
+    LockGuard statlk(statmtx_);
+    VLOG(6) << "VolumeSpace update from metaserver:" << metaserverId
+            << ", fsId=" << fsId_ << ", groups size=" << groups.size();
+
+    // update summary_ with latest groups
+    std::unordered_map<uint64_t, uint64_t> reportGroups;
+    for (auto &group : groups) {
+        auto offset = group.blockgroupoffset();
+        auto deallocatableSize = group.deallocatablesize();
+        reportGroups[offset] = deallocatableSize;
+
+        auto iter = summary_.find(offset);
+        if (iter == summary_.end()) {
+            summary_.emplace(offset, deallocatableSize);
+        } else {
+            iter->second += group.deallocatablesize();
+        }
+        VLOG(6) << "VolumeSpace update summary, fsId=" << fsId_
+                << ", blockGroupOffset=" << offset
+                << ", deallocatableSize=" << group.deallocatablesize();
+    }
+
+    // remove groups from last round of reporting and record latest in
+    // lastUpdate_
+    auto lastUpdateIter = lastUpdate_.find(metaserverId);
+    if (lastUpdateIter == lastUpdate_.end()) {
+        lastUpdate_[metaserverId] = groups;
+    } else {
+        for (auto &group : lastUpdateIter->second) {
+            auto offset = group.blockgroupoffset();
+            auto lastDeallocatableSize = group.deallocatablesize();
+            if (reportGroups.count(offset) == 0) {
+                continue;
+            }
+
+            summary_[offset] -= lastDeallocatableSize;
+            if (summary_[offset] == 0) {
+                summary_.erase(offset);
+                LOG(INFO) << "VolumeSpace remove block group from summary, no "
+                             "need deallocatable, "
+                             "fsId="
+                          << fsId_ << ", blcokGroupOffset=" << offset;
+            }
+
+            group.set_deallocatablesize(reportGroups[offset]);
+        }
+    }
+}
+
+void VolumeSpace::UpdateDeallocatingBlockGroup(
+    uint32_t metaserverId, const BlockGroupDeallcateStatusMap &stats) {
+    LockGuard lk(mtx_);
+
+    VLOG(6) << "VolumeSpace update deallocating block group from metaserver="
+            << metaserverId << ", fsId=" << fsId_
+            << ", stats size=" << stats.size();
+
+    // get completed deallocate blockgroup
+    std::vector<uint64_t> doneGroups;
+    for (auto &stat : stats) {
+        auto offset = stat.first;
+        auto status = stat.second;
+
+        auto iter = deallocatingGroups_.find(offset);
+        if (iter == deallocatingGroups_.end()) {
+            LOG(ERROR) << "VolumeSpace block group not found in "
+                          "deallocatingGroups_, fsId="
+                       << fsId_  << ", blockGroupOffset=" << offset;
+            continue;
+        }
+
+        VLOG(6) << "VolumeSpace get block group stat from metaserver="
+                << metaserverId << ", fsId=" << fsId_
+                << ", blockGroupOffset=" << offset
+                << ", status=" << BlockGroupDeallcateStatusCode_Name(status);
+
+        auto alreadyDeallocated =
+            std::find(iter->second.deallocated().begin(),
+                      iter->second.deallocated().end(), metaserverId);
+        if (status == BlockGroupDeallcateStatusCode::BGDP_DONE &&
+            alreadyDeallocated == iter->second.deallocated().end()) {
+            doneGroups.emplace_back(offset);
+            LOG(INFO) << "VolumeSpace block group is deallocated done, fsId: "
+                      << fsId_ << ", blcokGroupOffset: " << offset
+                      << ", metaserverId: " << metaserverId;
+        }
+    }
+
+    // update the metaserver from the deallocating state of the blockgroup to
+    // the deallocated state
+    for (auto offset : doneGroups) {
+        auto iter = deallocatingGroups_.find(offset);
+        assert(iter != deallocatingGroups_.end());
+
+        std::vector<uint32_t> newDeallocating;
+        for (auto id : iter->second.deallocating()) {
+            if (metaserverId == id) {
+                continue;
+            }
+            newDeallocating.emplace_back(id);
+        }
+
+        iter->second.add_deallocated(metaserverId);
+        auto mutableDeallocating = iter->second.mutable_deallocating();
+        mutableDeallocating->Resize(newDeallocating.size(), 0);
+        std::copy(newDeallocating.begin(), newDeallocating.end(),
+                  mutableDeallocating->begin());
+
+        auto err = PersistBlockGroup(iter->second);
+        // TODO(wuhanqing): handle error, and rollback if necessary
+        if (err != SpaceOk) {
+            LOG(ERROR) << "VolumeSpace put block group failed, fsId: " << fsId_
+                       << ", offset: " << offset
+                       << ", err: " << SpaceErrCode_Name(err);
+            return;
+        }
+    }
+}
+
+bool VolumeSpace::SelectBlockGroupForDeAllocate(uint32_t metaserverId,
+                                                uint64_t *issue) {
+    assert(issue != nullptr);
+    LockGuard lk(mtx_);
+
+    // TODO(ilixiaocui): support more groups to be issued
+    VLOG(3)
+        << "VolumeSpace select block group to be deallocate for metaserverId="
+        << metaserverId << ", fsId=" << fsId_
+        << ", waitDeallocateGroups_size=" << waitDeallocateGroups_.size();
+
+    if (!waitDeallocateGroups_.empty()) {
+        auto iter = waitDeallocateGroups_.begin();
+        auto offset = iter->first;
+        iter->second.add_deallocating(metaserverId);
+        deallocatingGroups_[offset] = std::move(iter->second);
+        waitDeallocateGroups_.erase(iter);
+        metric_.waitingDealloc << -1;
+
+        auto err = PersistBlockGroup(deallocatingGroups_[offset]);
+        // TODO(wuhanqing): handle error, and rollback if necessary
+        if (err != SpaceOk) {
+            LOG(ERROR) << "VolumeSpace put block group failed, fsId: " << fsId_
+                       << ", offset: " << offset
+                       << ", err: " << SpaceErrCode_Name(err);
+            return false;
+        }
+        *issue = offset;
+        LOG(INFO) << "VolumeSpace issue block group from "
+                     "waitDeallocateGroups_, fsId: "
+                  << fsId_ << ", offset: " << offset
+                  << ", to metaserverId: " << metaserverId;
+        return true;
+    }
+
+    auto deallocatingOne = deallocatingGroups_.begin();
+    if (deallocatingOne != deallocatingGroups_.end()) {
+        *issue = deallocatingOne->first;
+
+        auto alreadyIssued = std::find(
+            deallocatingOne->second.deallocating().begin(),
+            deallocatingOne->second.deallocating().end(), metaserverId);
+        if (alreadyIssued != deallocatingOne->second.deallocating().end()) {
+            return true;
+        }
+
+        auto alreadyDone = std::find(
+            deallocatingOne->second.deallocated().begin(),
+            deallocatingOne->second.deallocated().end(), metaserverId);
+        if (alreadyDone != deallocatingOne->second.deallocated().end()) {
+            return false;
+        }
+
+        deallocatingOne->second.add_deallocating(metaserverId);
+        auto err = PersistBlockGroup(deallocatingOne->second);
+        // TODO(wuhanqing): handle error, and rollback if necessary
+        if (err != SpaceOk) {
+            LOG(ERROR) << "VolumeSpace put block group failed, fsId: " << fsId_
+                       << ", offset: " << *issue
+                       << ", err: " << SpaceErrCode_Name(err);
+            return false;
+        }
+
+        LOG(INFO) << "VolumeSpace issue block group from "
+                     "deallocatingGroups_, fsId: "
+                  << fsId_ << ", offset: " << *issue
+                  << ", to metaserverId: " << metaserverId;
+        return true;
+    }
+
+    return false;
+}
+
 uint64_t ExtendedSize(uint64_t origin, double factor, uint64_t alignment) {
     return common::align_up(
         static_cast<uint64_t>(std::floor(static_cast<double>(origin) * factor)),

curvefs/src/mds/space/volume_space.h

@@ -36,13 +36,26 @@
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/space.pb.h"
+#include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/proto/heartbeat.pb.h"
 #include "curvefs/src/mds/fs_storage.h"
 #include "curvefs/src/mds/space/block_group_storage.h"
+#include "src/common/interruptible_sleeper.h"
 
 namespace curvefs {
 namespace mds {
 namespace space {
 
+using ::curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode;
+using ::curvefs::metaserver::DeallocatableBlockGroup;
+using ::curve::common::InterruptibleSleeper;
+
+using DeallocatableBlockGroupVec =
+    google::protobuf::RepeatedPtrField<DeallocatableBlockGroup>;
+using BlockGroupDeallcateStatusMap = ::google::protobuf::Map<
+    ::google::protobuf::uint64,
+    ::curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode>;
+
 class AbstractVolumeSpace {
  public:
     virtual ~AbstractVolumeSpace() = default;
@@ -56,8 +69,16 @@ class AbstractVolumeSpace {
                                            const std::string& owner,
                                            BlockGroup* group) = 0;
 
-    virtual SpaceErrCode ReleaseBlockGroups(
-        const std::vector<BlockGroup>& blockGroups) = 0;
+    virtual SpaceErrCode
+    ReleaseBlockGroups(const std::vector<BlockGroup> &blockGroups) = 0;
+
+    virtual SpaceErrCode ReleaseBlockGroups(const std::string &owner) = 0;
+
+    virtual bool
+    UpdateDeallocatableBlockGroup(uint32_t metaserverId, uint32_t metaserverNum,
+                                  const DeallocatableBlockGroupVec &groups,
+                                  const BlockGroupDeallcateStatusMap &stats,
+                                  uint64_t *issue) = 0;
 };
 
 using ::curvefs::common::BitmapLocation;
@@ -65,13 +86,13 @@ using ::curvefs::common::Volume;
 
 class VolumeSpace final : public AbstractVolumeSpace {
  public:
-    static std::unique_ptr<VolumeSpace> Create(uint32_t fsId,
-                                               const Volume& volume,
-                                               BlockGroupStorage* storage,
-                                               FsStorage* fsStorage);
+    static std::unique_ptr<VolumeSpace>
+    Create(uint32_t fsId, const Volume &volume, BlockGroupStorage *storage,
+           FsStorage *fsStorage, uint64_t calcIntervalSec);
 
     VolumeSpace(const VolumeSpace&) = delete;
     VolumeSpace& operator=(const VolumeSpace&) = delete;
+    ~VolumeSpace() { Stop(); }
 
     /**
      * @brief Allocate block groups
@@ -94,6 +115,11 @@ class VolumeSpace final : public AbstractVolumeSpace {
     SpaceErrCode ReleaseBlockGroups(
         const std::vector<BlockGroup>& blockGroups) override;
 
+    /**
+     * @brief Release block groups by owner
+     */
+    SpaceErrCode ReleaseBlockGroups(const std::string &owner) override;
+
     /**
      * @brief Remove all block groups and persistent records that belong to
      *        current volume
@@ -101,11 +127,23 @@ class VolumeSpace final : public AbstractVolumeSpace {
      */
     SpaceErrCode RemoveAllBlockGroups();
 
+
+    bool
+    UpdateDeallocatableBlockGroup(uint32_t metaserverId, uint32_t metaserverNum,
+                                  const DeallocatableBlockGroupVec &groups,
+                                  const BlockGroupDeallcateStatusMap &stats,
+                                  uint64_t *issue);
+
  private:
-    VolumeSpace(uint32_t fsId,
-                Volume volume,
-                BlockGroupStorage* storage,
-                FsStorage* fsStorage);
+    VolumeSpace(uint32_t fsId, Volume volume, BlockGroupStorage *storage,
+                FsStorage *fsStorage);
+
+    /**
+     * @brief Calculate block group that can be recycled
+     */
+    void Run();
+
+    void Stop();
 
  private:
     SpaceErrCode AllocateBlockGroupsInternal(
@@ -131,6 +169,24 @@ class VolumeSpace final : public AbstractVolumeSpace {
 
     void AddCleanGroups(uint64_t origin, uint64_t extended);
 
+    // pick out blockgroups that can be authorized for metaserver processing
+    void CalBlockGroupAvailableForDeAllocate();
+
+    // update the deallocatable space of blockgroup
+    void UpdateBlockGroupDeallocatableSpace(
+        uint32_t metaserverId, const DeallocatableBlockGroupVec &groups);
+
+    // update the deallocating progress reported by the metaserver
+    void
+    UpdateDeallocatingBlockGroup(uint32_t metaserverId,
+                                 const BlockGroupDeallcateStatusMap &stats);
+
+    // check whether there is currently a blockgroup that can be recycled and
+    // send it to the metaserver
+    bool SelectBlockGroupForDeAllocate(uint32_t metaserverId, uint64_t *issue);
+
+    FRIEND_TEST(VolumeSpaceTest, Test_CalBlockGroupAvailableForDeAllocate);
+
  private:
     // persist block group to backend storage
     SpaceErrCode PersistBlockGroup(const BlockGroup& group);
@@ -164,6 +220,13 @@ class VolumeSpace final : public AbstractVolumeSpace {
     // 3. clean
     //    these block groups' space is never used, and they can be allocated to
     //    other clients.
+    // 4. waitDeallocate
+    //    these block groups wait for being deallocated by
+    //    metaservers, and they can not allocate to clients.
+    // 5. deallocating
+    //    these block groups are being deallocated by metaservers, and they can
+    //    not allocate to clients.
+    //    these block groups are not persisted into storage.
 
     // key is block group offset
     std::unordered_map<uint64_t, BlockGroup> allocatedGroups_;
@@ -174,12 +237,55 @@ class VolumeSpace final : public AbstractVolumeSpace {
     // stores clean block groups' offset
     std::unordered_set<uint64_t> cleanGroups_;
 
+    // key is block group offset
+    std::unordered_map<uint64_t, BlockGroup> waitDeallocateGroups_;
+
+    // key is block group offset
+    std::unordered_map<uint64_t, BlockGroup> deallocatingGroups_;
+
+    mutable bthread::Mutex statmtx_;
+    // Summarize the deallocatable space of the blockgroup in this volume
+    // reported by all metaservers
+    //
+    // The metaserver reports the full amount of information instead of
+    // incremental information, so adding this data to the statistical data
+    // requires subtracting the last data
+    //
+    // - lastupdate_
+    //   metaserver report last time
+    // - summary_
+    //   summary add metaserver report info and remove lastupdate info
+
+    // key is metaserver id
+    std::unordered_map<uint32_t, DeallocatableBlockGroupVec> lastUpdate_;
+
+    // key is block group offset, value is deallocatable size
+    std::unordered_map<uint64_t, uint64_t> summary_;
+
     BlockGroupStorage* storage_;
 
     FsStorage* fsStorage_;
+
+    std::thread calThread_;
+    InterruptibleSleeper sleeper_;
+    int64_t calcIntervalSec_;
+
+    std::atomic<uint32_t> metaserverNum_;
+
+ private:
+    struct Metric {
+        bvar::Adder<uint64_t> dealloc;
+        bvar::Adder<uint64_t> waitingDealloc;
+
+        Metric()
+            : dealloc("mds_volume_space_dealloc"),
+              waitingDealloc("mds_volume_space_wait_dealloc") {}
+    };
+
+    Metric metric_;
 };
 
-// Calculate extended size based on origin with factor, and the result size is
+// Calculate extended size based on origin with factor, and the r esult size is
 // aligned with alignment
 uint64_t ExtendedSize(uint64_t origin, double factor, uint64_t alignment);
 

curvefs/src/metaserver/BUILD

@@ -29,6 +29,10 @@ cc_library(
         ["storage/*.cpp"],
     ) + glob(
         ["streaming/*.cpp"],
+    ) + glob(
+        ["space/*.cpp"],
+    ) + glob(
+        ["mds/*.cpp"],
     ),
     hdrs = glob(
         ["*.h"],
@@ -38,6 +42,10 @@ cc_library(
         ["storage/*.h"],
     ) + glob(
         ["streaming/*.h"],
+    ) + glob(
+        ["space/*.h"],
+    ) + glob(
+        ["mds/*.h"],
     ),
     copts = CURVE_DEFAULT_COPTS,
     visibility = ["//visibility:public"],
@@ -51,6 +59,8 @@ cc_library(
         "//curvefs/proto:mds_cc_proto",
         "//curvefs/src/common:curvefs_common",
         "//curvefs/src/metaserver/common:fs_metaserver_common",
+        "//curvefs/src/volume:volume",
+        "//curvefs/src/client:fuse_client_lib",
         "//external:braft",
         "//src/common:curve_common",
         "//src/fs:lfs",

curvefs/src/metaserver/common/types.h

@@ -24,6 +24,9 @@
 #define CURVEFS_SRC_METASERVER_COMMON_TYPES_H_
 
 #include <cstdint>
+#include <string>
+#include <sstream>
+#include <iomanip>
 
 namespace curvefs {
 namespace metaserver {
@@ -32,6 +35,14 @@ using PoolId = uint32_t;
 using CopysetId = uint32_t;
 using PartitionId = uint32_t;
 
+inline std::string StringToHex(const std::string &str) {
+    std::stringstream ss;
+    ss << std::hex << std::setfill('0');
+    for (const auto &c : str) {
+        ss << std::setw(2) << static_cast<int>(static_cast<unsigned char>(c));
+    }
+    return ss.str();
+}
 
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/copyset/copyset_node.cpp

@@ -31,6 +31,8 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include <map>
+#include <unordered_map>
 
 #include "absl/cleanup/cleanup.h"
 #include "absl/memory/memory.h"
@@ -574,24 +576,157 @@ void CopysetNode::ListPeers(std::vector<Peer>* peers) const {
 }
 
 // if copyset is loading, return false;
-// if copyset is not loading, and metastore returns fales, retry;
+// if copyset is not loading, and metastore returns false, retry;
 // if copyset is not loading, and metastore returns true, return true and get
 // partition info list success.
 bool CopysetNode::GetPartitionInfoList(
-                    std::list<PartitionInfo> *partitionInfoList) {
-    uint32_t retryCount = 0;
-    while (true) {
-        if (IsLoading()) {
-            LOG(INFO) << "Copyset is loading, return empty partition list";
-            return false;
-        }
-        bool ret = metaStore_->GetPartitionInfoList(partitionInfoList);
-        if (ret) {
-            return true;
-        }
-        LOG(WARNING) << "Copyset is not loading, but GetPartitionInfoList fail,"
-                     << " retryCount = " << retryCount++;
+    std::list<PartitionInfo> *partitionInfoList) {
+    if (IsLoading()) {
+        LOG(INFO) << "Copyset is loading, return empty partition list";
+        return false;
     }
+
+    uint32_t retryCount = 0;
+    do {
+        bool ret = metaStore_->GetPartitionInfoList(partitionInfoList);
+        if (!ret) {
+            LOG(WARNING)
+                << "Copyset is not loading, but GetPartitionInfoList fail,"
+                << " retryCount = " << retryCount++;
+            continue;
+        }
+
+        return true;
+    } while (true);
+}
+
+bool CopysetNode::GetBlockStatInfo(
+    std::map<uint32_t, BlockGroupStatInfo> *blockStatInfoMap) {
+    if (IsLoading()) {
+        LOG(INFO)
+            << "CopysetNode copyset is loading, return empty block stat info";
+        return false;
+    }
+
+    uint32_t retryCount = 0;
+    uint32_t blockGroupNum = 0;
+    do {
+        std::map<uint32_t, std::shared_ptr<Partition>> partitionSnap;
+        if (!metaStore_->GetPartitionSnap(&partitionSnap)) {
+            LOG(WARNING) << "CopysetNode get partition snap fail, retryCount = "
+                         << ++retryCount;
+            continue;
+        }
+
+        for (const auto &item : partitionSnap) {
+            auto partition = item.second;
+            VLOG(6) << "CopysetNode get block stat info from partition="
+                    << partition->GetPartitionId()
+                    << ", fsId=" << partition->GetFsId();
+            if (!AggregateBlockStatInfo(partition, blockStatInfoMap,
+                                        &blockGroupNum)) {
+                continue;
+            }
+        }
+
+        return true;
+    } while (true);
+}
+
+// NOTE:
+//  If the file system size is 1PB, the block size is 128MB
+//  - The maximum number of blockgroups in the file system is 838,8608.
+//
+//  The deallocatable space of the current blockgroup is counted according to
+//  the granularity of the partition. Now the maximum number of partitions on
+//  each copyset is configured to 128
+//  - In extreme cases, the number of statistical fragments of blockgroups in
+//  each copyset is 838,8608*128≈10billion.The number of fragments of the
+//  blockgroup that metaserver needs to carry in a heartbeat will be too large
+//
+//  Therefore, it is necessary to combine the information of these fragments to
+//  calculate.
+//
+// TODO(ilixiaocui): need more stat optimization
+bool CopysetNode::AggregateBlockStatInfo(
+    const std::shared_ptr<Partition> &partition,
+    std::map<uint32_t, BlockGroupStatInfo> *blockStatInfoMap,
+    uint32_t *blockGroupNum) {
+    uint32_t fsId = partition->GetFsId();
+    uint64_t partitionId = partition->GetPartitionId();
+
+    // get block group info in partition
+    std::vector<DeallocatableBlockGroup> deallocatableBlockGroupVec;
+    if (MetaStatusCode::OK !=
+        partition->GeAllBlockGroup(&deallocatableBlockGroupVec)) {
+        LOG(WARNING) << "CopysetNode get all blockgroup fail, partitionId= "
+                     << partitionId << ", fsId=" << fsId;
+        return false;
+    }
+
+    auto &blockGroupStatInfo = (*blockStatInfoMap)[fsId];
+    if (!blockGroupStatInfo.has_fsid()) {
+        blockGroupStatInfo.set_fsid(fsId);
+    }
+
+#define LIMITBLICKGROUPNUM 8192
+    // combine the information of the same blockgroup
+    auto blockGroups = blockGroupStatInfo.mutable_deallocatableblockgroups();
+    std::unordered_map<uint64_t, DeallocatableBlockGroup *> blockGroupMap;
+    for (auto &blockGroup : *blockGroups) {
+        blockGroupMap[blockGroup.blockgroupoffset()] = &blockGroup;
+    }
+
+    for (auto &blockGroup : deallocatableBlockGroupVec) {
+        auto it = blockGroupMap.find(blockGroup.blockgroupoffset());
+        if (it != blockGroupMap.end()) {
+            it->second->set_deallocatablesize(it->second->deallocatablesize() +
+                                              blockGroup.deallocatablesize());
+        } else {
+            blockGroup.clear_inodeidlist();
+            blockGroup.clear_inodeidunderdeallocate();
+            blockGroups->Add()->MergeFrom(blockGroup);
+            if (++(*blockGroupNum) > LIMITBLICKGROUPNUM) {
+                LOG(WARNING) << "CopysetNode get blockgroup num over limit, "
+                                "blockGroupNum = "
+                             << *blockGroupNum;
+                break;
+            }
+        }
+        VLOG(6) << "CopysetNode get block group info, fsId = " << fsId
+                << ", block info:" << blockGroup.DebugString();
+    }
+
+    VLOG(6) << "CopysetNode get block group num:" << *blockGroupNum;
+
+    return true;
+}
+
+void CopysetNode::Deallocate(uint64_t fsId, uint64_t blockGroupOffset) {
+    if (IsLoading()) {
+        LOG(INFO)
+            << "CopysetNode copyset is loading, return empty block stat info";
+        return;
+    }
+
+    uint32_t retryCount = 0;
+    do {
+        std::map<uint32_t, std::shared_ptr<Partition>> partitionSnap;
+        if (!metaStore_->GetPartitionSnap(&partitionSnap)) {
+            LOG(WARNING) << "CopysetNode get partition snap fail, retryCount = "
+                         << ++retryCount;
+            continue;
+        }
+
+        for (const auto &item : partitionSnap) {
+            auto partition = item.second;
+            if (partition->GetFsId() != fsId) {
+                continue;
+            }
+            item.second->SetVolumeDeallocate(fsId, blockGroupOffset);
+        }
+        return;
+    } while (true);
 }
 
 void CopysetNode::OnConfChangeComplete() {

curvefs/src/metaserver/copyset/copyset_node.h

@@ -24,11 +24,13 @@
 #define CURVEFS_SRC_METASERVER_COPYSET_COPYSET_NODE_H_
 
 #include <braft/raft.h>
+#include <gtest/gtest_prod.h>
 
 #include <list>
 #include <memory>
 #include <string>
 #include <vector>
+#include <map>
 
 #include "curvefs/src/metaserver/common/types.h"
 #include "curvefs/src/metaserver/copyset/apply_queue.h"
@@ -38,6 +40,7 @@
 #include "curvefs/src/metaserver/copyset/metric.h"
 #include "curvefs/src/metaserver/copyset/raft_node.h"
 #include "curvefs/src/metaserver/metastore.h"
+#include "curvefs/proto/heartbeat.pb.h"
 
 namespace curvefs {
 namespace metaserver {
@@ -47,6 +50,7 @@ using ::braft::PeerId;
 using ::curvefs::common::Peer;
 using ::curvefs::metaserver::MetaStore;
 using ::curve::mds::heartbeat::ConfigChangeType;
+using ::curvefs::mds::heartbeat::BlockGroupStatInfo;
 
 class CopysetNodeManager;
 
@@ -77,13 +81,13 @@ class CopysetNode : public braft::StateMachine {
 
     virtual bool IsLeaderTerm() const;
 
-    PoolId GetPoolId() const;
+    virtual PoolId GetPoolId() const;
 
-    const braft::PeerId& GetPeerId() const;
+    virtual const braft::PeerId& GetPeerId() const;
 
-    CopysetId GetCopysetId() const;
+    virtual CopysetId GetCopysetId() const;
 
-    PeerId GetLeaderId() const;
+    virtual PeerId GetLeaderId() const;
 
     MetaStore* GetMetaStore() const;
 
@@ -136,7 +140,7 @@ class CopysetNode : public braft::StateMachine {
     virtual void RemovePeer(const Peer& peer, braft::Closure* done = nullptr);
     virtual void ChangePeers(const std::vector<Peer>& newPeers,
                              braft::Closure* done = nullptr);
-    void GetConfChange(ConfigChangeType* type, Peer* alterPeer);
+    virtual void GetConfChange(ConfigChangeType* type, Peer* alterPeer);
     void OnConfChangeComplete();
 
  private:
@@ -175,9 +179,15 @@ class CopysetNode : public braft::StateMachine {
 
  public:
     // for heartbeat
-    bool GetPartitionInfoList(std::list<PartitionInfo> *partitionInfoList);
+    virtual bool
+    GetPartitionInfoList(std::list<PartitionInfo> *partitionInfoList);
 
-    bool IsLoading() const;
+    virtual bool IsLoading() const;
+
+    virtual bool
+    GetBlockStatInfo(std::map<uint32_t, BlockGroupStatInfo> *blockStatInfoMap);
+
+    virtual void Deallocate(uint64_t fsId, uint64_t blockGroupOffset);
 
  private:
     void InitRaftNodeOptions();
@@ -185,6 +195,13 @@ class CopysetNode : public braft::StateMachine {
     bool FetchLeaderStatus(const braft::PeerId& peerId,
                            braft::NodeStatus* leaderStatus);
 
+    bool AggregateBlockStatInfo(
+        const std::shared_ptr<Partition> &partition,
+        std::map<uint32_t, BlockGroupStatInfo> *blockStatInfoMap,
+        uint32_t *blockGroupNum);
+
+    FRIEND_TEST(CopysetNodeBlockGroupTest, Test_AggregateBlockStatInfo);
+
  private:
     const PoolId poolId_;
     const CopysetId copysetId_;

curvefs/src/metaserver/copyset/copyset_node_manager.cpp

@@ -246,7 +246,7 @@ bool CopysetNodeManager::CreateCopysetNode(PoolId poolId, CopysetId copysetId,
 }
 
 void CopysetNodeManager::GetAllCopysets(
-    std::vector<CopysetNode*>* nodes) const {
+    std::vector<CopysetNode *> *nodes) const {
     nodes->clear();
     ReadLockGuard lock(lock_);
     for (auto& copyset : copysets_) {

curvefs/src/metaserver/copyset/copyset_node_manager.h

@@ -74,7 +74,7 @@ class CopysetNodeManager {
 
     virtual bool PurgeCopysetNode(PoolId poolId, CopysetId copysetId);
 
-    void GetAllCopysets(std::vector<CopysetNode*>* nodes) const;
+    virtual void GetAllCopysets(std::vector<CopysetNode *> *nodes) const;
 
     virtual bool IsLoadFinished() const;
 

curvefs/src/metaserver/copyset/meta_operator.cpp

@@ -191,6 +191,7 @@ OPERATOR_ON_APPLY(CreatePartition);
 OPERATOR_ON_APPLY(DeletePartition);
 OPERATOR_ON_APPLY(PrepareRenameTx);
 OPERATOR_ON_APPLY(UpdateVolumeExtent);
+OPERATOR_ON_APPLY(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_ON_APPLY
 
@@ -269,7 +270,7 @@ void GetVolumeExtentOperator::OnApply(int64_t index,
     }
 
     // in streaming mode, swap slices out and send them by streaming
-    VolumeExtentList extents;
+    VolumeExtentSliceList extents;
     response->mutable_slices()->Swap(&extents);
     response->clear_slices();
 
@@ -316,6 +317,7 @@ OPERATOR_ON_APPLY_FROM_LOG(CreatePartition);
 OPERATOR_ON_APPLY_FROM_LOG(DeletePartition);
 OPERATOR_ON_APPLY_FROM_LOG(PrepareRenameTx);
 OPERATOR_ON_APPLY_FROM_LOG(UpdateVolumeExtent);
+OPERATOR_ON_APPLY_FROM_LOG(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_ON_APPLY_FROM_LOG
 
@@ -374,6 +376,7 @@ OPERATOR_REDIRECT(DeletePartition);
 OPERATOR_REDIRECT(PrepareRenameTx);
 OPERATOR_REDIRECT(GetVolumeExtent);
 OPERATOR_REDIRECT(UpdateVolumeExtent);
+OPERATOR_REDIRECT(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_REDIRECT
 
@@ -400,6 +403,7 @@ OPERATOR_ON_FAILED(DeletePartition);
 OPERATOR_ON_FAILED(PrepareRenameTx);
 OPERATOR_ON_FAILED(GetVolumeExtent);
 OPERATOR_ON_FAILED(UpdateVolumeExtent);
+OPERATOR_ON_FAILED(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_ON_FAILED
 
@@ -425,6 +429,8 @@ OPERATOR_HASH_CODE(PrepareRenameTx);
 OPERATOR_HASH_CODE(DeletePartition);
 OPERATOR_HASH_CODE(GetVolumeExtent);
 OPERATOR_HASH_CODE(UpdateVolumeExtent);
+OPERATOR_HASH_CODE(UpdateDeallocatableBlockGroup);
+
 
 #undef OPERATOR_HASH_CODE
 
@@ -462,6 +468,7 @@ OPERATOR_TYPE(CreatePartition);
 OPERATOR_TYPE(DeletePartition);
 OPERATOR_TYPE(GetVolumeExtent);
 OPERATOR_TYPE(UpdateVolumeExtent);
+OPERATOR_TYPE(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_TYPE
 

curvefs/src/metaserver/copyset/meta_operator.h

@@ -537,6 +537,25 @@ class UpdateVolumeExtentOperator : public MetaOperator {
     void OnFailed(MetaStatusCode code) override;
 };
 
+class UpdateDeallocatableBlockGroupOperator : public MetaOperator {
+ public:
+    using MetaOperator::MetaOperator;
+
+    void OnApply(int64_t index, google::protobuf::Closure *done,
+                 uint64_t startTimeUs) override;
+
+    void OnApplyFromLog(uint64_t startTimeUs) override;
+
+    uint64_t HashCode() const override;
+
+    OperatorType GetOperatorType() const override;
+
+ private:
+    void Redirect() override;
+
+    void OnFailed(MetaStatusCode code) override;
+};
+
 }  // namespace copyset
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/copyset/operator_type.cpp

@@ -66,6 +66,8 @@ const char* OperatorTypeName(OperatorType type) {
             return "GetVolumeExtent";
         case OperatorType::UpdateVolumeExtent:
             return "UpdateVolumeExtent";
+        case OperatorType::UpdateDeallocatableBlockGroup:
+            return "UpdateDeallocatableBlockGroup";
         // Add new case before `OperatorType::OperatorTypeMax`
         case OperatorType::OperatorTypeMax:
             break;

curvefs/src/metaserver/copyset/operator_type.h

@@ -52,6 +52,8 @@ enum class OperatorType : uint32_t {
     GetVolumeExtent = 15,
     UpdateVolumeExtent = 16,
     CreateManageInode = 17,
+    UpdateDeallocatableBlockGroup = 18,
+
     // NOTE:
     //   Add new operator before `OperatorTypeMax`
     //   And DO NOT recorder or delete previous types

curvefs/src/metaserver/copyset/raft_log_codec.cpp

@@ -165,6 +165,10 @@ std::unique_ptr<MetaOperator> RaftLogCodec::Decode(CopysetNode* node,
             return ParseFromRaftLog<UpdateVolumeExtentOperator,
                                     UpdateVolumeExtentRequest>(node, type,
                                                                meta);
+        case OperatorType::UpdateDeallocatableBlockGroup:
+            return ParseFromRaftLog<UpdateDeallocatableBlockGroupOperator,
+                                    UpdateDeallocatableBlockGroupRequest>(
+                node, type, meta);
         // Add new case before `OperatorType::OperatorTypeMax`
         case OperatorType::OperatorTypeMax:
             break;

curvefs/src/metaserver/heartbeat.cpp

@@ -40,19 +40,21 @@
 #include "curvefs/src/metaserver/copyset/utils.h"
 #include "curvefs/src/metaserver/storage/storage.h"
 #include "curvefs/src/metaserver/resource_statistic.h"
+#include "curvefs/src/metaserver/space/volume_deallocate_manager.h"
 
 namespace curvefs {
 namespace metaserver {
 
 using ::curve::mds::heartbeat::ConfigChangeType;
 using ::curvefs::mds::heartbeat::ConfigChangeInfo;
+using ::curvefs::mds::heartbeat::CopySetInfo;
 using ::curvefs::metaserver::copyset::CopysetService_Stub;
 using ::curvefs::metaserver::copyset::ToGroupIdString;
 
 namespace {
 
 int GatherCopysetConfChange(CopysetNode* node, ConfigChangeInfo* info) {
-    ConfigChangeType type;
+    ConfigChangeType type = ConfigChangeType::NONE;
     Peer peer;
 
     node->GetConfChange(&type, &peer);
@@ -161,7 +163,7 @@ int Heartbeat::Fini() {
     return 0;
 }
 
-void Heartbeat::BuildCopysetInfo(curvefs::mds::heartbeat::CopySetInfo *info,
+void Heartbeat::BuildCopysetInfo(CopySetInfo *info,
                                  CopysetNode *copyset) {
     int ret;
     PoolId poolId = copyset->GetPoolId();
@@ -213,6 +215,12 @@ void Heartbeat::BuildCopysetInfo(curvefs::mds::heartbeat::CopySetInfo *info,
     }
 }
 
+void Heartbeat::BuildBlockGroupStatInfo(CopysetNode *copyset,
+                             BlockGroupStatInfoMap *blockGroupStatInfoMap) {
+    copyset->GetBlockStatInfo(blockGroupStatInfoMap);
+}
+
+
 // TODO(@Wine93): now we use memory storage, so we gather disk usage bytes
 // which only has raft related capacity. If we use rocksdb storage, maybe
 // we should need more flexible strategy.
@@ -255,16 +263,50 @@ int Heartbeat::BuildRequest(HeartbeatRequest* req) {
 
     req->set_copysetcount(copysets.size());
     int leaders = 0;
-
+    BlockGroupStatInfoMap blockGroupStatInfoMap;
     for (auto copyset : copysets) {
-        curvefs::mds::heartbeat::CopySetInfo *info = req->add_copysetinfos();
-
+        // build copyset info
+        CopySetInfo *info = req->add_copysetinfos();
         BuildCopysetInfo(info, copyset);
 
         if (copyset->IsLeaderTerm()) {
+            // build block group info
+            VLOG(6) << "Heartbeat build block group stat info for copyset:"
+                    << info->DebugString();
+            BuildBlockGroupStatInfo(copyset, &blockGroupStatInfoMap);
             ++leaders;
         }
     }
+
+    // get deallocate task status
+    for (auto &item : blockGroupStatInfoMap) {
+        auto iterTask = taskExecutor_->deallocTask_.find(item.first);
+        if (iterTask != taskExecutor_->deallocTask_.end()) {
+            uint32_t fsId = iterTask->first;
+            uint64_t blockGroupOffset = iterTask->second;
+
+            bool doing = VolumeDeallocateManager::GetInstance().HasDeallocate();
+            auto status = item.second.mutable_blockgroupdeallocatestatus();
+            if (doing) {
+                status->insert(
+                    {blockGroupOffset,
+                     BlockGroupDeallcateStatusCode::BGDP_PROCESSING});
+            } else {
+                status->insert({blockGroupOffset,
+                                BlockGroupDeallcateStatusCode::BGDP_DONE});
+                taskExecutor_->deallocTask_.erase(iterTask);
+            }
+
+            VLOG(6) << "Heartbeat find fsId=" << fsId
+                    << " in deallocTask, blockgroupOffset=" << blockGroupOffset
+                    << ", status=" << status;
+        }
+
+        VLOG(6) << "Heartbeat find fsId=" << item.first
+                << " not in deallocTask";
+        *req->add_blockgroupstatinfos() = std::move(item.second);
+    }
+
     req->set_leadercount(leaders);
 
     MetaServerSpaceStatus* status = req->mutable_spacestatus();
@@ -277,21 +319,7 @@ int Heartbeat::BuildRequest(HeartbeatRequest* req) {
 }
 
 void Heartbeat::DumpHeartbeatRequest(const HeartbeatRequest& request) {
-    VLOG(6) << "Heartbeat request: Metaserver ID: " << request.metaserverid()
-             << ", IP = " << request.ip() << ", port = " << request.port()
-             << ", copyset count = " << request.copysetcount()
-             << ", leader count = " << request.leadercount()
-             << ", diskThresholdByte = "
-             << request.spacestatus().diskthresholdbyte()
-             << ", diskCopysetMinRequireByte = "
-             << request.spacestatus().diskcopysetminrequirebyte()
-             << ", diskUsedByte = "
-             << request.spacestatus().diskusedbyte()
-             << ", memoryThresholdByte = "
-             << request.spacestatus().memorythresholdbyte()
-             << ", memoryCopySetMinRequireByte = "
-             << request.spacestatus().memorycopysetminrequirebyte()
-             << ", memoryUsedByte = " << request.spacestatus().memoryusedbyte();
+    VLOG(6) << "Heartbeat reuqest: " << request.DebugString();
 
     for (int i = 0; i < request.copysetinfos_size(); i++) {
         const curvefs::mds::heartbeat::CopySetInfo &info =
@@ -314,9 +342,14 @@ void Heartbeat::DumpHeartbeatResponse(const HeartbeatResponse &response) {
     VLOG(3) << "Received heartbeat response, statusCode = "
             << response.statuscode();
 
-    for (auto& conf : response.needupdatecopysets()) {
+    for (auto &conf : response.needupdatecopysets()) {
         VLOG(3) << "need update copyset: " << conf.ShortDebugString();
     }
+
+    for (auto &issue : response.issuedblockgroups()) {
+        VLOG(3) << "issued block group fsid=" << issue.first
+                << ", blockgroupoffset=" << issue.second;
+    }
 }
 
 int Heartbeat::SendHeartbeat(const HeartbeatRequest &request,
@@ -406,10 +439,33 @@ HeartbeatTaskExecutor::HeartbeatTaskExecutor(CopysetNodeManager* mgr,
                                              const butil::EndPoint& endpoint)
     : copysetMgr_(mgr), ep_(endpoint) {}
 
-void HeartbeatTaskExecutor::ExecTasks(const HeartbeatResponse& response) {
-    for (auto& conf : response.needupdatecopysets()) {
+void HeartbeatTaskExecutor::ExecTasks(const HeartbeatResponse &response) {
+    for (auto &conf : response.needupdatecopysets()) {
         ExecOneTask(conf);
     }
+
+    std::vector<CopysetNode *> copysets;
+    copysetMgr_->GetAllCopysets(&copysets);
+    for (auto &issue : response.issuedblockgroups()) {
+        auto iter = deallocTask_.find(issue.first);
+        if (iter != deallocTask_.end()) {
+            VLOG(6) << "HeartbeatTaskExecutor dealloc task fsid=" << issue.first
+                    << ", blockgroupoffset=" << issue.second << " is excuting";
+            assert(iter->second == issue.second);
+            continue;
+        }
+
+        for (auto &copyset : copysets) {
+            if (copyset->IsLeaderTerm()) {
+                copyset->Deallocate(issue.first, issue.second);
+                VLOG(6) << "HeartbeatTaskExecutor issue dealloc task fsid="
+                        << issue.first << ", blockgroupoffset=" << issue.second
+                        << " to copyset " << copyset->Name();
+            }
+        }
+
+        deallocTask_.emplace(issue.first, issue.second);
+    }
 }
 
 void HeartbeatTaskExecutor::ExecOneTask(const CopySetConf& conf) {

curvefs/src/metaserver/heartbeat.h

@@ -25,6 +25,7 @@
 
 #include <braft/node.h>  // NodeImpl
 #include <braft/node_manager.h>
+#include <gtest/gtest_prod.h>
 
 #include <atomic>
 #include <map>
@@ -37,23 +38,26 @@
 #include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/wait_interval.h"
-
-using ::curve::common::Thread;
-using ::curvefs::metaserver::copyset::CopysetNode;
+#include "absl/types/optional.h"
 
 namespace curvefs {
 namespace metaserver {
+using curve::common::Thread;
+using curve::fs::LocalFileSystem;
+using curvefs::common::Peer;
+using curvefs::mds::heartbeat::BlockGroupStatInfo;
+using curvefs::mds::heartbeat::CopySetConf;
+using curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode;
+using curvefs::metaserver::copyset::CopysetNode;
+using curvefs::metaserver::copyset::CopysetNodeManager;
 
-using ::curve::fs::LocalFileSystem;
 using HeartbeatRequest = curvefs::mds::heartbeat::MetaServerHeartbeatRequest;
 using HeartbeatResponse = curvefs::mds::heartbeat::MetaServerHeartbeatResponse;
 using MetaServerSpaceStatus = curvefs::mds::heartbeat::MetaServerSpaceStatus;
-using ::curvefs::mds::heartbeat::CopySetConf;
-using TaskStatus = butil::Status;
 using CopysetNodePtr = std::shared_ptr<CopysetNode>;
-using curvefs::metaserver::copyset::CopysetNodeManager;
-using curvefs::common::Peer;
 using PeerId = braft::PeerId;
+using TaskStatus = butil::Status;
+using BlockGroupStatInfoMap = std::map<uint32_t, BlockGroupStatInfo>;
 
 class ResourceCollector;
 
@@ -104,6 +108,8 @@ class Heartbeat {
     int Run();
 
  private:
+    FRIEND_TEST(HeartbeatTest, Test_BuildRequest);
+
     /**
      * @brief stop heartbeat subsystem
      * @return 0:success; not 0: fail
@@ -118,6 +124,9 @@ class Heartbeat {
     void BuildCopysetInfo(curvefs::mds::heartbeat::CopySetInfo* info,
                          CopysetNode* copyset);
 
+    void BuildBlockGroupStatInfo(CopysetNode *copyset,
+                                 BlockGroupStatInfoMap *blockGroupStatInfoMap);
+
     int BuildRequest(HeartbeatRequest* request);
 
     int SendHeartbeat(const HeartbeatRequest& request,
@@ -136,11 +145,14 @@ class Heartbeat {
     bool GetMetaserverSpaceStatus(MetaServerSpaceStatus* status,
                                   uint64_t ncopysets);
 
+    // Handle heartbeat and send recyclable BlockGroup requests
+    void DeallocateBolckGroup(const HeartbeatResponse &response);
+
  private:
     friend class HeartbeatTest;
 
  private:
-     Thread hbThread_;
+    Thread hbThread_;
 
     std::atomic<bool> toStop_;
 
@@ -175,6 +187,11 @@ class HeartbeatTaskExecutor {
 
     void ExecTasks(const HeartbeatResponse& response);
 
+    // for unit test
+    void SetDeallocTask(uint32_t fsid, uint64_t offset) {
+        deallocTask_[fsid] = offset;
+    }
+
  private:
     void ExecOneTask(const CopySetConf& conf);
 
@@ -187,8 +204,13 @@ class HeartbeatTaskExecutor {
     bool NeedPurge(const CopySetConf& conf);
 
  private:
+    friend class Heartbeat;
+
     CopysetNodeManager* copysetMgr_;
     butil::EndPoint ep_;
+
+    // key is fsid, value is blockgroupOffset
+    std::map<uint32_t, uint64_t> deallocTask_;
 };
 
 }  // namespace metaserver

curvefs/src/metaserver/inode_manager.cpp

@@ -342,8 +342,12 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request) {
         needUpdate = true;
     }
 
+    bool fileNeedDeallocate =
+        (needAddTrash && (FsFileType::TYPE_FILE == old.type()));
+    bool s3NeedTrash = (needAddTrash && (FsFileType::TYPE_S3 == old.type()));
+
     if (needUpdate) {
-        ret = inodeStorage_->Update(old);
+        ret = inodeStorage_->Update(old, fileNeedDeallocate);
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "UpdateInode fail, " << request.ShortDebugString()
                        << ", ret: " << MetaStatusCode_Name(ret);
@@ -351,7 +355,7 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request) {
         }
     }
 
-    if (needAddTrash) {
+    if (s3NeedTrash) {
         trash_->Add(old.fsid(), old.inodeid(), old.dtime());
         --(*type2InodeNum_)[old.type()];
     }
@@ -556,7 +560,7 @@ MetaStatusCode InodeManager::UpdateVolumeExtentSlice(
 MetaStatusCode InodeManager::UpdateVolumeExtent(
     uint32_t fsId,
     uint64_t inodeId,
-    const VolumeExtentList &extents) {
+    const VolumeExtentSliceList &extents) {
     VLOG(6) << "UpdateInodeExtent, fsId: " << fsId << ", inodeId: " << inodeId;
     NameLockGuard guard(inodeLock_, GetInodeLockName(fsId, inodeId));
 
@@ -578,7 +582,7 @@ MetaStatusCode InodeManager::GetVolumeExtent(
     uint32_t fsId,
     uint64_t inodeId,
     const std::vector<uint64_t> &slices,
-    VolumeExtentList *extents) {
+    VolumeExtentSliceList *extents) {
     VLOG(6) << "GetInodeExtent, fsId: " << fsId << ", inodeId: " << inodeId;
 
     if (slices.empty()) {

curvefs/src/metaserver/inode_manager.h

@@ -113,7 +113,7 @@ class InodeManager {
     // Update one or more volume extent slice
     MetaStatusCode UpdateVolumeExtent(uint32_t fsId,
                                       uint64_t inodeId,
-                                      const VolumeExtentList &extents);
+                                      const VolumeExtentSliceList &extents);
 
     // Update only one volume extent slice
     MetaStatusCode UpdateVolumeExtentSlice(uint32_t fsId,
@@ -123,7 +123,7 @@ class InodeManager {
     MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                    uint64_t inodeId,
                                    const std::vector<uint64_t> &slices,
-                                   VolumeExtentList *extents);
+                                   VolumeExtentSliceList *extents);
 
  private:
     void GenerateInodeInternal(uint64_t inodeId, const InodeParam &param,

curvefs/src/metaserver/inode_storage.cpp

@@ -25,13 +25,16 @@
 #include <memory>
 #include <vector>
 #include <algorithm>
+#include <set>
 
 #include "src/common/concurrent/rw_lock.h"
 #include "src/common/string_util.h"
 #include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/proto/common.pb.h"
 #include "curvefs/src/metaserver/storage/status.h"
 #include "curvefs/src/metaserver/inode_storage.h"
 #include "curvefs/src/metaserver/storage/converter.h"
+#include "curvefs/src/metaserver/common/types.h"
 
 namespace curvefs {
 namespace metaserver {
@@ -39,6 +42,7 @@ namespace metaserver {
 using ::curve::common::ReadLockGuard;
 using ::curve::common::WriteLockGuard;
 using ::curve::common::StringStartWith;
+using ::curvefs::common::EmptyMsg;
 using ::curvefs::metaserver::storage::Status;
 using ::curvefs::metaserver::storage::KVStorage;
 using ::curvefs::metaserver::storage::Key4S3ChunkInfoList;
@@ -48,8 +52,8 @@ using ::curvefs::metaserver::storage::Prefix4ChunkIndexS3ChunkInfoList;
 using ::curvefs::metaserver::storage::Prefix4InodeS3ChunkInfoList;
 using ::curvefs::metaserver::storage::Prefix4AllInode;
 using ::curvefs::metaserver::storage::Key4InodeAuxInfo;
-
-using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+using ::curvefs::metaserver::storage::Key4DeallocatableBlockGroup;
+using ::curvefs::metaserver::storage::Prefix4AllDeallocatableBlockGroup;
 
 InodeStorage::InodeStorage(std::shared_ptr<KVStorage> kvStorage,
                            std::shared_ptr<NameGenerator> nameGenerator,
@@ -59,8 +63,11 @@ InodeStorage::InodeStorage(std::shared_ptr<KVStorage> kvStorage,
       table4S3ChunkInfo_(nameGenerator->GetS3ChunkInfoTableName()),
       table4VolumeExtent_(nameGenerator->GetVolumeExtentTableName()),
       table4InodeAuxInfo_(nameGenerator->GetInodeAuxInfoTableName()),
-      nInode_(nInode),
-      conv_() {}
+      table4DeallocatableInode_(
+          nameGenerator->GetDeallocatableInodeTableName()),
+      table4DeallocatableBlockGroup_(
+          nameGenerator->GetDeallocatableBlockGroupTableName()),
+      nInode_(nInode), conv_() {}
 
 MetaStatusCode InodeStorage::Insert(const Inode& inode) {
     WriteLockGuard lg(rwLock_);
@@ -181,18 +188,53 @@ MetaStatusCode InodeStorage::Delete(const Key4Inode& key) {
     return MetaStatusCode::STORAGE_INTERNAL_ERROR;
 }
 
-MetaStatusCode InodeStorage::Update(const Inode& inode) {
+MetaStatusCode InodeStorage::Update(const Inode &inode, bool inodeDeallocate) {
     WriteLockGuard lg(rwLock_);
     Key4Inode key(inode.fsid(), inode.inodeid());
     std::string skey = conv_.SerializeToString(key);
 
-    Status s = kvStorage_->HSet(table4Inode_, skey, inode);
-    if (s.ok()) {
-        return MetaStatusCode::OK;
+    // only update inodes
+    if (!inodeDeallocate) {
+        Status s = kvStorage_->HSet(table4Inode_, skey, inode);
+        if (s.ok()) {
+            return MetaStatusCode::OK;
+        }
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
     }
-    return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+
+    // update inode and update deallocatable inode list
+    EmptyMsg value;
+    auto txn = kvStorage_->BeginTransaction();
+    if (nullptr == txn) {
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+
+    std::string step = "update inode " + key.SerializeToString();
+
+    Status s = txn->HSet(table4Inode_, skey, inode);
+    if (s.ok()) {
+        s = txn->HSet(table4DeallocatableInode_, skey, value);
+        step = "add inode " + key.SerializeToString() +
+               " to inode deallocatable list";
+    }
+
+    if (!s.ok()) {
+        LOG(ERROR) << "txn is failed in " << step;
+        if (!txn->Rollback().ok()) {
+            LOG(ERROR) << "rollback transaction failed, inode="
+                       << key.SerializeToString();
+            return  MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        }
+    } else if (!txn->Commit().ok()) {
+        LOG(ERROR) << "commit transaction failed, inode="
+                   << key.SerializeToString();
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+
+    return MetaStatusCode::OK;
 }
 
+
 std::shared_ptr<Iterator> InodeStorage::GetAllInode() {
     ReadLockGuard lg(rwLock_);
     std::string sprefix = conv_.SerializeToString(Prefix4AllInode());
@@ -527,9 +569,9 @@ MetaStatusCode InodeStorage::UpdateVolumeExtentSlice(
                    : MetaStatusCode::STORAGE_INTERNAL_ERROR;
 }
 
-MetaStatusCode InodeStorage::GetAllVolumeExtent(uint32_t fsId,
-                                                uint64_t inodeId,
-                                                VolumeExtentList* extents) {
+MetaStatusCode
+InodeStorage::GetAllVolumeExtent(uint32_t fsId, uint64_t inodeId,
+                                 VolumeExtentSliceList *extents) {
     ReadLockGuard guard(rwLock_);
     auto key = conv_.SerializeToString(Prefix4InodeVolumeExtent{fsId, inodeId});
     auto iter = kvStorage_->SSeek(table4VolumeExtent_, key);
@@ -552,6 +594,13 @@ MetaStatusCode InodeStorage::GetAllVolumeExtent(uint32_t fsId,
     return MetaStatusCode::OK;
 }
 
+std::shared_ptr<Iterator> InodeStorage::GetAllVolumeExtent(uint32_t fsId,
+                                                           uint64_t inodeId) {
+    ReadLockGuard guard(rwLock_);
+    auto key = conv_.SerializeToString(Prefix4InodeVolumeExtent{fsId, inodeId});
+    return kvStorage_->SSeek(table4VolumeExtent_, key);
+}
+
 MetaStatusCode InodeStorage::GetVolumeExtentByOffset(uint32_t fsId,
                                                      uint64_t inodeId,
                                                      uint64_t offset,
@@ -571,5 +620,204 @@ MetaStatusCode InodeStorage::GetVolumeExtentByOffset(uint32_t fsId,
     return MetaStatusCode::STORAGE_INTERNAL_ERROR;
 }
 
+MetaStatusCode InodeStorage::GeAllBlockGroup(
+    std::vector<DeallocatableBlockGroup> *deallocatableBlockGroupVec) {
+    auto iter = kvStorage_->HGetAll(table4DeallocatableBlockGroup_);
+    if (iter->Status() != 0) {
+        LOG(ERROR) << "InodeStorage failed to get iterator for all "
+                      "deallocatable block group";
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+
+    uint32_t count = 0;
+    DeallocatableBlockGroup deallocatbleBlockGroup;
+    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+        if (!conv_.ParseFromString(iter->Value(), &deallocatbleBlockGroup)) {
+            LOG(ERROR) << "InodeStorage failed to parse deallocatable block "
+                          "group";
+            return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        }
+
+        deallocatableBlockGroupVec->emplace_back(
+            std::move(deallocatbleBlockGroup));
+        count++;
+    }
+
+    return count > 0 ? MetaStatusCode::OK : MetaStatusCode::NOT_FOUND;
+}
+
+MetaStatusCode InodeStorage::UpdateDeallocatableBlockGroup(
+    uint32_t fsId, const DeallocatableBlockGroupVec &update) {
+    auto txn = kvStorage_->BeginTransaction();
+
+    MetaStatusCode st = MetaStatusCode::OK;
+    std::string step;
+
+    for (auto &item : update) {
+        Key4DeallocatableBlockGroup key(fsId, item.blockgroupoffset());
+        std::string skey(key.SerializeToString());
+
+        DeallocatableBlockGroup out;
+        auto s = txn->HGet(table4DeallocatableBlockGroup_, skey, &out);
+
+        if (!s.ok() && !s.IsNotFound()) {
+            step = "get deallocatable group skey=" + skey + " failed";
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+            break;
+        }
+
+        if (item.has_increase()) {
+            if (!out.IsInitialized()) {
+                out.set_blockgroupoffset(item.blockgroupoffset());
+            }
+            st = Increase(txn, fsId, item.increase(), &out);
+        } else if (item.has_decrease()) {
+            st = Decrease(item.decrease(), &out);
+        } else if (item.has_mark()) {
+            st = Mark(item.mark(), &out);
+        }
+
+        s = txn->HSet(table4DeallocatableBlockGroup_, skey, out);
+        if (!s.ok()) {
+            step = "update deallocatable group skey=" + skey + " failed";
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+            break;
+        }
+    }
+
+    if (st != MetaStatusCode::OK) {
+        LOG(ERROR) << "UpdateDeallocatableBlockGroup txn is failed at " << step;
+        if (!txn->Rollback().ok()) {
+            LOG(ERROR) << "UpdateDeallocatableBlockGroup rollback transaction "
+                          "failed, fsId="
+                       << fsId;
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        }
+    } else if (!txn->Commit().ok()) {
+        LOG(ERROR)
+            << "UpdateDeallocatableBlockGroup commit transaction failed, fsId="
+            << fsId;
+        st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+
+    return st;
+}
+
+MetaStatusCode
+InodeStorage::Increase(Transaction txn, uint32_t fsId,
+                       const IncreaseDeallocatableBlockGroup &increase,
+                       DeallocatableBlockGroup *out) {
+    MetaStatusCode st = MetaStatusCode::OK;
+
+    // update DeallocatableBlockGroup
+    VLOG(6) << "InodeStorage handle increase=" << increase.DebugString();
+
+    uint64_t oldSize =
+        out->has_deallocatablesize() ? out->deallocatablesize() : 0;
+    out->set_deallocatablesize(oldSize + increase.increasedeallocatablesize());
+    out->mutable_inodeidlist()->MergeFrom(increase.inodeidlistadd());
+    std::set<uint64_t> unique_elements(out->inodeidlist().begin(),
+                                       out->inodeidlist().end());
+    out->mutable_inodeidlist()->Clear();
+    for (auto &elem : unique_elements) {
+        out->mutable_inodeidlist()->Add(elem);
+    }
+
+    VLOG(6) << "InodeStorage handle increase set out="
+            << out->DebugString();
+
+    // remove related inode in table4DeallocatableInode_
+    for (auto &inodeid : increase.inodeidlistadd()) {
+        auto s = txn->HDel(
+            table4DeallocatableInode_,
+            conv_.SerializeToString(Key4Inode{fsId, inodeid}));
+        if (!s.ok()) {
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+            VLOG(6) << "InodeStorage delete inodeid=" << inodeid << " from "
+                    << table4DeallocatableInode_ << " fail";
+            break;
+        }
+
+        VLOG(6) << "InodeStorage delete inodeid=" << inodeid << " from "
+                << StringToHex(table4DeallocatableInode_) << " success";
+    }
+
+    return st;
+}
+
+MetaStatusCode
+InodeStorage::Decrease(const DecreaseDeallocatableBlockGroup &decrease,
+                       DeallocatableBlockGroup *out) {
+    MetaStatusCode st = MetaStatusCode::OK;
+
+    VLOG(6) << "InodeStorage handle increase=" << decrease.DebugString();
+    if (!out->IsInitialized() || !out->has_deallocatablesize()) {
+        LOG(ERROR)
+            << "UpdateDeallocatableBlockGroup record missing required fields";
+        st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    } else {
+        uint64_t oldSize = out->deallocatablesize();
+        if (oldSize < decrease.decreasedeallocatablesize()) {
+            LOG(ERROR)
+                << "UpdateDeallocatableBlockGroup decrease size is too big, "
+                   "oldSize="
+                << oldSize
+                << ", decreasesize=" << decrease.decreasedeallocatablesize();
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        } else {
+            // update dallocatablesize
+            out->set_deallocatablesize(oldSize -
+                                       decrease.decreasedeallocatablesize());
+
+            // update deallocatableinode list
+            auto inodeidlist = out->mutable_inodeidunderdeallocate();
+            inodeidlist->erase(
+                std::remove_if(inodeidlist->begin(), inodeidlist->end(),
+                               [&decrease](uint64_t inodeid) {
+                                   auto search = decrease.inodeddeallocated();
+                                   return std::find(search.begin(),
+                                                    search.end(),
+                                                    inodeid) != search.end();
+                               }),
+                inodeidlist->end());
+
+            VLOG(6) << "InodeStorage handle decrease ok, and set out="
+                    << out->DebugString();
+        }
+    }
+
+    return st;
+}
+
+MetaStatusCode InodeStorage::Mark(const MarkDeallocatableBlockGroup &mark,
+                                  DeallocatableBlockGroup *out) {
+    MetaStatusCode st = MetaStatusCode::OK;
+
+    VLOG(6) << "InodeStorage handle mark=" << mark.DebugString();
+    if (!out->IsInitialized() || !out->has_deallocatablesize()) {
+        LOG(ERROR)
+            << "UpdateDeallocatableBlockGroup record missing required fields";
+        st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    } else {
+        // update inodeunderdeallocate
+        out->mutable_inodeidunderdeallocate()->MergeFrom(
+            mark.inodeidunderdeallocate());
+
+        // update inodeidlist
+        auto inodeidlist = out->mutable_inodeidlist();
+        inodeidlist->erase(
+            std::remove_if(inodeidlist->begin(), inodeidlist->end(),
+                           [&mark](uint64_t inodeid) {
+                               auto search = mark.inodeidunderdeallocate();
+                               return std::find(search.begin(), search.end(),
+                                                inodeid) != search.end();
+                           }),
+            inodeidlist->end());
+        VLOG(6) << "InodeStorage handle mark ok, and set out="
+                << out->DebugString();
+    }
+    return st;
+}
+
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/inode_storage.h

@@ -30,6 +30,7 @@
 #include <functional>
 #include <unordered_map>
 #include <unordered_set>
+#include <vector>
 
 #include "absl/container/btree_set.h"
 #include "absl/container/btree_map.h"
@@ -49,8 +50,12 @@ using ::curvefs::metaserver::storage::StorageTransaction;
 using ::curvefs::metaserver::storage::Key4Inode;
 using ::curvefs::metaserver::storage::Converter;
 using ::curvefs::metaserver::storage::NameGenerator;
+
 using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+using DeallocatableBlockGroupVec =
+    google::protobuf::RepeatedPtrField<DeallocatableBlockGroup>;
 using Transaction = std::shared_ptr<StorageTransaction>;
+
 class InodeStorage {
  public:
     InodeStorage(std::shared_ptr<KVStorage> kvStorage,
@@ -98,9 +103,10 @@ class InodeStorage {
     /**
      * @brief update inode from storage
      * @param[in] inode: the inode want to update
+     * @param[in] inodeDeallocate: Whether the inode needs to deallocate space
      * @return If inode not exist, return NOT_FOUND; else replace and return OK
      */
-    MetaStatusCode Update(const Inode& inode);
+    MetaStatusCode Update(const Inode& inode, bool inodeDeallocate = false);
 
     std::shared_ptr<Iterator> GetAllInode();
 
@@ -141,19 +147,24 @@ class InodeStorage {
 
     MetaStatusCode GetAllVolumeExtent(uint32_t fsId,
                                       uint64_t inodeId,
-                                      VolumeExtentList* extents);
+                                      VolumeExtentSliceList* extents);
+
+    std::shared_ptr<Iterator> GetAllVolumeExtent(uint32_t fsId,
+                                                 uint64_t inodeId);
 
     MetaStatusCode GetVolumeExtentByOffset(uint32_t fsId,
                                            uint64_t inodeId,
                                            uint64_t offset,
                                            VolumeExtentSlice* slice);
 
-    MetaStatusCode AddS3ChunkInfoList(
-        std::shared_ptr<StorageTransaction> txn,
-        uint32_t fsId,
-        uint64_t inodeId,
-        uint64_t chunkIndex,
-        const S3ChunkInfoList* list2add);
+    // use the transaction to delete {inodes} in the deallocatable_inode_list
+    // and update the statistics of each item of blockgroup_list
+    MetaStatusCode
+    UpdateDeallocatableBlockGroup(uint32_t fsId,
+                                  const DeallocatableBlockGroupVec &update);
+
+    MetaStatusCode GeAllBlockGroup(
+        std::vector<DeallocatableBlockGroup> *deallocatableBlockGroupVec);
 
  private:
     MetaStatusCode UpdateInodeS3MetaSize(Transaction txn, uint32_t fsId,
@@ -162,12 +173,25 @@ class InodeStorage {
 
     uint64_t GetInodeS3MetaSize(uint32_t fsId, uint64_t inodeId);
 
-    MetaStatusCode DelS3ChunkInfoList(
-        std::shared_ptr<StorageTransaction> txn,
-        uint32_t fsId,
-        uint64_t inodeId,
-        uint64_t chunkIndex,
-        const S3ChunkInfoList* list2del);
+    MetaStatusCode DelS3ChunkInfoList(Transaction txn,
+                                      uint32_t fsId, uint64_t inodeId,
+                                      uint64_t chunkIndex,
+                                      const S3ChunkInfoList *list2del);
+
+    MetaStatusCode AddS3ChunkInfoList(Transaction txn,
+                                      uint32_t fsId, uint64_t inodeId,
+                                      uint64_t chunkIndex,
+                                      const S3ChunkInfoList *list2add);
+
+    MetaStatusCode Increase(Transaction txn, uint32_t fsId,
+                            const IncreaseDeallocatableBlockGroup &increase,
+                            DeallocatableBlockGroup *out);
+
+    MetaStatusCode Decrease(const DecreaseDeallocatableBlockGroup &decrease,
+                            DeallocatableBlockGroup *out);
+
+    MetaStatusCode Mark(const MarkDeallocatableBlockGroup &mark,
+                        DeallocatableBlockGroup *out);
 
  private:
     // FIXME: please remove this lock, because we has locked each inode
@@ -179,6 +203,9 @@ class InodeStorage {
     std::string table4S3ChunkInfo_;
     std::string table4VolumeExtent_;
     std::string table4InodeAuxInfo_;
+    std::string table4DeallocatableBlockGroup_;
+    std::string table4DeallocatableInode_;
+
     size_t nInode_;
     Converter conv_;
 };

curvefs/src/metaserver/mds/fsinfo_manager.cpp

@@ -0,0 +1,50 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Wed Mar 22 10:39:52 CST 2023
+ * Author: lixiaocui
+ */
+
+#include "curvefs/src/metaserver/mds/fsinfo_manager.h"
+
+namespace curvefs {
+namespace metaserver {
+bool FsInfoManager::GetFsInfo(uint32_t fsId, FsInfo *fsInfo) {
+    std::lock_guard<std::mutex> lock(mtx_);
+    if (fsInfoMap_.find(fsId) == fsInfoMap_.end()) {
+        auto ret = mdsClient_->GetFsInfo(fsId, fsInfo);
+        if (ret != FSStatusCode::OK) {
+            if (FSStatusCode::NOT_FOUND == ret) {
+                LOG(ERROR) << "The fsName not exist, fsId = " << fsId;
+                return false;
+            } else {
+                LOG(ERROR) << "GetFsInfo failed, FSStatusCode = " << ret
+                           << ", FSStatusCode_Name = " << FSStatusCode_Name(ret)
+                           << ", fsId = " << fsId;
+                return false;
+            }
+        }
+        fsInfoMap_.insert({fsId, *fsInfo});
+    } else {
+        *fsInfo = fsInfoMap_.find(fsId)->second;
+    }
+
+    return true;
+}
+}  // namespace metaserver
+}  // namespace curvefs

curvefs/src/metaserver/mds/fsinfo_manager.h

@@ -0,0 +1,58 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Wed Mar 22 10:40:08 CST 2023
+ * Author: lixiaocui
+ */
+
+#ifndef CURVEFS_SRC_METASERVER_MDS_FSINFO_MANAGER_H_
+#define CURVEFS_SRC_METASERVER_MDS_FSINFO_MANAGER_H_
+
+#include <map>
+#include <memory>
+#include <mutex>
+#include "curvefs/src/client/rpcclient/mds_client.h"
+
+namespace curvefs {
+namespace metaserver {
+
+using ::curvefs::client::rpcclient::MdsClient;
+
+class FsInfoManager {
+ public:
+    static FsInfoManager &GetInstance() {
+        static FsInfoManager instance_;
+        return instance_;
+    }
+
+    void SetMdsClient(std::shared_ptr<MdsClient> mdsClient) {
+        mdsClient_ = mdsClient;
+    }
+
+    bool GetFsInfo(uint32_t fsId, FsInfo *fsInfo);
+
+ private:
+    std::shared_ptr<MdsClient> mdsClient_;
+    std::map<uint32_t, FsInfo> fsInfoMap_;
+
+    std::mutex mtx_;
+};
+}  // namespace metaserver
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_METASERVER_MDS_FSINFO_MANAGER_H_

curvefs/src/metaserver/metacli_manager.cpp

@@ -0,0 +1,49 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Fri Jun 30 10:55:44 CST 2023
+ * Author: lixiaocui
+ */
+
+#include "curvefs/src/metaserver/metacli_manager.h"
+
+namespace curvefs {
+namespace metaserver {
+
+using curvefs::client::rpcclient::MetaServerClientImpl;
+
+std::shared_ptr<MetaServerClient> MetaCliManager::GetMetaCli(uint32_t fsId) {
+    std::lock_guard<std::mutex> lock(mtx_);
+    auto out = metaCliMap_.find(fsId);
+
+    if (out == metaCliMap_.end()) {
+        auto metaCli = std::make_shared<MetaServerClientImpl>();
+        auto metaCache = std::make_shared<MetaCache>();
+
+        metaCache->Init(opt_.metaCacheOpt, opt_.cli2Cli, opt_.mdsCli);
+        metaCli->Init(opt_.executorOpt, opt_.internalOpt, metaCache,
+                      opt_.channelManager);
+        metaCliMap_.insert({fsId, metaCli});
+
+        return metaCli;
+    } else {
+        return out->second;
+    }
+}
+}  // namespace metaserver
+}  // namespace curvefs

curvefs/src/metaserver/metacli_manager.h

@@ -0,0 +1,70 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Fri Jun 30 10:56:51 CST 2023
+ * Author: lixiaocui
+ */
+
+#ifndef CURVEFS_SRC_METASERVER_METACLI_MANAGER_H_
+#define CURVEFS_SRC_METASERVER_METACLI_MANAGER_H_
+
+#include <map>
+#include "curvefs/src/client/rpcclient/mds_client.h"
+#include "curvefs/src/client/rpcclient/metaserver_client.h"
+
+namespace curvefs {
+namespace metaserver {
+
+using curvefs::client::rpcclient::ChannelManager;
+using curvefs::client::rpcclient::Cli2Client;
+using curvefs::client::rpcclient::MdsClient;
+using curvefs::client::rpcclient::MetaCache;
+using curvefs::client::rpcclient::MetaServerClient;
+
+struct MetaCliManagerOpt {
+    MetaCacheOpt metaCacheOpt;
+    ExcutorOpt executorOpt;
+    ExcutorOpt internalOpt;
+
+    std::shared_ptr<Cli2Client> cli2Cli;
+    std::shared_ptr<MdsClient> mdsCli;
+    std::shared_ptr<ChannelManager<MetaserverID>> channelManager;
+};
+
+class MetaCliManager {
+ public:
+    static MetaCliManager &GetInstance() {
+        static MetaCliManager instance_;
+        return instance_;
+    }
+
+    void Init(MetaCliManagerOpt &&Opt) {
+        opt_ = std::move(Opt);
+    }
+
+    std::shared_ptr<MetaServerClient> GetMetaCli(uint32_t fsId);
+
+ private:
+    std::mutex mtx_;
+    std::map<uint32_t, std::shared_ptr<MetaServerClient>> metaCliMap_;
+
+    MetaCliManagerOpt opt_;
+};
+}  // namespace metaserver
+}  // namespace curvefs
+#endif  // CURVEFS_SRC_METASERVER_METACLI_MANAGER_H_

curvefs/src/metaserver/metaserver.cpp

@@ -38,6 +38,7 @@
 #include "curvefs/src/metaserver/trash_manager.h"
 #include "curvefs/src/metaserver/storage/storage.h"
 #include "curvefs/src/metaserver/storage/rocksdb_perf.h"
+#include "curvefs/src/metaserver/mds/fsinfo_manager.h"
 #include "src/common/crc32.h"
 #include "src/common/curve_version.h"
 #include "src/common/s3_adapter.h"
@@ -46,6 +47,7 @@
 #include "src/common/uri_parser.h"
 #include "curvefs/src/metaserver/resource_statistic.h"
 #include "src/fs/ext4_filesystem_impl.h"
+#include "curvefs/src/metaserver/metacli_manager.h"
 
 namespace braft {
 
@@ -158,6 +160,16 @@ void Metaserver::InitRecycleManagerOption(
                                          &recycleManagerOption->scanLimit));
 }
 
+void Metaserver::InitVolumeDeallocateOption(
+    VolumeDeallocateWorkerQueueOption *queueOpt,
+    VolumeDeallocateExecuteOption *execOpt) {
+    conf_->GetValueFatalIfFail("volume.deallocate.enable", &queueOpt->enable);
+    conf_->GetValueFatalIfFail("volume.deallocate.workerNum",
+                               &queueOpt->workerNum);
+    conf_->GetValueFatalIfFail("volume.deallocate.batchClean",
+                               &execOpt->batchClean);
+}
+
 void InitExcutorOption(const std::shared_ptr<Configuration>& conf,
                        ExcutorOpt *opts, bool internal) {
     if (internal) {
@@ -206,6 +218,8 @@ void Metaserver::Init() {
     mdsClient_ = std::make_shared<MdsClientImpl>();
     mdsClient_->Init(mdsOptions_, mdsBase_);
 
+    FsInfoManager::GetInstance().SetMdsClient(mdsClient_);
+
     // init metaserver client for recycle
     InitMetaClient();
 
@@ -243,6 +257,20 @@ void Metaserver::Init() {
 
     S3CompactManager::GetInstance().Init(conf_);
 
+    VolumeSpaceManagerOptions spaceManagerOpt;
+    spaceManagerOpt.mdsClient = mdsClient_;
+    conf_->GetValueFatalIfFail("volume.sdk.confPath",
+                               &spaceManagerOpt.deviceOpt.configPath);
+    auto volumeSpaceMgr = std::make_shared<VolumeSpaceManager>();
+    volumeSpaceMgr->Init(spaceManagerOpt);
+
+    VolumeDeallocateWorkerQueueOption queueOpt;
+    VolumeDeallocateExecuteOption executeOpt;
+    executeOpt.metaClient = metaClient_;
+    executeOpt.volumeSpaceManager = std::move(volumeSpaceMgr);
+    InitVolumeDeallocateOption(&queueOpt, &executeOpt);
+    VolumeDeallocateManager::GetInstance().Init(queueOpt, executeOpt);
+
     PartitionCleanOption partitionCleanOption;
     InitPartitionOption(s3Adaptor_, mdsClient_, &partitionCleanOption);
     PartitionCleanManager::GetInstance().Init(partitionCleanOption);
@@ -264,6 +292,15 @@ void Metaserver::InitMetaClient() {
     InitExcutorOption(conf_, &excutorOpt, false);
     InitExcutorOption(conf_, &internalOpt, true);
     metaClient_->Init(excutorOpt, internalOpt, metaCache, channelManager);
+
+    MetaCliManagerOpt opt;
+    opt.metaCacheOpt = std::move(metaCacheOpt);
+    opt.executorOpt = std::move(excutorOpt);
+    opt.internalOpt = std::move(internalOpt);
+    opt.cli2Cli = cli2Client;
+    opt.mdsCli = mdsClient_;
+    opt.channelManager = channelManager;
+    MetaCliManager::GetInstance().Init(std::move(opt));
 }
 
 void Metaserver::GetMetaserverDataByLoadOrRegister() {
@@ -491,6 +528,9 @@ void Metaserver::Run() {
     LOG_IF(FATAL, S3CompactManager::GetInstance().Run() != 0);
     running_ = true;
 
+    // start volume deallocate manager
+    VolumeDeallocateManager::GetInstance().Run();
+
     // start copyset node manager
     LOG_IF(FATAL, !copysetNodeManager_->Start())
         << "Failed to start copyset node manager";
@@ -525,6 +565,7 @@ void Metaserver::Stop() {
 
     s3Adaptor_ = nullptr;
     S3CompactManager::GetInstance().Stop();
+    VolumeDeallocateManager::GetInstance().Stop();
     LOG(INFO) << "MetaServer stopped success";
 }
 

curvefs/src/metaserver/metaserver.h

@@ -46,6 +46,9 @@
 #include "src/fs/local_filesystem.h"
 #include "curvefs/src/metaserver/resource_statistic.h"
 #include "curvefs/src/metaserver/recycle_manager.h"
+#include "curvefs/src/metaserver/space/volume_deallocate_manager.h"
+#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h"
+#include "curvefs/src/metaserver/space/volume_space_manager.h"
 
 namespace curvefs {
 namespace metaserver {
@@ -97,6 +100,9 @@ class Metaserver {
                              PartitionCleanOption* partitionCleanOption);
     void InitRecycleManagerOption(
                 RecycleManagerOption* recycleManagerOption);
+
+    void InitVolumeDeallocateOption(VolumeDeallocateWorkerQueueOption *queueOpt,
+                                    VolumeDeallocateExecuteOption *execOpt);
     void GetMetaserverDataByLoadOrRegister();
     int PersistMetaserverMeta(std::string path, MetaServerMetadata* metadata);
     int LoadMetaserverMeta(const std::string& metaFilePath,

curvefs/src/metaserver/metaserver_service.cpp

@@ -52,6 +52,7 @@ using ::curvefs::metaserver::copyset::DeletePartitionOperator;
 using ::curvefs::metaserver::copyset::PrepareRenameTxOperator;
 using ::curvefs::metaserver::copyset::GetVolumeExtentOperator;
 using ::curvefs::metaserver::copyset::UpdateVolumeExtentOperator;
+using ::curvefs::metaserver::copyset::UpdateDeallocatableBlockGroupOperator;
 
 namespace {
 
@@ -298,5 +299,16 @@ void MetaServerServiceImpl::UpdateVolumeExtent(
                                                   request->copysetid());
 }
 
+void MetaServerServiceImpl::UpdateDeallocatableBlockGroup(
+    ::google::protobuf::RpcController *controller,
+    const UpdateDeallocatableBlockGroupRequest *request,
+    UpdateDeallocatableBlockGroupResponse *response,
+    ::google::protobuf::Closure *done) {
+    OperatorHelper helper(copysetNodeManager_, inflightThrottle_);
+    helper.operator()<UpdateDeallocatableBlockGroupOperator>(
+        controller, request, response, done, request->poolid(),
+        request->copysetid());
+}
+
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/metaserver_service.h

@@ -123,6 +123,12 @@ class MetaServerServiceImpl : public MetaServerService {
                             UpdateVolumeExtentResponse* response,
                             ::google::protobuf::Closure* done) override;
 
+    void UpdateDeallocatableBlockGroup(
+        ::google::protobuf::RpcController *controller,
+        const UpdateDeallocatableBlockGroupRequest *request,
+        UpdateDeallocatableBlockGroupResponse *response,
+        ::google::protobuf::Closure *done) override;
+
  private:
     CopysetNodeManager* copysetNodeManager_;
     InflightThrottle* inflightThrottle_;

curvefs/src/metaserver/metastore.cpp

@@ -204,6 +204,7 @@ bool MetaStoreImpl::ClearInternal() {
     for (auto it = partitionMap_.begin(); it != partitionMap_.end(); it++) {
         TrashManager::GetInstance().Remove(it->first);
         it->second->CancelS3Compact();
+        it->second->CancelVolumeDeallocate();
         PartitionCleanManager::GetInstance().Remove(it->first);
 
         if (!it->second->Clear()) {
@@ -276,6 +277,7 @@ MetaStoreImpl::DeletePartition(const DeletePartitionRequest *request,
         TrashManager::GetInstance().Remove(partitionId);
         RecycleManager::GetInstance().Remove(partitionId);
         it->second->CancelS3Compact();
+        it->second->CancelVolumeDeallocate();
         PartitionCleanManager::GetInstance().Remove(partitionId);
         partitionMap_.erase(it);
         response->set_statuscode(MetaStatusCode::OK);
@@ -294,6 +296,7 @@ MetaStoreImpl::DeletePartition(const DeletePartitionRequest *request,
         TrashManager::GetInstance().Remove(partitionId);
         RecycleManager::GetInstance().Remove(partitionId);
         it->second->CancelS3Compact();
+        it->second->CancelVolumeDeallocate();
     } else {
         LOG(INFO) << "DeletePartition, partition is already deleting"
                   << ", partitionId = " << partitionId;
@@ -316,27 +319,46 @@ bool MetaStoreImpl::GetPartitionInfoList(
         }
         rwLock_.Unlock();
         return true;
-    } else {
-        LOG(WARNING) << "metastore GetPartitionInfoList fail, it fail to get"
-                        " the rwLock_";
-        return false;
     }
+    LOG(WARNING) << "metastore GetPartitionInfoList fail, it fail to get"
+                    " the rwLock_";
+    return false;
+}
+
+bool MetaStoreImpl::GetPartitionSnap(
+    std::map<uint32_t, std::shared_ptr<Partition>> *partitionSnap) {
+    int ret = rwLock_.TryRDLock();
+    if (ret == 0) {
+        *partitionSnap = partitionMap_;
+        rwLock_.Unlock();
+        return true;
+    }
+
+    LOG(WARNING) << "MetaStoreImpl get partition snap fail, it fail to get"
+                    " the rwLock_";
+    return false;
 }
 
 std::shared_ptr<StreamServer> MetaStoreImpl::GetStreamServer() {
     return streamServer_;
 }
 
+#define GET_PARTITION(PARTITION)                                               \
+    PARTITION = GetPartition(request->partitionid());                          \
+    if (PARTITION == nullptr) {                                                \
+        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;           \
+        response->set_statuscode(status);                                      \
+        return status;                                                         \
+    }
+
+
 // dentry
 MetaStatusCode MetaStoreImpl::CreateDentry(const CreateDentryRequest *request,
                                            CreateDentryResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
+
     MetaStatusCode status = partition->CreateDentry(request->dentry());
     response->set_statuscode(status);
     return status;
@@ -349,12 +371,8 @@ MetaStatusCode MetaStoreImpl::GetDentry(const GetDentryRequest *request,
     const auto &name = request->name();
     auto txId = request->txid();
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     // handle by partition
     Dentry dentry;
@@ -378,12 +396,8 @@ MetaStatusCode MetaStoreImpl::DeleteDentry(const DeleteDentryRequest *request,
     std::string name = request->name();
     auto txId = request->txid();
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     // handle by partition
     Dentry dentry;
@@ -404,12 +418,8 @@ MetaStatusCode MetaStoreImpl::ListDentry(const ListDentryRequest *request,
     uint64_t parentInodeId = request->dirinodeid();
     auto txId = request->txid();
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     // handle by partition
     Dentry dentry;
@@ -440,16 +450,12 @@ MetaStoreImpl::PrepareRenameTx(const PrepareRenameTxRequest *request,
                                PrepareRenameTxResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
     MetaStatusCode rc;
-    auto partitionId = request->partitionid();
-    auto partition = GetPartition(partitionId);
-    if (nullptr == partition) {
-        rc = MetaStatusCode::PARTITION_NOT_FOUND;
-    } else {
-        std::vector<Dentry> dentrys{request->dentrys().begin(),
-                                    request->dentrys().end()};
-        rc = partition->HandleRenameTx(dentrys);
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
+    std::vector<Dentry> dentrys{request->dentrys().begin(),
+                                request->dentrys().end()};
+    rc = partition->HandleRenameTx(dentrys);
     response->set_statuscode(rc);
     return rc;
 }
@@ -487,12 +493,9 @@ MetaStatusCode MetaStoreImpl::CreateInode(const CreateInodeRequest *request,
     }
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
+
     MetaStatusCode status =
         partition->CreateInode(param, response->mutable_inode());
     response->set_statuscode(status);
@@ -521,12 +524,8 @@ MetaStoreImpl::CreateRootInode(const CreateRootInodeRequest *request,
     }
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     MetaStatusCode status = partition->CreateRootInode(param);
     response->set_statuscode(status);
@@ -560,12 +559,8 @@ MetaStoreImpl::CreateManageInode(const CreateManageInodeRequest *request,
     }
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     MetaStatusCode status = partition->CreateManageInode(
         param, request->managetype(), response->mutable_inode());
@@ -597,12 +592,8 @@ MetaStatusCode MetaStoreImpl::GetInode(const GetInodeRequest *request,
     uint64_t inodeId = request->inodeid();
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     Inode *inode = response->mutable_inode();
     MetaStatusCode rc = partition->GetInode(fsId, inodeId, inode);
@@ -633,12 +624,8 @@ MetaStatusCode
 MetaStoreImpl::BatchGetInodeAttr(const BatchGetInodeAttrRequest *request,
                                  BatchGetInodeAttrResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     uint32_t fsId = request->fsid();
     MetaStatusCode status = MetaStatusCode::OK;
@@ -658,12 +645,8 @@ MetaStoreImpl::BatchGetInodeAttr(const BatchGetInodeAttrRequest *request,
 MetaStatusCode MetaStoreImpl::BatchGetXAttr(const BatchGetXAttrRequest *request,
                                             BatchGetXAttrResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     uint32_t fsId = request->fsid();
     MetaStatusCode status = MetaStatusCode::OK;
@@ -686,12 +669,8 @@ MetaStatusCode MetaStoreImpl::DeleteInode(const DeleteInodeRequest *request,
     uint64_t inodeId = request->inodeid();
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     MetaStatusCode status = partition->DeleteInode(fsId, inodeId);
     response->set_statuscode(status);
@@ -702,12 +681,8 @@ MetaStatusCode MetaStoreImpl::UpdateInode(const UpdateInodeRequest *request,
                                           UpdateInodeResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
     VLOG(9) << "UpdateInode inode " << request->inodeid();
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     MetaStatusCode status = partition->UpdateInode(*request);
     response->set_statuscode(status);
@@ -720,12 +695,8 @@ MetaStatusCode MetaStoreImpl::GetOrModifyS3ChunkInfo(
     std::shared_ptr<Iterator> *iterator) {
     MetaStatusCode rc;
     ReadLockGuard readLockGuard(rwLock_);
-    auto partition = GetPartition(request->partitionid());
-    if (nullptr == partition) {
-        rc = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(rc);
-        return rc;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     uint32_t fsId = request->fsid();
     uint64_t inodeId = request->inodeid();
@@ -792,12 +763,8 @@ MetaStatusCode
 MetaStoreImpl::GetVolumeExtent(const GetVolumeExtentRequest *request,
                                GetVolumeExtentResponse *response) {
     ReadLockGuard guard(rwLock_);
-    auto partition = GetPartition(request->partitionid());
-    if (!partition) {
-        auto st = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(st);
-        return st;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     std::vector<uint64_t> slices(request->sliceoffsets().begin(),
                                  request->sliceoffsets().end());
@@ -815,12 +782,8 @@ MetaStatusCode
 MetaStoreImpl::UpdateVolumeExtent(const UpdateVolumeExtentRequest *request,
                                   UpdateVolumeExtentResponse *response) {
     ReadLockGuard guard(rwLock_);
-    auto partition = GetPartition(request->partitionid());
-    if (!partition) {
-        auto st = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(st);
-        return st;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     VLOG(9) << "UpdateVolumeExtent, request: " << request->ShortDebugString();
 
@@ -830,6 +793,22 @@ MetaStoreImpl::UpdateVolumeExtent(const UpdateVolumeExtentRequest *request,
     return st;
 }
 
+MetaStatusCode MetaStoreImpl::UpdateDeallocatableBlockGroup(
+    const UpdateDeallocatableBlockGroupRequest *request,
+    UpdateDeallocatableBlockGroupResponse *response) {
+    ReadLockGuard guard(rwLock_);
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
+
+    VLOG(9) << "UpdateDeallocatableBlockGroup, request: "
+            << request->ShortDebugString();
+
+    auto st = partition->UpdateDeallocatableBlockGroup(*request);
+    response->set_statuscode(st);
+
+    return MetaStatusCode::OK;
+}
+
 bool MetaStoreImpl::InitStorage() {
     if (storageOptions_.type == "memory") {
         kvStorage_ = std::make_shared<MemoryStorage>(storageOptions_);

curvefs/src/metaserver/metastore.h

@@ -86,6 +86,22 @@ using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 
 using ::curvefs::metaserver::storage::StorageOptions;
 
+// Dentry and inode related data will be stored in kvstorage
+//
+// 1. When kvstorage is rocksdb, the structure of the data is as follows:
+//    (table is related with partitionID)
+//                  column
+//                     |
+//       (table1     table2             table3)
+//           |         |                   |
+//         inode  inodedealloc  blockGroup-with-inodedealloc
+//
+//                  column
+//                     |
+//       (table1     table2         table3)
+//           |          |              |
+//         dentry  s3chunkinfo    volumnextent
+
 class MetaStore {
  public:
     MetaStore() = default;
@@ -107,6 +123,9 @@ class MetaStore {
     virtual bool GetPartitionInfoList(
                             std::list<PartitionInfo> *partitionInfoList) = 0;
 
+    virtual bool GetPartitionSnap(
+        std::map<uint32_t, std::shared_ptr<Partition>> *partitionSnap) = 0;
+
     virtual std::shared_ptr<StreamServer> GetStreamServer() = 0;
 
     // dentry
@@ -170,6 +189,10 @@ class MetaStore {
     virtual MetaStatusCode UpdateVolumeExtent(
         const UpdateVolumeExtentRequest* request,
         UpdateVolumeExtentResponse* response) = 0;
+
+    virtual MetaStatusCode UpdateDeallocatableBlockGroup(
+        const UpdateDeallocatableBlockGroupRequest *request,
+        UpdateDeallocatableBlockGroupResponse *response) = 0;
 };
 
 class MetaStoreImpl : public MetaStore {
@@ -193,6 +216,9 @@ class MetaStoreImpl : public MetaStore {
     bool GetPartitionInfoList(
                         std::list<PartitionInfo> *partitionInfoList) override;
 
+    bool GetPartitionSnap(
+        std::map<uint32_t, std::shared_ptr<Partition>> *partitionSnap) override;
+
     std::shared_ptr<StreamServer> GetStreamServer() override;
 
     // dentry
@@ -255,6 +281,11 @@ class MetaStoreImpl : public MetaStore {
         const UpdateVolumeExtentRequest* request,
         UpdateVolumeExtentResponse* response) override;
 
+    // block group
+    MetaStatusCode UpdateDeallocatableBlockGroup(
+        const UpdateDeallocatableBlockGroupRequest *request,
+        UpdateDeallocatableBlockGroupResponse *response) override;
+
  private:
     FRIEND_TEST(MetastoreTest, partition);
     FRIEND_TEST(MetastoreTest, test_inode);

curvefs/src/metaserver/metastore_fstream.cpp

@@ -80,8 +80,7 @@ bool MetaStoreFStream::LoadPartition(uint32_t partitionId,
         return false;
     }
 
-    LOG(INFO) << "Load partition, partition id: " << partitionId
-              << ", partition info: " << partitionInfo.ShortDebugString();
+    LOG(INFO) << "Load partition info: " << partitionInfo.ShortDebugString();
 
     // FIXME: partitionId is always 0 in some unittest,
     //        maybe this problem also exist in production code