curvefs: support space deallocate for curvebs volume as backend

Signed-off-by: ilixiaocui <ilixiaocui@163.com>

curvefs/conf/client.conf

@@ -58,10 +58,6 @@ executorOpt.maxRetryTimesBeforeConsiderSuspend=20
 # batch limit of get inode attr and xattr
 executorOpt.batchInodeAttrLimit=10000
 
-#### spaceserver
-spaceServer.spaceAddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space | join_peer(hostvars, "space_listen_port") }} __ANSIBLE_TEMPLATE__
-spaceServer.rpcTimeoutMs=1000
-
 #### bdev
 # curve client's config file
 bdev.confPath=/etc/curve/client.conf
@@ -132,6 +128,15 @@ volume.bitmapAllocator.smallAllocProportion=0.2
 # number of block groups that allocated once
 volume.blockGroup.allocateOnce=4
 
+## spaceserver
+# the space used by the blockgroup exceeds this percentage and can 
+# be returned to mds [0.8-1]
+volume.space.useThreshold=0.95
+
+# the background thread calculates the time interval for returning
+# the blockgroup to mds
+volume.space.releaseInterSec=300
+
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false

curvefs/conf/mds.conf

@@ -159,3 +159,6 @@ bs.mds.maxFailedTimesBeforeChangeMDS=2
 bs.mds.normalRetryTimesBeforeTriggerWait=3
 # sleep interval in ms for wait
 bs.mds.waitSleepMs=1000
+
+#### Options for volume space deallcatable setting
+mds.space.calIntervalSec=60

curvefs/conf/metaserver.conf

@@ -307,3 +307,9 @@ metaCacheOpt.metacacheGetLeaderRetry=3
 metaCacheOpt.metacacheRPCRetryIntervalUS=100000
 # RPC timeout of get leader
 metaCacheOpt.metacacheGetLeaderRPCTimeOutMS=1000
+
+#### volume deallocate
+volume.deallocate.enable=true
+volume.deallocate.workerNum=5
+volume.deallocate.batchClean=10
+volume.sdk.confPath=/etc/curvefs/curvebs_client.conf

curvefs/proto/BUILD

@@ -113,6 +113,7 @@ proto_library(
     name = "curvefs_heartbeat_proto",
     srcs = ["heartbeat.proto"],
     deps = [":curvefs_common_proto",
+            ":metaserver_proto",
             "//proto:heartbeat_proto"],
 )
 

curvefs/proto/common.proto

@@ -31,6 +31,13 @@ enum BitmapLocation {
     AtEnd = 2;
 }
 
+message EmptyMsg {}
+
+message BlockGroupID {
+    required uint64 fsId = 1;
+    required uint64 offset = 2;
+}
+
 // When creating fs, `volumeSize` and `extendAlignment` are fetched from the bs cluster
 message Volume {
     optional uint64 volumeSize = 1;

curvefs/proto/heartbeat.proto

@@ -16,6 +16,7 @@
 
 syntax = "proto2";
 import "curvefs/proto/common.proto";
+import "curvefs/proto/metaserver.proto";
 import "proto/heartbeat.proto";
 package curvefs.mds.heartbeat;
 option cc_generic_services = true;
@@ -27,6 +28,17 @@ option go_package = "curvefs/proto/heartbeat";
 //     required uint64 diskUsedByte = 2;  // the disk this copyset used
 // };
 
+enum BlockGroupDeallcateStatusCode {
+    BGDP_PROCESSING = 0;
+    BGDP_DONE = 1;
+}
+
+message BlockGroupStatInfo {
+    required uint32 fsId = 1;
+    repeated metaserver.DeallocatableBlockGroup deallocatableBlockGroups = 2;
+    map<uint64, BlockGroupDeallcateStatusCode> blockGroupDeallocateStatus = 3;
+}
+
 message CopySetInfo {
     required uint32 poolId = 1;
     required uint32 copysetId = 2;
@@ -81,6 +93,7 @@ message MetaServerHeartbeatRequest {
     required uint32 leaderCount = 7;
     required uint32 copysetCount = 8;
     required MetaServerSpaceStatus spaceStatus = 9;
+    repeated BlockGroupStatInfo blockGroupStatInfos = 10;
 };
 
 message CopySetConf {
@@ -107,11 +120,14 @@ enum HeartbeatStatusCode {
     hbMetaServerIpPortNotMatch = 2;
     hbMetaServerTokenNotMatch = 3;
     hbAnalyseCopysetError = 4;
+    hbMetaServerFSUnkown = 5;
 }
 
 message MetaServerHeartbeatResponse {
     required HeartbeatStatusCode statusCode = 1;
     repeated CopySetConf needUpdateCopysets = 2;
+    // key is fsid, value is blockgroup offset
+    map<uint64, uint64> issuedBlockGroups = 3;
 };
 
 service HeartbeatService {

curvefs/proto/metaserver.proto

@@ -167,6 +167,46 @@ enum FsFileType {
     TYPE_S3 = 4;
 };
 
+message DeallocatableBlockGroup {
+    required uint64 blockGroupOffset = 1;
+    optional uint64 deallocatableSize = 2;
+    repeated uint64 inodeIdlist = 3;
+    repeated uint64 inodeIdUnderDeallocate = 4;
+
+    oneof type {
+        IncreaseDeallocatableBlockGroup increase = 5;
+        DecreaseDeallocatableBlockGroup decrease = 6;
+        MarkDeallocatableBlockGroup mark = 7;
+    }
+}
+
+message IncreaseDeallocatableBlockGroup {
+    required uint64 increaseDeallocatableSize = 1;
+    repeated uint64 inodeIdlistAdd = 2;
+}
+
+message DecreaseDeallocatableBlockGroup {
+    required uint64 decreaseDeallocatableSize = 1;
+    repeated uint64 inodedDeallocated = 2;
+}
+
+message MarkDeallocatableBlockGroup {
+    repeated uint64 inodeIdUnderDeallocate = 2;
+}
+
+message UpdateDeallocatableBlockGroupRequest {
+    required uint32 poolId = 1;
+    required uint32 copysetId = 2;
+    required uint32 partitionId = 3;
+    required uint64 fsId = 4;
+    repeated DeallocatableBlockGroup update = 5;
+}
+
+message UpdateDeallocatableBlockGroupResponse {
+    required MetaStatusCode statusCode = 1; 
+    optional uint64 appliedIndex = 2;
+}
+
 message VolumeExtent {
     required uint64 fsOffset = 1;
     required uint64 volumeOffset = 2;
@@ -179,7 +219,7 @@ message VolumeExtentSlice {
     repeated VolumeExtent extents = 2;
 }
 
-message VolumeExtentList {
+message VolumeExtentSliceList {
     repeated VolumeExtentSlice slices = 1;
 }
 
@@ -317,7 +357,7 @@ message UpdateInodeRequest {
     map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
-    optional VolumeExtentList volumeExtents = 23;
+    optional VolumeExtentSliceList volumeExtents = 23;
 }
 
 message UpdateInodeResponse {
@@ -464,7 +504,7 @@ message GetVolumeExtentRequest {
 message GetVolumeExtentResponse {
     required MetaStatusCode statusCode = 1;
     optional uint64 appliedIndex = 2;
-    optional VolumeExtentList slices = 3;
+    optional VolumeExtentSliceList slices = 3;
 }
 
 message UpdateVolumeExtentRequest {
@@ -473,7 +513,7 @@ message UpdateVolumeExtentRequest {
     required uint32 partitionId = 3;
     required uint32 fsId = 4;
     required uint64 inodeId = 5;
-    required VolumeExtentList extents = 6;
+    required VolumeExtentSliceList extents = 6;
 }
 
 message UpdateVolumeExtentResponse {
@@ -508,4 +548,7 @@ service MetaServerService {
     // volume extent interface
     rpc GetVolumeExtent(GetVolumeExtentRequest) returns (GetVolumeExtentResponse);
     rpc UpdateVolumeExtent(UpdateVolumeExtentRequest) returns (UpdateVolumeExtentResponse);
+
+    // block group with deallocatable inode list interface
+    rpc UpdateDeallocatableBlockGroup(UpdateDeallocatableBlockGroupRequest) returns (UpdateDeallocatableBlockGroupResponse);
 }

curvefs/proto/space.proto

@@ -48,6 +48,10 @@ message BlockGroup {
     required common.BitmapLocation bitmaplocation = 4;
     // owner, who owns this block group
     optional string owner = 5;
+    // deallocating, metaserver who is deallocate this block group
+    repeated uint32 deallocating = 6;
+    // deallocated, metaserver who finish deallocate this block group
+    repeated uint32 deallocated = 7;
 }
 
 message AllocateBlockGroupRequest {

curvefs/src/client/async_request_closure.cpp

@@ -69,13 +69,20 @@ CURVEFS_ERROR UpdateVolumeExtentClosure::Wait() {
 void UpdateVolumeExtentClosure::Run() {
     auto st = GetStatusCode();
     if (!IsOK(st)) {
-        LOG(ERROR) << "UpdateVolumeExtent failed, error: "
-                   << MetaStatusCode_Name(st)
-                   << ", inodeid: " << inode_->GetInodeId();
-        inode_->MarkInodeError();
+        if (inode_ != nullptr) {
+            inode_->MarkInodeError();
+            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
+                       << MetaStatusCode_Name(st)
+                       << ", inodeid: " << inode_->GetInodeId();
+        } else {
+            LOG(ERROR) << "UpdateVolumeExtent failed, error: "
+                       << MetaStatusCode_Name(st);
+        }
     }
 
-    inode_->syncingVolumeExtentsMtx_.unlock();
+    if (inode_ != nullptr) {
+        inode_->syncingVolumeExtentsMtx_.unlock();
+    }
 
     if (sync_) {
         std::lock_guard<bthread::Mutex> lk(mtx_);

curvefs/src/client/common/common.h

@@ -60,6 +60,7 @@ enum class MetaServerOpType {
     GetVolumeExtent,
     UpdateVolumeExtent,
     CreateManageInode,
+    UpdateDeallocatableBlockGroup,
 };
 
 std::ostream &operator<<(std::ostream &os, MetaServerOpType optype);

curvefs/src/client/common/config.cpp

@@ -197,6 +197,10 @@ void InitVolumeOption(Configuration *conf, VolumeOption *volumeOpt) {
     conf->GetValueFatalIfFail("volume.fsBlockSize", &volumeOpt->fsBlockSize);
     conf->GetValueFatalIfFail("volume.allocator.type",
                               &volumeOpt->allocatorOption.type);
+    conf->GetValueFatalIfFail("volume.space.useThreshold",
+                              &volumeOpt->threshold);
+    conf->GetValueFatalIfFail("volume.space.releaseInterSec",
+                              &volumeOpt->releaseInterSec);
 
     conf->GetValueFatalIfFail(
         "volume.blockGroup.allocateOnce",

curvefs/src/client/common/config.h

@@ -73,11 +73,6 @@ struct LeaseOpt {
     uint32_t leaseTimeUs = 20000000;
 };
 
-struct SpaceAllocServerOption {
-    std::string spaceaddr;
-    uint64_t rpcTimeoutMs;
-};
-
 struct KVClientManagerOpt {
     int setThreadPooln = 4;
     int getThreadPooln = 4;
@@ -165,6 +160,9 @@ struct VolumeOption {
     uint64_t volBlockSize;
     uint64_t fsBlockSize;
     VolumeAllocatorOption allocatorOption;
+
+    double threshold{1.0};
+    uint64_t releaseInterSec{300};
 };
 
 struct ExtentManagerOption {
@@ -180,7 +178,6 @@ struct FuseClientOption {
     MetaCacheOpt metaCacheOpt;
     ExcutorOpt excutorOpt;
     ExcutorOpt excutorInternalOpt;
-    SpaceAllocServerOption spaceOpt;
     BlockDeviceClientOptions bdevOpt;
     S3Option s3Opt;
     ExtentManagerOption extentManagerOpt;

curvefs/src/client/fuse_volume_client.cpp

@@ -111,9 +111,12 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpInit(void *userdata,
         volOpts_.allocatorOption.bitmapAllocatorOption.sizePerBit;
     option.allocatorOption.bitmapAllocatorOption.smallAllocProportion =
         volOpts_.allocatorOption.bitmapAllocatorOption.smallAllocProportion;
+    option.threshold = volOpts_.threshold;
+    option.releaseInterSec = volOpts_.releaseInterSec;
 
     spaceManager_ = absl::make_unique<SpaceManagerImpl>(option, mdsClient_,
                                                         blockDeviceClient_);
+    spaceManager_->Run();
 
     storage_ = absl::make_unique<DefaultVolumeStorage>(
         spaceManager_.get(), blockDeviceClient_.get(), inodeManager_.get());

curvefs/src/client/inode_wrapper.cpp

@@ -583,7 +583,7 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
-    VolumeExtentList extents;
+    VolumeExtentSliceList extents;
     auto st = metaClient_->GetVolumeExtent(inode_.fsid(), inode_.inodeid(),
                                            true, &extents);
     VLOG(9) << "RefreshVolumeExtent, ino: " << inode_.inodeid()

curvefs/src/client/inode_wrapper.h

@@ -52,7 +52,7 @@ constexpr int kAccessTime = 1 << 0;
 constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;
 
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtentSliceList;
 
 enum class InodeStatus {
     kNormal = 0,

curvefs/src/client/metric/client_metric.h

@@ -92,13 +92,12 @@ struct MetaServerClientMetric {
     // volume extent
     InterfaceMetric updateVolumeExtent;
     InterfaceMetric getVolumeExtent;
+    InterfaceMetric updateDeallocatableBlockGroup;
 
     MetaServerClientMetric()
-        : getDentry(prefix, "getDentry"),
-          listDentry(prefix, "listDentry"),
+        : getDentry(prefix, "getDentry"), listDentry(prefix, "listDentry"),
           createDentry(prefix, "createDentry"),
-          deleteDentry(prefix, "deleteDentry"),
-          getInode(prefix, "getInode"),
+          deleteDentry(prefix, "deleteDentry"), getInode(prefix, "getInode"),
           batchGetInodeAttr(prefix, "batchGetInodeAttr"),
           batchGetXattr(prefix, "batchGetXattr"),
           createInode(prefix, "createInode"),
@@ -107,7 +106,9 @@ struct MetaServerClientMetric {
           appendS3ChunkInfo(prefix, "appendS3ChunkInfo"),
           prepareRenameTx(prefix, "prepareRenameTx"),
           updateVolumeExtent(prefix, "updateVolumeExtent"),
-          getVolumeExtent(prefix, "getVolumeExtent") {}
+          getVolumeExtent(prefix, "getVolumeExtent"),
+          updateDeallocatableBlockGroup(prefix,
+                                        "updateDeallocatableBlockGroup") {}
 };
 
 struct InflightGuard {

curvefs/src/client/rpcclient/metaserver_client.cpp

@@ -65,6 +65,7 @@ using BatchGetXAttrExcutor = TaskExecutor;
 using GetOrModifyS3ChunkInfoExcutor = TaskExecutor;
 using UpdateVolumeExtentExecutor = TaskExecutor;
 using GetVolumeExtentExecutor = TaskExecutor;
+using UpdateDeallocatableBlockGroupExcutor = TaskExecutor;
 
 using ::curvefs::common::LatencyUpdater;
 using ::curvefs::common::StreamConnection;
@@ -1465,7 +1466,7 @@ void UpdateVolumeExtentRpcDone::Run() {
     } while (0)
 
 void MetaServerClientImpl::AsyncUpdateVolumeExtent(
-    uint32_t fsId, uint64_t inodeId, const VolumeExtentList &extents,
+    uint32_t fsId, uint64_t inodeId, const VolumeExtentSliceList &extents,
     MetaServerClientDone *done) {
     auto task = AsyncRPCTask {
         (void)txId;
@@ -1473,7 +1474,7 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
         metric_.updateVolumeExtent.qps.count << 1;
         metaserver::UpdateVolumeExtentRequest request;
         SET_COMMON_FIELDS;
-        request.set_allocated_extents(new VolumeExtentList{extents});
+        request.set_allocated_extents(new VolumeExtentSliceList{extents});
 
         auto *rpcDone =
             new UpdateVolumeExtentRpcDone(taskExecutorDone, &metric_);
@@ -1493,7 +1494,8 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
 namespace {
 
 struct ParseVolumeExtentCallBack {
-    explicit ParseVolumeExtentCallBack(VolumeExtentList *ext) : extents(ext) {}
+    explicit ParseVolumeExtentCallBack(VolumeExtentSliceList *ext)
+        : extents(ext) {}
 
     bool operator()(butil::IOBuf *data) const {
         metaserver::VolumeExtentSlice slice;
@@ -1506,7 +1508,7 @@ struct ParseVolumeExtentCallBack {
         return true;
     }
 
-    VolumeExtentList *extents;
+    VolumeExtentSliceList *extents;
 };
 
 }  // namespace
@@ -1514,7 +1516,7 @@ struct ParseVolumeExtentCallBack {
 MetaStatusCode
 MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
                                       bool streaming,
-                                      VolumeExtentList *extents) {
+                                      VolumeExtentSliceList *extents) {
     auto task = RPCTask {
         (void)txId;
         (void)applyIndex;
@@ -1596,9 +1598,66 @@ MetaServerClientImpl::GetVolumeExtent(uint32_t fsId, uint64_t inodeId,
     return ConvertToMetaStatusCode(executor.DoRPCTask());
 }
 
-MetaStatusCode MetaServerClientImpl::GetInodeAttr(uint32_t fsId,
-                                                  uint64_t inodeid,
-                                                  InodeAttr *attr) {
+MetaStatusCode MetaServerClientImpl::UpdateDeallocatableBlockGroup(
+    uint32_t fsId, uint64_t inodeId, DeallocatableBlockGroupMap *statistic) {
+    auto task = RPCTask {
+        metric_.updateDeallocatableBlockGroup.qps.count << 1;
+        LatencyUpdater updater(&metric_.updateDeallocatableBlockGroup.latency);
+
+        metaserver::UpdateDeallocatableBlockGroupRequest request;
+        metaserver::UpdateDeallocatableBlockGroupResponse response;
+        request.set_poolid(poolID);
+        request.set_copysetid(copysetID);
+        request.set_partitionid(partitionID);
+        request.set_fsid(fsId);
+        auto *upadte = request.mutable_update();
+        for (auto &it : *statistic) {
+            upadte->Add()->CopyFrom(std::move(it.second));
+        }
+
+        curvefs::metaserver::MetaServerService_Stub stub(channel);
+        stub.UpdateDeallocatableBlockGroup(cntl, &request, &response, nullptr);
+
+        if (cntl->Failed()) {
+            metric_.updateDeallocatableBlockGroup.eps.count << 1;
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup failed"
+                         << ", errorCode = " << cntl->ErrorCode()
+                         << ", errorText = " << cntl->ErrorText()
+                         << ", logId = " << cntl->log_id();
+            return -cntl->ErrorCode();
+        }
+
+        auto rc = response.statuscode();
+        if (rc != MetaStatusCode::OK) {
+            metric_.updateDeallocatableBlockGroup.eps.count << 1;
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup: retCode = " << rc
+                         << ", message = " << MetaStatusCode_Name(rc);
+        } else if (response.has_appliedindex()) {
+            metaCache_->UpdateApplyIndex(CopysetGroupID(poolID, copysetID),
+                                         response.appliedindex());
+        } else {
+            LOG(WARNING) << "UpdateDeallocatableBlockGroup OK"
+                         << ", but applyIndex not set in response:"
+                         << response.DebugString();
+            return -1;
+        }
+
+        VLOG(6) << "UpdateDeallocatableBlockGroup done, request: "
+                << request.DebugString()
+                << "response: " << response.DebugString();
+        return rc;
+    };
+
+    auto taskCtx = std::make_shared<TaskContext>(
+        MetaServerOpType::UpdateDeallocatableBlockGroup, task, fsId, inodeId);
+
+    UpdateDeallocatableBlockGroupExcutor excutor(
+        opt_, metaCache_, channelManager_, std::move(taskCtx));
+    return ConvertToMetaStatusCode(excutor.DoRPCTask());
+}
+
+MetaStatusCode MetaServerClientImpl::GetInodeAttr(
+    uint32_t fsId, uint64_t inodeid, InodeAttr *attr) {
     std::set<uint64_t> inodeIds;
     inodeIds.insert(inodeid);
     std::list<InodeAttr> attrs;

curvefs/src/client/rpcclient/metaserver_client.h

@@ -30,6 +30,7 @@
 #include <set>
 #include <unordered_map>
 #include <utility>
+#include <map>
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/metaserver.pb.h"
@@ -49,21 +50,24 @@ using ::curvefs::metaserver::InodeAttr;
 using ::curvefs::metaserver::XAttr;
 using ::curvefs::metaserver::MetaStatusCode;
 using ::curvefs::metaserver::S3ChunkInfoList;
+using ::curvefs::metaserver::DeallocatableBlockGroup;
 using ::curvefs::common::StreamStatus;
 using ::curvefs::common::StreamClient;
-using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 using ::curvefs::metaserver::Time;
 
+using DeallocatableBlockGroupMap = std::map<uint64_t, DeallocatableBlockGroup>;
+using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+
 namespace curvefs {
 namespace client {
 namespace rpcclient {
 
 using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtentSliceList;
 
 struct DataIndices {
     absl::optional<S3ChunkInfoMap> s3ChunkInfoMap;
-    absl::optional<VolumeExtentList> volumeExtents;
+    absl::optional<VolumeExtentSliceList> volumeExtents;
 };
 
 class MetaServerClient {
@@ -161,13 +165,17 @@ class MetaServerClient {
 
     virtual void AsyncUpdateVolumeExtent(uint32_t fsId,
                                          uint64_t inodeId,
-                                         const VolumeExtentList &extents,
+                                         const VolumeExtentSliceList &extents,
                                          MetaServerClientDone *done) = 0;
 
     virtual MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                            uint64_t inodeId,
                                            bool streaming,
-                                           VolumeExtentList *extents) = 0;
+                                           VolumeExtentSliceList *extents) = 0;
+
+    virtual MetaStatusCode
+    UpdateDeallocatableBlockGroup(uint32_t fsId, uint64_t inodeId,
+                                  DeallocatableBlockGroupMap *statistic) = 0;
 };
 
 class MetaServerClientImpl : public MetaServerClient {
@@ -265,13 +273,17 @@ class MetaServerClientImpl : public MetaServerClient {
 
     void AsyncUpdateVolumeExtent(uint32_t fsId,
                                  uint64_t inodeId,
-                                 const VolumeExtentList &extents,
+                                 const VolumeExtentSliceList &extents,
                                  MetaServerClientDone *done) override;
 
     MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                    uint64_t inodeId,
                                    bool streaming,
-                                   VolumeExtentList *extents) override;
+                                   VolumeExtentSliceList *extents) override;
+
+    MetaStatusCode UpdateDeallocatableBlockGroup(
+        uint32_t fsId, uint64_t inodeId,
+        DeallocatableBlockGroupMap *statistic) override;
 
  private:
     MetaStatusCode UpdateInode(const UpdateInodeRequest &request,

curvefs/src/client/volume/extent_cache.cpp

@@ -228,7 +228,7 @@ void ExtentCache::SetOption(const ExtentCacheOption& option) {
     LOG(INFO) << "ExtentCacheOption: [" << option_ << "]";
 }
 
-void ExtentCache::Build(const VolumeExtentList &extents) {
+void ExtentCache::Build(const VolumeExtentSliceList &extents) {
     WriteLockGuard lk(lock_);
     slices_.clear();
     dirties_.clear();
@@ -238,8 +238,8 @@ void ExtentCache::Build(const VolumeExtentList &extents) {
     }
 }
 
-VolumeExtentList ExtentCache::GetDirtyExtents() {
-    VolumeExtentList result;
+VolumeExtentSliceList ExtentCache::GetDirtyExtents() {
+    VolumeExtentSliceList result;
     WriteLockGuard lk(lock_);
     for (const auto* slice : dirties_) {
         *result.add_slices() = slice->ToVolumeExtentSlice();

curvefs/src/client/volume/extent_cache.h

@@ -43,7 +43,7 @@ namespace client {
 using ::curvefs::volume::ReadPart;
 using ::curvefs::volume::WritePart;
 using ::curvefs::metaserver::VolumeExtentSlice;
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtentSliceList;
 
 struct ExtentCacheOption {
     // preallocation size if offset ~ length is not allocated
@@ -61,7 +61,7 @@ class ExtentCache {
 
     static void SetOption(const ExtentCacheOption& option);
 
-    void Build(const VolumeExtentList& extents);
+    void Build(const VolumeExtentSliceList& extents);
 
     void DivideForWrite(uint64_t offset,
                         uint64_t len,
@@ -81,7 +81,7 @@ class ExtentCache {
 
     bool HasDirtyExtents() const;
 
-    VolumeExtentList GetDirtyExtents();
+    VolumeExtentSliceList GetDirtyExtents();
 
     std::unordered_map<uint64_t, std::map<uint64_t, PExtent>>
     GetExtentsForTesting() const;

curvefs/src/mds/heartbeat/BUILD

@@ -27,6 +27,7 @@ cc_library(
         "//curvefs/proto:curvefs_topology_cc_proto",
         "//curvefs/proto:metaserver_cc_proto",
         "//curvefs/src/mds/topology:curvefs_topology",
+        "//curvefs/src/mds/space:curvefs_mds_space",
         "//external:brpc",
         "//external:gflags",
         "//external:glog",

curvefs/src/mds/heartbeat/heartbeat_manager.cpp

@@ -42,8 +42,9 @@ namespace mds {
 namespace heartbeat {
 HeartbeatManager::HeartbeatManager(
     const HeartbeatOption &option, const std::shared_ptr<Topology> &topology,
-    const std::shared_ptr<Coordinator> &coordinator)
-    : topology_(topology) {
+    const std::shared_ptr<Coordinator> &coordinator,
+    const std::shared_ptr<SpaceManager> &spaceManager)
+    : topology_(topology), spaceManager_(spaceManager) {
     healthyChecker_ =
         std::make_shared<MetaserverHealthyChecker>(option, topology);
 
@@ -86,24 +87,6 @@ void HeartbeatManager::Stop() {
     }
 }
 
-void HeartbeatManager::MetaServerHealthyChecker() {
-    while (sleeper_.wait_for(
-        std::chrono::milliseconds(metaserverHealthyCheckerRunInter_))) {
-        healthyChecker_->CheckHeartBeatInterval();
-    }
-}
-
-void HeartbeatManager::UpdateMetaServerSpace(
-    const MetaServerHeartbeatRequest &request) {
-    MetaServerSpace space(request.spacestatus());
-    TopoStatusCode ret =
-        topology_->UpdateMetaServerSpace(space, request.metaserverid());
-    if (ret != TopoStatusCode::TOPO_OK) {
-        LOG(ERROR) << "heartbeat UpdateMetaServerSpace fail, ret = "
-                   << TopoStatusCode_Name(ret);
-    }
-}
-
 void HeartbeatManager::MetaServerHeartbeat(
     const MetaServerHeartbeatRequest &request,
     MetaServerHeartbeatResponse *response) {
@@ -127,6 +110,42 @@ void HeartbeatManager::MetaServerHeartbeat(
     UpdateMetaServerSpace(request);
 
     // dealing with copysets included in the heartbeat request
+    Coordinate(request, response);
+
+    // update deallocatable block group info
+    UpdateDeallocatableBlockGroup(request, response);
+}
+
+void HeartbeatManager::MetaServerHealthyChecker() {
+    while (sleeper_.wait_for(
+        std::chrono::milliseconds(metaserverHealthyCheckerRunInter_))) {
+        healthyChecker_->CheckHeartBeatInterval();
+
+        auto metaservers =
+            topology_->GetMetaServerInCluster([](const MetaServer &ms) {
+                return ms.GetOnlineState() == OnlineState::ONLINE;
+            });
+
+        uint32_t currentOnlineMetaServerNum =
+            onlineMetaServerNum_.load(std::memory_order_acquire);
+        onlineMetaServerNum_.compare_exchange_strong(currentOnlineMetaServerNum,
+                                                     metaservers.size());
+    }
+}
+
+void HeartbeatManager::UpdateMetaServerSpace(
+    const MetaServerHeartbeatRequest &request) {
+    MetaServerSpace space(request.spacestatus());
+    TopoStatusCode ret =
+        topology_->UpdateMetaServerSpace(space, request.metaserverid());
+    if (ret != TopoStatusCode::TOPO_OK) {
+        LOG(ERROR) << "heartbeat UpdateMetaServerSpace fail, ret = "
+                   << TopoStatusCode_Name(ret);
+    }
+}
+
+void HeartbeatManager::Coordinate(const MetaServerHeartbeatRequest &request,
+                                  MetaServerHeartbeatResponse *response) {
     for (auto &value : request.copysetinfos()) {
         // convert copysetInfo from heartbeat format to topology format
         ::curvefs::mds::topology::CopySetInfo reportCopySetInfo;
@@ -168,6 +187,34 @@ void HeartbeatManager::MetaServerHeartbeat(
     }
 }
 
+void HeartbeatManager::UpdateDeallocatableBlockGroup(
+    const MetaServerHeartbeatRequest &request,
+    MetaServerHeartbeatResponse *response) {
+    uint32_t metaserverId = request.metaserverid();
+
+    for (auto &info : request.blockgroupstatinfos()) {
+        auto volumeSpace = spaceManager_->GetVolumeSpace(info.fsid());
+        if (volumeSpace == nullptr) {
+            LOG(ERROR) << "HeartbeatManager fsid=" << info.fsid()
+                       << " do not have volumeSpace manager";
+            response->set_statuscode(HeartbeatStatusCode::hbMetaServerFSUnkown);
+            return;
+        }
+
+        uint64_t issued = 0;
+        bool hasissued = volumeSpace->UpdateDeallocatableBlockGroup(
+            metaserverId, onlineMetaServerNum_, info.deallocatableblockgroups(),
+            info.blockgroupdeallocatestatus(), &issued);
+        if (hasissued) {
+            response->mutable_issuedblockgroups()->insert(
+                {info.fsid(), issued});
+            LOG(INFO) << "HeartbeatManager issue metaserverid=" << metaserverId
+                      << " blockgroup, fsid=" << info.fsid()
+                      << ", issued=" << issued;
+        }
+    }
+}
+
 HeartbeatStatusCode HeartbeatManager::CheckRequest(
     const MetaServerHeartbeatRequest &request) {
     MetaServer metaServer;

curvefs/src/mds/heartbeat/heartbeat_manager.h

@@ -36,6 +36,7 @@
 #include "curvefs/src/mds/heartbeat/topo_updater.h"
 #include "curvefs/src/mds/schedule/coordinator.h"
 #include "curvefs/src/mds/topology/topology.h"
+#include "curvefs/src/mds/space/manager.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/interruptible_sleeper.h"
 
@@ -43,6 +44,7 @@ using ::curvefs::mds::topology::PoolIdType;
 using ::curvefs::mds::topology::CopySetIdType;
 using ::curvefs::mds::topology::Topology;
 using ::curvefs::mds::schedule::Coordinator;
+using ::curvefs::mds::space::SpaceManager;
 
 using ::curve::common::Thread;
 using ::curve::common::Atomic;
@@ -65,7 +67,8 @@ class HeartbeatManager {
  public:
     HeartbeatManager(const HeartbeatOption &option,
                      const std::shared_ptr<Topology> &topology,
-                     const std::shared_ptr<Coordinator> &coordinator);
+                     const std::shared_ptr<Coordinator> &coordinator,
+                     const std::shared_ptr<SpaceManager> &spaceManager);
 
     ~HeartbeatManager() { Stop(); }
 
@@ -98,6 +101,7 @@ class HeartbeatManager {
                              MetaServerHeartbeatResponse *response);
 
  private:
+    FRIEND_TEST(TestHeartbeatManager, TEST_UpdateDeallocatableBlockGroup);
     /**
      * @brief Background thread for heartbeat timeout inspection
      */
@@ -140,10 +144,18 @@ class HeartbeatManager {
 
     void UpdateMetaServerSpace(const MetaServerHeartbeatRequest &request);
 
+    void Coordinate(const MetaServerHeartbeatRequest &request,
+                    MetaServerHeartbeatResponse *response);
+
+    void
+    UpdateDeallocatableBlockGroup(const MetaServerHeartbeatRequest &request,
+                                  MetaServerHeartbeatResponse *response);
+
  private:
     // Dependencies of heartbeat
     std::shared_ptr<Topology> topology_;
     std::shared_ptr<Coordinator> coordinator_;
+    std::shared_ptr<SpaceManager> spaceManager_;
 
     // healthyChecker_ health checker running in background thread
     std::shared_ptr<MetaserverHealthyChecker> healthyChecker_;
@@ -156,6 +168,8 @@ class HeartbeatManager {
     // Manage metaserverHealthyChecker threads
     Thread backEndThread_;
 
+    std::atomic<uint32_t> onlineMetaServerNum_;
+
     Atomic<bool> isStop_;
     InterruptibleSleeper sleeper_;
     int metaserverHealthyCheckerRunInter_;

curvefs/src/mds/mds.cpp

@@ -66,6 +66,8 @@ void MDS::InitOptions(std::shared_ptr<Configuration> conf) {
     conf_ = std::move(conf);
     conf_->GetValueFatalIfFail("mds.listen.addr", &options_.mdsListenAddr);
     conf_->GetValueFatalIfFail("mds.dummy.port", &options_.dummyPort);
+    conf_->GetValueFatalIfFail("mds.space.calIntervalSec",
+                               &options_.mdsSpaceCalIntervalSec);
 
     InitMetaServerOption(&options_.metaserverOptions);
     InitTopologyOption(&options_.topologyOptions);
@@ -169,8 +171,8 @@ void MDS::Init() {
     space::MdsProxyManager::SetProxyOptions(options_.bsMdsProxyOptions);
 
     fsStorage_ = std::make_shared<PersisKVStorage>(etcdClient_);
-    spaceManager_ =
-        std::make_shared<SpaceManagerImpl>(etcdClient_, fsStorage_);
+    spaceManager_ = std::make_shared<SpaceManagerImpl>(
+        etcdClient_, fsStorage_, options_.mdsSpaceCalIntervalSec);
     metaserverClient_ =
         std::make_shared<MetaserverClient>(options_.metaserverOptions);
     auto dlock = std::make_shared<DLock>(options_.dLockOptions, etcdClient_);
@@ -417,7 +419,7 @@ void MDS::InitHeartbeatManager() {
 
     heartbeatOption.mdsStartTime = steady_clock::now();
     heartbeatManager_ = std::make_shared<HeartbeatManager>(
-        heartbeatOption, topology_, coordinator_);
+        heartbeatOption, topology_, coordinator_, spaceManager_);
     heartbeatManager_->Init();
 }
 

curvefs/src/mds/mds.h

@@ -88,6 +88,8 @@ struct MDSOptions {
     MetaserverOptions metaserverOptions;
     // TODO(add EtcdConf): add etcd configure
 
+    uint64_t mdsSpaceCalIntervalSec;
+
     TopologyOption topologyOptions;
     HeartbeatOption heartbeatOption;
     ScheduleOption scheduleOption;

curvefs/src/mds/space/BUILD

@@ -27,6 +27,8 @@ cc_library(
     deps = [
         "//curvefs/proto:mds_cc_proto",
         "//curvefs/proto:space_cc_proto",
+        "//curvefs/proto:metaserver_cc_proto",
+        "//curvefs/proto:curvefs_heartbeat_cc_proto",
         "//curvefs/src/mds:curvefs_mds_fs_storage",
         "//curvefs/src/mds/codec:fs_mds_codec",
         "//external:brpc",

curvefs/src/mds/space/manager.cpp

@@ -59,8 +59,9 @@ SpaceErrCode SpaceManagerImpl::AddVolume(const FsInfo& fsInfo) {
         }
     }
 
-    auto space = VolumeSpace::Create(fsInfo.fsid(), fsInfo.detail().volume(),
-                                     storage_.get(), fsStorage_.get());
+    auto space =
+        VolumeSpace::Create(fsInfo.fsid(), fsInfo.detail().volume(),
+                            storage_.get(), fsStorage_.get(), calcIntervalSec_);
 
     if (!space) {
         LOG(ERROR) << "Create volume space failed, fsId: " << fsInfo.fsid();

curvefs/src/mds/space/manager.h

@@ -53,10 +53,10 @@ class SpaceManager {
 class SpaceManagerImpl final : public SpaceManager {
  public:
     SpaceManagerImpl(
-        const std::shared_ptr<curve::kvstorage::KVStorageClient>& kvstore,
-        std::shared_ptr<FsStorage> fsStorage)
+        const std::shared_ptr<curve::kvstorage::KVStorageClient> &kvstore,
+        std::shared_ptr<FsStorage> fsStorage, uint64_t calcIntervalSec)
         : storage_(new BlockGroupStorageImpl(kvstore)),
-          fsStorage_(std::move(fsStorage)) {}
+          fsStorage_(std::move(fsStorage)), calcIntervalSec_(calcIntervalSec) {}
 
     SpaceManagerImpl(const SpaceManagerImpl&) = delete;
     SpaceManagerImpl& operator=(const SpaceManagerImpl&) = delete;
@@ -80,6 +80,8 @@ class SpaceManagerImpl final : public SpaceManager {
     curve::common::GenericNameLock<Mutex> namelock_;
 
     std::shared_ptr<FsStorage> fsStorage_;
+
+    uint64_t calcIntervalSec_;
 };
 
 }  // namespace space

curvefs/src/mds/space/volume_space.cpp

@@ -65,9 +65,10 @@ BlockGroup BuildBlockGroupFromClean(uint64_t offset,
 }  // namespace
 
 std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
-                                                 const Volume& volume,
-                                                 BlockGroupStorage* storage,
-                                                 FsStorage* fsStorage) {
+                                                 const Volume &volume,
+                                                 BlockGroupStorage *storage,
+                                                 FsStorage *fsStorage,
+                                                 uint64_t calcIntervalSec) {
     if (!volume.has_volumesize()) {
         LOG(ERROR) << "Volume info doesn't have size";
         return nullptr;
@@ -94,7 +95,7 @@ std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
     // for allocated groups, client will send heartbeat to update usage
     uint64_t availableSize = 0;
     std::set<uint64_t> usedGroupOffsets;
-    for (auto& group : groups) {
+    for (auto &group : groups) {
         usedGroupOffsets.insert(group.offset());
         // availableSize += group.available();
 
@@ -102,8 +103,11 @@ std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
         assert(offset % blockGroupSize == 0);
         assert(group.size() == blockGroupSize);
         assert(group.bitmaplocation() == location);
+        assert((group.has_owner() && group.deallocating_size()) == 0);
         if (group.has_owner()) {
             space->allocatedGroups_.emplace(offset, std::move(group));
+        } else if (group.deallocating_size()) {
+            space->deallocatingGroups_.emplace(offset, std::move(group));
         } else {
             space->availableGroups_.emplace(offset, std::move(group));
             availableSize += group.available();
@@ -123,6 +127,9 @@ std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
 
     space->cleanGroups_ = std::move(cleanGroupOffsets);
 
+    space->calcIntervalSec_ = calcIntervalSec;
+    space->metaserverNum_ = 0;
+
     LOG(INFO) << "Init volume space success, fsId: " << fsId
               << ", size: " << volumeSize << ", available: " << availableSize
               << ", block size: " << blockSize
@@ -130,6 +137,7 @@ std::unique_ptr<VolumeSpace> VolumeSpace::Create(uint32_t fsId,
               << ", total groups: " << volumeSize / blockGroupSize
               << ", allocated groups: " << space->allocatedGroups_.size()
               << ", available groups: " << space->availableGroups_.size()
+              << ", deallocating groups: " << space->deallocatingGroups_.size()
               << ", clean groups: " << space->cleanGroups_.size();
 
     return space;
@@ -447,6 +455,84 @@ void VolumeSpace::AddCleanGroups(uint64_t origin, uint64_t extended) {
     }
 }
 
+void VolumeSpace::CalBlockGroupAvailableForDeAllocate() {
+    LockGuard lk(mtx_);
+    LockGuard statlk(statmtx_);
+    // check whether deallocatingGroups_ need move to availableGroups_
+    auto iter = deallocatingGroups_.begin();
+    while (iter != deallocatingGroups_.end()) {
+        uint32_t expectMetaserverNum =
+            metaserverNum_.load(std::memory_order_acquire);
+        uint32_t actualMetaserverNum = iter->second.deallocated().size();
+        if (actualMetaserverNum != expectMetaserverNum) {
+            ++iter;
+            continue;
+        }
+
+        LOG(INFO)
+            << "VolumeSpace move deallocatingGroups_ to availableGroups_, "
+               "fsId="
+            << fsId_ << ", block group offset=" << iter->first;
+
+        iter->second.clear_deallocated();
+        auto err = PersistBlockGroup(iter->second);
+        if (err != SpaceOk) {
+            LOG(ERROR) << "VolumeSpace put block group failed, fsId=" << fsId_
+                       << ", block group offset=" << iter->first
+                       << ", err=" << SpaceErrCode_Name(err);
+            continue;
+        }
+
+        availableGroups_.emplace(iter->first, std::move(iter->second));
+        iter = deallocatingGroups_.erase(iter);
+        metric_.dealloc << 1;
+    }
+
+    // check whether the cal conditions are met
+    if (!waitDeallocateGroups_.empty() || !deallocatingGroups_.empty() ||
+        availableGroups_.empty() || summary_.empty()) {
+        LOG_EVERY_N(INFO, 50)
+            << "VolumeSpace wait for cal, "
+               "waitDeallocateGroups_ size="
+            << waitDeallocateGroups_.size()
+            << ",deallocatingGroups_ size=" << deallocatingGroups_.size()
+            << ",availableGroups_ size=" << availableGroups_.size()
+            << ",summary_ size=" << summary_.size();
+        return;
+    }
+
+    // get the keys shared by availableGroups_ and summary_
+    std::vector<std::pair<uint64_t, uint64_t>> commonKeys;
+    for (const auto &item : summary_) {
+        if (availableGroups_.count(item.first)) {
+            commonKeys.push_back(item);
+        }
+    }
+
+    // sort
+    std::sort(commonKeys.begin(), commonKeys.end(),
+              [](const std::pair<uint64_t, uint64_t> &a,
+                 const std::pair<uint64_t, uint64_t> &b) {
+                  return a.second > b.second;
+              });
+    uint64_t selectKey = commonKeys[0].first;
+    LOG(INFO) << "VolumeSpace cal blockgroup=" << selectKey
+              << ",fsid=" << fsId_ << " wait for deallocate";
+
+    // move key from availableGroups_ to waitDeallocateGroups_
+    BlockGroup selectGroup;
+    auto it = availableGroups_.find(selectKey);
+    if (it != availableGroups_.end()) {
+        assert(!it->second.has_owner());
+        selectGroup = std::move(it->second);
+        selectGroup.clear_owner();
+        availableGroups_.erase(it);
+    }
+    assert(waitDeallocateGroups_.count(selectKey) == 0);
+    waitDeallocateGroups_.emplace(selectKey, std::move(selectGroup));
+    metric_.waitingDealloc << 1;
+}
+
 SpaceErrCode VolumeSpace::ExtendVolume() {
     if (!volume_.autoextend()) {
         LOG(WARNING) << "Auto extend is not supported, fsId: " << fsId_
@@ -461,7 +547,7 @@ SpaceErrCode VolumeSpace::ExtendVolume() {
     LOG(INFO) << "Going to extend volume size from " << volume_.volumesize()
               << " to " << extended;
 
-    auto* proxy = MdsProxyManager::GetInstance().GetOrCreateProxy(
+    auto *proxy = MdsProxyManager::GetInstance().GetOrCreateProxy(
         {volume_.cluster().begin(), volume_.cluster().end()});
     if (proxy == nullptr) {
         LOG(WARNING) << "Fail to get or create proxy";
@@ -487,6 +573,217 @@ SpaceErrCode VolumeSpace::ExtendVolume() {
     return SpaceOk;
 }
 
+void VolumeSpace::Run() {
+    calThread_ = std::thread([&] {
+        while (sleeper_.wait_for(std::chrono::seconds(calcIntervalSec_))) {
+            CalBlockGroupAvailableForDeAllocate();
+        }
+    });
+}
+
+void VolumeSpace::Stop() {
+    LOG(INFO) << "VolumeSpace stopping, fsid=" << fsId_;
+
+    sleeper_.interrupt();
+    calThread_.join();
+
+    LOG(INFO) << "VolumeSpace stopped, fsid=" << fsId_;
+}
+
+bool VolumeSpace::UpdateDeallocatableBlockGroup(
+    uint32_t metaserverId, uint32_t metaserverNum,
+    const DeallocatableBlockGroupVec &groups,
+    const BlockGroupDeallcateStatusMap &stats, uint64_t *issue) {
+    uint32_t current = metaserverNum_.load(std::memory_order_acquire);
+    metaserverNum_.compare_exchange_strong(current, metaserverNum,
+                                           std::memory_order_acq_rel);
+
+    UpdateBlockGroupDeallocatableSpace(metaserverId, groups);
+
+    UpdateDeallocatingBlockGroup(metaserverId, stats);
+
+    return SelectBlockGroupForDeAllocate(metaserverId, issue);
+}
+
+void VolumeSpace::UpdateBlockGroupDeallocatableSpace(
+    uint32_t metaserverId, const DeallocatableBlockGroupVec &groups) {
+    LockGuard statlk(statmtx_);
+
+    // update summary_ with latest groups
+    std::unordered_map<uint64_t, uint64_t> reportGroups;
+    for (auto &group : groups) {
+        auto offset = group.blockgroupoffset();
+        auto deallocatableSize = group.deallocatablesize();
+        reportGroups[offset] = deallocatableSize;
+
+        auto iter = summary_.find(offset);
+        if (iter == summary_.end()) {
+            summary_.emplace(offset, deallocatableSize);
+        } else {
+            iter->second += group.deallocatablesize();
+        }
+        VLOG(6) << "VolumeSpace update summary, fsId=" << fsId_
+                << ", blockGroupOffset=" << offset
+                << ", deallocatableSize=" << group.deallocatablesize();
+    }
+
+    // remove groups from last round of reporting and record latest in
+    // lastUpdate_
+    auto lastUpdateIter = lastUpdate_.find(metaserverId);
+    if (lastUpdateIter == lastUpdate_.end()) {
+        lastUpdate_[metaserverId] = groups;
+    } else {
+        for (auto &group : lastUpdateIter->second) {
+            auto offset = group.blockgroupoffset();
+            auto lastDeallocatableSize = group.deallocatablesize();
+            if (reportGroups.count(offset) == 0) {
+                continue;
+            }
+
+            summary_[offset] -= lastDeallocatableSize;
+            if (summary_[offset] == 0) {
+                summary_.erase(offset);
+                LOG(INFO) << "VolumeSpace remove block group from summary, no "
+                             "need deallocatable, "
+                             "fsId="
+                          << fsId_ << ", blcokGroupOffset=" << offset;
+            }
+
+            group.set_deallocatablesize(reportGroups[offset]);
+        }
+    }
+}
+
+void VolumeSpace::UpdateDeallocatingBlockGroup(
+    uint32_t metaserverId, const BlockGroupDeallcateStatusMap &stats) {
+    LockGuard lk(mtx_);
+
+    // get completed deallocate blockgroup
+    std::vector<uint64_t> doneGroups;
+    for (auto &stat : stats) {
+        auto offset = stat.first;
+        auto status = stat.second;
+
+        auto iter = deallocatingGroups_.find(offset);
+        if (iter == deallocatingGroups_.end()) {
+            LOG(ERROR) << "VolumeSpace block group not found in "
+                          "deallocatingGroups_, "
+                          "fsId: "
+                       << fsId_ << ", blcokGroupOffset: " << offset;
+            continue;
+        }
+
+        auto arlreadyDeallocated =
+            std::find(iter->second.deallocated().begin(),
+                      iter->second.deallocated().end(), metaserverId);
+        if (status == BlockGroupDeallcateStatusCode::BGDP_DONE &&
+            arlreadyDeallocated == iter->second.deallocated().end()) {
+            doneGroups.emplace_back(offset);
+            LOG(INFO) << "VolumeSpace block group is deallocated done, fsId: "
+                      << fsId_ << ", blcokGroupOffset: " << offset
+                      << ", metaserverId: " << metaserverId;
+        }
+    }
+
+    // update the metaserver from the deallocating state of the blockgroup to
+    // the deallocated state
+    for (auto offset : doneGroups) {
+        auto iter = deallocatingGroups_.find(offset);
+        assert(iter != deallocatingGroups_.end());
+
+        std::vector<uint32_t> newDeallocating;
+        for (auto id : iter->second.deallocating()) {
+            if (metaserverId == id) {
+                continue;
+            }
+            newDeallocating.emplace_back(id);
+        }
+
+        iter->second.add_deallocated(metaserverId);
+        auto mutableDeallocating = iter->second.mutable_deallocating();
+        mutableDeallocating->Resize(newDeallocating.size(), 0);
+        std::copy(newDeallocating.begin(), newDeallocating.end(),
+                  mutableDeallocating->begin());
+
+        auto err = PersistBlockGroup(iter->second);
+        // TODO(wuhanqing): handle error, and rollback if necessary
+        if (err != SpaceOk) {
+            LOG(ERROR) << "VolumeSpace put block group failed, fsId: " << fsId_
+                       << ", offset: " << offset
+                       << ", err: " << SpaceErrCode_Name(err);
+            return;
+        }
+    }
+}
+
+bool VolumeSpace::SelectBlockGroupForDeAllocate(uint32_t metaserverId,
+                                                uint64_t *issue) {
+    assert(issue != nullptr);
+    LockGuard lk(mtx_);
+
+    // TODO(ilixiaocui): support more groups to be issued
+    if (!waitDeallocateGroups_.empty()) {
+        auto iter = waitDeallocateGroups_.begin();
+        auto offset = iter->first;
+        iter->second.add_deallocating(metaserverId);
+        deallocatingGroups_[offset] = std::move(iter->second);
+        waitDeallocateGroups_.erase(iter);
+        metric_.waitingDealloc << -1;
+
+        auto err = PersistBlockGroup(deallocatingGroups_[offset]);
+        // TODO(wuhanqing): handle error, and rollback if necessary
+        if (err != SpaceOk) {
+            LOG(ERROR) << "VolumeSpace put block group failed, fsId: " << fsId_
+                       << ", offset: " << offset
+                       << ", err: " << SpaceErrCode_Name(err);
+            return false;
+        }
+        *issue = offset;
+        LOG(INFO) << "VolumeSpace issue block group from "
+                     "waitDeallocateGroups_, fsId: "
+                  << fsId_ << ", offset: " << offset
+                  << ", to metaserverId: " << metaserverId;
+        return true;
+    }
+
+    auto deallocatingOne = deallocatingGroups_.begin();
+    if (deallocatingOne != deallocatingGroups_.end()) {
+        *issue = deallocatingOne->first;
+
+        auto alreadyIssued = std::find(
+            deallocatingOne->second.deallocating().begin(),
+            deallocatingOne->second.deallocating().end(), metaserverId);
+        if (alreadyIssued != deallocatingOne->second.deallocating().end()) {
+            return true;
+        }
+
+        auto alreadyDone = std::find(
+            deallocatingOne->second.deallocated().begin(),
+            deallocatingOne->second.deallocated().end(), metaserverId);
+        if (alreadyDone != deallocatingOne->second.deallocated().end()) {
+            return false;
+        }
+
+        deallocatingOne->second.add_deallocating(metaserverId);
+        auto err = PersistBlockGroup(deallocatingOne->second);
+        // TODO(wuhanqing): handle error, and rollback if necessary
+        if (err != SpaceOk) {
+            LOG(ERROR) << "VolumeSpace put block group failed, fsId: " << fsId_
+                       << ", offset: " << *issue
+                       << ", err: " << SpaceErrCode_Name(err);
+            return false;
+        }
+
+        LOG(INFO) << "VolumeSpace issue block group from "
+                     "deallocatingGroups_, fsId: "
+                  << fsId_ << ", offset: " << *issue
+                  << ", to metaserverId: " << metaserverId;
+        return true;
+    }
+
+    return false;
+}
+
 uint64_t ExtendedSize(uint64_t origin, double factor, uint64_t alignment) {
     return common::align_up(
         static_cast<uint64_t>(std::floor(static_cast<double>(origin) * factor)),

curvefs/src/mds/space/volume_space.h

@@ -36,13 +36,26 @@
 
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/proto/space.pb.h"
+#include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/proto/heartbeat.pb.h"
 #include "curvefs/src/mds/fs_storage.h"
 #include "curvefs/src/mds/space/block_group_storage.h"
+#include "src/common/interruptible_sleeper.h"
 
 namespace curvefs {
 namespace mds {
 namespace space {
 
+using ::curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode;
+using ::curvefs::metaserver::DeallocatableBlockGroup;
+using ::curve::common::InterruptibleSleeper;
+
+using DeallocatableBlockGroupVec =
+    google::protobuf::RepeatedPtrField<DeallocatableBlockGroup>;
+using BlockGroupDeallcateStatusMap = ::google::protobuf::Map<
+    ::google::protobuf::uint64,
+    ::curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode>;
+
 class AbstractVolumeSpace {
  public:
     virtual ~AbstractVolumeSpace() = default;
@@ -58,6 +71,12 @@ class AbstractVolumeSpace {
 
     virtual SpaceErrCode ReleaseBlockGroups(
         const std::vector<BlockGroup>& blockGroups) = 0;
+
+    virtual bool
+    UpdateDeallocatableBlockGroup(uint32_t metaserverId, uint32_t metaserverNum,
+                                  const DeallocatableBlockGroupVec &groups,
+                                  const BlockGroupDeallcateStatusMap &stats,
+                                  uint64_t *issue) = 0;
 };
 
 using ::curvefs::common::BitmapLocation;
@@ -65,13 +84,13 @@ using ::curvefs::common::Volume;
 
 class VolumeSpace final : public AbstractVolumeSpace {
  public:
-    static std::unique_ptr<VolumeSpace> Create(uint32_t fsId,
-                                               const Volume& volume,
-                                               BlockGroupStorage* storage,
-                                               FsStorage* fsStorage);
+    static std::unique_ptr<VolumeSpace>
+    Create(uint32_t fsId, const Volume &volume, BlockGroupStorage *storage,
+           FsStorage *fsStorage, uint64_t calcIntervalSec);
 
     VolumeSpace(const VolumeSpace&) = delete;
     VolumeSpace& operator=(const VolumeSpace&) = delete;
+    ~VolumeSpace() {}
 
     /**
      * @brief Allocate block groups
@@ -101,6 +120,20 @@ class VolumeSpace final : public AbstractVolumeSpace {
      */
     SpaceErrCode RemoveAllBlockGroups();
 
+
+    bool
+    UpdateDeallocatableBlockGroup(uint32_t metaserverId, uint32_t metaserverNum,
+                                  const DeallocatableBlockGroupVec &groups,
+                                  const BlockGroupDeallcateStatusMap &stats,
+                                  uint64_t *issue);
+
+    /**
+     * @brief Calculate block group that can be recycled
+     */
+    void Run();
+
+    void Stop();
+
  private:
     VolumeSpace(uint32_t fsId,
                 Volume volume,
@@ -131,6 +164,24 @@ class VolumeSpace final : public AbstractVolumeSpace {
 
     void AddCleanGroups(uint64_t origin, uint64_t extended);
 
+    // pick out blockgroups that can be authorized for metaserver processing
+    void CalBlockGroupAvailableForDeAllocate();
+
+    // update the deallocatable space of blockgroup
+    void UpdateBlockGroupDeallocatableSpace(
+        uint32_t metaserverId, const DeallocatableBlockGroupVec &groups);
+
+    // update the deallocating progress reported by the metaserver
+    void
+    UpdateDeallocatingBlockGroup(uint32_t metaserverId,
+                                 const BlockGroupDeallcateStatusMap &stats);
+
+    // check whether there is currently a blockgroup that can be recycled and
+    // send it to the metaserver
+    bool SelectBlockGroupForDeAllocate(uint32_t metaserverId, uint64_t *issue);
+
+    FRIEND_TEST(VolumeSpaceTest, Test_CalBlockGroupAvailableForDeAllocate);
+
  private:
     // persist block group to backend storage
     SpaceErrCode PersistBlockGroup(const BlockGroup& group);
@@ -164,6 +215,13 @@ class VolumeSpace final : public AbstractVolumeSpace {
     // 3. clean
     //    these block groups' space is never used, and they can be allocated to
     //    other clients.
+    // 4. waitDeallocate
+    //    these block groups wait for being deallocated by
+    //    metaservers, and they can not allocate to clients.
+    // 5. deallocating
+    //    these block groups are being deallocated by metaservers, and they can
+    //    not allocate to clients.
+    //    these block groups are not persisted into storage.
 
     // key is block group offset
     std::unordered_map<uint64_t, BlockGroup> allocatedGroups_;
@@ -174,12 +232,55 @@ class VolumeSpace final : public AbstractVolumeSpace {
     // stores clean block groups' offset
     std::unordered_set<uint64_t> cleanGroups_;
 
+    // key is block group offset
+    std::unordered_map<uint64_t, BlockGroup> waitDeallocateGroups_;
+
+    // key is block group offset
+    std::unordered_map<uint64_t, BlockGroup> deallocatingGroups_;
+
+    mutable bthread::Mutex statmtx_;
+    // Summarize the deallocatable space of the blockgroup in this volume
+    // reported by all metaservers
+    //
+    // The metaserver reports the full amount of information instead of
+    // incremental information, so adding this data to the statistical data
+    // requires subtracting the last data
+    //
+    // - lastupdate_
+    //   metaserver report last time
+    // - summary_
+    //   summary add metaserver report info and remove lastupdate info
+
+    // key is metaserver id
+    std::unordered_map<uint32_t, DeallocatableBlockGroupVec> lastUpdate_;
+
+    // key is block group offset, value is deallocatable size
+    std::unordered_map<uint64_t, uint64_t> summary_;
+
     BlockGroupStorage* storage_;
 
     FsStorage* fsStorage_;
+
+    std::thread calThread_;
+    InterruptibleSleeper sleeper_;
+    int64_t calcIntervalSec_;
+
+    std::atomic<uint32_t> metaserverNum_;
+
+ private:
+    struct Metric {
+        bvar::Adder<uint64_t> dealloc;
+        bvar::Adder<uint64_t> waitingDealloc;
+
+        Metric()
+            : dealloc("mds_volume_space_dealloc"),
+              waitingDealloc("mds_volume_space_wait_dealloc") {}
+    };
+
+    Metric metric_;
 };
 
-// Calculate extended size based on origin with factor, and the result size is
+// Calculate extended size based on origin with factor, and the r esult size is
 // aligned with alignment
 uint64_t ExtendedSize(uint64_t origin, double factor, uint64_t alignment);
 

curvefs/src/metaserver/BUILD

@@ -29,6 +29,10 @@ cc_library(
         ["storage/*.cpp"],
     ) + glob(
         ["streaming/*.cpp"],
+    ) + glob(
+        ["space/*.cpp"],
+    ) + glob(
+        ["mds/*.cpp"],
     ),
     hdrs = glob(
         ["*.h"],
@@ -38,6 +42,10 @@ cc_library(
         ["storage/*.h"],
     ) + glob(
         ["streaming/*.h"],
+    ) + glob(
+        ["space/*.h"],
+    ) + glob(
+        ["mds/*.h"],
     ),
     copts = CURVE_DEFAULT_COPTS,
     visibility = ["//visibility:public"],
@@ -51,6 +59,8 @@ cc_library(
         "//curvefs/proto:mds_cc_proto",
         "//curvefs/src/common:curvefs_common",
         "//curvefs/src/metaserver/common:fs_metaserver_common",
+        "//curvefs/src/volume:volume",
+        "//curvefs/src/client:fuse_client_lib",
         "//external:braft",
         "//src/common:curve_common",
         "//src/fs:lfs",

curvefs/src/metaserver/copyset/copyset_node.cpp

@@ -31,6 +31,8 @@
 #include <string>
 #include <utility>
 #include <vector>
+#include <map>
+#include <unordered_map>
 
 #include "absl/cleanup/cleanup.h"
 #include "absl/memory/memory.h"
@@ -574,24 +576,149 @@ void CopysetNode::ListPeers(std::vector<Peer>* peers) const {
 }
 
 // if copyset is loading, return false;
-// if copyset is not loading, and metastore returns fales, retry;
+// if copyset is not loading, and metastore returns false, retry;
 // if copyset is not loading, and metastore returns true, return true and get
 // partition info list success.
 bool CopysetNode::GetPartitionInfoList(
-                    std::list<PartitionInfo> *partitionInfoList) {
-    uint32_t retryCount = 0;
-    while (true) {
-        if (IsLoading()) {
-            LOG(INFO) << "Copyset is loading, return empty partition list";
-            return false;
-        }
-        bool ret = metaStore_->GetPartitionInfoList(partitionInfoList);
-        if (ret) {
-            return true;
-        }
-        LOG(WARNING) << "Copyset is not loading, but GetPartitionInfoList fail,"
-                     << " retryCount = " << retryCount++;
+    std::list<PartitionInfo> *partitionInfoList) {
+    if (IsLoading()) {
+        LOG(INFO) << "Copyset is loading, return empty partition list";
+        return false;
     }
+
+    uint32_t retryCount = 0;
+    do {
+        bool ret = metaStore_->GetPartitionInfoList(partitionInfoList);
+        if (!ret) {
+            LOG(WARNING)
+                << "Copyset is not loading, but GetPartitionInfoList fail,"
+                << " retryCount = " << retryCount++;
+            continue;
+        }
+
+        return true;
+    } while (true);
+}
+
+bool CopysetNode::GetBlockStatInfo(
+    std::map<uint32_t, BlockGroupStatInfo> *blockStatInfoMap) {
+    if (IsLoading()) {
+        LOG(INFO)
+            << "CopysetNode copyset is loading, return empty block stat info";
+        return false;
+    }
+
+    uint32_t retryCount = 0;
+    uint32_t blockGroupNum = 0;
+    do {
+        std::map<uint32_t, std::shared_ptr<Partition>> partitionSnap;
+        if (!metaStore_->GetPartitionSnap(&partitionSnap)) {
+            LOG(WARNING) << "CopysetNode get partition snap fail, retryCount = "
+                         << ++retryCount;
+            continue;
+        }
+
+        for (const auto &item : partitionSnap) {
+            auto partition = item.second;
+            if (!AggregateBlockStatInfo(partition, blockStatInfoMap,
+                                        &blockGroupNum)) {
+                return false;
+            }
+        }
+
+        return true;
+    } while (true);
+}
+
+// NOTE:
+//  If the file system size is 1PB, the block size is 128MB
+//  - The maximum number of blockgroups in the file system is 838,8608.
+//
+//  The deallocatable space of the current blockgroup is counted according to
+//  the granularity of the partition. Now the maximum number of partitions on
+//  each copyset is configured to 128
+//  - In extreme cases, the number of statistical fragments of blockgroups in
+//  each copyset is 838,8608*128≈10billion.The number of fragments of the
+//  blockgroup that metaserver needs to carry in a heartbeat will be too large
+//
+//  Therefore, it is necessary to combine the information of these fragments to
+//  calculate.
+//
+// TODO(ilixiaocui): need more stat optimization
+bool CopysetNode::AggregateBlockStatInfo(
+    const std::shared_ptr<Partition> &partition,
+    std::map<uint32_t, BlockGroupStatInfo> *blockStatInfoMap,
+    uint32_t *blockGroupNum) {
+    uint32_t fsId = partition->GetFsId();
+
+    // get block group info in partition
+    std::vector<DeallocatableBlockGroup> deallocatableBlockGroupVec;
+    if (MetaStatusCode::OK !=
+        partition->GeAllBlockGroup(&deallocatableBlockGroupVec)) {
+        LOG(WARNING) << "CopysetNode get all blockgroup fail, partitionId = "
+                     << partition->GetPartitionId();
+        return false;
+    }
+
+    auto &blockGroupStatInfo = (*blockStatInfoMap)[fsId];
+    if (!blockGroupStatInfo.has_fsid()) {
+        blockGroupStatInfo.set_fsid(fsId);
+    }
+
+#define LIMITBLICKGROUPNUM 8192
+    // combine the information of the same blockgroup
+    auto blockGroups = blockGroupStatInfo.mutable_deallocatableblockgroups();
+    std::unordered_map<uint64_t, DeallocatableBlockGroup *> blockGroupMap;
+    for (auto &blockGroup : *blockGroups) {
+        blockGroupMap[blockGroup.blockgroupoffset()] = &blockGroup;
+    }
+
+    for (auto &blockGroup : deallocatableBlockGroupVec) {
+        auto it = blockGroupMap.find(blockGroup.blockgroupoffset());
+        if (it != blockGroupMap.end()) {
+            it->second->set_deallocatablesize(it->second->deallocatablesize() +
+                                              blockGroup.deallocatablesize());
+        } else {
+            blockGroup.clear_inodeidlist();
+            blockGroup.clear_inodeidunderdeallocate();
+            blockGroups->Add()->MergeFrom(blockGroup);
+            if (++(*blockGroupNum) > LIMITBLICKGROUPNUM) {
+                LOG(WARNING) << "CopysetNode get blockgroup num over limit, "
+                                "blockGroupNum = "
+                             << *blockGroupNum;
+                break;
+            }
+        }
+    }
+
+    return true;
+}
+
+void CopysetNode::Deallocate(uint64_t fsId, uint64_t blockGroupOffset) {
+    if (IsLoading()) {
+        LOG(INFO)
+            << "CopysetNode copyset is loading, return empty block stat info";
+        return;
+    }
+
+    uint32_t retryCount = 0;
+    do {
+        std::map<uint32_t, std::shared_ptr<Partition>> partitionSnap;
+        if (!metaStore_->GetPartitionSnap(&partitionSnap)) {
+            LOG(WARNING) << "CopysetNode get partition snap fail, retryCount = "
+                         << ++retryCount;
+            continue;
+        }
+
+        for (const auto &item : partitionSnap) {
+            auto partition = item.second;
+            if (partition->GetFsId() != fsId) {
+                continue;
+            }
+            item.second->SetVolumeDeallocate(fsId, blockGroupOffset);
+        }
+        return;
+    } while (true);
 }
 
 void CopysetNode::OnConfChangeComplete() {

curvefs/src/metaserver/copyset/copyset_node.h

@@ -24,11 +24,13 @@
 #define CURVEFS_SRC_METASERVER_COPYSET_COPYSET_NODE_H_
 
 #include <braft/raft.h>
+#include <gtest/gtest_prod.h>
 
 #include <list>
 #include <memory>
 #include <string>
 #include <vector>
+#include <map>
 
 #include "curvefs/src/metaserver/common/types.h"
 #include "curvefs/src/metaserver/copyset/apply_queue.h"
@@ -38,6 +40,7 @@
 #include "curvefs/src/metaserver/copyset/metric.h"
 #include "curvefs/src/metaserver/copyset/raft_node.h"
 #include "curvefs/src/metaserver/metastore.h"
+#include "curvefs/proto/heartbeat.pb.h"
 
 namespace curvefs {
 namespace metaserver {
@@ -47,6 +50,7 @@ using ::braft::PeerId;
 using ::curvefs::common::Peer;
 using ::curvefs::metaserver::MetaStore;
 using ::curve::mds::heartbeat::ConfigChangeType;
+using ::curvefs::mds::heartbeat::BlockGroupStatInfo;
 
 class CopysetNodeManager;
 
@@ -77,13 +81,13 @@ class CopysetNode : public braft::StateMachine {
 
     virtual bool IsLeaderTerm() const;
 
-    PoolId GetPoolId() const;
+    virtual PoolId GetPoolId() const;
 
-    const braft::PeerId& GetPeerId() const;
+    virtual const braft::PeerId& GetPeerId() const;
 
-    CopysetId GetCopysetId() const;
+    virtual CopysetId GetCopysetId() const;
 
-    PeerId GetLeaderId() const;
+    virtual PeerId GetLeaderId() const;
 
     MetaStore* GetMetaStore() const;
 
@@ -136,7 +140,7 @@ class CopysetNode : public braft::StateMachine {
     virtual void RemovePeer(const Peer& peer, braft::Closure* done = nullptr);
     virtual void ChangePeers(const std::vector<Peer>& newPeers,
                              braft::Closure* done = nullptr);
-    void GetConfChange(ConfigChangeType* type, Peer* alterPeer);
+    virtual void GetConfChange(ConfigChangeType* type, Peer* alterPeer);
     void OnConfChangeComplete();
 
  private:
@@ -175,9 +179,15 @@ class CopysetNode : public braft::StateMachine {
 
  public:
     // for heartbeat
-    bool GetPartitionInfoList(std::list<PartitionInfo> *partitionInfoList);
+    virtual bool
+    GetPartitionInfoList(std::list<PartitionInfo> *partitionInfoList);
 
-    bool IsLoading() const;
+    virtual bool IsLoading() const;
+
+    virtual bool
+    GetBlockStatInfo(std::map<uint32_t, BlockGroupStatInfo> *blockStatInfoMap);
+
+    virtual void Deallocate(uint64_t fsId, uint64_t blockGroupOffset);
 
  private:
     void InitRaftNodeOptions();
@@ -185,6 +195,13 @@ class CopysetNode : public braft::StateMachine {
     bool FetchLeaderStatus(const braft::PeerId& peerId,
                            braft::NodeStatus* leaderStatus);
 
+    bool AggregateBlockStatInfo(
+        const std::shared_ptr<Partition> &partition,
+        std::map<uint32_t, BlockGroupStatInfo> *blockStatInfoMap,
+        uint32_t *blockGroupNum);
+
+    FRIEND_TEST(CopysetNodeBlockGroupTest, Test_AggregateBlockStatInfo);
+
  private:
     const PoolId poolId_;
     const CopysetId copysetId_;

curvefs/src/metaserver/copyset/copyset_node_manager.cpp

@@ -246,7 +246,7 @@ bool CopysetNodeManager::CreateCopysetNode(PoolId poolId, CopysetId copysetId,
 }
 
 void CopysetNodeManager::GetAllCopysets(
-    std::vector<CopysetNode*>* nodes) const {
+    std::vector<CopysetNode *> *nodes) const {
     nodes->clear();
     ReadLockGuard lock(lock_);
     for (auto& copyset : copysets_) {

curvefs/src/metaserver/copyset/copyset_node_manager.h

@@ -74,7 +74,7 @@ class CopysetNodeManager {
 
     virtual bool PurgeCopysetNode(PoolId poolId, CopysetId copysetId);
 
-    void GetAllCopysets(std::vector<CopysetNode*>* nodes) const;
+    virtual void GetAllCopysets(std::vector<CopysetNode *> *nodes) const;
 
     virtual bool IsLoadFinished() const;
 

curvefs/src/metaserver/copyset/meta_operator.cpp

@@ -191,6 +191,7 @@ OPERATOR_ON_APPLY(CreatePartition);
 OPERATOR_ON_APPLY(DeletePartition);
 OPERATOR_ON_APPLY(PrepareRenameTx);
 OPERATOR_ON_APPLY(UpdateVolumeExtent);
+OPERATOR_ON_APPLY(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_ON_APPLY
 
@@ -269,7 +270,7 @@ void GetVolumeExtentOperator::OnApply(int64_t index,
     }
 
     // in streaming mode, swap slices out and send them by streaming
-    VolumeExtentList extents;
+    VolumeExtentSliceList extents;
     response->mutable_slices()->Swap(&extents);
     response->clear_slices();
 
@@ -316,6 +317,7 @@ OPERATOR_ON_APPLY_FROM_LOG(CreatePartition);
 OPERATOR_ON_APPLY_FROM_LOG(DeletePartition);
 OPERATOR_ON_APPLY_FROM_LOG(PrepareRenameTx);
 OPERATOR_ON_APPLY_FROM_LOG(UpdateVolumeExtent);
+OPERATOR_ON_APPLY_FROM_LOG(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_ON_APPLY_FROM_LOG
 
@@ -374,6 +376,7 @@ OPERATOR_REDIRECT(DeletePartition);
 OPERATOR_REDIRECT(PrepareRenameTx);
 OPERATOR_REDIRECT(GetVolumeExtent);
 OPERATOR_REDIRECT(UpdateVolumeExtent);
+OPERATOR_REDIRECT(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_REDIRECT
 
@@ -400,6 +403,7 @@ OPERATOR_ON_FAILED(DeletePartition);
 OPERATOR_ON_FAILED(PrepareRenameTx);
 OPERATOR_ON_FAILED(GetVolumeExtent);
 OPERATOR_ON_FAILED(UpdateVolumeExtent);
+OPERATOR_ON_FAILED(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_ON_FAILED
 
@@ -425,6 +429,8 @@ OPERATOR_HASH_CODE(PrepareRenameTx);
 OPERATOR_HASH_CODE(DeletePartition);
 OPERATOR_HASH_CODE(GetVolumeExtent);
 OPERATOR_HASH_CODE(UpdateVolumeExtent);
+OPERATOR_HASH_CODE(UpdateDeallocatableBlockGroup);
+
 
 #undef OPERATOR_HASH_CODE
 
@@ -462,6 +468,7 @@ OPERATOR_TYPE(CreatePartition);
 OPERATOR_TYPE(DeletePartition);
 OPERATOR_TYPE(GetVolumeExtent);
 OPERATOR_TYPE(UpdateVolumeExtent);
+OPERATOR_TYPE(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_TYPE
 

curvefs/src/metaserver/copyset/meta_operator.h

@@ -537,6 +537,25 @@ class UpdateVolumeExtentOperator : public MetaOperator {
     void OnFailed(MetaStatusCode code) override;
 };
 
+class UpdateDeallocatableBlockGroupOperator : public MetaOperator {
+ public:
+    using MetaOperator::MetaOperator;
+
+    void OnApply(int64_t index, google::protobuf::Closure *done,
+                 uint64_t startTimeUs) override;
+
+    void OnApplyFromLog(uint64_t startTimeUs) override;
+
+    uint64_t HashCode() const override;
+
+    OperatorType GetOperatorType() const override;
+
+ private:
+    void Redirect() override;
+
+    void OnFailed(MetaStatusCode code) override;
+};
+
 }  // namespace copyset
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/copyset/operator_type.cpp

@@ -66,6 +66,8 @@ const char* OperatorTypeName(OperatorType type) {
             return "GetVolumeExtent";
         case OperatorType::UpdateVolumeExtent:
             return "UpdateVolumeExtent";
+        case OperatorType::UpdateDeallocatableBlockGroup:
+            return "UpdateDeallocatableBlockGroup";
         // Add new case before `OperatorType::OperatorTypeMax`
         case OperatorType::OperatorTypeMax:
             break;

curvefs/src/metaserver/copyset/operator_type.h

@@ -52,6 +52,8 @@ enum class OperatorType : uint32_t {
     GetVolumeExtent = 15,
     UpdateVolumeExtent = 16,
     CreateManageInode = 17,
+    UpdateDeallocatableBlockGroup = 18,
+
     // NOTE:
     //   Add new operator before `OperatorTypeMax`
     //   And DO NOT recorder or delete previous types

curvefs/src/metaserver/copyset/raft_log_codec.cpp

@@ -165,6 +165,10 @@ std::unique_ptr<MetaOperator> RaftLogCodec::Decode(CopysetNode* node,
             return ParseFromRaftLog<UpdateVolumeExtentOperator,
                                     UpdateVolumeExtentRequest>(node, type,
                                                                meta);
+        case OperatorType::UpdateDeallocatableBlockGroup:
+            return ParseFromRaftLog<UpdateDeallocatableBlockGroupOperator,
+                                    UpdateDeallocatableBlockGroupRequest>(
+                node, type, meta);
         // Add new case before `OperatorType::OperatorTypeMax`
         case OperatorType::OperatorTypeMax:
             break;

curvefs/src/metaserver/heartbeat.cpp

@@ -40,19 +40,21 @@
 #include "curvefs/src/metaserver/copyset/utils.h"
 #include "curvefs/src/metaserver/storage/storage.h"
 #include "curvefs/src/metaserver/resource_statistic.h"
+#include "curvefs/src/metaserver/space/volume_deallocate_manager.h"
 
 namespace curvefs {
 namespace metaserver {
 
 using ::curve::mds::heartbeat::ConfigChangeType;
 using ::curvefs::mds::heartbeat::ConfigChangeInfo;
+using ::curvefs::mds::heartbeat::CopySetInfo;
 using ::curvefs::metaserver::copyset::CopysetService_Stub;
 using ::curvefs::metaserver::copyset::ToGroupIdString;
 
 namespace {
 
 int GatherCopysetConfChange(CopysetNode* node, ConfigChangeInfo* info) {
-    ConfigChangeType type;
+    ConfigChangeType type = ConfigChangeType::NONE;
     Peer peer;
 
     node->GetConfChange(&type, &peer);
@@ -161,7 +163,7 @@ int Heartbeat::Fini() {
     return 0;
 }
 
-void Heartbeat::BuildCopysetInfo(curvefs::mds::heartbeat::CopySetInfo *info,
+void Heartbeat::BuildCopysetInfo(CopySetInfo *info,
                                  CopysetNode *copyset) {
     int ret;
     PoolId poolId = copyset->GetPoolId();
@@ -213,6 +215,12 @@ void Heartbeat::BuildCopysetInfo(curvefs::mds::heartbeat::CopySetInfo *info,
     }
 }
 
+void Heartbeat::BuildBlockGroupStatInfo(CopysetNode *copyset,
+                             BlockGroupStatInfoMap *blockGroupStatInfoMap) {
+    copyset->GetBlockStatInfo(blockGroupStatInfoMap);
+}
+
+
 // TODO(@Wine93): now we use memory storage, so we gather disk usage bytes
 // which only has raft related capacity. If we use rocksdb storage, maybe
 // we should need more flexible strategy.
@@ -255,16 +263,38 @@ int Heartbeat::BuildRequest(HeartbeatRequest* req) {
 
     req->set_copysetcount(copysets.size());
     int leaders = 0;
-
+    BlockGroupStatInfoMap blockGroupStatInfoMap;
     for (auto copyset : copysets) {
-        curvefs::mds::heartbeat::CopySetInfo *info = req->add_copysetinfos();
-
+        // build copyset info
+        CopySetInfo *info = req->add_copysetinfos();
         BuildCopysetInfo(info, copyset);
 
         if (copyset->IsLeaderTerm()) {
+            // build block group info
+            BuildBlockGroupStatInfo(copyset, &blockGroupStatInfoMap);
             ++leaders;
         }
     }
+
+    // get deallocate task status
+    for (auto &item : blockGroupStatInfoMap) {
+        if (taskExecutor_->deallocfsid_.has_value() &&
+            item.second.fsid() == taskExecutor_->deallocfsid_.value()) {
+            bool doing = VolumeDeallocateManager::GetInstance().HasDeallocate();
+            auto status = item.second.mutable_blockgroupdeallocatestatus();
+            if (doing) {
+                status->insert(
+                    {item.second.fsid(),
+                     BlockGroupDeallcateStatusCode::BGDP_PROCESSING});
+            } else {
+                status->insert({item.second.fsid(),
+                                BlockGroupDeallcateStatusCode::BGDP_DONE});
+                taskExecutor_->deallocfsid_.reset();
+            }
+        }
+        *req->add_blockgroupstatinfos() = std::move(item.second);
+    }
+
     req->set_leadercount(leaders);
 
     MetaServerSpaceStatus* status = req->mutable_spacestatus();
@@ -314,9 +344,14 @@ void Heartbeat::DumpHeartbeatResponse(const HeartbeatResponse &response) {
     VLOG(3) << "Received heartbeat response, statusCode = "
             << response.statuscode();
 
-    for (auto& conf : response.needupdatecopysets()) {
+    for (auto &conf : response.needupdatecopysets()) {
         VLOG(3) << "need update copyset: " << conf.ShortDebugString();
     }
+
+    for (auto &issue : response.issuedblockgroups()) {
+        VLOG(3) << "issued block group fsid=" << issue.first
+                << ", blockgroupoffset=" << issue.second;
+    }
 }
 
 int Heartbeat::SendHeartbeat(const HeartbeatRequest &request,
@@ -410,6 +445,18 @@ void HeartbeatTaskExecutor::ExecTasks(const HeartbeatResponse& response) {
     for (auto& conf : response.needupdatecopysets()) {
         ExecOneTask(conf);
     }
+
+    std::vector<CopysetNode *> copysets;
+    copysetMgr_->GetAllCopysets(&copysets);
+    for (auto &issue : response.issuedblockgroups()) {
+        for (auto &copyset : copysets) {
+            if (copyset->IsLeaderTerm()) {
+                copyset->Deallocate(issue.first, issue.second);
+            }
+        }
+
+        deallocfsid_ = issue.first;
+    }
 }
 
 void HeartbeatTaskExecutor::ExecOneTask(const CopySetConf& conf) {

curvefs/src/metaserver/heartbeat.h

@@ -25,6 +25,7 @@
 
 #include <braft/node.h>  // NodeImpl
 #include <braft/node_manager.h>
+#include <gtest/gtest_prod.h>
 
 #include <atomic>
 #include <map>
@@ -37,23 +38,26 @@
 #include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
 #include "src/common/concurrent/concurrent.h"
 #include "src/common/wait_interval.h"
-
-using ::curve::common::Thread;
-using ::curvefs::metaserver::copyset::CopysetNode;
+#include "absl/types/optional.h"
 
 namespace curvefs {
 namespace metaserver {
+using curve::common::Thread;
+using curve::fs::LocalFileSystem;
+using curvefs::common::Peer;
+using curvefs::mds::heartbeat::BlockGroupStatInfo;
+using curvefs::mds::heartbeat::CopySetConf;
+using curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode;
+using curvefs::metaserver::copyset::CopysetNode;
+using curvefs::metaserver::copyset::CopysetNodeManager;
 
-using ::curve::fs::LocalFileSystem;
 using HeartbeatRequest = curvefs::mds::heartbeat::MetaServerHeartbeatRequest;
 using HeartbeatResponse = curvefs::mds::heartbeat::MetaServerHeartbeatResponse;
 using MetaServerSpaceStatus = curvefs::mds::heartbeat::MetaServerSpaceStatus;
-using ::curvefs::mds::heartbeat::CopySetConf;
-using TaskStatus = butil::Status;
 using CopysetNodePtr = std::shared_ptr<CopysetNode>;
-using curvefs::metaserver::copyset::CopysetNodeManager;
-using curvefs::common::Peer;
 using PeerId = braft::PeerId;
+using TaskStatus = butil::Status;
+using BlockGroupStatInfoMap = std::map<uint32_t, BlockGroupStatInfo>;
 
 class ResourceCollector;
 
@@ -104,6 +108,8 @@ class Heartbeat {
     int Run();
 
  private:
+    FRIEND_TEST(HeartbeatTest, Test_BuildRequest);
+
     /**
      * @brief stop heartbeat subsystem
      * @return 0:success; not 0: fail
@@ -118,6 +124,9 @@ class Heartbeat {
     void BuildCopysetInfo(curvefs::mds::heartbeat::CopySetInfo* info,
                          CopysetNode* copyset);
 
+    void BuildBlockGroupStatInfo(CopysetNode *copyset,
+                                 BlockGroupStatInfoMap *blockGroupStatInfoMap);
+
     int BuildRequest(HeartbeatRequest* request);
 
     int SendHeartbeat(const HeartbeatRequest& request,
@@ -136,11 +145,14 @@ class Heartbeat {
     bool GetMetaserverSpaceStatus(MetaServerSpaceStatus* status,
                                   uint64_t ncopysets);
 
+    // Handle heartbeat and send recyclable BlockGroup requests
+    void DeallocateBolckGroup(const HeartbeatResponse &response);
+
  private:
     friend class HeartbeatTest;
 
  private:
-     Thread hbThread_;
+    Thread hbThread_;
 
     std::atomic<bool> toStop_;
 
@@ -175,6 +187,10 @@ class HeartbeatTaskExecutor {
 
     void ExecTasks(const HeartbeatResponse& response);
 
+    void SetDeallocFsId(uint64_t fsid) {
+        deallocfsid_ = fsid;
+    }
+
  private:
     void ExecOneTask(const CopySetConf& conf);
 
@@ -187,8 +203,11 @@ class HeartbeatTaskExecutor {
     bool NeedPurge(const CopySetConf& conf);
 
  private:
+    friend class Heartbeat;
+
     CopysetNodeManager* copysetMgr_;
     butil::EndPoint ep_;
+    absl::optional<uint64_t> deallocfsid_;
 };
 
 }  // namespace metaserver

curvefs/src/metaserver/inode_manager.cpp

@@ -342,8 +342,12 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request) {
         needUpdate = true;
     }
 
+    bool fileNeedDeallocate =
+        (needAddTrash && (FsFileType::TYPE_FILE == old.type()));
+    bool s3NeedTrash = (needAddTrash && (FsFileType::TYPE_S3 == old.type()));
+
     if (needUpdate) {
-        ret = inodeStorage_->Update(old);
+        ret = inodeStorage_->Update(old, fileNeedDeallocate);
         if (ret != MetaStatusCode::OK) {
             LOG(ERROR) << "UpdateInode fail, " << request.ShortDebugString()
                        << ", ret: " << MetaStatusCode_Name(ret);
@@ -351,7 +355,7 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest& request) {
         }
     }
 
-    if (needAddTrash) {
+    if (s3NeedTrash) {
         trash_->Add(old.fsid(), old.inodeid(), old.dtime());
         --(*type2InodeNum_)[old.type()];
     }
@@ -556,7 +560,7 @@ MetaStatusCode InodeManager::UpdateVolumeExtentSlice(
 MetaStatusCode InodeManager::UpdateVolumeExtent(
     uint32_t fsId,
     uint64_t inodeId,
-    const VolumeExtentList &extents) {
+    const VolumeExtentSliceList &extents) {
     VLOG(6) << "UpdateInodeExtent, fsId: " << fsId << ", inodeId: " << inodeId;
     NameLockGuard guard(inodeLock_, GetInodeLockName(fsId, inodeId));
 
@@ -578,7 +582,7 @@ MetaStatusCode InodeManager::GetVolumeExtent(
     uint32_t fsId,
     uint64_t inodeId,
     const std::vector<uint64_t> &slices,
-    VolumeExtentList *extents) {
+    VolumeExtentSliceList *extents) {
     VLOG(6) << "GetInodeExtent, fsId: " << fsId << ", inodeId: " << inodeId;
 
     if (slices.empty()) {

curvefs/src/metaserver/inode_manager.h

@@ -113,7 +113,7 @@ class InodeManager {
     // Update one or more volume extent slice
     MetaStatusCode UpdateVolumeExtent(uint32_t fsId,
                                       uint64_t inodeId,
-                                      const VolumeExtentList &extents);
+                                      const VolumeExtentSliceList &extents);
 
     // Update only one volume extent slice
     MetaStatusCode UpdateVolumeExtentSlice(uint32_t fsId,
@@ -123,7 +123,7 @@ class InodeManager {
     MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                    uint64_t inodeId,
                                    const std::vector<uint64_t> &slices,
-                                   VolumeExtentList *extents);
+                                   VolumeExtentSliceList *extents);
 
  private:
     void GenerateInodeInternal(uint64_t inodeId, const InodeParam &param,

curvefs/src/metaserver/inode_storage.cpp

@@ -25,10 +25,12 @@
 #include <memory>
 #include <vector>
 #include <algorithm>
+#include <set>
 
 #include "src/common/concurrent/rw_lock.h"
 #include "src/common/string_util.h"
 #include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/proto/common.pb.h"
 #include "curvefs/src/metaserver/storage/status.h"
 #include "curvefs/src/metaserver/inode_storage.h"
 #include "curvefs/src/metaserver/storage/converter.h"
@@ -39,6 +41,7 @@ namespace metaserver {
 using ::curve::common::ReadLockGuard;
 using ::curve::common::WriteLockGuard;
 using ::curve::common::StringStartWith;
+using ::curvefs::common::EmptyMsg;
 using ::curvefs::metaserver::storage::Status;
 using ::curvefs::metaserver::storage::KVStorage;
 using ::curvefs::metaserver::storage::Key4S3ChunkInfoList;
@@ -48,8 +51,9 @@ using ::curvefs::metaserver::storage::Prefix4ChunkIndexS3ChunkInfoList;
 using ::curvefs::metaserver::storage::Prefix4InodeS3ChunkInfoList;
 using ::curvefs::metaserver::storage::Prefix4AllInode;
 using ::curvefs::metaserver::storage::Key4InodeAuxInfo;
-
-using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+using ::curvefs::metaserver::storage::Key4DeallocatableBlockGroup;
+using ::curvefs::metaserver::storage::Key4DeallocatableInode;
+using ::curvefs::metaserver::storage::Prefix4AllDeallocatableBlockGroup;
 
 InodeStorage::InodeStorage(std::shared_ptr<KVStorage> kvStorage,
                            std::shared_ptr<NameGenerator> nameGenerator,
@@ -59,8 +63,11 @@ InodeStorage::InodeStorage(std::shared_ptr<KVStorage> kvStorage,
       table4S3ChunkInfo_(nameGenerator->GetS3ChunkInfoTableName()),
       table4VolumeExtent_(nameGenerator->GetVolumeExtentTableName()),
       table4InodeAuxInfo_(nameGenerator->GetInodeAuxInfoTableName()),
-      nInode_(nInode),
-      conv_() {}
+      table4DeallocatableInode_(
+          nameGenerator->GetDeallocatableInodeTableName()),
+      table4DeallocatableBlockGroup_(
+          nameGenerator->GetDeallocatableBlockGroupTableName()),
+      nInode_(nInode), conv_() {}
 
 MetaStatusCode InodeStorage::Insert(const Inode& inode) {
     WriteLockGuard lg(rwLock_);
@@ -181,18 +188,53 @@ MetaStatusCode InodeStorage::Delete(const Key4Inode& key) {
     return MetaStatusCode::STORAGE_INTERNAL_ERROR;
 }
 
-MetaStatusCode InodeStorage::Update(const Inode& inode) {
+MetaStatusCode InodeStorage::Update(const Inode &inode, bool inodeDeallocate) {
     WriteLockGuard lg(rwLock_);
     Key4Inode key(inode.fsid(), inode.inodeid());
     std::string skey = conv_.SerializeToString(key);
 
-    Status s = kvStorage_->HSet(table4Inode_, skey, inode);
-    if (s.ok()) {
-        return MetaStatusCode::OK;
+    // only update inodes
+    if (!inodeDeallocate) {
+        Status s = kvStorage_->HSet(table4Inode_, skey, inode);
+        if (s.ok()) {
+            return MetaStatusCode::OK;
+        }
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
     }
-    return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+
+    // update inode and update deallocatable inode list
+    EmptyMsg value;
+    auto txn = kvStorage_->BeginTransaction();
+    if (nullptr == txn) {
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+
+    std::string step = "update inode " + key.SerializeToString();
+
+    Status s = txn->HSet(table4Inode_, skey, inode);
+    if (s.ok()) {
+        s = txn->HSet(table4DeallocatableInode_, skey, value);
+        step = "add inode " + key.SerializeToString() +
+               " to inode deallocatable list";
+    }
+
+    if (!s.ok()) {
+        LOG(ERROR) << "txn is failed in " << step;
+        if (!txn->Rollback().ok()) {
+            LOG(ERROR) << "rollback transaction failed, inode="
+                       << key.SerializeToString();
+            return  MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        }
+    } else if (!txn->Commit().ok()) {
+        LOG(ERROR) << "commit transaction failed, inode="
+                   << key.SerializeToString();
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+
+    return MetaStatusCode::OK;
 }
 
+
 std::shared_ptr<Iterator> InodeStorage::GetAllInode() {
     ReadLockGuard lg(rwLock_);
     std::string sprefix = conv_.SerializeToString(Prefix4AllInode());
@@ -527,9 +569,9 @@ MetaStatusCode InodeStorage::UpdateVolumeExtentSlice(
                    : MetaStatusCode::STORAGE_INTERNAL_ERROR;
 }
 
-MetaStatusCode InodeStorage::GetAllVolumeExtent(uint32_t fsId,
-                                                uint64_t inodeId,
-                                                VolumeExtentList* extents) {
+MetaStatusCode
+InodeStorage::GetAllVolumeExtent(uint32_t fsId, uint64_t inodeId,
+                                 VolumeExtentSliceList *extents) {
     ReadLockGuard guard(rwLock_);
     auto key = conv_.SerializeToString(Prefix4InodeVolumeExtent{fsId, inodeId});
     auto iter = kvStorage_->SSeek(table4VolumeExtent_, key);
@@ -552,6 +594,13 @@ MetaStatusCode InodeStorage::GetAllVolumeExtent(uint32_t fsId,
     return MetaStatusCode::OK;
 }
 
+std::shared_ptr<Iterator> InodeStorage::GetAllVolumeExtent(uint32_t fsId,
+                                                           uint64_t inodeId) {
+    ReadLockGuard guard(rwLock_);
+    auto key = conv_.SerializeToString(Prefix4InodeVolumeExtent{fsId, inodeId});
+    return kvStorage_->SSeek(table4VolumeExtent_, key);
+}
+
 MetaStatusCode InodeStorage::GetVolumeExtentByOffset(uint32_t fsId,
                                                      uint64_t inodeId,
                                                      uint64_t offset,
@@ -571,5 +620,190 @@ MetaStatusCode InodeStorage::GetVolumeExtentByOffset(uint32_t fsId,
     return MetaStatusCode::STORAGE_INTERNAL_ERROR;
 }
 
+MetaStatusCode InodeStorage::GeAllBlockGroup(
+    std::vector<DeallocatableBlockGroup> *deallocatableBlockGroupVec) {
+    std::string sprefix =
+        conv_.SerializeToString(Prefix4AllDeallocatableBlockGroup());
+
+    auto iter = kvStorage_->HGetAll(table4DeallocatableBlockGroup_);
+    if (iter->Status() != 0) {
+        LOG(ERROR) << "InodeStorage failed to get iterator for all "
+                      "deallocatable block group";
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+
+    uint32_t count = 0;
+    DeallocatableBlockGroup deallocatbleBlockGroup;
+    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+        if (!conv_.ParseFromString(iter->Value(), &deallocatbleBlockGroup)) {
+            LOG(ERROR) << "InodeStorage failed to parse deallocatable block "
+                          "group";
+            return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        }
+
+        deallocatableBlockGroupVec->emplace_back(
+            std::move(deallocatbleBlockGroup));
+        count++;
+    }
+
+    return count > 0 ? MetaStatusCode::OK : MetaStatusCode::NOT_FOUND;
+}
+
+MetaStatusCode InodeStorage::UpdateDeallocatableBlockGroup(
+    uint32_t fsId, const DeallocatableBlockGroupVec &update) {
+    auto txn = kvStorage_->BeginTransaction();
+
+    MetaStatusCode st = MetaStatusCode::OK;
+    std::string step;
+
+    for (auto &item : update) {
+        Key4DeallocatableBlockGroup key(fsId, item.blockgroupoffset());
+        std::string skey(key.SerializeToString());
+
+        DeallocatableBlockGroup out;
+        auto s = txn->HGet(table4DeallocatableBlockGroup_, skey, &out);
+
+        if (!s.ok() && !s.IsNotFound()) {
+            step = "get deallocatable group skey=" + skey + " failed";
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+            break;
+        }
+
+        if (item.has_increase()) {
+            if (!out.IsInitialized()) {
+                out.set_blockgroupoffset(item.blockgroupoffset());
+            }
+            st = Increase(txn, fsId, item.increase(), &out);
+        } else if (item.has_decrease()) {
+            st = Decrease(item.decrease(), &out);
+        } else if (item.has_mark()) {
+            st = Mark(item.mark(), &out);
+        }
+
+        s = txn->HSet(table4DeallocatableBlockGroup_, skey, out);
+        if (!s.ok()) {
+            step = "update deallocatable group skey=" + skey + " failed";
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+            break;
+        }
+    }
+
+    if (st != MetaStatusCode::OK) {
+        LOG(ERROR) << "UpdateDeallocatableBlockGroup txn is failed at " << step;
+        if (!txn->Rollback().ok()) {
+            LOG(ERROR) << "UpdateDeallocatableBlockGroup rollback transaction "
+                          "failed, fsId="
+                       << fsId;
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        }
+    } else if (!txn->Commit().ok()) {
+        LOG(ERROR)
+            << "UpdateDeallocatableBlockGroup commit transaction failed, fsId="
+            << fsId;
+        st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+
+    return st;
+}
+
+MetaStatusCode
+InodeStorage::Increase(Transaction txn, uint32_t fsId,
+                       const IncreaseDeallocatableBlockGroup &increase,
+                       DeallocatableBlockGroup *out) {
+    MetaStatusCode st = MetaStatusCode::OK;
+
+    // update DeallocatableBlockGroup
+    uint64_t oldSize =
+        out->has_deallocatablesize() ? out->deallocatablesize() : 0;
+    out->set_deallocatablesize(oldSize + increase.increasedeallocatablesize());
+    out->mutable_inodeidlist()->MergeFrom(increase.inodeidlistadd());
+    std::set<uint64_t> unique_elements(out->inodeidlist().begin(),
+                                       out->inodeidlist().end());
+    out->mutable_inodeidlist()->Clear();
+    for (auto &elem : unique_elements) {
+        out->mutable_inodeidlist()->Add(elem);
+    }
+
+    // remove related inode in table4DeallocatableInode_
+    for (auto &inodeid : increase.inodeidlistadd()) {
+        auto s = txn->HDel(
+            table4DeallocatableInode_,
+            conv_.SerializeToString(Key4DeallocatableInode{fsId, inodeid}));
+        if (!s.ok()) {
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        }
+        break;
+    }
+
+    return st;
+}
+
+MetaStatusCode
+InodeStorage::Decrease(const DecreaseDeallocatableBlockGroup &decrease,
+                       DeallocatableBlockGroup *out) {
+    MetaStatusCode st = MetaStatusCode::OK;
+
+    if (!out->IsInitialized() || !out->has_deallocatablesize()) {
+        LOG(ERROR)
+            << "UpdateDeallocatableBlockGroup record missing required fields";
+        st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    } else {
+        uint64_t oldSize = out->deallocatablesize();
+        if (oldSize < decrease.decreasedeallocatablesize()) {
+            LOG(ERROR)
+                << "UpdateDeallocatableBlockGroup decrease size is too big, "
+                   "oldSize="
+                << oldSize
+                << ", decreasesize=" << decrease.decreasedeallocatablesize();
+            st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+        } else {
+            // update dallocatablesize
+            out->set_deallocatablesize(oldSize -
+                                       decrease.decreasedeallocatablesize());
+
+            // update deallocatableinode list
+            auto inodeidlist = out->mutable_inodeidunderdeallocate();
+            inodeidlist->erase(
+                std::remove_if(inodeidlist->begin(), inodeidlist->end(),
+                               [&decrease](uint64_t inodeid) {
+                                   auto search = decrease.inodeddeallocated();
+                                   return std::find(search.begin(),
+                                                    search.end(),
+                                                    inodeid) != search.end();
+                               }),
+                inodeidlist->end());
+        }
+    }
+
+    return st;
+}
+
+MetaStatusCode InodeStorage::Mark(const MarkDeallocatableBlockGroup &mark,
+                                  DeallocatableBlockGroup *out) {
+    MetaStatusCode st = MetaStatusCode::OK;
+
+    if (!out->IsInitialized() || !out->has_deallocatablesize()) {
+        LOG(ERROR)
+            << "UpdateDeallocatableBlockGroup record missing required fields";
+        st = MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    } else {
+        // update inodeunderdeallocate
+        out->mutable_inodeidunderdeallocate()->MergeFrom(
+            mark.inodeidunderdeallocate());
+
+        // update inodeidlist
+        auto inodeidlist = out->mutable_inodeidlist();
+        inodeidlist->erase(
+            std::remove_if(inodeidlist->begin(), inodeidlist->end(),
+                           [&mark](uint64_t inodeid) {
+                               auto search = mark.inodeidunderdeallocate();
+                               return std::find(search.begin(), search.end(),
+                                                inodeid) != search.end();
+                           }),
+            inodeidlist->end());
+    }
+    return st;
+}
+
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/inode_storage.h

@@ -30,6 +30,7 @@
 #include <functional>
 #include <unordered_map>
 #include <unordered_set>
+#include <vector>
 
 #include "absl/container/btree_set.h"
 #include "absl/container/btree_map.h"
@@ -49,8 +50,12 @@ using ::curvefs::metaserver::storage::StorageTransaction;
 using ::curvefs::metaserver::storage::Key4Inode;
 using ::curvefs::metaserver::storage::Converter;
 using ::curvefs::metaserver::storage::NameGenerator;
+
 using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+using DeallocatableBlockGroupVec =
+    google::protobuf::RepeatedPtrField<DeallocatableBlockGroup>;
 using Transaction = std::shared_ptr<StorageTransaction>;
+
 class InodeStorage {
  public:
     InodeStorage(std::shared_ptr<KVStorage> kvStorage,
@@ -98,9 +103,10 @@ class InodeStorage {
     /**
      * @brief update inode from storage
      * @param[in] inode: the inode want to update
+     * @param[in] inodeDeallocate: Whether the inode needs to deallocate space
      * @return If inode not exist, return NOT_FOUND; else replace and return OK
      */
-    MetaStatusCode Update(const Inode& inode);
+    MetaStatusCode Update(const Inode& inode, bool inodeDeallocate = false);
 
     std::shared_ptr<Iterator> GetAllInode();
 
@@ -141,19 +147,24 @@ class InodeStorage {
 
     MetaStatusCode GetAllVolumeExtent(uint32_t fsId,
                                       uint64_t inodeId,
-                                      VolumeExtentList* extents);
+                                      VolumeExtentSliceList* extents);
+
+    std::shared_ptr<Iterator> GetAllVolumeExtent(uint32_t fsId,
+                                                 uint64_t inodeId);
 
     MetaStatusCode GetVolumeExtentByOffset(uint32_t fsId,
                                            uint64_t inodeId,
                                            uint64_t offset,
                                            VolumeExtentSlice* slice);
 
-    MetaStatusCode AddS3ChunkInfoList(
-        std::shared_ptr<StorageTransaction> txn,
-        uint32_t fsId,
-        uint64_t inodeId,
-        uint64_t chunkIndex,
-        const S3ChunkInfoList* list2add);
+    // use the transaction to delete {inodes} in the deallocatable_inode_list
+    // and update the statistics of each item of blockgroup_list
+    MetaStatusCode
+    UpdateDeallocatableBlockGroup(uint32_t fsId,
+                                  const DeallocatableBlockGroupVec &update);
+
+    MetaStatusCode GeAllBlockGroup(
+        std::vector<DeallocatableBlockGroup> *deallocatableBlockGroupVec);
 
  private:
     MetaStatusCode UpdateInodeS3MetaSize(Transaction txn, uint32_t fsId,
@@ -162,12 +173,25 @@ class InodeStorage {
 
     uint64_t GetInodeS3MetaSize(uint32_t fsId, uint64_t inodeId);
 
-    MetaStatusCode DelS3ChunkInfoList(
-        std::shared_ptr<StorageTransaction> txn,
-        uint32_t fsId,
-        uint64_t inodeId,
-        uint64_t chunkIndex,
-        const S3ChunkInfoList* list2del);
+    MetaStatusCode DelS3ChunkInfoList(Transaction txn,
+                                      uint32_t fsId, uint64_t inodeId,
+                                      uint64_t chunkIndex,
+                                      const S3ChunkInfoList *list2del);
+
+    MetaStatusCode AddS3ChunkInfoList(Transaction txn,
+                                      uint32_t fsId, uint64_t inodeId,
+                                      uint64_t chunkIndex,
+                                      const S3ChunkInfoList *list2add);
+
+    MetaStatusCode Increase(Transaction txn, uint32_t fsId,
+                            const IncreaseDeallocatableBlockGroup &increase,
+                            DeallocatableBlockGroup *out);
+
+    MetaStatusCode Decrease(const DecreaseDeallocatableBlockGroup &decrease,
+                            DeallocatableBlockGroup *out);
+
+    MetaStatusCode Mark(const MarkDeallocatableBlockGroup &mark,
+                        DeallocatableBlockGroup *out);
 
  private:
     // FIXME: please remove this lock, because we has locked each inode
@@ -179,6 +203,9 @@ class InodeStorage {
     std::string table4S3ChunkInfo_;
     std::string table4VolumeExtent_;
     std::string table4InodeAuxInfo_;
+    std::string table4DeallocatableBlockGroup_;
+    std::string table4DeallocatableInode_;
+
     size_t nInode_;
     Converter conv_;
 };

curvefs/src/metaserver/mds/fsinfo_manager.cpp

@@ -0,0 +1,50 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Wed Mar 22 10:39:52 CST 2023
+ * Author: lixiaocui
+ */
+
+#include "curvefs/src/metaserver/mds/fsinfo_manager.h"
+
+namespace curvefs {
+namespace metaserver {
+bool FsInfoManager::GetFsInfo(uint32_t fsId, FsInfo *fsInfo) {
+    std::lock_guard<std::mutex> lock(mtx_);
+    if (fsInfoMap_.find(fsId) == fsInfoMap_.end()) {
+        auto ret = mdsClient_->GetFsInfo(fsId, fsInfo);
+        if (ret != FSStatusCode::OK) {
+            if (FSStatusCode::NOT_FOUND == ret) {
+                LOG(ERROR) << "The fsName not exist, fsId = " << fsId;
+                return false;
+            } else {
+                LOG(ERROR) << "GetFsInfo failed, FSStatusCode = " << ret
+                           << ", FSStatusCode_Name = " << FSStatusCode_Name(ret)
+                           << ", fsId = " << fsId;
+                return false;
+            }
+        }
+        fsInfoMap_.insert({fsId, *fsInfo});
+    } else {
+        *fsInfo = fsInfoMap_.find(fsId)->second;
+    }
+
+    return true;
+}
+}  // namespace metaserver
+}  // namespace curvefs

curvefs/src/metaserver/mds/fsinfo_manager.h

@@ -0,0 +1,58 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Wed Mar 22 10:40:08 CST 2023
+ * Author: lixiaocui
+ */
+
+#ifndef CURVEFS_SRC_METASERVER_MDS_FSINFO_MANAGER_H_
+#define CURVEFS_SRC_METASERVER_MDS_FSINFO_MANAGER_H_
+
+#include <map>
+#include <memory>
+#include <mutex>
+#include "curvefs/src/client/rpcclient/mds_client.h"
+
+namespace curvefs {
+namespace metaserver {
+
+using ::curvefs::client::rpcclient::MdsClient;
+
+class FsInfoManager {
+ public:
+    static FsInfoManager &GetInstance() {
+        static FsInfoManager instance_;
+        return instance_;
+    }
+
+    void SetMdsClient(std::shared_ptr<MdsClient> mdsClient) {
+        mdsClient_ = mdsClient;
+    }
+
+    bool GetFsInfo(uint32_t fsId, FsInfo *fsInfo);
+
+ private:
+    std::shared_ptr<MdsClient> mdsClient_;
+    std::map<uint32_t, FsInfo> fsInfoMap_;
+
+    std::mutex mtx_;
+};
+}  // namespace metaserver
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_METASERVER_MDS_FSINFO_MANAGER_H_

curvefs/src/metaserver/metaserver.cpp

@@ -38,6 +38,7 @@
 #include "curvefs/src/metaserver/trash_manager.h"
 #include "curvefs/src/metaserver/storage/storage.h"
 #include "curvefs/src/metaserver/storage/rocksdb_perf.h"
+#include "curvefs/src/metaserver/mds/fsinfo_manager.h"
 #include "src/common/crc32.h"
 #include "src/common/curve_version.h"
 #include "src/common/s3_adapter.h"
@@ -158,6 +159,16 @@ void Metaserver::InitRecycleManagerOption(
                                          &recycleManagerOption->scanLimit));
 }
 
+void Metaserver::InitVolumeDeallocateOption(
+    VolumeDeallocateWorkerQueueOption *queueOpt,
+    VolumeDeallocateExecuteOption *execOpt) {
+    conf_->GetValueFatalIfFail("volume.deallocate.enable", &queueOpt->enable);
+    conf_->GetValueFatalIfFail("volume.deallocate.workerNum",
+                               &queueOpt->workerNum);
+    conf_->GetValueFatalIfFail("volume.deallocate.batchClean",
+                               &execOpt->batchClean);
+}
+
 void InitExcutorOption(const std::shared_ptr<Configuration>& conf,
                        ExcutorOpt *opts, bool internal) {
     if (internal) {
@@ -206,6 +217,8 @@ void Metaserver::Init() {
     mdsClient_ = std::make_shared<MdsClientImpl>();
     mdsClient_->Init(mdsOptions_, mdsBase_);
 
+    FsInfoManager::GetInstance().SetMdsClient(mdsClient_);
+
     // init metaserver client for recycle
     InitMetaClient();
 
@@ -243,6 +256,20 @@ void Metaserver::Init() {
 
     S3CompactManager::GetInstance().Init(conf_);
 
+    VolumeSpaceManagerOptions spaceManagerOpt;
+    spaceManagerOpt.mdsClient = mdsClient_;
+    conf_->GetValueFatalIfFail("volume.sdk.confPath",
+                               &spaceManagerOpt.deviceOpt.configPath);
+    auto volumeSpaceMgr = std::make_shared<VolumeSpaceManager>();
+    volumeSpaceMgr->Init(spaceManagerOpt);
+
+    VolumeDeallocateWorkerQueueOption queueOpt;
+    VolumeDeallocateExecuteOption executeOpt;
+    executeOpt.metaClient = metaClient_;
+    executeOpt.volumeSpaceManager = std::move(volumeSpaceMgr);
+    InitVolumeDeallocateOption(&queueOpt, &executeOpt);
+    VolumeDeallocateManager::GetInstance().Init(queueOpt, executeOpt);
+
     PartitionCleanOption partitionCleanOption;
     InitPartitionOption(s3Adaptor_, mdsClient_, &partitionCleanOption);
     PartitionCleanManager::GetInstance().Init(partitionCleanOption);
@@ -491,6 +518,9 @@ void Metaserver::Run() {
     LOG_IF(FATAL, S3CompactManager::GetInstance().Run() != 0);
     running_ = true;
 
+    // start volume deallocate manager
+    VolumeDeallocateManager::GetInstance().Run();
+
     // start copyset node manager
     LOG_IF(FATAL, !copysetNodeManager_->Start())
         << "Failed to start copyset node manager";
@@ -525,6 +555,7 @@ void Metaserver::Stop() {
 
     s3Adaptor_ = nullptr;
     S3CompactManager::GetInstance().Stop();
+    VolumeDeallocateManager::GetInstance().Stop();
     LOG(INFO) << "MetaServer stopped success";
 }
 

curvefs/src/metaserver/metaserver.h

@@ -46,6 +46,9 @@
 #include "src/fs/local_filesystem.h"
 #include "curvefs/src/metaserver/resource_statistic.h"
 #include "curvefs/src/metaserver/recycle_manager.h"
+#include "curvefs/src/metaserver/space/volume_deallocate_manager.h"
+#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h"
+#include "curvefs/src/metaserver/space/volume_space_manager.h"
 
 namespace curvefs {
 namespace metaserver {
@@ -97,6 +100,9 @@ class Metaserver {
                              PartitionCleanOption* partitionCleanOption);
     void InitRecycleManagerOption(
                 RecycleManagerOption* recycleManagerOption);
+
+    void InitVolumeDeallocateOption(VolumeDeallocateWorkerQueueOption *queueOpt,
+                                    VolumeDeallocateExecuteOption *execOpt);
     void GetMetaserverDataByLoadOrRegister();
     int PersistMetaserverMeta(std::string path, MetaServerMetadata* metadata);
     int LoadMetaserverMeta(const std::string& metaFilePath,

curvefs/src/metaserver/metaserver_service.cpp

@@ -52,6 +52,7 @@ using ::curvefs::metaserver::copyset::DeletePartitionOperator;
 using ::curvefs::metaserver::copyset::PrepareRenameTxOperator;
 using ::curvefs::metaserver::copyset::GetVolumeExtentOperator;
 using ::curvefs::metaserver::copyset::UpdateVolumeExtentOperator;
+using ::curvefs::metaserver::copyset::UpdateDeallocatableBlockGroupOperator;
 
 namespace {
 
@@ -298,5 +299,16 @@ void MetaServerServiceImpl::UpdateVolumeExtent(
                                                   request->copysetid());
 }
 
+void MetaServerServiceImpl::UpdateDeallocatableBlockGroup(
+    ::google::protobuf::RpcController *controller,
+    const UpdateDeallocatableBlockGroupRequest *request,
+    UpdateDeallocatableBlockGroupResponse *response,
+    ::google::protobuf::Closure *done) {
+    OperatorHelper helper(copysetNodeManager_, inflightThrottle_);
+    helper.operator()<UpdateDeallocatableBlockGroupOperator>(
+        controller, request, response, done, request->poolid(),
+        request->copysetid());
+}
+
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/metaserver_service.h

@@ -123,6 +123,12 @@ class MetaServerServiceImpl : public MetaServerService {
                             UpdateVolumeExtentResponse* response,
                             ::google::protobuf::Closure* done) override;
 
+    void UpdateDeallocatableBlockGroup(
+        ::google::protobuf::RpcController *controller,
+        const UpdateDeallocatableBlockGroupRequest *request,
+        UpdateDeallocatableBlockGroupResponse *response,
+        ::google::protobuf::Closure *done) override;
+
  private:
     CopysetNodeManager* copysetNodeManager_;
     InflightThrottle* inflightThrottle_;

curvefs/src/metaserver/metastore.cpp

@@ -204,6 +204,7 @@ bool MetaStoreImpl::ClearInternal() {
     for (auto it = partitionMap_.begin(); it != partitionMap_.end(); it++) {
         TrashManager::GetInstance().Remove(it->first);
         it->second->CancelS3Compact();
+        it->second->CancelVolumeDeallocate();
         PartitionCleanManager::GetInstance().Remove(it->first);
 
         if (!it->second->Clear()) {
@@ -276,6 +277,7 @@ MetaStoreImpl::DeletePartition(const DeletePartitionRequest *request,
         TrashManager::GetInstance().Remove(partitionId);
         RecycleManager::GetInstance().Remove(partitionId);
         it->second->CancelS3Compact();
+        it->second->CancelVolumeDeallocate();
         PartitionCleanManager::GetInstance().Remove(partitionId);
         partitionMap_.erase(it);
         response->set_statuscode(MetaStatusCode::OK);
@@ -294,6 +296,7 @@ MetaStoreImpl::DeletePartition(const DeletePartitionRequest *request,
         TrashManager::GetInstance().Remove(partitionId);
         RecycleManager::GetInstance().Remove(partitionId);
         it->second->CancelS3Compact();
+        it->second->CancelVolumeDeallocate();
     } else {
         LOG(INFO) << "DeletePartition, partition is already deleting"
                   << ", partitionId = " << partitionId;
@@ -316,27 +319,46 @@ bool MetaStoreImpl::GetPartitionInfoList(
         }
         rwLock_.Unlock();
         return true;
-    } else {
-        LOG(WARNING) << "metastore GetPartitionInfoList fail, it fail to get"
-                        " the rwLock_";
-        return false;
     }
+    LOG(WARNING) << "metastore GetPartitionInfoList fail, it fail to get"
+                    " the rwLock_";
+    return false;
+}
+
+bool MetaStoreImpl::GetPartitionSnap(
+    std::map<uint32_t, std::shared_ptr<Partition>> *partitionSnap) {
+    int ret = rwLock_.TryRDLock();
+    if (ret == 0) {
+        *partitionSnap = partitionMap_;
+        rwLock_.Unlock();
+        return true;
+    }
+
+    LOG(WARNING) << "MetaStoreImpl get partition snap fail, it fail to get"
+                    " the rwLock_";
+    return false;
 }
 
 std::shared_ptr<StreamServer> MetaStoreImpl::GetStreamServer() {
     return streamServer_;
 }
 
+#define GET_PARTITION(PARTITION)                                               \
+    PARTITION = GetPartition(request->partitionid());                          \
+    if (PARTITION == nullptr) {                                                \
+        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;           \
+        response->set_statuscode(status);                                      \
+        return status;                                                         \
+    }
+
+
 // dentry
 MetaStatusCode MetaStoreImpl::CreateDentry(const CreateDentryRequest *request,
                                            CreateDentryResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
+
     MetaStatusCode status = partition->CreateDentry(request->dentry());
     response->set_statuscode(status);
     return status;
@@ -349,12 +371,8 @@ MetaStatusCode MetaStoreImpl::GetDentry(const GetDentryRequest *request,
     const auto &name = request->name();
     auto txId = request->txid();
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     // handle by partition
     Dentry dentry;
@@ -378,12 +396,8 @@ MetaStatusCode MetaStoreImpl::DeleteDentry(const DeleteDentryRequest *request,
     std::string name = request->name();
     auto txId = request->txid();
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     // handle by partition
     Dentry dentry;
@@ -404,12 +418,8 @@ MetaStatusCode MetaStoreImpl::ListDentry(const ListDentryRequest *request,
     uint64_t parentInodeId = request->dirinodeid();
     auto txId = request->txid();
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     // handle by partition
     Dentry dentry;
@@ -440,16 +450,12 @@ MetaStoreImpl::PrepareRenameTx(const PrepareRenameTxRequest *request,
                                PrepareRenameTxResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
     MetaStatusCode rc;
-    auto partitionId = request->partitionid();
-    auto partition = GetPartition(partitionId);
-    if (nullptr == partition) {
-        rc = MetaStatusCode::PARTITION_NOT_FOUND;
-    } else {
-        std::vector<Dentry> dentrys{request->dentrys().begin(),
-                                    request->dentrys().end()};
-        rc = partition->HandleRenameTx(dentrys);
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
+    std::vector<Dentry> dentrys{request->dentrys().begin(),
+                                request->dentrys().end()};
+    rc = partition->HandleRenameTx(dentrys);
     response->set_statuscode(rc);
     return rc;
 }
@@ -487,12 +493,9 @@ MetaStatusCode MetaStoreImpl::CreateInode(const CreateInodeRequest *request,
     }
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
+
     MetaStatusCode status =
         partition->CreateInode(param, response->mutable_inode());
     response->set_statuscode(status);
@@ -521,12 +524,8 @@ MetaStoreImpl::CreateRootInode(const CreateRootInodeRequest *request,
     }
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     MetaStatusCode status = partition->CreateRootInode(param);
     response->set_statuscode(status);
@@ -560,12 +559,8 @@ MetaStoreImpl::CreateManageInode(const CreateManageInodeRequest *request,
     }
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     MetaStatusCode status = partition->CreateManageInode(
         param, request->managetype(), response->mutable_inode());
@@ -597,12 +592,8 @@ MetaStatusCode MetaStoreImpl::GetInode(const GetInodeRequest *request,
     uint64_t inodeId = request->inodeid();
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     Inode *inode = response->mutable_inode();
     MetaStatusCode rc = partition->GetInode(fsId, inodeId, inode);
@@ -633,12 +624,8 @@ MetaStatusCode
 MetaStoreImpl::BatchGetInodeAttr(const BatchGetInodeAttrRequest *request,
                                  BatchGetInodeAttrResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     uint32_t fsId = request->fsid();
     MetaStatusCode status = MetaStatusCode::OK;
@@ -658,12 +645,8 @@ MetaStoreImpl::BatchGetInodeAttr(const BatchGetInodeAttrRequest *request,
 MetaStatusCode MetaStoreImpl::BatchGetXAttr(const BatchGetXAttrRequest *request,
                                             BatchGetXAttrResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     uint32_t fsId = request->fsid();
     MetaStatusCode status = MetaStatusCode::OK;
@@ -686,12 +669,8 @@ MetaStatusCode MetaStoreImpl::DeleteInode(const DeleteInodeRequest *request,
     uint64_t inodeId = request->inodeid();
 
     ReadLockGuard readLockGuard(rwLock_);
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     MetaStatusCode status = partition->DeleteInode(fsId, inodeId);
     response->set_statuscode(status);
@@ -702,12 +681,8 @@ MetaStatusCode MetaStoreImpl::UpdateInode(const UpdateInodeRequest *request,
                                           UpdateInodeResponse *response) {
     ReadLockGuard readLockGuard(rwLock_);
     VLOG(9) << "UpdateInode inode " << request->inodeid();
-    std::shared_ptr<Partition> partition = GetPartition(request->partitionid());
-    if (partition == nullptr) {
-        MetaStatusCode status = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(status);
-        return status;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     MetaStatusCode status = partition->UpdateInode(*request);
     response->set_statuscode(status);
@@ -720,12 +695,8 @@ MetaStatusCode MetaStoreImpl::GetOrModifyS3ChunkInfo(
     std::shared_ptr<Iterator> *iterator) {
     MetaStatusCode rc;
     ReadLockGuard readLockGuard(rwLock_);
-    auto partition = GetPartition(request->partitionid());
-    if (nullptr == partition) {
-        rc = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(rc);
-        return rc;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     uint32_t fsId = request->fsid();
     uint64_t inodeId = request->inodeid();
@@ -792,12 +763,8 @@ MetaStatusCode
 MetaStoreImpl::GetVolumeExtent(const GetVolumeExtentRequest *request,
                                GetVolumeExtentResponse *response) {
     ReadLockGuard guard(rwLock_);
-    auto partition = GetPartition(request->partitionid());
-    if (!partition) {
-        auto st = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(st);
-        return st;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     std::vector<uint64_t> slices(request->sliceoffsets().begin(),
                                  request->sliceoffsets().end());
@@ -815,12 +782,8 @@ MetaStatusCode
 MetaStoreImpl::UpdateVolumeExtent(const UpdateVolumeExtentRequest *request,
                                   UpdateVolumeExtentResponse *response) {
     ReadLockGuard guard(rwLock_);
-    auto partition = GetPartition(request->partitionid());
-    if (!partition) {
-        auto st = MetaStatusCode::PARTITION_NOT_FOUND;
-        response->set_statuscode(st);
-        return st;
-    }
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
 
     VLOG(9) << "UpdateVolumeExtent, request: " << request->ShortDebugString();
 
@@ -830,6 +793,22 @@ MetaStoreImpl::UpdateVolumeExtent(const UpdateVolumeExtentRequest *request,
     return st;
 }
 
+MetaStatusCode MetaStoreImpl::UpdateDeallocatableBlockGroup(
+    const UpdateDeallocatableBlockGroupRequest *request,
+    UpdateDeallocatableBlockGroupResponse *response) {
+    ReadLockGuard guard(rwLock_);
+    std::shared_ptr<Partition> partition;
+    GET_PARTITION(partition);
+
+    VLOG(9) << "UpdateDeallocatableBlockGroup, request: "
+            << request->ShortDebugString();
+
+    auto st = partition->UpdateDeallocatableBlockGroup(*request);
+    response->set_statuscode(st);
+
+    return MetaStatusCode::OK;
+}
+
 bool MetaStoreImpl::InitStorage() {
     if (storageOptions_.type == "memory") {
         kvStorage_ = std::make_shared<MemoryStorage>(storageOptions_);

curvefs/src/metaserver/metastore.h

@@ -86,6 +86,22 @@ using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 
 using ::curvefs::metaserver::storage::StorageOptions;
 
+// Dentry and inode related data will be stored in kvstorage
+//
+// 1. When kvstorage is rocksdb, the structure of the data is as follows:
+//    (table is related with partitionID)
+//                  column
+//                     |
+//       (table1     table2             table3)
+//           |         |                   |
+//         inode  inodedealloc  blockGroup-with-inodedealloc
+//
+//                  column
+//                     |
+//       (table1     table2         table3)
+//           |          |              |
+//         dentry  s3chunkinfo    volumnextent
+
 class MetaStore {
  public:
     MetaStore() = default;
@@ -107,6 +123,9 @@ class MetaStore {
     virtual bool GetPartitionInfoList(
                             std::list<PartitionInfo> *partitionInfoList) = 0;
 
+    virtual bool GetPartitionSnap(
+        std::map<uint32_t, std::shared_ptr<Partition>> *partitionSnap) = 0;
+
     virtual std::shared_ptr<StreamServer> GetStreamServer() = 0;
 
     // dentry
@@ -170,6 +189,10 @@ class MetaStore {
     virtual MetaStatusCode UpdateVolumeExtent(
         const UpdateVolumeExtentRequest* request,
         UpdateVolumeExtentResponse* response) = 0;
+
+    virtual MetaStatusCode UpdateDeallocatableBlockGroup(
+        const UpdateDeallocatableBlockGroupRequest *request,
+        UpdateDeallocatableBlockGroupResponse *response) = 0;
 };
 
 class MetaStoreImpl : public MetaStore {
@@ -193,6 +216,9 @@ class MetaStoreImpl : public MetaStore {
     bool GetPartitionInfoList(
                         std::list<PartitionInfo> *partitionInfoList) override;
 
+    bool GetPartitionSnap(
+        std::map<uint32_t, std::shared_ptr<Partition>> *partitionSnap) override;
+
     std::shared_ptr<StreamServer> GetStreamServer() override;
 
     // dentry
@@ -255,6 +281,11 @@ class MetaStoreImpl : public MetaStore {
         const UpdateVolumeExtentRequest* request,
         UpdateVolumeExtentResponse* response) override;
 
+    // block group
+    MetaStatusCode UpdateDeallocatableBlockGroup(
+        const UpdateDeallocatableBlockGroupRequest *request,
+        UpdateDeallocatableBlockGroupResponse *response) override;
+
  private:
     FRIEND_TEST(MetastoreTest, partition);
     FRIEND_TEST(MetastoreTest, test_inode);

curvefs/src/metaserver/partition.cpp

@@ -33,6 +33,10 @@
 #include "curvefs/src/metaserver/trash_manager.h"
 #include "curvefs/src/metaserver/storage/converter.h"
 #include "curvefs/src/metaserver/s3compact.h"
+#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h"
+#include "curvefs/src/metaserver/space/volume_deallocate_manager.h"
+#include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
+#include "curvefs/src/metaserver/mds/fsinfo_manager.h"
 
 namespace curvefs {
 namespace metaserver {
@@ -41,28 +45,30 @@ using ::curvefs::metaserver::storage::NameGenerator;
 
 Partition::Partition(PartitionInfo partition,
                      std::shared_ptr<KVStorage> kvStorage,
-                     bool startCompact) {
+                     bool startCompact, bool startVolumeDeallocate) {
     assert(partition.start() <= partition.end());
     partitionInfo_ = std::move(partition);
+    uint32_t partitionId = partitionInfo_.partitionid();
+
+    kvStorage_ = kvStorage;
+    nameGen_ = std::make_shared<NameGenerator>(partitionId);
 
     uint64_t nInode = 0;
     uint64_t nDentry = 0;
-    uint32_t partitionId = partitionInfo_.partitionid();
     if (partitionInfo_.has_inodenum()) {
         nInode = partitionInfo_.inodenum();
     }
     if (partitionInfo_.has_dentrynum()) {
         nDentry = partitionInfo_.dentrynum();
     }
-    auto tableName = std::make_shared<NameGenerator>(partitionId);
-    inodeStorage_ = std::make_shared<InodeStorage>(
-        kvStorage, tableName, nInode);
-    dentryStorage_ = std::make_shared<DentryStorage>(
-        kvStorage, tableName, nDentry);
+    inodeStorage_ =
+        std::make_shared<InodeStorage>(kvStorage_, nameGen_, nInode);
+    dentryStorage_ =
+        std::make_shared<DentryStorage>(kvStorage_, nameGen_, nDentry);
 
-    trash_ = std::make_shared<TrashImpl>(inodeStorage_);
+    auto trash = std::make_shared<TrashImpl>(inodeStorage_);
     inodeManager_ = std::make_shared<InodeManager>(
-        inodeStorage_, trash_, partitionInfo_.mutable_filetype2inodenum());
+        inodeStorage_, trash, partitionInfo_.mutable_filetype2inodenum());
     txManager_ = std::make_shared<TxManager>(dentryStorage_);
     dentryManager_ =
         std::make_shared<DentryManager>(dentryStorage_, txManager_);
@@ -72,21 +78,39 @@ Partition::Partition(PartitionInfo partition,
     }
 
     if (partitionInfo_.status() != PartitionStatus::DELETING) {
-        TrashManager::GetInstance().Add(partitionInfo_.partitionid(), trash_);
+        TrashManager::GetInstance().Add(partitionInfo_.partitionid(), trash);
         if (startCompact) {
             StartS3Compact();
         }
+        if (startVolumeDeallocate) {
+            StartVolumeDeallocate();
+        }
     }
 }
 
-MetaStatusCode Partition::CreateDentry(const Dentry& dentry) {
-    if (!IsInodeBelongs(dentry.fsid(), dentry.parentinodeid())) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
+#define PRECHECK(fsId, inodeId)                                                \
+    do {                                                                       \
+        if (!IsInodeBelongs((fsId), (inodeId))) {            \
+            return MetaStatusCode::PARTITION_ID_MISSMATCH;                     \
+        }                                                                      \
+        if (GetStatus() == PartitionStatus::DELETING) {                        \
+            return MetaStatusCode::PARTITION_DELETING;                         \
+        }                                                                      \
+    } while (0)
+
+#define PRECHECKFSID(fsId)                                                     \
+    do {                                                                       \
+        if (!IsInodeBelongs((fsId))) {                                         \
+            return MetaStatusCode::PARTITION_ID_MISSMATCH;                     \
+        }                                                                      \
+        if (GetStatus() == PartitionStatus::DELETING) {                        \
+            return MetaStatusCode::PARTITION_DELETING;                         \
+        }                                                                      \
+    } while (0)
+
+MetaStatusCode Partition::CreateDentry(const Dentry& dentry) {
+    PRECHECK(dentry.fsid(), dentry.parentinodeid());
 
-    if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
     MetaStatusCode ret = dentryManager_->CreateDentry(dentry);
     if (MetaStatusCode::OK == ret) {
         if (dentry.has_type()) {
@@ -107,11 +131,8 @@ MetaStatusCode Partition::CreateDentry(const Dentry& dentry) {
 
 MetaStatusCode Partition::LoadDentry(const DentryVec& vec, bool merge) {
     auto dentry = vec.dentrys(0);
-    if (!IsInodeBelongs(dentry.fsid(), dentry.parentinodeid())) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    } else if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
+
+    PRECHECK(dentry.fsid(), dentry.parentinodeid());
 
     MetaStatusCode rc = dentryManager_->CreateDentry(vec, merge);
     if (rc == MetaStatusCode::OK ||
@@ -122,9 +143,7 @@ MetaStatusCode Partition::LoadDentry(const DentryVec& vec, bool merge) {
 }
 
 MetaStatusCode Partition::DeleteDentry(const Dentry& dentry) {
-    if (!IsInodeBelongs(dentry.fsid(), dentry.parentinodeid())) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
+    PRECHECK(dentry.fsid(), dentry.parentinodeid());
 
     MetaStatusCode ret = dentryManager_->DeleteDentry(dentry);
     if (MetaStatusCode::OK == ret) {
@@ -143,14 +162,7 @@ MetaStatusCode Partition::DeleteDentry(const Dentry& dentry) {
 }
 
 MetaStatusCode Partition::GetDentry(Dentry* dentry) {
-    if (!IsInodeBelongs(dentry->fsid(), dentry->parentinodeid())) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
-    if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
-
+    PRECHECK(dentry->fsid(), dentry->parentinodeid());
     return dentryManager_->GetDentry(dentry);
 }
 
@@ -158,14 +170,7 @@ MetaStatusCode Partition::ListDentry(const Dentry& dentry,
                                      std::vector<Dentry>* dentrys,
                                      uint32_t limit,
                                      bool onlyDir) {
-    if (!IsInodeBelongs(dentry.fsid(), dentry.parentinodeid())) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
-    if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
-
+    PRECHECK(dentry.fsid(), dentry.parentinodeid());
     return dentryManager_->ListDentry(dentry, dentrys, limit, onlyDir);
 }
 
@@ -174,14 +179,8 @@ void Partition::ClearDentry() {
 }
 
 MetaStatusCode Partition::HandleRenameTx(const std::vector<Dentry>& dentrys) {
-    for (const auto& it : dentrys) {
-        if (!IsInodeBelongs(it.fsid(), it.parentinodeid())) {
-            return MetaStatusCode::PARTITION_ID_MISSMATCH;
-        }
-    }
-
-    if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
+    for (const auto &it : dentrys) {
+        PRECHECK(it.fsid(), it.parentinodeid());
     }
 
     return dentryManager_->HandleRenameTx(dentrys);
@@ -196,10 +195,6 @@ bool Partition::InsertPendingTx(const PrepareRenameTxRequest& pendingTx) {
         }
     }
 
-    if (GetStatus() == PartitionStatus::DELETING) {
-        return false;
-    }
-
     auto renameTx = RenameTx(dentrys, dentryStorage_);
     return txManager_->InsertPendingTx(renameTx);
 }
@@ -247,74 +242,42 @@ MetaStatusCode Partition::CreateInode(const InodeParam &param,
 }
 
 MetaStatusCode Partition::CreateRootInode(const InodeParam &param) {
-    if (!IsInodeBelongs(param.fsId)) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
-    if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
-
+    PRECHECKFSID(param.fsId);
     return inodeManager_->CreateRootInode(param);
 }
 
 MetaStatusCode Partition::CreateManageInode(const InodeParam &param,
                                             ManageInodeType manageType,
                                             Inode* inode) {
-    if (!IsInodeBelongs(param.fsId)) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
-    if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
-
+    PRECHECKFSID(param.fsId);
     return inodeManager_->CreateManageInode(param, manageType, inode);
 }
 
 MetaStatusCode Partition::GetInode(uint32_t fsId, uint64_t inodeId,
                                    Inode* inode) {
-    if (!IsInodeBelongs(fsId, inodeId)) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
+    PRECHECK(fsId, inodeId);
     return inodeManager_->GetInode(fsId, inodeId, inode);
 }
 
 MetaStatusCode Partition::GetInodeAttr(uint32_t fsId, uint64_t inodeId,
                                        InodeAttr* attr) {
-    if (!IsInodeBelongs(fsId, inodeId)) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
+    PRECHECK(fsId, inodeId);
     return inodeManager_->GetInodeAttr(fsId, inodeId, attr);
 }
 
 MetaStatusCode Partition::GetXAttr(uint32_t fsId, uint64_t inodeId,
                                    XAttr* xattr) {
-    if (!IsInodeBelongs(fsId, inodeId)) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
+    PRECHECK(fsId, inodeId);
     return inodeManager_->GetXAttr(fsId, inodeId, xattr);
 }
 
 MetaStatusCode Partition::DeleteInode(uint32_t fsId, uint64_t inodeId) {
-    if (!IsInodeBelongs(fsId, inodeId)) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
+    PRECHECK(fsId, inodeId);
     return inodeManager_->DeleteInode(fsId, inodeId);
 }
 
 MetaStatusCode Partition::UpdateInode(const UpdateInodeRequest& request) {
-    if (!IsInodeBelongs(request.fsid(), request.inodeid())) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
-    if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
-
+    PRECHECK(request.fsid(), request.inodeid());
     return inodeManager_->UpdateInode(request);
 }
 
@@ -325,12 +288,7 @@ MetaStatusCode Partition::GetOrModifyS3ChunkInfo(
     const S3ChunkInfoMap& map2del,
     bool returnS3ChunkInfoMap,
     std::shared_ptr<Iterator>* iterator) {
-    if (!IsInodeBelongs(fsId, inodeId)) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    } else if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
-
+    PRECHECK(fsId, inodeId);
     return inodeManager_->GetOrModifyS3ChunkInfo(
         fsId, inodeId, map2add, map2del, returnS3ChunkInfoMap, iterator);
 }
@@ -339,19 +297,12 @@ MetaStatusCode Partition::PaddingInodeS3ChunkInfo(int32_t fsId,
                                                   uint64_t inodeId,
                                                   S3ChunkInfoMap* m,
                                                   uint64_t limit) {
-    if (!IsInodeBelongs(fsId, inodeId)) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    } else if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
+    PRECHECK(fsId, inodeId);
     return inodeManager_->PaddingInodeS3ChunkInfo(fsId, inodeId, m, limit);
 }
 
 MetaStatusCode Partition::InsertInode(const Inode& inode) {
-    if (!IsInodeBelongs(inode.fsid(), inode.inodeid())) {
-        return MetaStatusCode::PARTITION_ID_MISSMATCH;
-    }
-
+    PRECHECK(inode.fsid(), inode.inodeid());
     return inodeManager_->InsertInode(inode);
 }
 
@@ -374,7 +325,7 @@ bool Partition::IsDeletable() {
     return true;
 }
 
-bool Partition::IsInodeBelongs(uint32_t fsId, uint64_t inodeId) {
+bool Partition::IsInodeBelongs(uint32_t fsId, uint64_t inodeId) const {
     if (fsId != partitionInfo_.fsid()) {
         LOG(WARNING) << "partition fsid mismatch, fsId = " << fsId
                      << ", inodeId = " << inodeId
@@ -394,7 +345,7 @@ bool Partition::IsInodeBelongs(uint32_t fsId, uint64_t inodeId) {
     return true;
 }
 
-bool Partition::IsInodeBelongs(uint32_t fsId) {
+bool Partition::IsInodeBelongs(uint32_t fsId) const {
     if (fsId != partitionInfo_.fsid()) {
         return false;
     }
@@ -402,10 +353,6 @@ bool Partition::IsInodeBelongs(uint32_t fsId) {
     return true;
 }
 
-uint32_t Partition::GetPartitionId() const {
-    return partitionInfo_.partitionid();
-}
-
 PartitionInfo Partition::GetPartitionInfo() {
     partitionInfo_.set_inodenum(GetInodeNum());
     partitionInfo_.set_dentrynum(GetDentryNum());
@@ -481,19 +428,9 @@ std::string Partition::GetDentryTablename() {
     return oss.str();
 }
 
-#define PRECHECK(fsId, inodeId)                            \
-    do {                                                   \
-        if (!IsInodeBelongs((fsId), (inodeId))) {          \
-            return MetaStatusCode::PARTITION_ID_MISSMATCH; \
-        }                                                  \
-        if (GetStatus() == PartitionStatus::DELETING) {    \
-            return MetaStatusCode::PARTITION_DELETING;     \
-        }                                                  \
-    } while (0)
-
-MetaStatusCode Partition::UpdateVolumeExtent(uint32_t fsId,
-                                             uint64_t inodeId,
-                                             const VolumeExtentList& extents) {
+MetaStatusCode
+Partition::UpdateVolumeExtent(uint32_t fsId, uint64_t inodeId,
+                              const VolumeExtentSliceList &extents) {
     PRECHECK(fsId, inodeId);
     return inodeManager_->UpdateVolumeExtent(fsId, inodeId, extents);
 }
@@ -509,11 +446,23 @@ MetaStatusCode Partition::UpdateVolumeExtentSlice(
 MetaStatusCode Partition::GetVolumeExtent(uint32_t fsId,
                                           uint64_t inodeId,
                                           const std::vector<uint64_t>& slices,
-                                          VolumeExtentList* extents) {
+                                          VolumeExtentSliceList* extents) {
     PRECHECK(fsId, inodeId);
     return inodeManager_->GetVolumeExtent(fsId, inodeId, slices, extents);
 }
 
+MetaStatusCode Partition::UpdateDeallocatableBlockGroup(
+    const UpdateDeallocatableBlockGroupRequest &request) {
+    PRECHECKFSID(request.fsid());
+    return inodeStorage_->UpdateDeallocatableBlockGroup(request.fsid(),
+                                                        request.update());
+}
+
+MetaStatusCode Partition::GeAllBlockGroup(
+    std::vector<DeallocatableBlockGroup> *deallocatableBlockGroupVec) {
+    return inodeStorage_->GeAllBlockGroup(deallocatableBlockGroupVec);
+}
+
 void Partition::StartS3Compact() {
     S3CompactManager::GetInstance().Register(
         S3Compact{inodeManager_, partitionInfo_});
@@ -523,5 +472,54 @@ void Partition::CancelS3Compact() {
     S3CompactManager::GetInstance().Cancel(partitionInfo_.partitionid());
 }
 
+void Partition::StartVolumeDeallocate() {
+    FsInfo fsInfo;
+    bool ok =
+        FsInfoManager::GetInstance().GetFsInfo(partitionInfo_.fsid(), &fsInfo);
+    if (!ok) {
+        LOG(ERROR)
+            << "Partition start volume deallocate fail, get fsinfo fail. fsid="
+            << partitionInfo_.fsid();
+        return;
+    }
+
+    if (!fsInfo.detail().has_volume()) {
+        LOG(INFO) << "Partition not belong to volume, do not need start "
+                     "deallocate. partitionInfo="
+                  << partitionInfo_.DebugString();
+        return;
+    }
+
+    VolumeDeallocateCalOption calOpt;
+    calOpt.kvStorage = kvStorage_;
+    calOpt.inodeStorage = inodeStorage_;
+    calOpt.nameGen = nameGen_;
+    auto copysetNode =
+        copyset::CopysetNodeManager::GetInstance().GetSharedCopysetNode(
+            partitionInfo_.poolid(), partitionInfo_.copysetid());
+    if (copysetNode == nullptr) {
+        LOG(ERROR) << "Partition get copyset node failed. poolid="
+                   << partitionInfo_.poolid()
+                   << ", copysetid=" << partitionInfo_.copysetid();
+        return;
+    }
+
+    InodeVolumeSpaceDeallocate task(partitionInfo_.fsid(),
+                                    partitionInfo_.partitionid(), copysetNode);
+    task.Init(calOpt);
+
+    VolumeDeallocateManager::GetInstance().Register(std::move(task));
+}
+
+void Partition::CancelVolumeDeallocate() {
+    VolumeDeallocateManager::GetInstance().Cancel(partitionInfo_.partitionid());
+}
+
+void Partition::SetVolumeDeallocate(uint64_t fsId, uint64_t blockGroupOffset) {
+    assert(fsId == partitionInfo_.fsid());
+    VolumeDeallocateManager::GetInstance().Deallocate(
+        partitionInfo_.partitionid(), blockGroupOffset);
+}
+
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/partition.h

@@ -52,7 +52,8 @@ class Partition {
  public:
     Partition(PartitionInfo partition,
               std::shared_ptr<KVStorage> kvStorage,
-              bool startCompact = true);
+              bool startCompact = true, bool startVolumeDeallocate = true);
+    Partition() = default;
 
     // dentry
     MetaStatusCode CreateDentry(const Dentry& dentry);
@@ -111,7 +112,7 @@ class Partition {
 
     MetaStatusCode UpdateVolumeExtent(uint32_t fsId,
                                       uint64_t inodeId,
-                                      const VolumeExtentList& extents);
+                                      const VolumeExtentSliceList& extents);
 
     MetaStatusCode UpdateVolumeExtentSlice(uint32_t fsId,
                                            uint64_t inodeId,
@@ -120,7 +121,14 @@ class Partition {
     MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                    uint64_t inodeId,
                                    const std::vector<uint64_t>& slices,
-                                   VolumeExtentList* extents);
+                                   VolumeExtentSliceList* extents);
+
+
+    MetaStatusCode UpdateDeallocatableBlockGroup(
+        const UpdateDeallocatableBlockGroupRequest &request);
+
+    virtual MetaStatusCode GeAllBlockGroup(
+        std::vector<DeallocatableBlockGroup> *deallocatableBlockGroupVec);
 
     MetaStatusCode InsertInode(const Inode& inode);
 
@@ -130,18 +138,20 @@ class Partition {
     bool IsDeletable();
 
     // check if fsid matchs and inode range belongs to this partition
-    bool IsInodeBelongs(uint32_t fsId, uint64_t inodeId);
+    bool IsInodeBelongs(uint32_t fsId, uint64_t inodeId) const;
 
     // check if fsid match this partition
-    bool IsInodeBelongs(uint32_t fsId);
+    bool IsInodeBelongs(uint32_t fsId) const;
 
-    uint32_t GetPartitionId() const;
+    virtual uint32_t GetPartitionId() const {
+        return partitionInfo_.partitionid();
+    }
 
-    uint32_t GetPoolId() { return partitionInfo_.poolid(); }
+    virtual uint32_t GetPoolId() const { return partitionInfo_.poolid(); }
 
-    uint32_t GetCopySetId() { return partitionInfo_.copysetid(); }
+    virtual uint32_t GetCopySetId() const { return partitionInfo_.copysetid(); }
 
-    uint32_t GetFsId() { return partitionInfo_.fsid(); }
+    virtual uint32_t GetFsId() const { return partitionInfo_.fsid(); }
 
     PartitionInfo GetPartitionInfo();
 
@@ -165,6 +175,12 @@ class Partition {
 
     void CancelS3Compact();
 
+    void StartVolumeDeallocate();
+
+    void CancelVolumeDeallocate();
+
+    void SetVolumeDeallocate(uint64_t fsId, uint64_t blockGroupOffset);
+
     std::string GetInodeTablename();
 
     std::string GetDentryTablename();
@@ -190,10 +206,13 @@ class Partition {
     }
 
  private:
+    std::shared_ptr<KVStorage> kvStorage_;
+    std::shared_ptr<NameGenerator> nameGen_;
+
     std::shared_ptr<InodeStorage> inodeStorage_;
     std::shared_ptr<DentryStorage> dentryStorage_;
+
     std::shared_ptr<InodeManager> inodeManager_;
-    std::shared_ptr<TrashImpl> trash_;
     std::shared_ptr<DentryManager> dentryManager_;
     std::shared_ptr<TxManager> txManager_;
 

curvefs/src/metaserver/partition_cleaner.cpp

@@ -19,10 +19,10 @@
  * @Date: 2021-12-15 10:54:37
  * @Author: chenwei
  */
-#include "curvefs/src/metaserver/partition_cleaner.h"
 
 #include <list>
-
+#include "curvefs/src/metaserver/partition_cleaner.h"
+#include "curvefs/src/metaserver/mds/fsinfo_manager.h"
 #include "curvefs/src/metaserver/copyset/meta_operator.h"
 
 namespace curvefs {
@@ -87,26 +87,15 @@ MetaStatusCode PartitionCleaner::CleanDataAndDeleteInode(const Inode& inode) {
     // TODO(cw123) : consider FsFileType::TYPE_FILE
     if (FsFileType::TYPE_S3 == inode.type()) {
          // get s3info from mds
-        FsInfo fsInfo;
-        if (fsInfoMap_.find(inode.fsid()) == fsInfoMap_.end()) {
-            auto ret = mdsClient_->GetFsInfo(inode.fsid(), &fsInfo);
-            if (ret != FSStatusCode::OK) {
-                if (FSStatusCode::NOT_FOUND == ret) {
-                    LOG(ERROR)
-                        << "The fsName not exist, fsId = " << inode.fsid();
-                    return MetaStatusCode::S3_DELETE_ERR;
-                } else {
-                    LOG(ERROR)
-                        << "GetFsInfo failed, FSStatusCode = " << ret
-                        << ", FSStatusCode_Name = " << FSStatusCode_Name(ret)
-                        << ", fsId = " << inode.fsid();
-                    return MetaStatusCode::S3_DELETE_ERR;
-                }
-            }
-            fsInfoMap_.insert({inode.fsid(), fsInfo});
-        } else {
-            fsInfo = fsInfoMap_.find(inode.fsid())->second;
-        }
+         FsInfo fsInfo;
+         auto ret =
+             FsInfoManager::GetInstance().GetFsInfo(inode.fsid(), &fsInfo);
+         if (!ret) {
+             LOG(ERROR) << "PartitionCleaner get fsinfo failed, fsId = "
+                        << inode.fsid();
+             return MetaStatusCode::S3_DELETE_ERR;
+         }
+
         const auto& s3Info = fsInfo.detail().s3info();
         // reinit s3 adaptor
         S3ClientAdaptorOption clientAdaptorOption;

curvefs/src/metaserver/partition_cleaner.h

@@ -83,7 +83,6 @@ class PartitionCleaner {
     std::shared_ptr<MdsClient> mdsClient_;
     bool isStop_;
     uint32_t inodeDeletePeriodMs_;
-    std::unordered_map<uint32_t, FsInfo> fsInfoMap_;
 };
 
 class PartitionCleanerClosure : public google::protobuf::Closure {

curvefs/src/metaserver/space/inode_volume_space_deallocate.cpp

@@ -0,0 +1,310 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Wed Mar 22 10:38:14 CST 2023
+ * Author: lixiaocui
+ */
+
+#include <glog/logging.h>
+#include <algorithm>
+#include <iterator>
+#include <unordered_map>
+#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h"
+#include "curvefs/src/client/async_request_closure.h"
+
+using curvefs::client::UpdateVolumeExtentClosure;
+
+namespace curvefs {
+namespace metaserver {
+
+void InodeVolumeSpaceDeallocate::CalDeallocatableSpace() {
+    // get all deallocatable inode
+    auto iter = calOpt_.kvStorage->HGetAll(
+        calOpt_.nameGen->GetDeallocatableInodeTableName());
+    if (iter->Status() != 0) {
+        LOG(ERROR) << "InodeVolumeSpaceDeallocate failed to get iterator for "
+                      "all deallocatable indoe";
+        return;
+    } else {
+        iter->SeekToFirst();
+    }
+
+    // key is volume offset
+    DeallocatableBlockGroupMap increase;
+    while (iter->Valid()) {
+        Key4Inode key;
+        bool ok = calOpt_.conv.ParseFromString(iter->Key(), &key);
+        uint64_t typicalInode = key.inodeId;
+        iter->Next();
+        if (!ok) {
+            LOG(ERROR) << "InodeVolumeSpaceDeallocate parse inode key from "
+                       << iter->Key() << "fail";
+            continue;
+        }
+
+        // fill IncreaseDeallocatableBlockGroup
+        ok = DeallocatableSapceForInode(key, &increase);
+        if (!ok) {
+            continue;
+        }
+
+        if (increase.size() >= executeOpt_.batchClean || !iter->Valid()) {
+            // update according to metaserver client
+            MetaStatusCode st =
+                executeOpt_.metaClient->UpdateDeallocatableBlockGroup(
+                    fsId_, typicalInode, &increase);
+            if (st != MetaStatusCode::OK) {
+                LOG(ERROR) << "InodeVolumeSpaceDeallocate update "
+                           << "deallocatable block group fail";
+            }
+            LOG(INFO) << "InodeVolumeSpaceDeallocate cal success this round";
+            increase.clear();
+        }
+    }
+}
+
+MetaStatusCode
+InodeVolumeSpaceDeallocate::DeallocateOneBlockGroup(uint64_t blockGroupOffset) {
+    // get block group
+    auto skey = calOpt_.conv.SerializeToString(
+        Key4DeallocatableBlockGroup{fsId_, blockGroupOffset});
+
+    DeallocatableBlockGroup out;
+    auto s = calOpt_.kvStorage->HGet(
+        calOpt_.nameGen->GetDeallocatableBlockGroupTableName(), skey, &out);
+    if (s.IsNotFound()) {
+        LOG(INFO) << "InodeVolumeSpaceDeallocate do not record deallocatable "
+                     "blockgroup, "
+                     "fsId="
+                  << fsId_ << ", blockGroupOffset=" << blockGroupOffset;
+        return MetaStatusCode::OK;
+    }
+    if (!s.ok()) {
+        LOG(ERROR) << "InodeVolumeSpaceDeallocate deallocate blockgroup fail, "
+                      "fsId="
+                   << fsId_ << ", blockGroupOffset=" << blockGroupOffset
+                   << ", partitionId=" << partitionId_
+                   << ", status=" << s.ToString();
+        return MetaStatusCode::STORAGE_INTERNAL_ERROR;
+    }
+
+    // batch processing of inodelists involving space deallocatable
+    DeallocatableBlockGroupMap mark;
+    auto &onemark = mark[blockGroupOffset];
+    onemark.set_blockgroupoffset(blockGroupOffset);
+
+    auto iter = out.inodeidlist().begin();
+    auto boundary = out.inodeidlist().end();
+    while (iter != boundary) {
+        onemark.mutable_mark()->add_inodeidunderdeallocate(*iter);
+        iter++;
+
+        auto size = onemark.mark().inodeidunderdeallocate_size();
+        if (size >= executeOpt_.batchClean || (iter == boundary && size > 0)) {
+            ProcessSepcifyInodeList(blockGroupOffset, &mark);
+            onemark.mutable_mark()->clear_inodeidunderdeallocate();
+        }
+    }
+
+    return MetaStatusCode::OK;
+}
+
+bool InodeVolumeSpaceDeallocate::DeallocatableSapceForInode(
+    const Key4Inode &key, DeallocatableBlockGroupMap *increaseMap) {
+
+    // Calculate the deallocatable space of the blockgroup corresponding to the
+    // extent in the inode
+    auto iter = calOpt_.inodeStorage->GetAllVolumeExtent(key.fsId, key.inodeId);
+
+    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+        VolumeExtentSlice slice;
+        if (!slice.ParseFromString(iter->Value())) {
+            LOG(ERROR) << "InodeVolumeSpaceDeallocate parse VolumeExtentSlice "
+                          "failed, fsId="
+                       << key.fsId << ", inodeId=" << key.inodeId;
+            return false;
+        }
+
+        DeallocatbleSpaceForVolumeExtent(slice, key, increaseMap);
+    }
+
+    return true;
+}
+
+void InodeVolumeSpaceDeallocate::DeallocatbleSpaceForVolumeExtent(
+    const VolumeExtentSlice &slice, const Key4Inode &key,
+    DeallocatableBlockGroupMap *increaseMap) {
+    std::unordered_map<uint64_t, bool> relatedBlockGroup;
+
+    for (const auto &next : slice.extents()) {
+        // |           BlockGroup         |           BlockGroup          |
+        //  -----------                    -----------
+        // | 4k bitmap |                  | 4k bitmap |
+        //  -----------                    -----------
+        // by default, 4K bitmap is not pre-allocated, so the extent will not
+        // cross block group.
+        auto blockGroupOffset =
+            (next.volumeoffset() / blockGroupSize_) * blockGroupSize_;
+
+        auto &exist = (*increaseMap)[blockGroupOffset];
+        auto increase = exist.mutable_increase();
+
+        if (relatedBlockGroup.count(blockGroupOffset)) {
+            auto oldSize = increase->increasedeallocatablesize();
+            increase->set_increasedeallocatablesize(oldSize + next.length());
+        } else {
+            relatedBlockGroup[blockGroupOffset] = true;
+            exist.set_blockgroupoffset(blockGroupOffset);
+            increase->set_increasedeallocatablesize(next.length());
+        }
+    }
+
+    for (const auto &item : relatedBlockGroup) {
+        auto &exist = (*increaseMap)[item.first];
+        exist.mutable_increase()->add_inodeidlistadd(key.inodeId);
+    }
+}
+
+// NOTE:
+//  1. update the status of DeallocatableBlockGroup: add inodeunderdeallocate
+//  and remove related inode to inodeidlist
+//
+//  2. dallocate the space of the block according to the fetched inode
+//
+//  3. update the deallocatable status of DeallocatableBlockGroup: decrease
+//  deallocatableSize
+void InodeVolumeSpaceDeallocate::ProcessSepcifyInodeList(
+    uint64_t blockGroupOffset, DeallocatableBlockGroupMap *markMap) {
+
+    // 1
+    uint64_t typicalInode =
+        (*markMap)[blockGroupOffset].mark().inodeidunderdeallocate(0);
+    MetaStatusCode st = executeOpt_.metaClient->UpdateDeallocatableBlockGroup(
+        fsId_, typicalInode, markMap);
+    if (st != MetaStatusCode::OK) {
+        LOG(ERROR) << "InodeVolumeSpaceDeallocate mark inodelist to be "
+                      "deallocatable failed, fsId="
+                   << fsId_ << ", blockGroupOffset=" << blockGroupOffset;
+        return;
+    }
+    auto &onemark = (*markMap)[blockGroupOffset];
+
+    // 2
+    uint64_t decreaseSize = 0;
+    auto inodeUnderDeallocate =
+        onemark.mutable_mark()->mutable_inodeidunderdeallocate();
+    bool ok =
+        DeallocateInode(blockGroupOffset, *inodeUnderDeallocate, &decreaseSize);
+    if (!ok) {
+        LOG(ERROR) << "InodeVolumeSpaceDeallocate an error occurred while "
+                      "actually deallocate the volume space, fsId="
+                   << fsId_ << ", blockGroupOffset=" << blockGroupOffset;
+        return;
+    }
+
+    // 3
+    DeallocatableBlockGroupMap decrease;
+    auto &onedecrease = decrease[blockGroupOffset];
+    onedecrease.set_blockgroupoffset(blockGroupOffset);
+    onedecrease.mutable_decrease()->set_decreasedeallocatablesize(decreaseSize);
+    onedecrease.mutable_decrease()->mutable_inodeddeallocated()->Swap(
+        inodeUnderDeallocate);
+
+    st = executeOpt_.metaClient->UpdateDeallocatableBlockGroup(
+        fsId_, typicalInode, &decrease);
+    if (st != MetaStatusCode::OK) {
+        LOG(ERROR) << "InodeVolumeSpaceDeallocate update deallocatable size "
+                      "failed, fsId="
+                   << fsId_ << ", blockGroupOffset=" << blockGroupOffset;
+        return;
+    }
+}
+
+// NOTE:
+// 1. update inode extent
+// 2. deallocate the corresponding space in volume
+//
+// According to this order, one is that the corresponding volume space will not
+// be deleted repeatedly, but there is a disadvantage, when the process hangs
+// up, this part of the space will not be recovered. And may need to be
+// deallocate by means of fscheck, etc.
+bool InodeVolumeSpaceDeallocate::DeallocateInode(uint64_t blockGroupOffset,
+                                                 const Uint64Vec &inodelist,
+                                                 uint64_t *decrease) {
+    std::vector<Extent> deallocatableVolumeSpace;
+
+    for (const auto &inodeId : inodelist) {
+        VolumeExtentSliceList sliceList;
+        MetaStatusCode st = calOpt_.inodeStorage->GetAllVolumeExtent(
+            fsId_, inodeId, &sliceList);
+        if (st != MetaStatusCode::OK) {
+            LOG(ERROR) << "InodeVolumeSpaceDeallocate get inode extent "
+                          "failed, fsId="
+                       << fsId_ << ", inodeId=" << inodeId;
+            return false;
+        }
+
+        // 1
+        UpdateDeallocateInodeExtentSlice(blockGroupOffset, inodeId, decrease,
+                                         &sliceList, &deallocatableVolumeSpace);
+        auto closure = new UpdateVolumeExtentClosure(nullptr, true);
+        executeOpt_.metaClient->AsyncUpdateVolumeExtent(fsId_, inodeId,
+                                                        sliceList, closure);
+        closure->Wait();
+        // TODO(ilixiaocui): AsyncUpdateVolumeExtent update failed
+    }
+
+    // 2
+    bool ok = executeOpt_.volumeSpaceManager->DeallocVolumeSpace(
+        fsId_, blockGroupOffset, deallocatableVolumeSpace);
+    if (!ok) {
+        // TODO(ilixiaocui): this part of the non-recyclable space should be
+        // included in the metric statistics
+    }
+
+    return ok;
+}
+
+void InodeVolumeSpaceDeallocate::UpdateDeallocateInodeExtentSlice(
+    uint64_t blockGroupOffset, uint64_t inodeId, uint64_t *decrease,
+    VolumeExtentSliceList *sliceList,
+    std::vector<Extent> *deallocatableVolumeSpace) {
+    uint64_t boundary =
+        (blockGroupOffset / blockGroupSize_ + 1) * blockGroupSize_;
+
+    for (auto &slice : *sliceList->mutable_slices()) {
+        auto extents = slice.mutable_extents();
+        extents->erase(
+            std::remove_if(extents->begin(), extents->end(),
+                           [&](const VolumeExtent &extent) {
+                               bool res = (extent.volumeoffset() >=
+                                           blockGroupOffset) &&
+                                          (extent.volumeoffset() < boundary);
+                               if (res) {
+                                   *decrease += extent.length();
+                                   deallocatableVolumeSpace->emplace_back(
+                                       extent.volumeoffset(), extent.length());
+                               }
+                               return res;
+                           }),
+            extents->end());
+    }
+}
+
+}  // namespace metaserver
+}  // namespace curvefs

curvefs/src/metaserver/space/inode_volume_space_deallocate.h

@@ -0,0 +1,178 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Wed Mar 22 10:39:04 CST 2023
+ * Author: lixiaocui
+ */
+
+#ifndef CURVEFS_SRC_METASERVER_SPACE_INODE_VOLUME_SPACE_DEALLOCATE_H_
+#define CURVEFS_SRC_METASERVER_SPACE_INODE_VOLUME_SPACE_DEALLOCATE_H_
+
+#include <gtest/gtest_prod.h>
+
+#include <map>
+#include <string>
+#include <vector>
+#include <utility>
+#include <memory>
+
+#include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/proto/common.pb.h"
+#include "curvefs/src/metaserver/storage/converter.h"
+#include "curvefs/src/metaserver/storage/storage.h"
+#include "curvefs/src/metaserver/inode_storage.h"
+#include "curvefs/src/metaserver/space/volume_space_manager.h"
+#include "curvefs/src/client/rpcclient/metaserver_client.h"
+#include "curvefs/src/volume/common.h"
+#include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
+
+namespace curvefs {
+namespace metaserver {
+
+using curvefs::client::rpcclient::MetaServerClient;
+using curvefs::common::BlockGroupID;
+using curvefs::metaserver::storage::Converter;
+using curvefs::metaserver::storage::Key4DeallocatableBlockGroup;
+using curvefs::metaserver::storage::Key4Inode;
+using curvefs::metaserver::storage::KVStorage;
+using curvefs::metaserver::storage::Prefix4InodeVolumeExtent;
+using curvefs::volume::Extent;
+
+using Uint64Vec = google::protobuf::RepeatedField<google::protobuf::uint64>;
+using DeallocatableBlockGroupMap = std::map<uint64_t, DeallocatableBlockGroup>;
+
+struct VolumeDeallocateCalOption {
+    Converter conv;
+    std::shared_ptr<NameGenerator> nameGen;
+    std::shared_ptr<KVStorage> kvStorage;
+    std::shared_ptr<InodeStorage> inodeStorage;
+};
+
+struct VolumeDeallocateExecuteOption {
+    std::shared_ptr<VolumeSpaceManager> volumeSpaceManager;
+    std::shared_ptr<MetaServerClient> metaClient;
+
+    uint32_t batchClean;
+};
+
+class InodeVolumeSpaceDeallocate {
+ public:
+    InodeVolumeSpaceDeallocate(
+        uint64_t fsId, uint32_t partitionId,
+        std::shared_ptr<copyset::CopysetNode> copysetNode)
+        : fsId_(fsId), partitionId_(partitionId), copysetNode_(copysetNode) {}
+
+    void Init(VolumeDeallocateCalOption calOpt) { calOpt_ = std::move(calOpt); }
+
+    void Init(const VolumeDeallocateExecuteOption &executeOpt) {
+        executeOpt_.volumeSpaceManager = executeOpt.volumeSpaceManager;
+        executeOpt_.metaClient = executeOpt.metaClient;
+        executeOpt_.batchClean = executeOpt.batchClean;
+
+        blockGroupSize_ =
+            executeOpt_.volumeSpaceManager->GetBlockGroupSize(fsId_);
+    }
+
+    // used to traverse the list of inodes to be deleted, and count the
+    // deallocatable space of BlockGroup
+    void CalDeallocatableSpace();
+
+    // Return the reclaimable space of each BlockGroup
+    //  deallocatablespace: key=str(fsid+offset) value=(deallocatabe space size)
+    void
+    GetDeallocatableSpace(std::map<std::string, uint64_t> *deallocatablespaces);
+
+    // DeAllocate the space of the specified BlockGroup
+    MetaStatusCode DeallocateOneBlockGroup(uint64_t blockGroupOffset);
+
+    uint32_t GetPartitionID() const { return partitionId_; }
+
+    void SetCanceled() { canceled_ = true; }
+
+    bool IsCanceled() const { return canceled_; }
+
+    bool HasDeallocateTask() {
+        return waitingBlockGroupOffset_.has_value();
+    }
+
+    uint64_t GetDeallocateTask() {
+        return waitingBlockGroupOffset_.value();
+    }
+
+    void ResetDeallocateTask() {
+        waitingBlockGroupOffset_.reset();
+    }
+
+    void SetDeallocateTask(uint64_t blockGroupOffset) {
+        waitingBlockGroupOffset_ = blockGroupOffset;
+    }
+
+    bool CanStart() {
+        bool leader = copysetNode_->IsLeaderTerm();
+        if (!leader) {
+            LOG(WARNING) << "InodeVolumeSpaceDeallocate copyset="
+                         << copysetNode_->Name() << " is not leader, skip";
+        }
+        return leader;
+    }
+
+ private:
+    FRIEND_TEST(InodeVolumeSpaceDeallocateTest,
+                Test_DeallocatableSapceForInode);
+    FRIEND_TEST(InodeVolumeSpaceDeallocateTest, Test_ProcessSepcifyInodeList);
+    FRIEND_TEST(InodeVolumeSpaceDeallocateTest, Test_DeallocateInode);
+    FRIEND_TEST(InodeVolumeSpaceDeallocateTest,
+                Test_UpdateDeallocateInodeExtentSlice);
+
+    bool DeallocatableSapceForInode(const Key4Inode &key,
+                                    DeallocatableBlockGroupMap *increaseMap);
+
+    void
+    DeallocatbleSpaceForVolumeExtent(const VolumeExtentSlice &slice,
+                                     const Key4Inode &key,
+                                     DeallocatableBlockGroupMap *increaseMap);
+
+    void ProcessSepcifyInodeList(uint64_t blockGroupOffset,
+                                 DeallocatableBlockGroupMap *markMap);
+    bool DeallocateInode(uint64_t blockGroupOffset, const Uint64Vec &inodelist,
+                         uint64_t *decrease);
+
+ private:
+    void UpdateDeallocateInodeExtentSlice(
+        uint64_t blockGroupOffset, uint64_t inodeId, uint64_t *decrease,
+        VolumeExtentSliceList *sliceList,
+        std::vector<Extent> *deallocatableVolumeSpace);
+
+ private:
+    VolumeDeallocateCalOption calOpt_;
+    VolumeDeallocateExecuteOption executeOpt_;
+
+    uint64_t fsId_;
+    uint32_t partitionId_;
+    uint64_t blockGroupSize_;
+
+    absl::optional<uint64_t> waitingBlockGroupOffset_;
+
+    bool canceled_{false};
+
+    std::shared_ptr<copyset::CopysetNode> copysetNode_;
+};
+}  // namespace metaserver
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_METASERVER_SPACE_INODE_VOLUME_SPACE_DEALLOCATE_H_

curvefs/src/metaserver/space/volume_deallocate_manager.cpp

@@ -0,0 +1,144 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Tue Apr 25 15:46:58 CST 2023
+ * Author: lixiaocui
+ */
+
+#include <utility>
+#include "curvefs/src/metaserver/space/volume_deallocate_manager.h"
+
+namespace curvefs {
+namespace metaserver {
+void VolumeDeallocateManager::Register(InodeVolumeSpaceDeallocate task) {
+    {
+        std::lock_guard<std::mutex> lk(workerCtx_.mtx);
+        task.Init(executeOpt_);
+        workerCtx_.allTasks.emplace_back(std::move(task));
+    }
+
+    workerCtx_.cond.notify_one();
+}
+
+void VolumeDeallocateManager::Cancel(uint32_t partitionId) {
+    std::lock_guard<std::mutex> lk(workerCtx_.mtx);
+
+    // cancle from tasking
+    auto it = workerCtx_.tasking.find(partitionId);
+    if (it != workerCtx_.tasking.end()) {
+        it->second->Cancel(partitionId);
+        LOG(INFO) << "VolumeDeallocateManager cancel task, partitionId="
+                  << partitionId;
+        return;
+    }
+
+    // cancel from waiting
+    workerCtx_.allTasks.remove_if([partitionId](
+                                      const InodeVolumeSpaceDeallocate &task) {
+        bool ret = (task.GetPartitionID() == partitionId);
+        if (ret) {
+            LOG(INFO)
+                << "VolumeDeallocateManager cancel waiting task, partitionId="
+                << partitionId;
+        }
+        return ret;
+    });
+}
+
+void VolumeDeallocateManager::Deallocate(uint32_t partitioId,
+                                         uint64_t blockGroupOffset) {
+    std::lock_guard<std::mutex> lk(workerCtx_.mtx);
+
+    // set from waiting
+    {
+        auto it =
+            std::find_if(workerCtx_.allTasks.begin(), workerCtx_.allTasks.end(),
+                         [partitioId](const InodeVolumeSpaceDeallocate &task) {
+                             return task.GetPartitionID() == partitioId;
+                         });
+        if (it != workerCtx_.allTasks.end()) {
+            it->SetDeallocateTask(blockGroupOffset);
+            return;
+        }
+    }
+
+    // set from tasking
+    {
+        auto it = workerCtx_.tasking.find(partitioId);
+        if (it != workerCtx_.tasking.end()) {
+            it->second->SetDeallocate(blockGroupOffset);
+            return;
+        }
+    }
+
+    LOG(ERROR) << "VolumeDeallocateManager do not find deallocate handler for "
+                  "partitionId="
+               << partitioId;
+}
+
+bool VolumeDeallocateManager::HasDeallocate() {
+    std::lock_guard<std::mutex> lk(workerCtx_.mtx);
+
+    // view from waiting
+    for (auto &task : workerCtx_.allTasks) {
+        if (task.HasDeallocateTask()) {
+            return true;
+        }
+    }
+
+    // view from tasking
+    for (auto &task : workerCtx_.tasking) {
+        if (task.second->HasDeallocate()) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
+void VolumeDeallocateManager::Run() {
+    if (!workerOpt_.enable) {
+        LOG(INFO) << "VolumeDeallocateManager not enable";
+        return;
+    }
+
+    if (!workerCtx_.running.exchange(true)) {
+        for (uint16_t i = 0; i < workerOpt_.workerNum; i++) {
+            workers_.emplace_back(
+                absl::make_unique<VolumeDeallocateWorker>(&workerCtx_));
+            workers_.back()->Run();
+        }
+        LOG(INFO) << "VolumeDeallocateManager start all workers ok";
+    } else {
+        LOG(INFO) << "VolumeDeallocateManager already run";
+    }
+}
+
+void VolumeDeallocateManager::Stop() {
+    if (!workerCtx_.running.exchange(false)) {
+        return;
+    }
+
+    workerCtx_.cond.notify_all();
+    for (auto &worker : workers_) {
+        worker->Stop();
+    }
+}
+
+}  // namespace metaserver
+}  // namespace curvefs

curvefs/src/metaserver/space/volume_deallocate_manager.h

@@ -0,0 +1,81 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Tue Apr 25 15:36:29 CST 2023
+ * Author: lixiaocui
+ */
+
+#ifndef CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_MANAGER_H_
+#define CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_MANAGER_H_
+
+#include <vector>
+#include <memory>
+
+#include "curvefs/src/metaserver/space/volume_deallocate_worker.h"
+#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h"
+
+namespace curvefs {
+namespace metaserver {
+
+struct VolumeDeallocateWorkerQueueOption {
+    bool enable;
+    uint32_t workerNum;
+};
+
+class VolumeDeallocateManager {
+ public:
+    static VolumeDeallocateManager &GetInstance() {
+        static VolumeDeallocateManager instance;
+        return instance;
+    }
+
+    void Init(const VolumeDeallocateWorkerQueueOption &option,
+              const VolumeDeallocateExecuteOption &executeOpt) {
+        workerOpt_.enable = option.enable;
+        workerOpt_.workerNum = option.workerNum;
+
+        executeOpt_.volumeSpaceManager = executeOpt.volumeSpaceManager;
+        executeOpt_.metaClient = executeOpt.metaClient;
+        executeOpt_.batchClean = executeOpt.batchClean;
+    }
+
+    void Register(InodeVolumeSpaceDeallocate task);
+    void Cancel(uint32_t partitionId);
+    void Deallocate(uint32_t partitioId, uint64_t blockGroupOffset);
+    bool HasDeallocate();
+
+    void Run();
+    void Stop();
+
+
+
+ private:
+    VolumeDeallocateManager() = default;
+    ~VolumeDeallocateManager() = default;
+
+    VolumeDeallocateWorkerQueueOption workerOpt_;
+    VolumeDeallocateExecuteOption executeOpt_;
+
+    VolumeDeallocateWorkerContext workerCtx_;
+    std::vector<std::unique_ptr<VolumeDeallocateWorker>> workers_;
+};
+
+}  // namespace metaserver
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_MANAGER_H_

curvefs/src/metaserver/space/volume_deallocate_worker.cpp

@@ -0,0 +1,137 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Tue Apr 25 16:52:55 CST 2023
+ * Author: lixiaocui
+ */
+
+#include "absl/cleanup/cleanup.h"
+
+#include "curvefs/src/common/threading.h"
+#include "curvefs/src/metaserver/space/volume_deallocate_worker.h"
+
+namespace curvefs {
+namespace metaserver {
+void VolumeDeallocateWorker::Run() {
+    calWorkingThread_ =
+        std::thread(&VolumeDeallocateWorker::DeallocatetWorker, this);
+}
+
+void VolumeDeallocateWorker::Stop() {
+    sleeper_.interrupt();
+
+    if (calWorkingThread_.joinable()) {
+        calWorkingThread_.join();
+    }
+
+    LOG(INFO) << "VolumeDeallocateWorker stopped";
+}
+
+void VolumeDeallocateWorker::Cancel(uint32_t partitionId) {
+    assert(current_->GetPartitionID() == partitionId);
+    current_->SetCanceled();
+    sleeper_.interrupt();
+}
+
+void VolumeDeallocateWorker::SetDeallocate(uint64_t blockGroupOffset) {
+    current_->SetDeallocateTask(blockGroupOffset);
+}
+
+bool VolumeDeallocateWorker::HasDeallocate() {
+    return current_->HasDeallocateTask();
+}
+
+bool VolumeDeallocateWorker::WaitDeallocate() {
+    // wait for task arrive
+    std::unique_lock<std::mutex> lock(ctx_->mtx);
+    if (ctx_->allTasks.empty()) {
+        ctx_->cond.wait(
+            lock, [this] { return !ctx_->allTasks.empty() || !ctx_->running; });
+    }
+
+    // cancel by manager
+    if (!ctx_->running) {
+        return false;
+    }
+
+    // wait ok
+    current_ = std::move(ctx_->allTasks.front());
+    ctx_->allTasks.pop_front();
+    ctx_->tasking.emplace(current_->GetPartitionID(), this);
+
+    VLOG(1) << "VolumeDeallocateWorker get task, partitionId = "
+            << current_->GetPartitionID();
+
+    sleeper_.init();
+    return true;
+}
+
+void VolumeDeallocateWorker::Cleanup() {
+    std::lock_guard<std::mutex> lock(ctx_->mtx);
+
+    ctx_->tasking.erase(current_->GetPartitionID());
+
+    if (current_->IsCanceled()) {
+        LOG(INFO) << "VolumeDeallocateWorker task canceled, partitionId="
+                  << current_->GetPartitionID();
+        return;
+    }
+
+    ctx_->allTasks.emplace_back(std::move(current_.value()));
+    ctx_->cond.notify_one();
+
+    current_.reset();
+}
+
+void VolumeDeallocateWorker::DeallocatetWorker() {
+    common::SetThreadName("CalVolumeDeallocateWorker");
+    while (ctx_->running) {
+        if (!WaitDeallocate()) {
+            break;
+        }
+
+        CHECK(current_.has_value());
+        VLOG(1) << "VolumeDeallocateWorker Going to statistic deallocatable "
+                   "space for block group";
+
+        auto cleanup = absl::MakeCleanup([&, this] { Cleanup(); });
+
+        if (!current_->CanStart()) {
+            if (current_->HasDeallocateTask()) {
+                current_->ResetDeallocateTask();
+            }
+            break;
+        }
+
+        if (current_->HasDeallocateTask()) {
+            current_->DeallocateOneBlockGroup(current_->GetDeallocateTask());
+            VLOG(1) << "VolumeDeallocateWorker deallocate for "
+                       "partitionId="
+                    << current_->GetPartitionID() << " ok";
+            current_->ResetDeallocateTask();
+        } else {
+            current_->CalDeallocatableSpace();
+            VLOG(1) << "VolumeDeallocateWorker cal deallocatable space for "
+                       "partitionId="
+                    << current_->GetPartitionID() << " ok";
+        }
+    }
+}
+
+}  // namespace metaserver
+}  // namespace curvefs

curvefs/src/metaserver/space/volume_deallocate_worker.h

@@ -0,0 +1,80 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Tue Apr 25 16:13:06 CST 2023
+ * Author: lixiaocui
+ */
+
+#ifndef CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_WORKER_H_
+#define CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_WORKER_H_
+
+#include <condition_variable>
+#include <thread>
+#include <list>
+#include <unordered_map>
+
+#include "absl/types/optional.h"
+
+#include "src/common/interruptible_sleeper.h"
+#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h"
+
+namespace curvefs {
+namespace metaserver {
+
+class VolumeDeallocateWorker;
+
+struct VolumeDeallocateWorkerContext {
+    std::atomic<bool> running{false};
+
+    std::mutex mtx;
+    std::condition_variable cond;
+    std::list<InodeVolumeSpaceDeallocate> allTasks;
+    std::unordered_map<uint32_t, VolumeDeallocateWorker *> tasking;
+};
+
+class VolumeDeallocateWorker {
+ public:
+    explicit VolumeDeallocateWorker(VolumeDeallocateWorkerContext *context)
+        : ctx_(context) {}
+
+    void Run();
+    void Stop();
+    void Cancel(uint32_t partitionId);
+    void SetDeallocate(uint64_t blockGroupOffset);
+    bool HasDeallocate();
+
+ private:
+    bool WaitDeallocate();
+    void Cleanup();
+    void DeallocatetWorker();
+    void ExecuteDeallocateWorker();
+
+ private:
+    VolumeDeallocateWorkerContext *ctx_;
+    std::thread calWorkingThread_;
+
+    // current cal
+    absl::optional<InodeVolumeSpaceDeallocate> current_;
+
+    curve::common::InterruptibleSleeper sleeper_;
+};
+
+}  // namespace metaserver
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_METASERVER_SPACE_VOLUME_DEALLOCATE_WORKER_H_

curvefs/src/metaserver/space/volume_space_manager.cpp

@@ -0,0 +1,122 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Tue Mar 28 19:55:27 CST 2023
+ * Author: lixiaocui
+ */
+
+#include "curvefs/src/metaserver/space/volume_space_manager.h"
+#include "curvefs/src/metaserver/mds/fsinfo_manager.h"
+
+namespace curvefs {
+namespace metaserver {
+
+using curvefs::volume::SpaceManagerImpl;
+
+void VolumeSpaceManager::Uninit() {
+    std::lock_guard<std::mutex> lock(mutex_);
+    for (auto &item : spaceManagers_) {
+        item.second->Shutdown();
+    }
+}
+
+void VolumeSpaceManager::Destroy(uint32_t fsId) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto it = spaceManagers_.find(fsId);
+    if (it != spaceManagers_.end()) {
+        spaceManagers_.erase(it);
+    }
+}
+
+uint64_t VolumeSpaceManager::GetBlockGroupSize(uint32_t fsId) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto it = spaceManagers_.find(fsId);
+    if (it == spaceManagers_.end()) {
+        return 0;
+    }
+
+    return it->second->GetBlockGroupSize();
+}
+
+bool VolumeSpaceManager::DeallocVolumeSpace(
+    uint32_t fsId, uint64_t blockGroupOffset,
+    const std::vector<Extent> &deallocatableVolumeSpace) {
+    auto spaceManager = GetSpaceManager(fsId);
+    if (spaceManager == nullptr) {
+        LOG(ERROR) << "VolumeSpaceManager dealloc volume space failed, could "
+                      "not get space manager, fsId "
+                   << fsId << " not exist";
+        return false;
+    }
+    return spaceManager->DeAlloc(blockGroupOffset, deallocatableVolumeSpace);
+}
+
+std::shared_ptr<SpaceManager>
+VolumeSpaceManager::GetSpaceManager(uint32_t fsId) {
+    std::lock_guard<std::mutex> lock(mutex_);
+    auto it = spaceManagers_.find(fsId);
+    if (it == spaceManagers_.end()) {
+        if (!GenerateSpaceManager(fsId)) {
+            return nullptr;
+        }
+    }
+    return spaceManagers_[fsId];
+}
+
+bool VolumeSpaceManager::GenerateSpaceManager(uint32_t fsId) {
+    FsInfo fsInfo;
+    bool ret = FsInfoManager::GetInstance().GetFsInfo(fsId, &fsInfo);
+    if (!ret) {
+        LOG(ERROR) << "VolumeSpaceManager generate space manager failed, fsId "
+                   << fsId << " not exist";
+        return false;
+    }
+
+    // init block device client
+    auto blockDeviceClient = std::make_shared<BlockDeviceClientImpl>();
+    ret = blockDeviceClient->Init(options_.deviceOpt);
+    if (!ret) {
+        LOG(ERROR) << "VolumeSpaceManager init block device failed";
+        return false;
+    }
+    const auto &vol = fsInfo.detail().volume();
+    const auto &volName = vol.volumename();
+    const auto &user = vol.user();
+    ret = blockDeviceClient->Open(volName, user);
+    if (!ret) {
+        LOG(ERROR) << "VolumeSpaceManager open failed, volName=" << volName
+                   << ", user=" << user;
+        return false;
+    }
+
+    // init space manager option
+    SpaceManagerOption spaceManagerOpt;
+    spaceManagerOpt.blockGroupManagerOption.fsId = fsInfo.fsid();
+    spaceManagerOpt.blockGroupManagerOption.blockGroupSize =
+        fsInfo.detail().volume().blockgroupsize();
+    spaceManagerOpt.blockGroupManagerOption.blockSize =
+        fsInfo.detail().volume().blocksize();
+
+    spaceManagers_[fsId] = std::make_shared<SpaceManagerImpl>(
+        spaceManagerOpt, options_.mdsClient, blockDeviceClient);
+
+    return true;
+}
+
+}  // namespace metaserver
+}  // namespace curvefs

curvefs/src/metaserver/space/volume_space_manager.h

@@ -0,0 +1,75 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Fri Mar 24 17:09:28 CST 2023
+ * Author: lixiaocui
+ */
+
+#ifndef CURVEFS_SRC_METASERVER_SPACE_VOLUME_SPACE_MANAGER_H_
+#define CURVEFS_SRC_METASERVER_SPACE_VOLUME_SPACE_MANAGER_H_
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "curvefs/src/volume/space_manager.h"
+
+namespace curvefs {
+namespace metaserver {
+using curvefs::client::rpcclient::MdsClient;
+using curvefs::volume::BlockDeviceClientOptions;
+using curvefs::volume::BlockDeviceClientImpl;
+using curvefs::volume::Extent;
+using curvefs::volume::SpaceManager;
+using curvefs::volume::SpaceManagerOption;
+
+struct VolumeSpaceManagerOptions {
+    std::shared_ptr<MdsClient> mdsClient;
+    BlockDeviceClientOptions deviceOpt;
+};
+
+class VolumeSpaceManager {
+ public:
+    void Init(VolumeSpaceManagerOptions options) { options_ = options; }
+
+    void Uninit();
+
+    virtual bool
+    DeallocVolumeSpace(uint32_t fsId, uint64_t volumeOffset,
+                       const std::vector<Extent> &deallocatableVolumeSpace);
+
+    virtual void Destroy(uint32_t fsId);
+
+    virtual uint64_t GetBlockGroupSize(uint32_t fsId);
+
+ private:
+    std::shared_ptr<SpaceManager> GetSpaceManager(uint32_t fsId);
+
+    bool GenerateSpaceManager(uint32_t fsId);
+
+ private:
+    std::mutex mutex_;
+    std::map<uint32_t, std::shared_ptr<SpaceManager>> spaceManagers_;
+    VolumeSpaceManagerOptions options_;
+};
+
+}  // namespace metaserver
+}  // namespace curvefs
+
+#endif  // CURVEFS_SRC_METASERVER_SPACE_VOLUME_SPACE_MANAGER_H_

curvefs/src/metaserver/storage/converter.cpp

@@ -54,6 +54,10 @@ static bool CompareType(const std::string& str, KEY_TYPE keyType) {
 
 NameGenerator::NameGenerator(uint32_t partitionId)
     : tableName4Inode_(Format(kTypeInode, partitionId)),
+      tableName4DeallocatableIndoe_(
+          Format(kTypeDeallocatableInode, partitionId)),
+      tableName4DeallocatableBlockGroup_(
+          Format(kTypeDeallocatableBlockGroup, partitionId)),
       tableName4S3ChunkInfo_(Format(kTypeS3ChunkInfo, partitionId)),
       tableName4Dentry_(Format(kTypeDentry, partitionId)),
       tableName4VolumeExtent_(Format(kTypeVolumeExtent, partitionId)),
@@ -63,6 +67,14 @@ std::string NameGenerator::GetInodeTableName() const {
     return tableName4Inode_;
 }
 
+std::string NameGenerator::GetDeallocatableInodeTableName() const {
+    return tableName4DeallocatableIndoe_;
+}
+
+std::string NameGenerator::GetDeallocatableBlockGroupTableName() const {
+    return tableName4DeallocatableBlockGroup_;
+}
+
 std::string NameGenerator::GetS3ChunkInfoTableName() const {
     return tableName4S3ChunkInfo_;
 }
@@ -340,6 +352,40 @@ bool Key4InodeAuxInfo::ParseFromString(const std::string& value) {
            StringToUl(items[1], &fsId) && StringToUll(items[2], &inodeId);
 }
 
+std::string Key4DeallocatableBlockGroup::SerializeToString() const {
+    return absl::StrCat(keyType_, kDelimiter, fsId, kDelimiter, volumeOffset);
+}
+
+
+bool Key4DeallocatableBlockGroup::ParseFromString(const std::string &value) {
+    std::vector<std::string> items;
+    SplitString(value, kDelimiter, &items);
+    return items.size() == 3 && CompareType(items[0], keyType_) &&
+           StringToUl(items[1], &fsId) && StringToUll(items[2], &volumeOffset);
+}
+
+std::string Key4DeallocatableInode::SerializeToString() const {
+    return absl::StrCat(keyType_, kDelimiter, fsId, kDelimiter, inodeId);
+}
+
+bool Key4DeallocatableInode::ParseFromString(const std::string &value) {
+    std::vector<std::string> items;
+    SplitString(value, kDelimiter, &items);
+    return items.size() == 3 && CompareType(items[0], keyType_) &&
+           StringToUl(items[1], &fsId) && StringToUll(items[2], &inodeId);
+}
+
+std::string Prefix4AllDeallocatableBlockGroup::SerializeToString() const {
+    return absl::StrCat(keyType_, ":");
+}
+
+bool Prefix4AllDeallocatableBlockGroup::ParseFromString(
+    const std::string &value) {
+    std::vector<std::string> items;
+    SplitString(value, ":", &items);
+    return items.size() == 1 && CompareType(items[0], keyType_);
+}
+
 std::string Converter::SerializeToString(const StorageKey& key) {
     return key.SerializeToString();
 }
@@ -352,7 +398,6 @@ bool Converter::SerializeToString(const google::protobuf::Message& entry,
     }
     return entry.SerializeToString(value);
 }
-
 }  // namespace storage
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/storage/converter.h

@@ -43,6 +43,9 @@ enum KEY_TYPE : unsigned char {
     kTypeDentry = 3,
     kTypeVolumeExtent = 4,
     kTypeInodeAuxInfo = 5,
+    kTypeBlockGroup = 6,
+    kTypeDeallocatableBlockGroup = 7,
+    kTypeDeallocatableInode = 8,
 };
 
 // NOTE: you must generate all table name by NameGenerator class for
@@ -54,6 +57,8 @@ class NameGenerator {
 
     std::string GetInodeTableName() const;
 
+    std::string GetDeallocatableInodeTableName() const;
+
     std::string GetS3ChunkInfoTableName() const;
 
     std::string GetDentryTableName() const;
@@ -62,6 +67,10 @@ class NameGenerator {
 
     std::string GetInodeAuxInfoTableName() const;
 
+    std::string GetBlockGroupStatisticTableName() const;
+
+    std::string GetDeallocatableBlockGroupTableName() const;
+
     static size_t GetFixedLength();
 
  private:
@@ -69,6 +78,8 @@ class NameGenerator {
 
  private:
     std::string tableName4Inode_;
+    std::string tableName4DeallocatableIndoe_;
+    std::string tableName4DeallocatableBlockGroup_;
     std::string tableName4S3ChunkInfo_;
     std::string tableName4Dentry_;
     std::string tableName4VolumeExtent_;
@@ -84,7 +95,7 @@ class StorageKey {
 };
 
 /* rules for key serialization:
- *   Key4Inode                        : kTypeInode:fsId:InodeId
+ *   Key4Inode                        : kTypeInode:fsId:inodeId
  *   Prefix4AllInode                  : kTypeInode:
  *   Key4S3ChunkInfoList              : kTypeS3ChunkInfo:fsId:inodeId:chunkIndex:firstChunkId:lastChunkId  // NOLINT
  *   Prefix4ChunkIndexS3ChunkInfoList : kTypeS3ChunkInfo:fsId:inodeId:chunkIndex:  // NOLINT
@@ -93,10 +104,13 @@ class StorageKey {
  *   Key4Dentry                       : kTypeDentry:parentInodeId:name
  *   Prefix4SameParentDentry          : kTypeDentry:parentInodeId:
  *   Prefix4AllDentry                 : kTypeDentry:
- *   Key4VolumeExtentSlice            : kTypeExtent:fsId:InodeId:SliceOffset
- *   Prefix4InodeVolumeExtent         : kTypeExtent:fsId:InodeId:
+ *   Key4VolumeExtentSlice            : kTypeExtent:fsId:inodeId:sliceOffset
+ *   Prefix4InodeVolumeExtent         : kTypeExtent:fsId:inodeId:
  *   Prefix4AllVolumeExtent           : kTypeExtent:
  *   Key4InodeAuxInfo                 : kTypeInodeAuxInfo:fsId:inodeId
+ *   Key4DeallocatableBlockGroup      : kTypeBlockGroup:fsId:volumeOffset
+ *   Prefix4AllDeallocatableBlockGroup: kTypeBlockGroup:
+ *   Key4DeallocatableInode           : kTypeDeallocatableInode:fsId:inodeId
  */
 
 class Key4Inode : public StorageKey {
@@ -324,6 +338,56 @@ class Key4InodeAuxInfo : public StorageKey {
     static constexpr KEY_TYPE keyType_ = kTypeInodeAuxInfo;
 };
 
+class Key4DeallocatableBlockGroup : public StorageKey {
+ public:
+    Key4DeallocatableBlockGroup() = default;
+
+    Key4DeallocatableBlockGroup(uint32_t fsId, uint64_t volumeOffset)
+        : fsId(fsId), volumeOffset(volumeOffset) {}
+
+    std::string SerializeToString() const override;
+
+    bool ParseFromString(const std::string &value) override;
+
+ public:
+    uint32_t fsId;
+    uint64_t volumeOffset;
+
+ private:
+    static constexpr KEY_TYPE keyType_ = kTypeDeallocatableBlockGroup;
+};
+
+class Key4DeallocatableInode : public StorageKey {
+ public:
+    Key4DeallocatableInode() = default;
+
+    Key4DeallocatableInode(uint32_t fsId, uint64_t inodeId)
+        : fsId(fsId), inodeId(inodeId) {}
+
+    std::string SerializeToString() const override;
+
+    bool ParseFromString(const std::string &value) override;
+
+ public:
+    uint32_t fsId;
+    uint64_t inodeId;
+
+ private:
+    static constexpr KEY_TYPE keyType_ = kTypeDeallocatableInode;
+};
+
+class Prefix4AllDeallocatableBlockGroup : public StorageKey {
+ public:
+    Prefix4AllDeallocatableBlockGroup() = default;
+
+    std::string SerializeToString() const override;
+
+    bool ParseFromString(const std::string &value) override;
+
+ public:
+    static const KEY_TYPE keyType_ = kTypeDeallocatableBlockGroup;
+};
+
 // converter
 class Converter {
  public:

curvefs/src/metaserver/storage/iterator.h

@@ -72,6 +72,7 @@ class MergeIterator : public Iterator {
         for (const auto &child : children_) {
             size += child->Size();
         }
+
         return size;
     }
 

curvefs/src/metaserver/streaming_utils.cpp

@@ -31,7 +31,7 @@ namespace curvefs {
 namespace metaserver {
 
 MetaStatusCode StreamingSendVolumeExtent(StreamConnection* connection,
-                                         const VolumeExtentList& extents) {
+                                         const VolumeExtentSliceList& extents) {
     VLOG(9) << "StreamingSendVolumeExtent, extents: "
             << extents.ShortDebugString();
     for (const auto& slice : extents.slices()) {

curvefs/src/metaserver/streaming_utils.h

@@ -32,7 +32,7 @@ namespace metaserver {
 using ::curvefs::common::StreamConnection;
 
 MetaStatusCode StreamingSendVolumeExtent(StreamConnection* connection,
-                                         const VolumeExtentList& extents);
+                                         const VolumeExtentSliceList& extents);
 
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/volume/block_group_loader.h

@@ -36,6 +36,7 @@ namespace curvefs {
 namespace volume {
 
 using ::curvefs::common::BitmapLocation;
+using ::curvefs::mds::space::BlockGroup;
 
 class BlockDeviceClient;
 
@@ -48,20 +49,15 @@ struct AllocatorAndBitmapUpdater {
 // load bitmap for each block group
 class BlockGroupBitmapLoader {
  public:
-    BlockGroupBitmapLoader(BlockDeviceClient* client,
-                           uint32_t blockSize,
-                           uint64_t offset,
-                           uint64_t blockGroupSize,
-                           BitmapLocation location,
-                           bool clean,
-                           const AllocatorOption& option)
-        : blockDev_(client),
-          offset_(offset),
-          blockGroupSize_(blockGroupSize),
-          blockSize_(blockSize),
-          bitmapLocation_(location),
-          clean_(clean),
-          allocatorOption_(option) {}
+    BlockGroupBitmapLoader(BlockDeviceClient *client, uint32_t blockSize,
+                           const AllocatorOption &option,
+                           const BlockGroup &blockGroup)
+        : blockDev_(client), blockSize_(blockSize), allocatorOption_(option) {
+        offset_ = blockGroup.offset();
+        blockGroupSize_ = blockGroup.size();
+        bitmapLocation_ = blockGroup.bitmaplocation();
+        clean_ = (blockGroup.size() == blockGroup.available());
+    }
 
     BlockGroupBitmapLoader(const BlockGroupBitmapLoader&) = delete;
     BlockGroupBitmapLoader& operator=(const BlockGroupBitmapLoader&) = delete;

curvefs/src/volume/block_group_manager.cpp

@@ -35,6 +35,7 @@ namespace volume {
 using ::curve::common::Bitmap;
 using ::curve::common::BITMAP_UNIT_SIZE;
 using ::curve::common::BitRange;
+using ::curve::common::WriteLockGuard;
 using ::curvefs::common::BitmapLocation;
 using ::curvefs::mds::space::BlockGroup;
 using ::curvefs::mds::space::SpaceErrCode_Name;
@@ -66,14 +67,14 @@ bool BlockGroupManagerImpl::AllocateBlockGroup(
         return false;
     }
 
-    for (const auto& group : groups) {
+    WriteLockGuard lk(groupsLock_);
+    for (auto& group : groups) {
         VLOG(9) << "load group: " << group.ShortDebugString();
         AllocatorAndBitmapUpdater res;
         res.blockGroupOffset = group.offset();
-        BlockGroupBitmapLoader loader(
-            blockDeviceClient_.get(), option_.blockSize, group.offset(),
-            group.size(), group.bitmaplocation(),
-            group.size() == group.available(), allocatorOption_);
+        BlockGroupBitmapLoader loader(blockDeviceClient_.get(),
+                                      option_.blockSize, allocatorOption_,
+                                      group);
         auto ret = loader.Load(&res);
         if (!ret) {
             LOG(ERROR) << "Create allocator for block group failed";
@@ -81,13 +82,81 @@ bool BlockGroupManagerImpl::AllocateBlockGroup(
         } else {
             out->push_back(std::move(res));
         }
+
+        groups_.emplace_back(std::move(group));
     }
 
     return true;
 }
 
-// TODO(wuhanqing): implement this function
+bool BlockGroupManagerImpl::AcquireBlockGroup(uint64_t blockGroupOffset,
+                                              AllocatorAndBitmapUpdater *out) {
+    BlockGroup group;
+    SpaceErrCode err = mdsClient_->AcquireVolumeBlockGroup(
+        option_.fsId, blockGroupOffset, option_.owner, &group);
+    if (err != SpaceErrCode::SpaceOk) {
+        LOG(WARNING) << "Acquire volume block group failed, err: "
+                     << SpaceErrCode_Name(err);
+        return false;
+    }
+
+    AllocatorAndBitmapUpdater res;
+    res.blockGroupOffset = group.offset();
+    BlockGroupBitmapLoader loader(blockDeviceClient_.get(), option_.blockSize,
+                                  allocatorOption_, group);
+    auto ret = loader.Load(&res);
+    if (!ret) {
+        LOG(ERROR) << "Create allocator for block group failed";
+        return false;
+    }
+
+    *out = std::move(res);
+    return true;
+}
+
+
+bool BlockGroupManagerImpl::ReleaseBlockGroup(uint64_t blockGroupOffset) {
+    WriteLockGuard lk(groupsLock_);
+    auto iter = std::find_if(groups_.begin(), groups_.end(),
+                             [blockGroupOffset](const BlockGroup &group) {
+                                 return group.offset() == blockGroupOffset;
+                             });
+    if (iter == groups_.end()) {
+        LOG(ERROR) << "BlockGroupManagerImpl find group block: "
+                   << blockGroupOffset << " failed";
+        return false;
+    }
+
+    SpaceErrCode err = mdsClient_->ReleaseVolumeBlockGroup(
+        option_.fsId, option_.owner, {*iter});
+    if (err != SpaceErrCode::SpaceOk) {
+        LOG(WARNING) << "BlockGroupManagerImpl release block group: "
+                     << blockGroupOffset
+                     << "failed, err: " << SpaceErrCode_Name(err)
+                     << ", blockGroupOffset: " << blockGroupOffset;
+        return false;
+    }
+
+    groups_.erase(iter);
+    LOG(INFO) << "BlockGroupManagerImpl release one volume block group "
+                 "successfully";
+    return true;
+}
+
 bool BlockGroupManagerImpl::ReleaseAllBlockGroups() {
+    WriteLockGuard lk(groupsLock_);
+    SpaceErrCode err = mdsClient_->ReleaseVolumeBlockGroup(
+        option_.fsId, option_.owner, groups_);
+    if (err != SpaceErrCode::SpaceOk) {
+        LOG(WARNING) << "BlockGroupManagerImpl release all volume block groups "
+                        "failed, err: "
+                     << SpaceErrCode_Name(err) << "";
+        return false;
+    }
+
+    groups_.clear();
+    LOG(INFO) << "BlockGroupManagerImpl release all volume block groups "
+                 "successfully";
     return true;
 }
 

curvefs/src/volume/block_group_manager.h

@@ -49,6 +49,11 @@ class BlockGroupManager {
         std::vector<AllocatorAndBitmapUpdater>* out) = 0;
 
     virtual bool ReleaseAllBlockGroups() = 0;
+
+    virtual bool ReleaseBlockGroup(uint64_t blockGroupOffset) = 0;
+
+    virtual bool AcquireBlockGroup(uint64_t blockGroupOffset,
+                                   AllocatorAndBitmapUpdater *out) = 0;
 };
 
 class BlockGroupManagerImpl final : public BlockGroupManager {
@@ -62,12 +67,15 @@ class BlockGroupManagerImpl final : public BlockGroupManager {
     bool AllocateBlockGroup(
         std::vector<AllocatorAndBitmapUpdater>* out) override;
 
-    void AcquireBlockGroup();
+    bool AcquireBlockGroup(uint64_t blockGroupOffset,
+                           AllocatorAndBitmapUpdater *out) override;
 
     void AllocateBlockGroupAsync();
 
     bool ReleaseAllBlockGroups() override;
 
+    bool ReleaseBlockGroup(uint64_t blockGroupOffset) override;
+
  private:
     SpaceManager* spaceManager_;
     std::shared_ptr<MdsClient> mdsClient_;
@@ -75,6 +83,9 @@ class BlockGroupManagerImpl final : public BlockGroupManager {
 
     BlockGroupManagerOption option_;
     AllocatorOption allocatorOption_;
+
+    curve::common::RWLock groupsLock_;
+    std::vector<BlockGroup> groups_;
 };
 
 }  // namespace volume

curvefs/src/volume/option.h

@@ -54,6 +54,9 @@ struct AllocatorOption {
 struct SpaceManagerOption {
     AllocatorOption allocatorOption;
     BlockGroupManagerOption blockGroupManagerOption;
+
+    double threshold{1.0};
+    uint64_t releaseInterSec{300};
 };
 
 }  // namespace volume

curvefs/src/volume/space_manager.cpp

@@ -41,20 +41,17 @@ using ::curve::common::ReadLockGuard;
 using ::curve::common::WriteLockGuard;
 
 SpaceManagerImpl::SpaceManagerImpl(
-    const SpaceManagerOption& option,
-    const std::shared_ptr<MdsClient>& mdsClient,
-    const std::shared_ptr<BlockDeviceClient>& blockDev)
-    : totalBytes_(0),
-      availableBytes_(0),
+    const SpaceManagerOption &option,
+    const std::shared_ptr<MdsClient> &mdsClient,
+    const std::shared_ptr<BlockDeviceClient> &blockDev)
+    : totalBytes_(0), availableBytes_(0),
       blockSize_(option.blockGroupManagerOption.blockSize),
       blockGroupSize_(option.blockGroupManagerOption.blockGroupSize),
-      blockGroupManager_(
-          new BlockGroupManagerImpl(this,
-                                    mdsClient,
-                                    blockDev,
-                                    option.blockGroupManagerOption,
-                                    option.allocatorOption)),
-      allocating_(false) {}
+      blockGroupManager_(new BlockGroupManagerImpl(
+          this, mdsClient, blockDev, option.blockGroupManagerOption,
+          option.allocatorOption)),
+      allocating_(false), threshold_(option.threshold),
+      releaseInterSec_(option.releaseInterSec) {}
 
 bool SpaceManagerImpl::Alloc(uint32_t size,
                              const AllocateHint& hint,
@@ -88,7 +85,7 @@ bool SpaceManagerImpl::Alloc(uint32_t size,
         left -= allocated;
     }
 
-    auto ret = UpdateBitmap(*extents);
+    auto ret = UpdateBitmap(*extents, BlockGroupBitmapUpdater::Set);
     if (!ret) {
         LOG(ERROR) << "Update bitmap failed";
         metric_.errorCount << 1;
@@ -104,9 +101,34 @@ bool SpaceManagerImpl::Alloc(uint32_t size,
     return true;
 }
 
-bool SpaceManagerImpl::DeAlloc(const std::vector<Extent>& extents) {
-    // TODO(wuhanqing): fix
-    (void)extents;
+bool SpaceManagerImpl::DeAlloc(uint64_t blockGroupOffset,
+                               const std::vector<Extent> &extents) {
+    butil::Timer timer;
+    timer.start();
+
+    if (!AcquireBlockGroup(blockGroupOffset)) {
+        LOG(WARNING) << "Acquire block group failed, block group offset: "
+                     << blockGroupOffset;
+        return false;
+    }
+
+    bool ret = UpdateBitmap(extents, BlockGroupBitmapUpdater::Op::Clear);
+    if (!ret) {
+        LOG(ERROR) << "DeAlloc update bitmap failed";
+        metric_.errorCount << 1;
+        return false;
+    }
+
+    uint64_t size = 0;
+    for (const auto &ext : extents) {
+        size += ext.len;
+    }
+
+    timer.stop();
+    metric_.deallocLatency << timer.u_elapsed();
+    metric_.allocSize << -size;
+
+    VLOG(9) << "Dealloc success, " << extents;
     return true;
 }
 
@@ -162,13 +184,14 @@ int64_t SpaceManagerImpl::AllocInternal(int64_t size,
     return size - left;
 }
 
-bool SpaceManagerImpl::UpdateBitmap(const std::vector<Extent>& exts) {
+bool SpaceManagerImpl::UpdateBitmap(const std::vector<Extent> &exts,
+                                    BlockGroupBitmapUpdater::Op op) {
     ReadLockGuard lk(updatersLock_);
 
     std::unordered_set<BlockGroupBitmapUpdater*> dirty;
     for (const auto& ext : exts) {
         BlockGroupBitmapUpdater* updater = FindBitmapUpdater(ext);
-        updater->Update(ext, BlockGroupBitmapUpdater::Set);
+        updater->Update(ext, op);
         dirty.insert(updater);
     }
 
@@ -191,47 +214,106 @@ BlockGroupBitmapUpdater* SpaceManagerImpl::FindBitmapUpdater(
     return it->second.get();
 }
 
-bool SpaceManagerImpl::Shutdown() {
-    WriteLockGuard allocLk(allocatorsLock_);
-    WriteLockGuard updaterLk(updatersLock_);
+void SpaceManagerImpl::Run() {
+    releaseT_ = std::thread(&SpaceManagerImpl::ReleaseFullBlockGroups, this);
+    running_ = true;
+}
 
-    // sync all bitmap updater
-    bool ret = false;
-    for (auto& updater : bitmapUpdaters_) {
-        ret = updater.second->Sync();
-        if (!ret) {
-            LOG(ERROR) << "Sync bitmap updater failed";
-            return false;
+
+void SpaceManagerImpl::ReleaseFullBlockGroups() {
+    while (sleeper_.wait_for(std::chrono::seconds(releaseInterSec_))) {
+        std::vector<uint64_t> selectBlockGroups;
+
+        // find the blockgroup whose space usage ratio is greater than a certain
+        // threshold
+        {
+            ReadLockGuard lk(allocatorsLock_);
+            for (auto &alloc : allocators_) {
+                if (alloc.second->Total() == 0) {
+                    continue;
+                }
+
+                double total = alloc.second->Total();
+                double available = alloc.second->AvailableSize();
+                double usedPer = 1.0 - available / total;
+
+                if (usedPer > threshold_) {
+                    selectBlockGroups.push_back(alloc.first);
+                }
+            }
+        }
+
+        if (selectBlockGroups.empty()) {
+            LOG(INFO) << "No block group need to release";
+            continue;
+        }
+
+        // release selected blockgroups
+        {
+            WriteLockGuard allocLk(allocatorsLock_);
+            WriteLockGuard updaterLk(updatersLock_);
+            for (auto &id : selectBlockGroups) {
+                auto iter = allocators_.find(id);
+                assert(iter != allocators_.end());
+
+                allocators_.erase(id);
+                bitmapUpdaters_.erase(id);
+
+                availableBytes_.fetch_sub(iter->second->AvailableSize(),
+                                          std::memory_order_relaxed);
+                totalBytes_.fetch_sub(iter->second->AvailableSize(),
+                                      std::memory_order_release);
+
+                bool ret = blockGroupManager_->ReleaseBlockGroup(id);
+                if (ret) {
+                    LOG(INFO) << "Release block group success, id: " << id;
+                } else {
+                    LOG(ERROR) << "Release block group failed, id: " << id;
+                }
+            }
         }
     }
+}
 
-    // release all block group
-    ret = blockGroupManager_->ReleaseAllBlockGroups();
-    LOG_IF(ERROR, !ret) << "Release all block groups failed";
+bool SpaceManagerImpl::Shutdown() {
+    bool ret = false;
+
+    {
+        WriteLockGuard allocLk(allocatorsLock_);
+        WriteLockGuard updaterLk(updatersLock_);
+
+        // sync all bitmap updater
+        for (auto &updater : bitmapUpdaters_) {
+            ret = updater.second->Sync();
+            if (!ret) {
+                LOG(ERROR) << "Sync bitmap updater failed";
+                return false;
+            }
+        }
+
+        // release all block group
+        ret = blockGroupManager_->ReleaseAllBlockGroups();
+        LOG_IF(ERROR, !ret) << "Release all block groups failed";
+    }
+
+    if (running_) {
+        sleeper_.interrupt();
+        releaseT_.join();
+        LOG(INFO) << "SpaceManagerImpl stop thread ok";
+    }
     return ret;
 }
 
 bool SpaceManagerImpl::AllocateBlockGroup(uint64_t hint) {
-    {
-        std::unique_lock<std::mutex> lk(mtx_);
-        if (allocating_) {
-            cond_.wait(lk);
-        }
-
-        if (availableBytes_.load(std::memory_order_relaxed) >= hint) {
-            return true;
-        }
+    std::unique_lock<std::mutex> lk(mtx_);
+    if (availableBytes_.load(std::memory_order_relaxed) >= hint) {
+        return true;
     }
 
-    std::lock_guard<std::mutex> lk(mtx_);
-    allocating_ = true;
-
     std::vector<AllocatorAndBitmapUpdater> out;
     auto ret = blockGroupManager_->AllocateBlockGroup(&out);
     if (!ret) {
         LOG(ERROR) << "Allocate block group failed";
-        allocating_ = false;
-        cond_.notify_all();
         return false;
     }
 
@@ -239,7 +321,7 @@ bool SpaceManagerImpl::AllocateBlockGroup(uint64_t hint) {
     uint64_t total = 0;
     WriteLockGuard allocLk(allocatorsLock_);
     WriteLockGuard updaterLk(updatersLock_);
-    for (auto& d : out) {
+    for (auto &d : out) {
         VLOG(9) << "add allocator, offset: " << d.blockGroupOffset
                 << ", available: " << d.allocator->AvailableSize()
                 << ", total: " << d.allocator->Total();
@@ -251,9 +333,28 @@ bool SpaceManagerImpl::AllocateBlockGroup(uint64_t hint) {
 
     availableBytes_.fetch_add(available, std::memory_order_release);
     totalBytes_.fetch_add(total, std::memory_order_release);
+    return true;
+}
 
-    allocating_ = false;
-    cond_.notify_all();
+bool SpaceManagerImpl::AcquireBlockGroup(uint64_t blockGroupOffset) {
+    std::unique_lock<std::mutex> lk(mtx_);
+    if (bitmapUpdaters_.find(blockGroupOffset) != bitmapUpdaters_.end()) {
+        return true;
+    }
+
+    AllocatorAndBitmapUpdater out;
+    bool ret = blockGroupManager_->AcquireBlockGroup(blockGroupOffset, &out);
+    if (!ret) {
+        LOG(WARNING) << "Acquire block group failed, offset: "
+                     << blockGroupOffset;
+        return false;
+    }
+
+    WriteLockGuard allocLk(allocatorsLock_);
+    WriteLockGuard updaterLk(updatersLock_);
+
+    allocators_.emplace(blockGroupOffset, std::move(out.allocator));
+    bitmapUpdaters_.emplace(blockGroupOffset, std::move(out.bitmapUpdater));
     return true;
 }
 

curvefs/src/volume/space_manager.h

@@ -27,6 +27,7 @@
 #include <memory>
 #include <vector>
 
+#include "src/common/interruptible_sleeper.h"
 #include "curvefs/proto/common.pb.h"
 #include "curvefs/src/client/rpcclient/mds_client.h"
 #include "curvefs/src/volume/allocator.h"
@@ -39,6 +40,7 @@ namespace curvefs {
 namespace volume {
 
 using ::curvefs::client::rpcclient::MdsClient;
+using ::curve::common::InterruptibleSleeper;
 
 class SpaceManager {
  public:
@@ -48,9 +50,14 @@ class SpaceManager {
                        const AllocateHint& hint,
                        std::vector<Extent>* extents) = 0;
 
-    virtual bool DeAlloc(const std::vector<Extent>& extents) = 0;
+    virtual bool DeAlloc(uint64_t blockGroupOffset,
+                         const std::vector<Extent> &extents) = 0;
+
+    virtual void Run() = 0;
 
     virtual bool Shutdown() = 0;
+
+    virtual uint64_t GetBlockGroupSize() = 0;
 };
 
 class SpaceManagerImpl final : public SpaceManager {
@@ -67,13 +74,18 @@ class SpaceManagerImpl final : public SpaceManager {
                const AllocateHint& hint,
                std::vector<Extent>* extents) override;
 
-    bool DeAlloc(const std::vector<Extent>& extents) override;
+    bool DeAlloc(uint64_t blockGroupOffset,
+                 const std::vector<Extent> &extents) override;
+
+    void Run() override;
 
     /**
      * @brief Shutdown space manager, release all blockgroups' rights
      */
     bool Shutdown() override;
 
+    uint64_t GetBlockGroupSize() override { return blockGroupSize_; }
+
  private:
     int64_t AllocInternal(int64_t size,
                           const AllocateHint& hint,
@@ -87,7 +99,10 @@ class SpaceManagerImpl final : public SpaceManager {
      */
     BlockGroupBitmapUpdater* FindBitmapUpdater(const Extent& ext);
 
-    bool UpdateBitmap(const std::vector<Extent>& exts);
+    bool UpdateBitmap(const std::vector<Extent> &exts,
+                      BlockGroupBitmapUpdater::Op op);
+
+    void ReleaseFullBlockGroups();
 
  private:
     bool AllocateBlockGroup(uint64_t hint);
@@ -115,16 +130,25 @@ class SpaceManagerImpl final : public SpaceManager {
     std::mutex mtx_;
     std::condition_variable cond_;
 
+    // releaseT_ is periodically release the allocated blockgroup
+    double threshold_;
+    uint64_t releaseInterSec_;
+    InterruptibleSleeper sleeper_;
+    bool running_{false};
+    std::thread releaseT_;
+
+
  private:
     struct Metric {
         bvar::LatencyRecorder allocLatency;
+        bvar::LatencyRecorder deallocLatency;
         bvar::LatencyRecorder allocSize;
         bvar::Adder<uint64_t> errorCount;
 
         Metric()
             : allocLatency("space_alloc_latency"),
-              allocSize("space_alloc_size"),
-              errorCount("space_alloc_error") {}
+              deallocLatency("space_dealloc_latency"),
+              allocSize("space_alloc_size"), errorCount("space_alloc_error") {}
     };
 
     Metric metric_;

curvefs/test/client/mock_metaserver_client.h

@@ -152,11 +152,15 @@ class MockMetaServerClient : public MetaServerClient {
     MOCK_METHOD4(AsyncUpdateVolumeExtent,
                  void(uint32_t,
                       uint64_t,
-                      const VolumeExtentList &,
+                      const VolumeExtentSliceList &,
                       MetaServerClientDone *));
 
-    MOCK_METHOD4(GetVolumeExtent,
-                 MetaStatusCode(uint32_t, uint64_t, bool, VolumeExtentList *));
+    MOCK_METHOD4(GetVolumeExtent, MetaStatusCode(uint32_t, uint64_t, bool,
+                                                 VolumeExtentSliceList *));
+
+    MOCK_METHOD3(UpdateDeallocatableBlockGroup,
+                 MetaStatusCode(uint32_t, uint64_t,
+                                DeallocatableBlockGroupMap *));
 };
 
 }  // namespace rpcclient

curvefs/test/client/rpcclient/metaserver_client_test.cpp

@@ -61,6 +61,8 @@ using ::curvefs::metaserver::BatchGetInodeAttrRequest;
 using ::curvefs::metaserver::BatchGetInodeAttrResponse;
 using ::curvefs::metaserver::BatchGetXAttrRequest;
 using ::curvefs::metaserver::BatchGetXAttrResponse;
+using ::curvefs::metaserver::UpdateDeallocatableBlockGroupRequest;
+using ::curvefs::metaserver::UpdateDeallocatableBlockGroupResponse;
 using ::curvefs::common::StreamServer;
 using ::curvefs::common::StreamOptions;
 using ::curvefs::common::StreamConnection;
@@ -1337,7 +1339,7 @@ TEST_F(MetaServerClientImplTest, TestGetVolumeExtent) {
     const uint64_t applyIndex = 10;
 
     for (auto streaming : {true, false}) {
-        metaserver::VolumeExtentList out;
+        metaserver::VolumeExtentSliceList out;
 
         EXPECT_CALL(*mockMetacache_, GetTarget(_, _, _, _, _))
             .WillRepeatedly(
@@ -1372,6 +1374,62 @@ TEST_F(MetaServerClientImplTest, TestGetVolumeExtent) {
     }
 }
 
+TEST_F(MetaServerClientImplTest, TestUpdateDeallocatableBlockGroup) {
+    const uint32_t fsid = 1;
+    const uint64_t ino = 1;
+    const uint64_t blockgroupoffset = 0;
+    const uint64_t applyIndex = 10;
+    DeallocatableBlockGroupMap statistic;
+    auto &item = statistic[0];
+    item.set_blockgroupoffset(blockgroupoffset);
+    auto increase = item.mutable_increase();
+    increase->set_increasedeallocatablesize(1024 * 1024);
+    increase->add_inodeidlistadd(ino);
+
+    // set response
+    UpdateDeallocatableBlockGroupResponse response;
+    response.set_statuscode(MetaStatusCode::OK);
+    response.set_appliedindex(applyIndex);
+
+    {
+        // test rpc ok
+        EXPECT_CALL(*mockMetacache_.get(), GetTarget(_, _, _, _, _))
+            .WillOnce(DoAll(SetArgPointee<2>(target_),
+                                  SetArgPointee<3>(applyIndex), Return(true)));
+        EXPECT_CALL(mockMetaServerService_,
+                    UpdateDeallocatableBlockGroup(_, _, _, _))
+            .WillOnce(DoAll(
+                SetArgPointee<2>(response),
+                Invoke(SetRpcService<UpdateDeallocatableBlockGroupRequest,
+                                     UpdateDeallocatableBlockGroupResponse>)));
+
+        EXPECT_CALL(*mockMetacache_.get(), UpdateApplyIndex(_, _)).Times(1);
+        ASSERT_EQ(MetaStatusCode::OK,
+                  metaserverCli_.UpdateDeallocatableBlockGroup(fsid, ino,
+                                                               &statistic));
+    }
+
+    {
+        // test rpc error
+        EXPECT_CALL(*mockMetacache_.get(), GetTarget(_, _, _, _, _))
+            .WillRepeatedly(DoAll(SetArgPointee<2>(target_),
+                                  SetArgPointee<3>(applyIndex), Return(true)));
+        EXPECT_CALL(mockMetaServerService_,
+                    UpdateDeallocatableBlockGroup(_, _, _, _))
+            .WillRepeatedly(
+                Invoke(SetRpcService<UpdateDeallocatableBlockGroupRequest,
+                                     UpdateDeallocatableBlockGroupResponse>));
+        EXPECT_CALL(*mockMetacache_.get(), GetTargetLeader(_, _, _))
+            .WillRepeatedly(Return(true));
+        EXPECT_CALL(*mockMetacache_.get(), GetTarget(_, _, _, _, _))
+            .WillRepeatedly(DoAll(SetArgPointee<2>(target_),
+                                  SetArgPointee<3>(applyIndex), Return(true)));
+        ASSERT_EQ(MetaStatusCode::RPC_ERROR,
+                  metaserverCli_.UpdateDeallocatableBlockGroup(fsid, ino,
+                                                               &statistic));
+    }
+}
+
 }  // namespace rpcclient
 }  // namespace client
 }  // namespace curvefs

curvefs/test/client/rpcclient/mock_metaserver_service.h

@@ -116,6 +116,15 @@ class MockMetaServerService : public curvefs::metaserver::MetaServerService {
              const ::curvefs::metaserver::UpdateVolumeExtentRequest *request,
              ::curvefs::metaserver::UpdateVolumeExtentResponse *response,
              ::google::protobuf::Closure *done));
+
+    MOCK_METHOD4(
+        UpdateDeallocatableBlockGroup,
+        void(::google::protobuf::RpcController *controller,
+             const ::curvefs::metaserver::UpdateDeallocatableBlockGroupRequest
+                 *request,
+             ::curvefs::metaserver::UpdateDeallocatableBlockGroupResponse
+                 *response,
+             ::google::protobuf::Closure *done));
 };
 }  // namespace rpcclient
 }  // namespace client

curvefs/test/client/test_inodeWrapper.cpp

@@ -198,7 +198,7 @@ TEST_F(TestInodeWrapper, TestFlushVolumeExtent) {
     EXPECT_CALL(*metaClient_, UpdateInodeAttrWithOutNlink(_, _, _, _, _))
         .Times(0);
     EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(_, _, _, _))
-        .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentList&,
+        .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentSliceList&,
                             MetaServerClientDone* done) {
             done->SetMetaStatusCode(MetaStatusCode::OK);
             done->Run();

curvefs/test/client/volume/default_volume_storage_test.cpp

@@ -85,10 +85,10 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadError) {
     Inode inode;
     inode.set_type(FsFileType::TYPE_FILE);
 
-    VolumeExtentList exts;
-    auto* slice = exts.add_slices();
+    VolumeExtentSliceList exts;
+    auto *slice = exts.add_slices();
     slice->set_offset(0);
-    auto* ext = slice->add_extents();
+    auto *ext = slice->add_extents();
     ext->set_fsoffset(0);
     ext->set_length(4096);
     ext->set_volumeoffset(8192);
@@ -97,8 +97,8 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadError) {
     auto inodeWrapper = std::make_shared<InodeWrapper>(inode, metaServerCli_);
 
     EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _))
-        .WillOnce(
-            Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) {
+        .WillOnce(Invoke(
+            [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) {
                 *extents = exts;
                 return MetaStatusCode::OK;
             }));
@@ -106,13 +106,12 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadError) {
     ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent());
 
     EXPECT_CALL(inodeCacheMgr_, GetInode(_, _))
-        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper>& out) {
+        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper> &out) {
             out = inodeWrapper;
             return CURVEFS_ERROR::OK;
         }));
 
-    EXPECT_CALL(blockDev_, Readv(_))
-        .WillOnce(Return(-1));
+    EXPECT_CALL(blockDev_, Readv(_)).WillOnce(Return(-1));
 
     uint64_t ino = 1;
     off_t offset = 0;
@@ -127,10 +126,10 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadSuccess) {
     Inode inode;
     inode.set_type(FsFileType::TYPE_FILE);
 
-    VolumeExtentList exts;
-    auto* slice = exts.add_slices();
+    VolumeExtentSliceList exts;
+    auto *slice = exts.add_slices();
     slice->set_offset(0);
-    auto* ext = slice->add_extents();
+    auto *ext = slice->add_extents();
     ext->set_fsoffset(0);
     ext->set_length(4096);
     ext->set_volumeoffset(8192);
@@ -139,8 +138,8 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadSuccess) {
     auto inodeWrapper = std::make_shared<InodeWrapper>(inode, metaServerCli_);
 
     EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _))
-        .WillOnce(
-            Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) {
+        .WillOnce(Invoke(
+            [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) {
                 *extents = exts;
                 return MetaStatusCode::OK;
             }));
@@ -148,7 +147,7 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadSuccess) {
     ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent());
 
     EXPECT_CALL(inodeCacheMgr_, GetInode(_, _))
-        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper>& out) {
+        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper> &out) {
             out = inodeWrapper;
             return CURVEFS_ERROR::OK;
         }));
@@ -158,11 +157,9 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadSuccess) {
     size_t len = 4096;
     std::unique_ptr<char[]> data(new char[len]);
 
-    EXPECT_CALL(blockDev_, Readv(_))
-        .WillOnce(Return(len));
+    EXPECT_CALL(blockDev_, Readv(_)).WillOnce(Return(len));
 
-    EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper))
-        .Times(1);
+    EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)).Times(1);
 
     ASSERT_EQ(CURVEFS_ERROR::OK, storage_.Read(ino, offset, len, data.get()));
 }
@@ -171,10 +168,10 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadHoleSuccess) {
     Inode inode;
     inode.set_type(FsFileType::TYPE_FILE);
 
-    VolumeExtentList exts;
-    auto* slice = exts.add_slices();
+    VolumeExtentSliceList exts;
+    auto *slice = exts.add_slices();
     slice->set_offset(0);
-    auto* ext = slice->add_extents();
+    auto *ext = slice->add_extents();
     ext->set_fsoffset(0);
     ext->set_length(4096);
     ext->set_volumeoffset(8192);
@@ -183,8 +180,8 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadHoleSuccess) {
     auto inodeWrapper = std::make_shared<InodeWrapper>(inode, metaServerCli_);
 
     EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _))
-        .WillOnce(
-            Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) {
+        .WillOnce(Invoke(
+            [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) {
                 *extents = exts;
                 return MetaStatusCode::OK;
             }));
@@ -192,7 +189,7 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadHoleSuccess) {
     ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent());
 
     EXPECT_CALL(inodeCacheMgr_, GetInode(_, _))
-        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper>& out) {
+        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper> &out) {
             out = inodeWrapper;
             return CURVEFS_ERROR::OK;
         }));
@@ -204,11 +201,9 @@ TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadHoleSuccess) {
 
     memset(data.get(), 'x', len);
 
-    EXPECT_CALL(blockDev_, Readv(_))
-        .Times(0);
+    EXPECT_CALL(blockDev_, Readv(_)).Times(0);
 
-    EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper))
-        .Times(1);
+    EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)).Times(1);
 
     ASSERT_EQ(CURVEFS_ERROR::OK, storage_.Read(ino, offset, len, data.get()));
 
@@ -224,8 +219,8 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_PrepareError) {
     auto inodeWrapper = std::make_shared<InodeWrapper>(inode, metaServerCli_);
 
     EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _))
-        .WillOnce(
-            Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) {
+        .WillOnce(Invoke(
+            [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) {
                 extents->clear_slices();
                 return MetaStatusCode::OK;
             }));
@@ -233,13 +228,12 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_PrepareError) {
     ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent());
 
     EXPECT_CALL(inodeCacheMgr_, GetInode(_, _))
-        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper>& out) {
+        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper> &out) {
             out = inodeWrapper;
             return CURVEFS_ERROR::OK;
         }));
 
-    EXPECT_CALL(spaceMgr_, Alloc(_, _, _))
-        .WillOnce(Return(false));
+    EXPECT_CALL(spaceMgr_, Alloc(_, _, _)).WillOnce(Return(false));
 
     uint64_t ino = 1;
     off_t offset = 0;
@@ -257,8 +251,8 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteError) {
     auto inodeWrapper = std::make_shared<InodeWrapper>(inode, metaServerCli_);
 
     EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _))
-        .WillOnce(
-            Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) {
+        .WillOnce(Invoke(
+            [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) {
                 extents->clear_slices();
                 return MetaStatusCode::OK;
             }));
@@ -266,7 +260,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteError) {
     ASSERT_EQ(CURVEFS_ERROR::OK, inodeWrapper->RefreshVolumeExtent());
 
     EXPECT_CALL(inodeCacheMgr_, GetInode(_, _))
-        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper>& out) {
+        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper> &out) {
             out = inodeWrapper;
             return CURVEFS_ERROR::OK;
         }));
@@ -277,7 +271,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteError) {
 
     EXPECT_CALL(spaceMgr_, Alloc(_, _, _))
         .WillOnce(Invoke(
-            [](uint32_t size, const AllocateHint&, std::vector<Extent>* exts) {
+            [](uint32_t size, const AllocateHint &, std::vector<Extent> *exts) {
                 exts->emplace_back(size, size);
                 return true;
             }));
@@ -287,8 +281,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteError) {
     size_t len = 4096;
     std::unique_ptr<char[]> data(new char[len]);
 
-    EXPECT_CALL(blockDev_, Writev(_))
-        .WillOnce(Return(-1));
+    EXPECT_CALL(blockDev_, Writev(_)).WillOnce(Return(-1));
 
     ASSERT_EQ(CURVEFS_ERROR::IO_ERROR,
               storage_.Write(ino, offset, len, data.get()));
@@ -301,8 +294,8 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) {
     auto inodeWrapper = std::make_shared<InodeWrapper>(inode, metaServerCli_);
 
     EXPECT_CALL(*metaServerCli_, GetVolumeExtent(_, _, _, _))
-        .WillOnce(
-            Invoke([&](uint32_t, uint64_t, bool, VolumeExtentList* extents) {
+        .WillOnce(Invoke(
+            [&](uint32_t, uint64_t, bool, VolumeExtentSliceList *extents) {
                 extents->clear_slices();
                 return MetaStatusCode::OK;
             }));
@@ -311,7 +304,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) {
 
 
     EXPECT_CALL(inodeCacheMgr_, GetInode(_, _))
-        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper>& out) {
+        .WillOnce(Invoke([&](uint64_t, std::shared_ptr<InodeWrapper> &out) {
             out = inodeWrapper;
             return CURVEFS_ERROR::OK;
         }));
@@ -322,7 +315,7 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) {
 
     EXPECT_CALL(spaceMgr_, Alloc(_, _, _))
         .WillOnce(Invoke(
-            [](uint32_t size, const AllocateHint&, std::vector<Extent>* exts) {
+            [](uint32_t size, const AllocateHint &, std::vector<Extent> *exts) {
                 exts->emplace_back(size, size);
                 return true;
             }));
@@ -332,11 +325,9 @@ TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) {
     size_t len = 4096;
     std::unique_ptr<char[]> data(new char[len]);
 
-    EXPECT_CALL(blockDev_, Writev(_))
-        .WillOnce(Return(len));
+    EXPECT_CALL(blockDev_, Writev(_)).WillOnce(Return(len));
 
-    EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper))
-        .Times(1);
+    EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)).Times(1);
 
     ASSERT_EQ(CURVEFS_ERROR::OK, storage_.Write(ino, offset, len, data.get()));
 

curvefs/test/mds/heartbeat/heartbeat_manager_test.cpp

@@ -29,14 +29,19 @@
 #include "curvefs/src/mds/heartbeat/metaserver_healthy_checker.h"
 #include "curvefs/test/mds/mock/mock_topology.h"
 #include "curvefs/test/mds/mock/mock_coordinator.h"
+#include "curvefs/test/mds/mock/mock_space_manager.h"
+#include "curvefs/test/mds/mock/mock_volume_space.h"
 #include "src/common/timeutility.h"
 
 using ::curvefs::mds::topology::MockIdGenerator;
 using ::curvefs::mds::topology::MockStorage;
 using ::curvefs::mds::topology::MockTokenGenerator;
 using ::curvefs::mds::topology::MockTopology;
+using ::curvefs::mds::space::MockSpaceManager;
+using ::curvefs::mds::space::MockVolumeSpace;
 using ::curvefs::mds::topology::TopoStatusCode;
 using ::curve::mds::heartbeat::ConfigChangeType;
+
 using ::testing::_;
 using ::testing::DoAll;
 using ::testing::Return;
@@ -58,8 +63,9 @@ class TestHeartbeatManager : public ::testing::Test {
         topology_ = std::make_shared<MockTopology>(idGenerator_,
                                                    tokenGenerator_, storage_);
         coordinator_ = std::make_shared<MockCoordinator>();
-        heartbeatManager_ =
-            std::make_shared<HeartbeatManager>(option, topology_, coordinator_);
+        spaceManager_ = std::make_shared<MockSpaceManager>();
+        heartbeatManager_ = std::make_shared<HeartbeatManager>(
+            option, topology_, coordinator_, spaceManager_);
     }
 
     void TearDown() override {}
@@ -69,6 +75,7 @@ class TestHeartbeatManager : public ::testing::Test {
     std::shared_ptr<MockTokenGenerator> tokenGenerator_;
     std::shared_ptr<MockStorage> storage_;
     std::shared_ptr<MockTopology> topology_;
+    std::shared_ptr<MockSpaceManager> spaceManager_;
     std::shared_ptr<HeartbeatManager> heartbeatManager_;
     std::shared_ptr<Coordinator> coordinator_;
 };
@@ -219,7 +226,6 @@ TEST_F(TestHeartbeatManager, test_updatespace_get_metaserver_fail2) {
     ASSERT_EQ(HeartbeatStatusCode::hbOK, response.statuscode());
 }
 
-
 TEST_F(TestHeartbeatManager, test_updatespace_get_server_fail) {
     ::curvefs::mds::topology::MetaServer metaServer(
         1, "hostname", "hello", 1, "192.168.10.1", 9000, "", 9000);
@@ -918,6 +924,58 @@ TEST_F(TestHeartbeatManager, test_update_partition) {
     heartbeatManager_->MetaServerHeartbeat(request, &response);
     ASSERT_EQ(HeartbeatStatusCode::hbOK, response.statuscode());
 }
+
+TEST_F(TestHeartbeatManager, TEST_UpdateDeallocatableBlockGroup) {
+    auto request = GetMetaServerHeartbeatRequestForTest();
+    MetaServerHeartbeatResponse response;
+
+    // prepare request
+    {
+        auto blockGroupStatInfo = request.add_blockgroupstatinfos();
+        blockGroupStatInfo->set_fsid(1);
+        auto deallocatbleBlockGroup =
+            blockGroupStatInfo->add_deallocatableblockgroups();
+        deallocatbleBlockGroup->set_blockgroupoffset(0);
+        deallocatbleBlockGroup->set_deallocatablesize(128 * 1024);
+    }
+
+    // 1. get volumespace fail
+    {
+        EXPECT_CALL(*spaceManager_, GetVolumeSpace(1))
+            .WillOnce(Return(nullptr));
+
+        heartbeatManager_->UpdateDeallocatableBlockGroup(request, &response);
+        ASSERT_EQ(HeartbeatStatusCode::hbMetaServerFSUnkown,
+                  response.statuscode());
+    }
+
+    // 2. no issued block group
+    {
+        MockVolumeSpace volumeSpace;
+        EXPECT_CALL(*spaceManager_, GetVolumeSpace(1))
+            .WillOnce(Return(&volumeSpace));
+        EXPECT_CALL(volumeSpace, UpdateDeallocatableBlockGroup(_, _, _, _, _))
+            .WillOnce(Return(false));
+
+        heartbeatManager_->UpdateDeallocatableBlockGroup(request, &response);
+        ASSERT_EQ(response.issuedblockgroups_size(), 0);
+    }
+
+    // 3. has issued block group
+    {
+        MockVolumeSpace volumeSpace;
+        EXPECT_CALL(*spaceManager_, GetVolumeSpace(1))
+            .WillOnce(Return(&volumeSpace));
+        EXPECT_CALL(volumeSpace, UpdateDeallocatableBlockGroup(_, _, _, _, _))
+            .WillOnce(DoAll(SetArgPointee<4>(1024 * 1024), Return(true)));
+
+        heartbeatManager_->UpdateDeallocatableBlockGroup(request, &response);
+        ASSERT_EQ(response.issuedblockgroups_size(), 1);
+        ASSERT_EQ(response.issuedblockgroups().at(1), 1024 * 1024);
+    }
+}
+
 }  // namespace heartbeat
 }  // namespace mds
 }  // namespace curvefs
+

curvefs/test/mds/mock/mock_volume_space.h

@@ -48,6 +48,12 @@ class MockVolumeSpace : public AbstractVolumeSpace {
 
     MOCK_METHOD1(ReleaseBlockGroups,
                  SpaceErrCode(const std::vector<BlockGroup>& blockGroups));
+
+    MOCK_METHOD5(UpdateDeallocatableBlockGroup,
+                 bool(uint32_t metaserverId, uint32_t metaserverNum,
+                      const DeallocatableBlockGroupVec &groups,
+                      const BlockGroupDeallcateStatusMap &stats,
+                      uint64_t *issue));
 };
 
 }  // namespace space

curvefs/test/mds/space/space_manager_test.cpp

@@ -71,7 +71,7 @@ class SpaceManagerTest : public ::testing::Test {
         etcdclient_ = std::make_shared<MockEtcdClientImpl>();
         fsStorage_ = std::make_shared<MockFsStorage>();
         spaceManager_ =
-            absl::make_unique<SpaceManagerImpl>(etcdclient_, fsStorage_);
+            absl::make_unique<SpaceManagerImpl>(etcdclient_, fsStorage_, 1);
     }
 
     void TearDown() override {}

curvefs/test/mds/space/volume_space_test.cpp

@@ -26,6 +26,7 @@
 #include <brpc/server.h>
 #include <gtest/gtest.h>
 #include <cassert>
+#include <utility>
 
 #include "absl/memory/memory.h"
 #include "curvefs/proto/common.pb.h"
@@ -42,8 +43,8 @@ namespace space {
 
 using ::testing::_;
 using ::testing::Invoke;
-using ::testing::Return;
 using ::testing::Matcher;
+using ::testing::Return;
 
 static constexpr uint32_t kFsId = 1;
 static constexpr uint64_t kVolumeSize = 10ULL * 1024 * 1024 * 1024;
@@ -53,7 +54,7 @@ static constexpr curvefs::common::BitmapLocation kBitmapLocation =
     curvefs::common::BitmapLocation::AtStart;
 static constexpr uint32_t kAllocateOnce = 4;
 static constexpr double kExtendFactor = 1.5;
-static const char* kOwner = "test";
+static const char *kOwner = "test";
 
 static unsigned int seed = time(nullptr);
 
@@ -72,19 +73,20 @@ class VolumeSpaceTest : public ::testing::Test {
 
     std::unique_ptr<VolumeSpace> CreateOneEmptyVolumeSpace() {
         EXPECT_CALL(*storage_, ListBlockGroups(_, _))
-            .WillOnce(Invoke([](uint32_t, std::vector<BlockGroup>* groups) {
+            .WillOnce(Invoke([](uint32_t, std::vector<BlockGroup> *groups) {
                 groups->clear();
                 return SpaceOk;
             }));
 
         auto space = VolumeSpace::Create(kFsId, volume_, storage_.get(),
-                                         fsStorage_.get());
+                                         fsStorage_.get(), 1);
         EXPECT_NE(nullptr, space);
         return space;
     }
 
-    std::unique_ptr<VolumeSpace> CreateVolumeSpaceWithTwoGroups(
-        bool allocated = true, std::vector<BlockGroup>* out = nullptr) {
+    std::unique_ptr<VolumeSpace>
+    CreateVolumeSpaceWithTwoGroups(bool allocated = true,
+                                   std::vector<BlockGroup> *out = nullptr) {
         BlockGroup group1;
         group1.set_offset(0);
         group1.set_size(kBlockGroupSize);
@@ -99,7 +101,6 @@ class VolumeSpaceTest : public ::testing::Test {
         group2.set_size(kBlockGroupSize);
         group2.set_available(kBlockGroupSize / 2);
         group2.set_bitmaplocation(kBitmapLocation);
-        group2.set_owner(kOwner);
         if (allocated) {
             group2.set_owner(kOwner);
         }
@@ -112,17 +113,75 @@ class VolumeSpaceTest : public ::testing::Test {
 
         EXPECT_CALL(*storage_, ListBlockGroups(_, _))
             .WillOnce(
-                Invoke([&exist](uint32_t, std::vector<BlockGroup>* groups) {
+                Invoke([&exist](uint32_t, std::vector<BlockGroup> *groups) {
                     *groups = exist;
                     return SpaceOk;
                 }));
 
         auto space = VolumeSpace::Create(kFsId, volume_, storage_.get(),
-                                         fsStorage_.get());
+                                         fsStorage_.get(), 1);
         EXPECT_NE(nullptr, space);
         return space;
     }
 
+    void
+    UpdateBlockGroupSpaceInfo(uint32_t metaserverId,
+                              std::unique_ptr<VolumeSpace> &space,  // NOLINT
+                              bool needIssued = false,
+                              uint64_t expectIssued = 0) {
+        BlockGroupDeallcateStatusMap stats;
+        DeallocatableBlockGroupVec groups;
+        {
+            DeallocatableBlockGroup group1;
+            group1.set_blockgroupoffset(0);
+            group1.set_deallocatablesize(kBlockGroupSize / 2);
+            groups.Add()->CopyFrom(group1);
+
+
+            DeallocatableBlockGroup group2;
+            group2.set_blockgroupoffset(kBlockGroupSize);
+            group2.set_deallocatablesize(kBlockGroupSize / 4);
+            groups.Add()->CopyFrom(group2);
+        }
+
+        uint64_t issued = 0;
+        if (needIssued) {
+            ASSERT_TRUE(space->UpdateDeallocatableBlockGroup(
+                metaserverId, 2, groups, stats, &issued));
+            ASSERT_EQ(issued, expectIssued);
+
+        } else {
+            ASSERT_FALSE(space->UpdateDeallocatableBlockGroup(
+                metaserverId, 2, groups, stats, &issued));
+        }
+    }
+
+    void
+    UpdateBlockGroupIssueStat(uint32_t metaserverId,
+                              std::unique_ptr<VolumeSpace> &space,  // NOLINT
+                              uint64_t blockoffset,
+                              BlockGroupDeallcateStatusCode status,
+                              bool needIssue = true) {
+        DeallocatableBlockGroupVec groups;
+        BlockGroupDeallcateStatusMap stats;
+        stats[blockoffset] = status;
+        if (status == BlockGroupDeallcateStatusCode::BGDP_DONE) {
+            DeallocatableBlockGroup group;
+            group.set_blockgroupoffset(blockoffset);
+            group.set_deallocatablesize(0);
+            groups.Add()->CopyFrom(group);
+        }
+
+        uint64_t issued = 0;
+        if (needIssue) {
+            ASSERT_TRUE(space->UpdateDeallocatableBlockGroup(
+                metaserverId, 2, groups, stats, &issued));
+        } else {
+            ASSERT_FALSE(space->UpdateDeallocatableBlockGroup(
+                metaserverId, 2, groups, stats, &issued));
+        }
+    }
+
  protected:
     std::unique_ptr<MockBlockGroupStorage> storage_;
     std::unique_ptr<MockFsStorage> fsStorage_;
@@ -133,20 +192,20 @@ TEST_F(VolumeSpaceTest, TestCreate_ListError) {
     EXPECT_CALL(*storage_, ListBlockGroups(_, _))
         .WillOnce(Return(SpaceErrStorage));
 
-    auto space =
-        VolumeSpace::Create(kFsId, volume_, storage_.get(), fsStorage_.get());
+    auto space = VolumeSpace::Create(kFsId, volume_, storage_.get(),
+                                     fsStorage_.get(), 1);
     EXPECT_EQ(nullptr, space);
 }
 
 TEST_F(VolumeSpaceTest, TestCreate_Success) {
     EXPECT_CALL(*storage_, ListBlockGroups(_, _))
-        .WillOnce(Invoke([](uint32_t, std::vector<BlockGroup>* groups) {
+        .WillOnce(Invoke([](uint32_t, std::vector<BlockGroup> *groups) {
             groups->clear();
             return SpaceOk;
         }));
 
-    auto space =
-        VolumeSpace::Create(kFsId, volume_, storage_.get(), fsStorage_.get());
+    auto space = VolumeSpace::Create(kFsId, volume_, storage_.get(),
+                                     fsStorage_.get(), 1);
     EXPECT_NE(nullptr, space);
 }
 
@@ -186,7 +245,7 @@ TEST_F(VolumeSpaceTest, TestAllocateBlockGroups) {
     }
 
     std::set<uint64_t> offsets;
-    for (auto& group : groups) {
+    for (auto &group : groups) {
         ASSERT_EQ(kOwner, group.owner());
         offsets.insert(group.offset());
     }
@@ -225,10 +284,10 @@ TEST_F(VolumeSpaceTest, TestAutoExtendVolume_ExtendError) {
 namespace {
 class FakeCurveFSService : public curve::mds::CurveFSService {
  public:
-    void ExtendFile(::google::protobuf::RpcController* controller,
-                    const ::curve::mds::ExtendFileRequest* request,
-                    ::curve::mds::ExtendFileResponse* response,
-                    ::google::protobuf::Closure* done) override {
+    void ExtendFile(::google::protobuf::RpcController *controller,
+                    const ::curve::mds::ExtendFileRequest *request,
+                    ::curve::mds::ExtendFileResponse *response,
+                    ::google::protobuf::Closure *done) override {
         brpc::ClosureGuard guard(done);
         if (request->newsize() % kBlockGroupSize != 0) {
             response->set_statuscode(curve::mds::kParaError);
@@ -304,16 +363,14 @@ TEST_F(VolumeSpaceTest, TestAutoExtendVolumeSuccess) {
 
     EXPECT_CALL(*fsStorage_, Get(Matcher<uint64_t>(_), _))
         .WillOnce(Return(FSStatusCode::OK));
-    EXPECT_CALL(*fsStorage_, Update(_))
-        .WillOnce(Return(FSStatusCode::OK));
+    EXPECT_CALL(*fsStorage_, Update(_)).WillOnce(Return(FSStatusCode::OK));
 
     EXPECT_CALL(*storage_, PutBlockGroup(_, _, _))
         .Times(1)
         .WillRepeatedly(Return(SpaceOk));
 
     std::vector<BlockGroup> newGroups;
-    ASSERT_EQ(SpaceOk,
-              space->AllocateBlockGroups(1, kOwner, &newGroups));
+    ASSERT_EQ(SpaceOk, space->AllocateBlockGroups(1, kOwner, &newGroups));
 
     ASSERT_EQ(totalGroups + 1, space->allocatedGroups_.size());
     ASSERT_TRUE(space->availableGroups_.empty());
@@ -439,9 +496,7 @@ TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsNotUsed_ButClearError) {
     ASSERT_EQ(SpaceErrStorage, space->ReleaseBlockGroups(exists));
 }
 
-MATCHER(NoOwner, "") {
-    return !arg.has_owner();
-}
+MATCHER(NoOwner, "") { return !arg.has_owner(); }
 
 TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsPartialUsed) {
     std::vector<BlockGroup> exists;
@@ -470,6 +525,131 @@ TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsPartialUsed_PutError) {
     ASSERT_EQ(SpaceErrStorage, space->ReleaseBlockGroups(exists));
 }
 
+TEST_F(VolumeSpaceTest, Test_CalBlockGroupAvailableForDeAllocate) {
+    // 1. test cal none
+    {
+        auto space = CreateOneEmptyVolumeSpace();
+        space->Run();
+        sleep(1);
+        space->Stop();
+        ASSERT_EQ(0, space->waitDeallocateGroups_.size());
+
+        space = CreateVolumeSpaceWithTwoGroups(false);
+        space->Run();
+        sleep(1);
+        space->Stop();
+        ASSERT_EQ(0, space->waitDeallocateGroups_.size());
+        ASSERT_EQ(2, space->availableGroups_.size());
+    }
+
+    // 2. test cal one and issue
+    {
+        // cal one
+        uint32_t metaserverId1 = 1;
+        std::vector<BlockGroup> exists;
+        auto space = CreateVolumeSpaceWithTwoGroups(false, &exists);
+        UpdateBlockGroupSpaceInfo(metaserverId1, space);
+        space->CalBlockGroupAvailableForDeAllocate();
+        ASSERT_EQ(1, space->waitDeallocateGroups_.size());
+        ASSERT_EQ(exists[0].offset(), space->waitDeallocateGroups_[0].offset());
+        ASSERT_EQ(1, space->availableGroups_.size());
+        ASSERT_EQ(exists[0].offset(), space->availableGroups_[0].offset());
+        ASSERT_EQ(kBlockGroupSize / 2, space->summary_[exists[0].offset()]);
+        ASSERT_EQ(kBlockGroupSize / 4, space->summary_[exists[1].offset()]);
+
+        // issue to metaserver1
+        EXPECT_CALL(*storage_, PutBlockGroup(_, _, _))
+            .Times(1)
+            .WillOnce(Return(SpaceOk));
+        UpdateBlockGroupSpaceInfo(metaserverId1, space, true,
+                                  exists[0].offset());
+        ASSERT_TRUE(space->waitDeallocateGroups_.empty());
+        ASSERT_EQ(1, space->deallocatingGroups_.size());
+        ASSERT_EQ(
+            metaserverId1,
+            space->deallocatingGroups_[exists[0].offset()].deallocating()[0]);
+
+        // issue to metaserver2
+        uint32_t metaserverId2 = 2;
+        EXPECT_CALL(*storage_, PutBlockGroup(_, _, _))
+            .Times(1)
+            .WillOnce(Return(SpaceOk));
+        UpdateBlockGroupSpaceInfo(metaserverId2, space, true,
+                                  exists[0].offset());
+        ASSERT_EQ(
+            metaserverId1,
+            space->deallocatingGroups_[exists[0].offset()].deallocating()[0]);
+
+        ASSERT_EQ(
+            metaserverId2,
+            space->deallocatingGroups_[exists[0].offset()].deallocating()[1]);
+        ASSERT_EQ(kBlockGroupSize, space->summary_[exists[0].offset()]);
+        ASSERT_EQ(kBlockGroupSize / 2, space->summary_[exists[1].offset()]);
+
+        // metaserver2 report, update summary and be issued again
+        UpdateBlockGroupSpaceInfo(metaserverId2, space, true,
+                                  exists[0].offset());
+        ASSERT_EQ(kBlockGroupSize, space->summary_[exists[0].offset()]);
+        ASSERT_EQ(kBlockGroupSize / 2, space->summary_[exists[1].offset()]);
+
+        // metaserver1 report issued stat: BGDP_PROCESSING
+        UpdateBlockGroupIssueStat(
+            metaserverId1, space, exists[0].offset(),
+            BlockGroupDeallcateStatusCode::BGDP_PROCESSING);
+        ASSERT_EQ(2, space->deallocatingGroups_[exists[0].offset()]
+                         .deallocating()
+                         .size());
+        ASSERT_EQ(2, space->summary_.size());
+
+        // metaserver1 report issued stat: BGDP_DONE
+        EXPECT_CALL(*storage_, PutBlockGroup(_, _, _))
+            .Times(1)
+            .WillOnce(Return(SpaceOk));
+        UpdateBlockGroupIssueStat(metaserverId1, space, exists[0].offset(),
+                                  BlockGroupDeallcateStatusCode::BGDP_DONE,
+                                  false);
+        ASSERT_EQ(
+            metaserverId2,
+            space->deallocatingGroups_[exists[0].offset()].deallocating()[0]);
+        ASSERT_EQ(
+            metaserverId1,
+            space->deallocatingGroups_[exists[0].offset()].deallocated()[0]);
+        ASSERT_EQ(2, space->summary_.size());
+
+        // metaserver2 report issued stat: BGDP_DONE
+        EXPECT_CALL(*storage_, PutBlockGroup(_, _, _))
+            .Times(1)
+            .WillOnce(Return(SpaceOk));
+        UpdateBlockGroupIssueStat(metaserverId2, space, exists[0].offset(),
+                                  BlockGroupDeallcateStatusCode::BGDP_DONE,
+                                  false);
+        ASSERT_EQ(
+            metaserverId1,
+            space->deallocatingGroups_[exists[0].offset()].deallocated()[0]);
+        ASSERT_EQ(
+            metaserverId2,
+            space->deallocatingGroups_[exists[0].offset()].deallocated()[1]);
+        ASSERT_EQ(1, space->summary_.size());
+        ASSERT_EQ(kBlockGroupSize / 2, space->summary_[exists[1].offset()]);
+
+        // metaserver1 and metaserver2 process done
+        EXPECT_CALL(*storage_, PutBlockGroup(_, exists[0].offset(), _))
+            .Times(1)
+            .WillOnce(Return(SpaceOk));
+        space->CalBlockGroupAvailableForDeAllocate();
+        ASSERT_EQ(1, space->waitDeallocateGroups_.count(exists[1].offset()));
+        ASSERT_EQ(1, space->waitDeallocateGroups_.size());
+        ASSERT_EQ(1, space->availableGroups_.count(exists[0].offset()));
+        ASSERT_EQ(1, space->availableGroups_.size());
+    }
+}
+
+// TEST_F(VolumeSpaceTest, Test_UpdateBlockGroupDeallocatableSpace) {}
+
+// TEST_F(VolumeSpaceTest, Test_UpdnaateDeallocatingBlockGroup) {}
+
+// TEST_F(VolumeSpaceTest, Test_UpdateDeallocatingBlockGroupError) {}
+
 }  // namespace space
 }  // namespace mds
 }  // namespace curvefs

curvefs/test/metaserver/copyset/copyset_node_block_group_test.cpp

@@ -0,0 +1,139 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Mon Apr 24 14:30:20 CST 2023
+ * Author: lixiaocui
+ */
+
+#include <gtest/gtest.h>
+
+#include "curvefs/src/metaserver/copyset/copyset_node.h"
+#include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
+#include "curvefs/test/metaserver/mock/mock_partition.h"
+#include "curvefs/test/metaserver/mock/mock_metastore.h"
+#include "src/common/uuid.h"
+#include "test/fs/mock_local_filesystem.h"
+
+namespace curvefs {
+namespace metaserver {
+namespace copyset {
+
+using ::curve::common::UUIDGenerator;
+using ::curve::fs::MockLocalFileSystem;
+using ::curvefs::metaserver::mock::MockPartition;
+using ::testing::_;
+using ::testing::Return;
+using ::testing::SetArgPointee;
+
+class CopysetNodeBlockGroupTest : public testing::Test {
+ protected:
+    void SetUp() override {
+        dataPath_ = "./runlog/" + UUIDGenerator{}.GenerateUUID();
+        trashPath_ = dataPath_ + "_trash";
+        mockfs_ = absl::make_unique<MockLocalFileSystem>();
+        nodeManager_ = &CopysetNodeManager::GetInstance();
+        mockMetaStore_ = absl::make_unique<mock::MockMetaStore>();
+
+        poolId_ = time(nullptr) % 12345;
+        copysetId_ = time(nullptr) % reinterpret_cast<uint64_t>(this);
+
+        options_.ip = "127.0.0.1";
+        options_.port = 6666;
+
+        options_.dataUri = "local://" + dataPath_;
+        options_.raftNodeOptions.log_uri  = "local://" + dataPath_;
+        options_.raftNodeOptions.raft_meta_uri = "local://" + dataPath_;
+        options_.raftNodeOptions.snapshot_uri = "local://" + dataPath_;
+
+        // disable raft snapshot
+        options_.raftNodeOptions.snapshot_interval_s = -1;
+
+        options_.localFileSystem = mockfs_.get();
+
+        options_.trashOptions.trashUri = "local://" + trashPath_;
+        options_.storageOptions.type = "memory";
+
+        butil::ip_t ip;
+        ASSERT_EQ(0, butil::str2ip(options_.ip.c_str(), &ip));
+        butil::EndPoint listenAddr(ip, 6666);
+
+        ASSERT_TRUE(nodeManager_->Init(options_));
+    }
+
+    void TearDown() {
+        system(std::string("rm -rf " + dataPath_).c_str());
+    }
+
+ protected:
+    PoolId poolId_;
+    CopysetId copysetId_;
+    std::string dataPath_;
+    std::string trashPath_;
+
+    CopysetNodeOptions options_;
+    CopysetNodeManager *nodeManager_;
+
+    std::unique_ptr<MockLocalFileSystem> mockfs_;
+    std::unique_ptr<mock::MockMetaStore> mockMetaStore_;
+};
+
+TEST_F(CopysetNodeBlockGroupTest, Test_AggregateBlockStatInfo) {
+    auto partition = std::make_shared<MockPartition>();
+    std::map<uint32_t, BlockGroupStatInfo> blockStatInfoMap;
+    uint32_t blockGroupNum = 0;
+    uint32_t fsId = 1;
+    uint32_t partitionId = 1;
+
+    auto copyset = nodeManager_->GetCopysetNode(poolId_, copysetId_);
+
+    // get fail
+    {
+        EXPECT_CALL(*partition, GetFsId()).WillOnce(Return(fsId));
+        EXPECT_CALL(*partition, GeAllBlockGroup(_))
+            .WillOnce(Return(MetaStatusCode::PARAM_ERROR));
+        EXPECT_CALL(*partition, GetPartitionId()).WillOnce(Return(partitionId));
+        ASSERT_FALSE(copyset->AggregateBlockStatInfo(
+            partition, &blockStatInfoMap, &blockGroupNum));
+        ASSERT_EQ(0, blockGroupNum);
+    }
+
+    // get success
+    {
+        std::vector<DeallocatableBlockGroup> in;
+        DeallocatableBlockGroup bg;
+        bg.set_blockgroupoffset(0);
+        bg.set_deallocatablesize(1024);
+        bg.add_inodeidlist(1);
+        in.emplace_back(bg);
+
+        EXPECT_CALL(*partition, GetFsId()).WillOnce(Return(fsId));
+        EXPECT_CALL(*partition, GeAllBlockGroup(_))
+            .WillOnce(DoAll(SetArgPointee<0>(in), Return(MetaStatusCode::OK)));
+        ASSERT_TRUE(copyset->AggregateBlockStatInfo(
+            partition, &blockStatInfoMap, &blockGroupNum));
+        ASSERT_EQ(1, blockGroupNum);
+        ASSERT_EQ(1, blockStatInfoMap.size());
+        auto &out = blockStatInfoMap[fsId];
+        ASSERT_EQ(fsId, out.fsid());
+        ASSERT_EQ(1, out.deallocatableblockgroups_size());
+    }
+}
+
+}  // namespace copyset
+}  // namespace metaserver
+}  // namespace curvefs

curvefs/test/metaserver/copyset/copyset_node_snapshot_test.cpp

@@ -402,7 +402,15 @@ void RunGetPartitionInfoList(CopysetNode* node, uint32_t sleepSec,
     ASSERT_EQ(node->GetPartitionInfoList(&partitionInfoList), expectedValue);
 }
 
-// get partition list while copyset is loading
+void RunGetBlockStatInfo(CopysetNode *node, uint32_t sleepSec,
+                         bool expectedValue) {
+    sleep(sleepSec);
+    std::map<uint32_t, BlockGroupStatInfo> blockStatInfoMap;
+
+    ASSERT_EQ(node->GetBlockStatInfo(&blockStatInfoMap), expectedValue);
+}
+
+// get partition list / get block statinfo while copyset is loading
 TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadSuccess1) {
     ASSERT_TRUE(CreateOneCopyset());
 
@@ -431,8 +439,10 @@ TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadSuccess1) {
 
     std::thread thread1(RunOnSnapshotLoad, node, &reader, 0);
     std::thread thread2(RunGetPartitionInfoList, node, 3, false);
+    std::thread thread3(RunGetBlockStatInfo, node, 3, false);
     thread1.join();
     thread2.join();
+    thread3.join();
 
     node->SetMetaStore(nullptr);
 }
@@ -465,11 +475,18 @@ TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadSuccess2) {
         .Times(1);
     EXPECT_CALL(*mockMetaStore, GetPartitionInfoList(_))
         .WillOnce(Return(true));
+    EXPECT_CALL(*mockMetaStore, GetPartitionSnap(_))
+        .Times(2)
+        .WillOnce(Return(false))
+        .WillOnce(Return(true));
 
     std::thread thread1(RunOnSnapshotLoad, node, &reader, 0);
     std::thread thread2(RunGetPartitionInfoList, node, 3, true);
+    std::thread thread3(RunGetBlockStatInfo, node, 5, true);
+
     thread1.join();
     thread2.join();
+    thread3.join();
 
     node->SetMetaStore(nullptr);
 }
@@ -510,6 +527,7 @@ TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadSuccess3) {
 
     node->SetMetaStore(nullptr);
 }
+
 }  // namespace copyset
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/test/metaserver/copyset/meta_operator_test.cpp

@@ -146,6 +146,8 @@ TEST_F(MetaOperatorTest, OperatorTypeTest) {
     TEST_OPERATOR_TYPE(CreatePartition);
     TEST_OPERATOR_TYPE(DeletePartition);
     TEST_OPERATOR_TYPE(PrepareRenameTx);
+    TEST_OPERATOR_TYPE(UpdateDeallocatableBlockGroup);
+
 
 #undef TEST_OPERATOR_TYPE
 }

curvefs/test/metaserver/copyset/mock/mock_copyset_node.h

@@ -26,6 +26,8 @@
 #include <gmock/gmock.h>
 
 #include <vector>
+#include <list>
+#include <map>
 
 #include "curvefs/src/metaserver/copyset/copyset_node.h"
 
@@ -45,6 +47,17 @@ class MockCopysetNode : public CopysetNode {
     MOCK_CONST_METHOD1(ListPeers, void(std::vector<Peer>*));
     MOCK_CONST_METHOD0(IsLeaderTerm, bool());
     MOCK_METHOD1(Propose, void(const braft::Task& task));
+    MOCK_METHOD2(GetConfChange, void(ConfigChangeType *type, Peer *alterPeer));
+
+    MOCK_CONST_METHOD0(GetPoolId, PoolId());
+    MOCK_CONST_METHOD0(GetPeerId, const braft::PeerId &());
+    MOCK_CONST_METHOD0(GetLeaderId, PeerId());
+    MOCK_CONST_METHOD0(GetCopysetId, CopysetId());
+
+    MOCK_METHOD1(GetPartitionInfoList, bool(std::list<PartitionInfo> *));
+    MOCK_CONST_METHOD0(IsLoading, bool());
+    MOCK_METHOD1(GetBlockStatInfo,
+                 bool(std::map<uint32_t, BlockGroupStatInfo> *));
 };
 
 }  // namespace copyset

curvefs/test/metaserver/copyset/mock/mock_copyset_node_manager.h

@@ -24,7 +24,7 @@
 #define CURVEFS_TEST_METASERVER_COPYSET_MOCK_MOCK_COPYSET_NODE_MANAGER_H_
 
 #include <gmock/gmock.h>
-
+#include <vector>
 #include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
 
 namespace curvefs {
@@ -35,6 +35,7 @@ class MockCopysetNodeManager : public CopysetNodeManager {
  public:
     MOCK_METHOD2(GetCopysetNode, CopysetNode*(PoolId, CopysetId));
     MOCK_METHOD2(PurgeCopysetNode, bool(PoolId, CopysetId));
+    MOCK_CONST_METHOD1(GetAllCopysets, void(std::vector<CopysetNode *> *));
     MOCK_CONST_METHOD0(IsLoadFinished, bool());
 };
 

curvefs/test/metaserver/fsinfo_manager_test.cpp

@@ -0,0 +1,56 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Fri May 05 11:42:53 CST 2023
+ * Author: lixiaocui
+ */
+
+#include <gtest/gtest.h>
+
+#include "curvefs/src/metaserver/mds/fsinfo_manager.h"
+#include "curvefs/test/client/rpcclient/mock_mds_client.h"
+
+using ::curvefs::client::rpcclient::MockMdsClient;
+
+using ::testing::_;
+
+namespace curvefs {
+namespace metaserver {
+
+TEST(FsInfoManagerTest, TestGetFsInfo) {
+    auto &manager = FsInfoManager::GetInstance();
+
+    auto mdsCli = std::make_shared<MockMdsClient>();
+    manager.SetMdsClient(mdsCli);
+    uint32_t fsId = 1;
+
+    EXPECT_CALL(*mdsCli, GetFsInfo(fsId, _))
+        .WillOnce(Return(FSStatusCode::NOT_FOUND));
+    FsInfo fsInfo;
+    ASSERT_FALSE(manager.GetFsInfo(fsId, &fsInfo));
+
+    EXPECT_CALL(*mdsCli, GetFsInfo(fsId, _))
+        .WillOnce(Return(FSStatusCode::UNKNOWN_ERROR));
+    ASSERT_FALSE(manager.GetFsInfo(fsId, &fsInfo));
+
+    EXPECT_CALL(*mdsCli, GetFsInfo(fsId, _)).WillOnce(Return(FSStatusCode::OK));
+    ASSERT_TRUE(manager.GetFsInfo(fsId, &fsInfo));
+}
+
+}  // namespace metaserver
+}  // namespace curvefs

curvefs/test/metaserver/heartbeat_test.cpp

@@ -22,9 +22,12 @@
 #include <gtest/gtest.h>
 #include <gmock/gmock.h>
 #include "curvefs/src/metaserver/heartbeat.h"
+#include "curvefs/proto/heartbeat.pb.h"
 #include "curvefs/test/metaserver/mock_heartbeat_service.h"
 #include "curvefs/src/metaserver/storage/storage.h"
 #include "curvefs/src/metaserver/resource_statistic.h"
+#include "curvefs/test/metaserver/copyset/mock/mock_copyset_node_manager.h"
+#include "curvefs/test/metaserver/copyset/mock/mock_copyset_node.h"
 
 using ::testing::AtLeast;
 using ::testing::StrEq;
@@ -42,6 +45,9 @@ using ::curvefs::mds::heartbeat::HeartbeatStatusCode;
 using ::curve::fs::FileSystemType;
 using ::curve::fs::LocalFsFactory;
 using ::curvefs::metaserver::storage::StorageOptions;
+using ::curvefs::metaserver::copyset::MockCopysetNodeManager;
+using ::curvefs::metaserver::copyset::MockCopysetNode;
+using ::curvefs::mds::heartbeat::BlockGroupDeallcateStatusCode;
 
 namespace curvefs {
 namespace metaserver {
@@ -56,6 +62,8 @@ class HeartbeatTest : public ::testing::Test {
         resourceCollector_ = absl::make_unique<ResourceCollector>(
             options_.maxDiskQuotaBytes, options_.maxMemoryQuotaBytes,
             options_.dataDir);
+
+        GetHeartbeatOption(&hbopts_);
     }
 
     bool GetMetaserverSpaceStatus(MetaServerSpaceStatus* status,
@@ -67,9 +75,25 @@ class HeartbeatTest : public ::testing::Test {
         return heartbeat.GetMetaserverSpaceStatus(status, ncopysets);
     }
 
+    void GetHeartbeatOption(HeartbeatOptions *opt) {
+        opt->metaserverId = 1;
+        opt->metaserverToken = "token";
+        opt->intervalSec = 2;
+        opt->timeout = 1000;
+        opt->ip = "127.0.0.1";
+        opt->port = 6000;
+        opt->mdsListenAddr = "127.0.0.1:6710";
+        opt->copysetNodeManager = &mockCopysetManager_;
+        opt->storeUri = "local://./metaserver_data/copysets";
+        opt->fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, "");
+        opt->resourceCollector = resourceCollector_.get();
+    }
+
  protected:
     StorageOptions options_;
     std::unique_ptr<ResourceCollector> resourceCollector_;
+    HeartbeatOptions hbopts_;
+    MockCopysetNodeManager mockCopysetManager_;
 };
 
 template <typename RpcRequestType, typename RpcResponseType,
@@ -107,43 +131,18 @@ TEST_F(HeartbeatTest, testInitFail) {
 }
 
 TEST_F(HeartbeatTest, test1) {
-    HeartbeatOptions options;
     Heartbeat heartbeat;
 
-    options.metaserverId = 1;
-    options.metaserverToken = "token";
-    options.intervalSec = 1;
-    options.timeout = 1000;
-    options.ip = "127.0.0.1";
-    options.port = 6000;
-    options.mdsListenAddr = "127.0.0.1:6710";
-    options.copysetNodeManager = &CopysetNodeManager::GetInstance();
-    options.storeUri = "local://./metaserver_data/copysets";
-    options.fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, "");
-
     // mds service not start
-    ASSERT_EQ(heartbeat.Init(options), 0);
+    ASSERT_EQ(heartbeat.Init(hbopts_), 0);
     ASSERT_EQ(heartbeat.Run(), 0);
     sleep(2);
     ASSERT_EQ(heartbeat.Fini(), 0);
 }
 
 TEST_F(HeartbeatTest, test_ok) {
-    HeartbeatOptions options;
     Heartbeat heartbeat;
 
-    options.metaserverId = 1;
-    options.metaserverToken = "token";
-    options.intervalSec = 2;
-    options.timeout = 1000;
-    options.ip = "127.0.0.1";
-    options.port = 6000;
-    options.mdsListenAddr = "127.0.0.1:6710";
-    options.copysetNodeManager = &CopysetNodeManager::GetInstance();
-    options.storeUri = "local://./metaserver_data/copysets";
-    options.fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, "");
-    options.resourceCollector = resourceCollector_.get();
-
     // send heartbeat ok
     brpc::Server server;
     MockHeartbeatService mockHeartbeatService;
@@ -159,7 +158,7 @@ TEST_F(HeartbeatTest, test_ok) {
                         Invoke(RpcService<MetaServerHeartbeatRequest,
                                           MetaServerHeartbeatResponse>)));
 
-    ASSERT_EQ(heartbeat.Init(options), 0);
+    ASSERT_EQ(heartbeat.Init(hbopts_), 0);
     ASSERT_EQ(heartbeat.Run(), 0);
     sleep(5);
     ASSERT_EQ(heartbeat.Fini(), 0);
@@ -169,20 +168,8 @@ TEST_F(HeartbeatTest, test_ok) {
 }
 
 TEST_F(HeartbeatTest, test_fail) {
-    HeartbeatOptions options;
     Heartbeat heartbeat;
 
-    options.metaserverId = 1;
-    options.metaserverToken = "token";
-    options.intervalSec = 2;
-    options.timeout = 1000;
-    options.ip = "127.0.0.1";
-    options.port = 6000;
-    options.mdsListenAddr = "127.0.0.1:6710";
-    options.copysetNodeManager = &CopysetNodeManager::GetInstance();
-    options.storeUri = "local://./metaserver_data/copysets";
-    options.fs = LocalFsFactory::CreateFs(FileSystemType::EXT4, "");
-
     // send heartbeat ok
     brpc::Server server;
     MockHeartbeatService mockHeartbeatService;
@@ -198,7 +185,7 @@ TEST_F(HeartbeatTest, test_fail) {
                         Invoke(RpcService<MetaServerHeartbeatRequest,
                                           MetaServerHeartbeatResponse>)));
 
-    ASSERT_EQ(heartbeat.Init(options), 0);
+    ASSERT_EQ(heartbeat.Init(hbopts_), 0);
     ASSERT_EQ(heartbeat.Run(), 0);
     sleep(5);
     ASSERT_EQ(heartbeat.Fini(), 0);
@@ -229,5 +216,66 @@ TEST_F(HeartbeatTest, GetMetaServerSpaceStatusTest) {
               status.memoryusedbyte());
 }
 
+TEST_F(HeartbeatTest, Test_BuildRequest) {
+    Heartbeat heartbeat;
+    ASSERT_EQ(heartbeat.Init(hbopts_), 0);
+
+    MockCopysetNode mockCopysetNode;
+    std::vector<CopysetNode *> copysetVec;
+    copysetVec.emplace_back(&mockCopysetNode);
+
+    BlockGroupStatInfoMap blockGroupStatInfoMap;
+    uint32_t fsId = 1;
+    auto &statInfo = blockGroupStatInfoMap[fsId];
+    statInfo.set_fsid(1);
+    auto de = statInfo.add_deallocatableblockgroups();
+    de->set_blockgroupoffset(0);
+    de->set_deallocatablesize(1024);
+
+    EXPECT_CALL(mockCopysetManager_, GetAllCopysets(_))
+        .WillOnce(SetArgPointee<0>(copysetVec));
+    EXPECT_CALL(mockCopysetNode, GetPoolId()).WillOnce(Return(1));
+    EXPECT_CALL(mockCopysetNode, GetCopysetId()).WillOnce(Return(1));
+    EXPECT_CALL(mockCopysetNode, ListPeers(_)).Times(1);
+    EXPECT_CALL(mockCopysetNode, GetLeaderId()).Times(1);
+    EXPECT_CALL(mockCopysetNode, IsLoading()).WillRepeatedly(Return(false));
+    EXPECT_CALL(mockCopysetNode, GetPartitionInfoList(_))
+        .WillOnce(Return(true));
+    EXPECT_CALL(mockCopysetNode, GetConfChange(_, _)).Times(1);
+    EXPECT_CALL(mockCopysetNode, GetConfEpoch()).WillOnce(Return(1));
+    EXPECT_CALL(mockCopysetNode, IsLeaderTerm()).WillOnce(Return(true));
+    EXPECT_CALL(mockCopysetNode, GetBlockStatInfo(_))
+        .WillOnce(DoAll(SetArgPointee<0>(blockGroupStatInfoMap), Return(true)));
+
+    HeartbeatRequest req;
+    heartbeat.taskExecutor_->SetDeallocFsId(1);
+    heartbeat.BuildRequest(&req);
+
+    // assert block group stat info
+    {
+        // assert deallocatableBlockGroups
+        auto outStatInfos = req.blockgroupstatinfos();
+        ASSERT_EQ(outStatInfos.size(), 1);
+
+        auto outOne = outStatInfos[0];
+        ASSERT_EQ(outOne.fsid(), fsId);
+
+        auto outDes = outOne.deallocatableblockgroups();
+        ASSERT_EQ(outDes.size(), 1);
+
+        auto outDesOne = outDes[0];
+        ASSERT_EQ(outDesOne.blockgroupoffset(), de->blockgroupoffset());
+        ASSERT_EQ(outDesOne.deallocatablesize(), de->deallocatablesize());
+        ASSERT_TRUE(outDesOne.inodeidlist().empty());
+        ASSERT_TRUE(outDesOne.inodeidunderdeallocate().empty());
+
+        // assert blockGroupDeallocateStatus
+        auto outStatus = outOne.blockgroupdeallocatestatus();
+        ASSERT_EQ(outStatus.size(), 1);
+
+        auto outStatusOne = outStatus[fsId];
+        ASSERT_EQ(outStatusOne, BlockGroupDeallcateStatusCode::BGDP_DONE);
+    }
+}
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/test/metaserver/inode_storage_test.cpp

@@ -31,6 +31,7 @@
 #include <random>
 #include <string>
 #include "curvefs/proto/metaserver.pb.h"
+#include "curvefs/proto/common.pb.h"
 #include "curvefs/src/metaserver/inode_storage.h"
 #include "curvefs/src/common/define.h"
 
@@ -44,20 +45,22 @@
 #include "curvefs/test/metaserver/mock/mock_kv_storage.h"
 #include "src/fs/ext4_filesystem_impl.h"
 
-using ::testing::AtLeast;
-using ::testing::StrEq;
 using ::testing::_;
+using ::testing::AtLeast;
+using ::testing::DoAll;
 using ::testing::Return;
 using ::testing::ReturnArg;
-using ::testing::DoAll;
-using ::testing::SetArgPointee;
 using ::testing::SaveArg;
+using ::testing::SetArgPointee;
+using ::testing::StrEq;
 
-using ::curvefs::metaserver::storage::KVStorage;
-using ::curvefs::metaserver::storage::StorageOptions;
-using ::curvefs::metaserver::storage::RocksDBStorage;
+using ::curvefs::common::EmptyMsg;
+using ::curvefs::metaserver::storage::Key4DeallocatableBlockGroup;
 using ::curvefs::metaserver::storage::Key4S3ChunkInfoList;
+using ::curvefs::metaserver::storage::KVStorage;
 using ::curvefs::metaserver::storage::RandomStoragePath;
+using ::curvefs::metaserver::storage::RocksDBStorage;
+using ::curvefs::metaserver::storage::StorageOptions;
 
 namespace curvefs {
 namespace metaserver {
@@ -85,7 +88,7 @@ class InodeStorageTest : public ::testing::Test {
         ASSERT_EQ(output.size(), 0);
     }
 
-    std::string execShell(const std::string& cmd) {
+    std::string execShell(const std::string &cmd) {
         std::array<char, 128> buffer;
         std::string result;
         std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"),
@@ -128,7 +131,7 @@ class InodeStorageTest : public ::testing::Test {
                                        uint64_t lastChunkId) {
         S3ChunkInfoList list;
         for (uint64_t id = firstChunkId; id <= lastChunkId; id++) {
-            S3ChunkInfo* info = list.add_s3chunks();
+            S3ChunkInfo *info = list.add_s3chunks();
             info->set_chunkid(id);
             info->set_compaction(0);
             info->set_offset(0);
@@ -139,17 +142,15 @@ class InodeStorageTest : public ::testing::Test {
         return list;
     }
 
-    bool EqualS3ChunkInfo(const S3ChunkInfo& lhs, const S3ChunkInfo& rhs) {
+    bool EqualS3ChunkInfo(const S3ChunkInfo &lhs, const S3ChunkInfo &rhs) {
         return lhs.chunkid() == rhs.chunkid() &&
-            lhs.compaction() == rhs.compaction() &&
-            lhs.offset() == rhs.offset() &&
-            lhs.len() == rhs.len() &&
-            lhs.size() == rhs.size() &&
-            lhs.zero() == rhs.zero();
+               lhs.compaction() == rhs.compaction() &&
+               lhs.offset() == rhs.offset() && lhs.len() == rhs.len() &&
+               lhs.size() == rhs.size() && lhs.zero() == rhs.zero();
     }
 
-    bool EqualS3ChunkInfoList(const S3ChunkInfoList& lhs,
-                              const S3ChunkInfoList& rhs) {
+    bool EqualS3ChunkInfoList(const S3ChunkInfoList &lhs,
+                              const S3ChunkInfoList &rhs) {
         size_t size = lhs.s3chunks_size();
         if (size != rhs.s3chunks_size()) {
             return false;
@@ -163,8 +164,8 @@ class InodeStorageTest : public ::testing::Test {
         return true;
     }
 
-    void CHECK_INODE_S3CHUNKINFOLIST(InodeStorage* storage,
-                                     uint32_t fsId, uint64_t inodeId,
+    void CHECK_INODE_S3CHUNKINFOLIST(InodeStorage *storage, uint32_t fsId,
+                                     uint64_t inodeId,
                                      const std::vector<uint64_t> chunkIndexs,
                                      const std::vector<S3ChunkInfoList> lists) {
         ASSERT_EQ(chunkIndexs.size(), lists.size());
@@ -219,8 +220,7 @@ TEST_F(InodeStorageTest, test1) {
     // delete
     ASSERT_EQ(storage.Delete(Key4Inode(inode1)), MetaStatusCode::OK);
     ASSERT_EQ(storage.Size(), 2);
-    ASSERT_EQ(storage.Get(Key4Inode(inode1), &temp),
-        MetaStatusCode::NOT_FOUND);
+    ASSERT_EQ(storage.Get(Key4Inode(inode1), &temp), MetaStatusCode::NOT_FOUND);
     ASSERT_EQ(storage.Delete(Key4Inode(inode1)), MetaStatusCode::OK);
 
     // update
@@ -261,7 +261,7 @@ TEST_F(InodeStorageTest, testGetAttrNotFound) {
     ASSERT_EQ(storage.Insert(inode), MetaStatusCode::OK);
     InodeAttr attr;
     ASSERT_EQ(storage.GetAttr(Key4Inode(1, 2), &attr),
-        MetaStatusCode::NOT_FOUND);
+              MetaStatusCode::NOT_FOUND);
 }
 
 TEST_F(InodeStorageTest, testGetAttr) {
@@ -359,8 +359,8 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
     {
         LOG(INFO) << "CASE 2: append one s3chunkinfo";
         ASSERT_EQ(storage.Clear(), MetaStatusCode::OK);
-        std::vector<uint64_t> chunkIndexs{ 1 };
-        std::vector<S3ChunkInfoList> lists2add{ GenS3ChunkInfoList(1, 1) };
+        std::vector<uint64_t> chunkIndexs{1};
+        std::vector<S3ChunkInfoList> lists2add{GenS3ChunkInfoList(1, 1)};
 
         for (size_t i = 0; i < chunkIndexs.size(); i++) {
             MetaStatusCode rc = storage.ModifyInodeS3ChunkInfoList(
@@ -368,15 +368,15 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
             ASSERT_EQ(rc, MetaStatusCode::OK);
         }
 
-        CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId,
-                                    chunkIndexs, lists2add);
+        CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, chunkIndexs,
+                                    lists2add);
     }
 
     // CASE 3: append multi s3chunkinfos
     {
         LOG(INFO) << "CASE 3: append multi s3chunkinfos";
         ASSERT_EQ(storage.Clear(), MetaStatusCode::OK);
-        std::vector<uint64_t> chunkIndexs{ 1, 2, 3 };
+        std::vector<uint64_t> chunkIndexs{1, 2, 3};
         std::vector<S3ChunkInfoList> lists2add{
             GenS3ChunkInfoList(1, 1),
             GenS3ChunkInfoList(2, 2),
@@ -389,15 +389,15 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
             ASSERT_EQ(rc, MetaStatusCode::OK);
         }
 
-        CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId,
-                                    chunkIndexs, lists2add);
+        CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId, chunkIndexs,
+                                    lists2add);
     }
 
     // CASE 4: check order for s3chunkinfo's chunk index
     {
         LOG(INFO) << "CASE 4: check order for s3chunkinfo's chunk index";
         ASSERT_EQ(storage.Clear(), MetaStatusCode::OK);
-        std::vector<uint64_t> chunkIndexs{ 2, 1, 3 };
+        std::vector<uint64_t> chunkIndexs{2, 1, 3};
         std::vector<S3ChunkInfoList> lists2add{
             GenS3ChunkInfoList(2, 2),
             GenS3ChunkInfoList(1, 1),
@@ -411,24 +411,22 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
         }
 
         CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId,
-            std::vector<uint64_t>{ 1, 2, 3 },
-            std::vector<S3ChunkInfoList>{
-                GenS3ChunkInfoList(1, 1),
-                GenS3ChunkInfoList(2, 2),
-                GenS3ChunkInfoList(3, 3),
-            });
+                                    std::vector<uint64_t>{1, 2, 3},
+                                    std::vector<S3ChunkInfoList>{
+                                        GenS3ChunkInfoList(1, 1),
+                                        GenS3ChunkInfoList(2, 2),
+                                        GenS3ChunkInfoList(3, 3),
+                                    });
     }
 
     // CASE 5: check order for s3chunkinfo's chunk id
     {
         LOG(INFO) << "CASE 5: check order for s3chunkinfo's chunk id";
         ASSERT_EQ(storage.Clear(), MetaStatusCode::OK);
-        std::vector<uint64_t> chunkIndexs{ 2, 1, 3, 1, 2 };
+        std::vector<uint64_t> chunkIndexs{2, 1, 3, 1, 2};
         std::vector<S3ChunkInfoList> lists2add{
-            GenS3ChunkInfoList(200, 210),
-            GenS3ChunkInfoList(120, 130),
-            GenS3ChunkInfoList(300, 310),
-            GenS3ChunkInfoList(100, 110),
+            GenS3ChunkInfoList(200, 210), GenS3ChunkInfoList(120, 130),
+            GenS3ChunkInfoList(300, 310), GenS3ChunkInfoList(100, 110),
             GenS3ChunkInfoList(220, 230),
         };
 
@@ -439,14 +437,14 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
         }
 
         CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId,
-            std::vector<uint64_t>{ 1, 1, 2, 2, 3 },
-            std::vector<S3ChunkInfoList>{
-                GenS3ChunkInfoList(100, 110),
-                GenS3ChunkInfoList(120, 130),
-                GenS3ChunkInfoList(200, 210),
-                GenS3ChunkInfoList(220, 230),
-                GenS3ChunkInfoList(300, 310),
-            });
+                                    std::vector<uint64_t>{1, 1, 2, 2, 3},
+                                    std::vector<S3ChunkInfoList>{
+                                        GenS3ChunkInfoList(100, 110),
+                                        GenS3ChunkInfoList(120, 130),
+                                        GenS3ChunkInfoList(200, 210),
+                                        GenS3ChunkInfoList(220, 230),
+                                        GenS3ChunkInfoList(300, 310),
+                                    });
     }
 
 
@@ -457,7 +455,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
         // step1: add s3chunkinfo
         {
-            std::vector<uint64_t> chunkIndexs{ 1, 2, 3 };
+            std::vector<uint64_t> chunkIndexs{1, 2, 3};
             std::vector<S3ChunkInfoList> lists2add{
                 GenS3ChunkInfoList(100, 199),
                 GenS3ChunkInfoList(200, 299),
@@ -473,7 +471,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
         // step2: compaction
         {
-            std::vector<uint64_t> chunkIndexs{ 1, 2, 3 };
+            std::vector<uint64_t> chunkIndexs{1, 2, 3};
             std::vector<S3ChunkInfoList> lists2add{
                 GenS3ChunkInfoList(199, 199),
                 GenS3ChunkInfoList(299, 299),
@@ -488,8 +486,8 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
             for (size_t i = 0; i < chunkIndexs.size(); i++) {
                 MetaStatusCode rc = storage.ModifyInodeS3ChunkInfoList(
-                    fsId, inodeId, chunkIndexs[i],
-                    &lists2add[i], &lists2del[i]);
+                    fsId, inodeId, chunkIndexs[i], &lists2add[i],
+                    &lists2del[i]);
                 ASSERT_EQ(rc, MetaStatusCode::STORAGE_INTERNAL_ERROR);
             }
         }
@@ -502,7 +500,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
         // step1: add s3chunkinfo
         {
-            std::vector<uint64_t> chunkIndexs{ 1, 2, 3 };
+            std::vector<uint64_t> chunkIndexs{1, 2, 3};
             std::vector<S3ChunkInfoList> lists2add{
                 GenS3ChunkInfoList(100, 199),
                 GenS3ChunkInfoList(200, 299),
@@ -518,7 +516,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
         // step2: compaction
         {
-            std::vector<uint64_t> chunkIndexs{ 1, 2, 3 };
+            std::vector<uint64_t> chunkIndexs{1, 2, 3};
             std::vector<S3ChunkInfoList> lists2add{
                 GenS3ChunkInfoList(199, 199),
                 GenS3ChunkInfoList(299, 299),
@@ -533,20 +531,20 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
             for (size_t i = 0; i < chunkIndexs.size(); i++) {
                 MetaStatusCode rc = storage.ModifyInodeS3ChunkInfoList(
-                    fsId, inodeId, chunkIndexs[i],
-                    &lists2add[i], &lists2del[i]);
+                    fsId, inodeId, chunkIndexs[i], &lists2add[i],
+                    &lists2del[i]);
                 ASSERT_EQ(rc, MetaStatusCode::OK);
             }
         }
 
         // step3: check result
         CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId,
-            std::vector<uint64_t>{ 1, 2, 3 },
-            std::vector<S3ChunkInfoList>{
-                GenS3ChunkInfoList(199, 199),
-                GenS3ChunkInfoList(299, 299),
-                GenS3ChunkInfoList(399, 399),
-        });
+                                    std::vector<uint64_t>{1, 2, 3},
+                                    std::vector<S3ChunkInfoList>{
+                                        GenS3ChunkInfoList(199, 199),
+                                        GenS3ChunkInfoList(299, 299),
+                                        GenS3ChunkInfoList(399, 399),
+                                    });
     }
 
 
@@ -557,7 +555,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
         // step1: add s3chunkinfo
         {
-            std::vector<uint64_t> chunkIndexs{ 1, 2, 3 };
+            std::vector<uint64_t> chunkIndexs{1, 2, 3};
             std::vector<S3ChunkInfoList> lists2add{
                 GenS3ChunkInfoList(100, 199),
                 GenS3ChunkInfoList(1000, 1999),
@@ -573,7 +571,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
         // step2: add s3chunkinfo again
         {
-            std::vector<uint64_t> chunkIndexs{ 1, 2, 3 };
+            std::vector<uint64_t> chunkIndexs{1, 2, 3};
             std::vector<S3ChunkInfoList> lists2add{
                 GenS3ChunkInfoList(200, 299),
                 GenS3ChunkInfoList(2000, 2999),
@@ -589,7 +587,7 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
         // step2: compaction
         {
-            std::vector<uint64_t> chunkIndexs{ 1, 2, 3 };
+            std::vector<uint64_t> chunkIndexs{1, 2, 3};
             std::vector<S3ChunkInfoList> lists2add{
                 GenS3ChunkInfoList(199, 199),
                 GenS3ChunkInfoList(2999, 2999),
@@ -604,22 +602,22 @@ TEST_F(InodeStorageTest, ModifyInodeS3ChunkInfoList) {
 
             for (size_t i = 0; i < chunkIndexs.size(); i++) {
                 MetaStatusCode rc = storage.ModifyInodeS3ChunkInfoList(
-                    fsId, inodeId, chunkIndexs[i],
-                    &lists2add[i], &lists2del[i]);
+                    fsId, inodeId, chunkIndexs[i], &lists2add[i],
+                    &lists2del[i]);
                 ASSERT_EQ(rc, MetaStatusCode::OK);
             }
         }
 
         // step3: check result
         CHECK_INODE_S3CHUNKINFOLIST(&storage, fsId, inodeId,
-            std::vector<uint64_t>{ 1, 1, 2, 3, 3 },
-            std::vector<S3ChunkInfoList>{
-                GenS3ChunkInfoList(199, 199),
-                GenS3ChunkInfoList(200, 299),
-                GenS3ChunkInfoList(2999, 2999),
-                GenS3ChunkInfoList(19999, 19999),
-                GenS3ChunkInfoList(20000, 29999),
-        });
+                                    std::vector<uint64_t>{1, 1, 2, 3, 3},
+                                    std::vector<S3ChunkInfoList>{
+                                        GenS3ChunkInfoList(199, 199),
+                                        GenS3ChunkInfoList(200, 299),
+                                        GenS3ChunkInfoList(2999, 2999),
+                                        GenS3ChunkInfoList(19999, 19999),
+                                        GenS3ChunkInfoList(20000, 29999),
+                                    });
     }
 }
 
@@ -634,12 +632,10 @@ TEST_F(InodeStorageTest, PaddingInodeS3ChunkInfo) {
     ASSERT_EQ(storage.Insert(inode), MetaStatusCode::OK);
 
     // step2: append s3chunkinfo
-    std::vector<uint64_t> chunkIndexs{ 1, 3, 2, 1, 2 };
+    std::vector<uint64_t> chunkIndexs{1, 3, 2, 1, 2};
     std::vector<S3ChunkInfoList> lists2add{
-        GenS3ChunkInfoList(100, 109),
-        GenS3ChunkInfoList(300, 310),
-        GenS3ChunkInfoList(200, 209),
-        GenS3ChunkInfoList(110, 120),
+        GenS3ChunkInfoList(100, 109), GenS3ChunkInfoList(300, 310),
+        GenS3ChunkInfoList(200, 209), GenS3ChunkInfoList(110, 120),
         GenS3ChunkInfoList(210, 220),
     };
 
@@ -715,8 +711,8 @@ TEST_F(InodeStorageTest, GetAllS3ChunkInfoList) {
     size_t size = 0;
     Key4S3ChunkInfoList key;
     S3ChunkInfoList list4get;
-    std::vector<uint32_t> fsIds{ 1, 2 };
-    std::vector<uint64_t> inodeIds{ 1, 2 };
+    std::vector<uint32_t> fsIds{1, 2};
+    std::vector<uint64_t> inodeIds{1, 2};
     auto iterator = storage.GetAllS3ChunkInfoList();
     for (iterator->SeekToFirst(); iterator->Valid(); iterator->Next()) {
         ASSERT_TRUE(conv_->ParseFromString(iterator->Key(), &key));
@@ -734,11 +730,10 @@ TEST_F(InodeStorageTest, TestUpdateVolumeExtentSlice) {
     using storage::Status;
 
     const std::vector<std::pair<MetaStatusCode, Status>> cases{
-        {                    MetaStatusCode::OK,            Status::OK()},
-        {MetaStatusCode::STORAGE_INTERNAL_ERROR, Status::InternalError()}
-    };
+        {MetaStatusCode::OK, Status::OK()},
+        {MetaStatusCode::STORAGE_INTERNAL_ERROR, Status::InternalError()}};
 
-    for (const auto& test : cases) {
+    for (const auto &test : cases) {
         auto kvStorage = std::make_shared<storage::MockKVStorage>();
         InodeStorage storage(kvStorage, nameGenerator_, 0);
 
@@ -758,7 +753,7 @@ TEST_F(InodeStorageTest, TestUpdateVolumeExtentSlice) {
     }
 }
 
-static void RandomSetExtent(VolumeExtent* ext) {
+static void RandomSetExtent(VolumeExtent *ext) {
     std::random_device rd;
 
     ext->set_fsoffset(rd());
@@ -767,10 +762,9 @@ static void RandomSetExtent(VolumeExtent* ext) {
     ext->set_isused(rd() & 1);
 }
 
-static bool PrepareGetAllVolumeExtentTest(InodeStorage* storage,
-                                          uint32_t fsId,
+static bool PrepareGetAllVolumeExtentTest(InodeStorage *storage, uint32_t fsId,
                                           uint64_t inodeId,
-                                          std::vector<VolumeExtentSlice>* out) {
+                                          std::vector<VolumeExtentSlice> *out) {
     VolumeExtentSlice slice1;
     slice1.set_offset(0);
 
@@ -806,22 +800,22 @@ static bool PrepareGetAllVolumeExtentTest(InodeStorage* storage,
     return true;
 }
 
-static bool operator==(const VolumeExtentList& list,
-                       const std::vector<VolumeExtentSlice>& slices) {
+static bool operator==(const VolumeExtentSliceList &list,
+                       const std::vector<VolumeExtentSlice> &slices) {
     std::vector<VolumeExtentSlice> clist(list.slices().begin(),
                                          list.slices().end());
 
     auto copy = slices;
     std::sort(copy.begin(), copy.end(),
-              [](const VolumeExtentSlice& s1, const VolumeExtentSlice& s2) {
+              [](const VolumeExtentSlice &s1, const VolumeExtentSlice &s2) {
                   return s1.offset() < s2.offset();
               });
 
     return true;
 }
 
-static bool operator==(const VolumeExtentSlice& s1,
-                       const VolumeExtentSlice& s2) {
+static bool operator==(const VolumeExtentSlice &s1,
+                       const VolumeExtentSlice &s2) {
     return google::protobuf::util::MessageDifferencer::Equals(s1, s2);
 }
 
@@ -833,7 +827,7 @@ TEST_F(InodeStorageTest, TestGetAllVolumeExtent) {
         std::make_shared<storage::MemoryStorage>(opts);
     std::shared_ptr<KVStorage> kvStore = kvStorage_;
 
-    for (auto& store : {memStore, kvStore}) {
+    for (auto &store : {memStore, kvStore}) {
         InodeStorage storage(store, nameGenerator_, 0);
         const uint32_t fsId = 1;
         const uint64_t inodeId = 2;
@@ -842,20 +836,28 @@ TEST_F(InodeStorageTest, TestGetAllVolumeExtent) {
         ASSERT_TRUE(
             PrepareGetAllVolumeExtentTest(&storage, fsId, inodeId, &slices));
 
-        VolumeExtentList list;
+        VolumeExtentSliceList list;
         ASSERT_EQ(MetaStatusCode::OK,
                   storage.GetAllVolumeExtent(fsId, inodeId, &list));
         ASSERT_EQ(2, list.slices().size());
         ASSERT_EQ(list, slices);
+
+        auto iter = storage.GetAllVolumeExtent(fsId, inodeId);
+        iter->SeekToFirst();
+        ASSERT_TRUE(iter->Valid());
+        iter->Next();
+        ASSERT_TRUE(iter->Valid());
+        iter->Next();
+        ASSERT_FALSE(iter->Valid());
     }
 }
 
 static std::string ToString(storage::STORAGE_TYPE type) {
     switch (type) {
-        case storage::STORAGE_TYPE::MEMORY_STORAGE:
-            return "memory";
-        case storage::STORAGE_TYPE::ROCKSDB_STORAGE:
-            return "rocksdb";
+    case storage::STORAGE_TYPE::MEMORY_STORAGE:
+        return "memory";
+    case storage::STORAGE_TYPE::ROCKSDB_STORAGE:
+        return "rocksdb";
     }
 
     return "unknown";
@@ -869,7 +871,7 @@ TEST_F(InodeStorageTest, TestGetVolumeExtentByOffset) {
         std::make_shared<storage::MemoryStorage>(opts);
     std::shared_ptr<KVStorage> kvStore = kvStorage_;
 
-    for (auto& store : {kvStorage_, memStore}) {
+    for (auto &store : {kvStorage_, memStore}) {
         InodeStorage storage(store, nameGenerator_, 0);
         const uint32_t fsId = 1;
         const uint64_t inodeId = 2;
@@ -896,5 +898,104 @@ TEST_F(InodeStorageTest, TestGetVolumeExtentByOffset) {
     }
 }
 
+TEST_F(InodeStorageTest, Test_UpdateInodeWithDeallocate) {
+    auto inode = GenInode(1, 1);
+    InodeStorage storage(kvStorage_, nameGenerator_, 0);
+    ASSERT_EQ(storage.Insert(inode), MetaStatusCode::OK);
+
+    ASSERT_EQ(MetaStatusCode::OK, storage.Update(inode, true));
+
+    auto tableName = nameGenerator_->GetDeallocatableInodeTableName();
+    Key4Inode key(1, 1);
+    auto skey = key.SerializeToString();
+    EmptyMsg value;
+    auto s = kvStorage_->HGet(tableName, skey, &value);
+    ASSERT_TRUE(s.ok());
+}
+
+TEST_F(InodeStorageTest, Test_UpdateDeallocatableBlockGroup) {
+    uint64_t blockGroupOffset = 0;
+    uint32_t fsId = 1;
+    auto tableName = nameGenerator_->GetDeallocatableBlockGroupTableName();
+    InodeStorage storage(kvStorage_, nameGenerator_, 0);
+    uint64_t increaseSize = 100 * 1024;
+    uint64_t decreaseSize = 4 * 1024;
+    Key4DeallocatableBlockGroup key(fsId, blockGroupOffset);
+    std::vector<DeallocatableBlockGroup> deallocatableBlockGroupVec;
+    ASSERT_EQ(MetaStatusCode::NOT_FOUND,
+              storage.GeAllBlockGroup(&deallocatableBlockGroupVec));
+
+    // test increase
+    {
+        DeallocatableBlockGroupVec inputVec;
+
+        DeallocatableBlockGroup blockGroup;
+        auto increase = blockGroup.mutable_increase();
+        increase->set_increasedeallocatablesize(increaseSize);
+        increase->add_inodeidlistadd(1);
+        inputVec.Add()->CopyFrom(blockGroup);
+
+        // increase first time
+        ASSERT_EQ(MetaStatusCode::OK,
+                  storage.UpdateDeallocatableBlockGroup(fsId, inputVec));
+        DeallocatableBlockGroup out;
+        auto st = kvStorage_->HGet(tableName, key.SerializeToString(), &out);
+        ASSERT_TRUE(st.ok());
+        ASSERT_EQ(increaseSize, out.deallocatablesize());
+        ASSERT_EQ(1, out.inodeidlist_size());
+        ASSERT_EQ(blockGroupOffset, out.blockgroupoffset());
+        ASSERT_EQ(0, out.inodeidunderdeallocate_size());
+
+        // increase second time
+        ASSERT_EQ(MetaStatusCode::OK,
+                  storage.UpdateDeallocatableBlockGroup(fsId, inputVec));
+        st = kvStorage_->HGet(tableName, key.SerializeToString(), &out);
+        ASSERT_TRUE(st.ok());
+        ASSERT_EQ(increaseSize * 2, out.deallocatablesize());
+        ASSERT_EQ(1, out.inodeidlist_size());
+        ASSERT_EQ(blockGroupOffset, out.blockgroupoffset());
+        ASSERT_EQ(0, out.inodeidunderdeallocate_size());
+    }
+
+    // test mark
+    {
+        DeallocatableBlockGroupVec inputVec;
+
+        DeallocatableBlockGroup blockGroup;
+        auto mark = blockGroup.mutable_mark();
+        mark->add_inodeidunderdeallocate(1);
+        inputVec.Add()->CopyFrom(blockGroup);
+
+        ASSERT_EQ(MetaStatusCode::OK,
+                  storage.UpdateDeallocatableBlockGroup(fsId, inputVec));
+        DeallocatableBlockGroup out;
+        auto st = kvStorage_->HGet(tableName, key.SerializeToString(), &out);
+        ASSERT_TRUE(st.ok());
+        ASSERT_EQ(increaseSize * 2, out.deallocatablesize());
+        ASSERT_EQ(0, out.inodeidlist_size());
+        ASSERT_EQ(1, out.inodeidunderdeallocate_size());
+    }
+
+    // test decrease
+    {
+        DeallocatableBlockGroupVec inputVec;
+
+        DeallocatableBlockGroup blockGroup;
+        auto decrease = blockGroup.mutable_decrease();
+        decrease->set_decreasedeallocatablesize(decreaseSize);
+        inputVec.Add()->CopyFrom(blockGroup);
+        ASSERT_EQ(MetaStatusCode::OK,
+                  storage.UpdateDeallocatableBlockGroup(fsId, inputVec));
+        DeallocatableBlockGroup out;
+        auto st = kvStorage_->HGet(tableName, key.SerializeToString(), &out);
+        ASSERT_TRUE(st.ok());
+        ASSERT_EQ(increaseSize * 2 - decreaseSize, out.deallocatablesize());
+    }
+
+    ASSERT_EQ(MetaStatusCode::OK,
+              storage.GeAllBlockGroup(&deallocatableBlockGroupVec));
+    ASSERT_EQ(1, deallocatableBlockGroupVec.size());
+}
+
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/test/metaserver/metaserver_service_test2.cpp

@@ -92,6 +92,10 @@ TEST_F(MetaServerServiceTest2, ServiceOverload) {
     TEST_SERVICE_OVERLOAD(CreatePartition);
     TEST_SERVICE_OVERLOAD(DeletePartition);
     TEST_SERVICE_OVERLOAD(PrepareRenameTx);
+    TEST_SERVICE_OVERLOAD(GetVolumeExtent);
+    TEST_SERVICE_OVERLOAD(UpdateVolumeExtent);
+    TEST_SERVICE_OVERLOAD(UpdateDeallocatableBlockGroup);
+
 
 #undef TEST_SERVICE_OVERLOAD
 }
@@ -126,6 +130,9 @@ TEST_F(MetaServerServiceTest2, CopysetNodeNotFound) {
     TEST_COPYSETNODE_NOTFOUND(CreatePartition);
     TEST_COPYSETNODE_NOTFOUND(DeletePartition);
     TEST_COPYSETNODE_NOTFOUND(PrepareRenameTx);
+    TEST_COPYSETNODE_NOTFOUND(GetVolumeExtent);
+    TEST_COPYSETNODE_NOTFOUND(UpdateVolumeExtent);
+    TEST_COPYSETNODE_NOTFOUND(UpdateDeallocatableBlockGroup);
 
 #undef TEST_COPYSETNODE_NOTFOUND
 }

curvefs/test/metaserver/mock/mock_metastore.h

@@ -28,6 +28,7 @@
 #include <string>
 #include <list>
 #include <memory>
+#include <map>
 
 #include "curvefs/src/metaserver/metastore.h"
 
@@ -47,6 +48,8 @@ class MockMetaStore : public curvefs::metaserver::MetaStore {
     MOCK_METHOD2(DeletePartition, MetaStatusCode(const DeletePartitionRequest*,
                                                  DeletePartitionResponse*));
     MOCK_METHOD1(GetPartitionInfoList, bool(std::list<PartitionInfo> *));
+    MOCK_METHOD1(GetPartitionSnap,
+                 bool(std::map<uint32_t, std::shared_ptr<Partition>> *));
 
     MOCK_METHOD2(CreateDentry, MetaStatusCode(const CreateDentryRequest*,
                                               CreateDentryResponse*));
@@ -98,6 +101,10 @@ class MockMetaStore : public curvefs::metaserver::MetaStore {
     MOCK_METHOD2(UpdateVolumeExtent,
                  MetaStatusCode(const UpdateVolumeExtentRequest*,
                                 UpdateVolumeExtentResponse*));
+
+    MOCK_METHOD2(UpdateDeallocatableBlockGroup,
+                 MetaStatusCode(const UpdateDeallocatableBlockGroupRequest *,
+                                UpdateDeallocatableBlockGroupResponse *));
 };
 
 }  // namespace mock

curvefs/test/metaserver/mock/mock_partition.h

@@ -0,0 +1,45 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Mon Apr 24 20:04:24 CST 2023
+ * Author: lixiaocui
+ */
+
+#ifndef CURVEFS_TEST_METASERVER_MOCK_MOCK_PARTITION_H_
+#define CURVEFS_TEST_METASERVER_MOCK_MOCK_PARTITION_H_
+
+#include <gmock/gmock.h>
+#include <vector>
+#include "curvefs/src/metaserver/partition.h"
+
+namespace curvefs {
+namespace metaserver {
+namespace mock {
+class MockPartition : public curvefs::metaserver::Partition {
+ public:
+    MockPartition() : Partition() {}
+    MOCK_METHOD1(GeAllBlockGroup,
+                 MetaStatusCode(std::vector<DeallocatableBlockGroup> *));
+    MOCK_CONST_METHOD0(GetPartitionId, uint32_t());
+    MOCK_CONST_METHOD0(GetFsId, uint32_t());
+};
+}  // namespace mock
+}  // namespace metaserver
+}  // namespace curvefs
+
+#endif  // CURVEFS_TEST_METASERVER_MOCK_MOCK_PARTITION_H_

curvefs/test/metaserver/partition_clean_test.cpp

@@ -30,19 +30,20 @@
 #include "curvefs/src/metaserver/storage/storage.h"
 #include "curvefs/src/metaserver/storage/rocksdb_storage.h"
 #include "curvefs/test/metaserver/storage/utils.h"
+#include "curvefs/src/metaserver/mds/fsinfo_manager.h"
 #include "src/fs/ext4_filesystem_impl.h"
 
-using ::curvefs::mds::FSStatusCode;
 using ::curvefs::client::rpcclient::MockMdsClient;
+using ::curvefs::mds::FSStatusCode;
 using ::testing::_;
 using ::testing::Invoke;
 using ::testing::Matcher;
 using ::testing::Return;
 
 using ::curvefs::metaserver::storage::KVStorage;
-using ::curvefs::metaserver::storage::StorageOptions;
-using ::curvefs::metaserver::storage::RocksDBStorage;
 using ::curvefs::metaserver::storage::RandomStoragePath;
+using ::curvefs::metaserver::storage::RocksDBStorage;
+using ::curvefs::metaserver::storage::StorageOptions;
 
 namespace curvefs {
 namespace metaserver {
@@ -60,6 +61,9 @@ class PartitionCleanManagerTest : public testing::Test {
         options.localFileSystem = localfs.get();
         kvStorage_ = std::make_shared<RocksDBStorage>(options);
         ASSERT_TRUE(kvStorage_->Open());
+
+        mdsCli_ = std::make_shared<MockMdsClient>();
+        FsInfoManager::GetInstance().SetMdsClient(mdsCli_);
     }
 
     void TearDown() override {
@@ -68,7 +72,7 @@ class PartitionCleanManagerTest : public testing::Test {
         ASSERT_EQ(output.size(), 0);
     }
 
-    std::string execShell(const string& cmd) {
+    std::string execShell(const string &cmd) {
         std::array<char, 128> buffer;
         std::string result;
         std::unique_ptr<FILE, decltype(&pclose)> pipe(popen(cmd.c_str(), "r"),
@@ -85,21 +89,20 @@ class PartitionCleanManagerTest : public testing::Test {
  protected:
     std::string dataDir_;
     std::shared_ptr<KVStorage> kvStorage_;
+    std::shared_ptr<MockMdsClient> mdsCli_;
 };
 
 TEST_F(PartitionCleanManagerTest, test1) {
     ASSERT_TRUE(true);
-    PartitionCleanManager* manager = &PartitionCleanManager::GetInstance();
+    PartitionCleanManager *manager = &PartitionCleanManager::GetInstance();
 
     PartitionCleanOption option;
     option.scanPeriodSec = 1;
     option.inodeDeletePeriodMs = 500;
     std::shared_ptr<MockS3ClientAdaptor> s3Adaptor =
-                            std::make_shared<MockS3ClientAdaptor>();
+        std::make_shared<MockS3ClientAdaptor>();
     option.s3Adaptor = s3Adaptor;
-    std::shared_ptr<MockMdsClient> mdsclient =
-        std::make_shared<MockMdsClient>();
-    option.mdsClient = mdsclient;
+    option.mdsClient = mdsCli_;
     manager->Init(option);
     manager->Run();
     uint32_t partitionId = 1;
@@ -110,7 +113,7 @@ TEST_F(PartitionCleanManagerTest, test1) {
     partitionInfo.set_start(0);
     partitionInfo.set_end(2000);
     std::shared_ptr<Partition> partition =
-                std::make_shared<Partition>(partitionInfo, kvStorage_);
+        std::make_shared<Partition>(partitionInfo, kvStorage_);
     Dentry dentry;
     dentry.set_fsid(fsId);
     dentry.set_parentinodeid(1);
@@ -140,10 +143,10 @@ TEST_F(PartitionCleanManagerTest, test1) {
     ASSERT_EQ(partition->GetDentryNum(), 1);
     Inode inode2;
     ASSERT_EQ(partition->GetInode(fsId, ROOTINODEID, &inode2),
-                MetaStatusCode::OK);
+              MetaStatusCode::OK);
 
     std::shared_ptr<PartitionCleaner> partitionCleaner =
-                std::make_shared<PartitionCleaner>(partition);
+        std::make_shared<PartitionCleaner>(partition);
 
     copyset::MockCopysetNode copysetNode;
 
@@ -152,28 +155,27 @@ TEST_F(PartitionCleanManagerTest, test1) {
         .WillRepeatedly(Return(true));
 
     EXPECT_CALL(copysetNode, Propose(_))
-        .WillOnce(Invoke([partition, fsId](const braft::Task& task) {
+        .WillOnce(Invoke([partition, fsId](const braft::Task &task) {
             ASSERT_EQ(partition->DeleteInode(fsId, ROOTINODEID),
                       MetaStatusCode::OK);
             LOG(INFO) << "Partition DeleteInode, fsId = " << fsId
                       << ", inodeId = " << ROOTINODEID;
             task.done->Run();
         }))
-        .WillOnce(Invoke([partition, fsId, inode1](const braft::Task& task) {
+        .WillOnce(Invoke([partition, fsId, inode1](const braft::Task &task) {
             ASSERT_EQ(partition->DeleteInode(fsId, inode1.inodeid()),
                       MetaStatusCode::OK);
             LOG(INFO) << "Partition DeleteInode, fsId = " << fsId
                       << ", inodeId = " << inode1.inodeid();
             task.done->Run();
         }))
-        .WillOnce(Invoke([partition](const braft::Task& task) {
+        .WillOnce(Invoke([partition](const braft::Task &task) {
             LOG(INFO) << "Partition deletePartition";
             task.done->Run();
         }));
 
-    EXPECT_CALL(*s3Adaptor, Delete(_))
-        .WillOnce(Return(0));
-    EXPECT_CALL(*mdsclient, GetFsInfo(Matcher<uint32_t>(_), _))
+    EXPECT_CALL(*s3Adaptor, Delete(_)).WillOnce(Return(0));
+    EXPECT_CALL(*mdsCli_, GetFsInfo(Matcher<uint32_t>(_), _))
         .WillOnce(Return(FSStatusCode::OK));
     EXPECT_CALL(*s3Adaptor, GetS3ClientAdaptorOption(_));
     EXPECT_CALL(*s3Adaptor, Reinit(_, _, _, _, _));
@@ -187,14 +189,12 @@ TEST_F(PartitionCleanManagerTest, test1) {
 
 TEST_F(PartitionCleanManagerTest, fsinfo_not_found) {
     PartitionInfo partition;
-    PartitionCleaner cleaner(std::make_shared<Partition>(
-        partition, kvStorage_));
-    std::shared_ptr<MockMdsClient> mdsClient =
-        std::make_shared<MockMdsClient>();
-    cleaner.SetMdsClient(mdsClient);
+    PartitionCleaner cleaner(
+        std::make_shared<Partition>(partition, kvStorage_));
+    cleaner.SetMdsClient(mdsCli_);
     Inode inode;
     inode.set_type(FsFileType::TYPE_S3);
-    EXPECT_CALL(*mdsClient, GetFsInfo(Matcher<uint32_t>(_), _))
+    EXPECT_CALL(*mdsCli_, GetFsInfo(Matcher<uint32_t>(_), _))
         .WillOnce(Return(FSStatusCode::NOT_FOUND));
     ASSERT_EQ(cleaner.CleanDataAndDeleteInode(inode),
               MetaStatusCode::S3_DELETE_ERR);
@@ -202,14 +202,12 @@ TEST_F(PartitionCleanManagerTest, fsinfo_not_found) {
 
 TEST_F(PartitionCleanManagerTest, fsinfo_other_error) {
     PartitionInfo partition;
-    PartitionCleaner cleaner(std::make_shared<Partition>(
-        partition, kvStorage_));
-    std::shared_ptr<MockMdsClient> mdsClient =
-        std::make_shared<MockMdsClient>();
-    cleaner.SetMdsClient(mdsClient);
+    PartitionCleaner cleaner(
+        std::make_shared<Partition>(partition, kvStorage_));
+    cleaner.SetMdsClient(mdsCli_);
     Inode inode;
     inode.set_type(FsFileType::TYPE_S3);
-    EXPECT_CALL(*mdsClient, GetFsInfo(Matcher<uint32_t>(_), _))
+    EXPECT_CALL(*mdsCli_, GetFsInfo(Matcher<uint32_t>(_), _))
         .WillOnce(Return(FSStatusCode::UNKNOWN_ERROR));
     ASSERT_EQ(cleaner.CleanDataAndDeleteInode(inode),
               MetaStatusCode::S3_DELETE_ERR);

curvefs/test/metaserver/space/BUILD

@@ -0,0 +1,40 @@
+#
+#  Copyright (c) 2023 NetEase Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+load("//:copts.bzl", "CURVE_TEST_COPTS")
+
+cc_test(
+    name = "space_test",
+    srcs = glob(
+        [
+            "*.cpp",
+            "*.h",
+        ],
+    ),
+    copts = CURVE_TEST_COPTS,
+    defines = ["UNIT_TEST"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//curvefs/src/metaserver:curvefs_metaserver",
+        "//curvefs/test/metaserver/storage:metaserver_storage_test_utils",
+        "//curvefs/test/client:mock",
+        "//curvefs/test/metaserver/copyset/mock:metaserver_copyset_test_mock",
+        "//external:gflags",
+        "@com_google_googletest//:gtest",
+        "@com_google_googletest//:gtest_main",
+        "@com_google_googletest//:gtest_prod",
+    ],
+)

curvefs/test/metaserver/space/inode_volume_space_deallocate_test.cpp

@@ -0,0 +1,377 @@
+/*
+ *  Copyright (c) 2023 NetEase Inc.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+/*
+ * Project: curve
+ * Date: Tue Apr 18 11:47:54 CST 2023
+ * Author: lixiaocui
+ */
+
+#include <gtest/gtest.h>
+
+#include "curvefs/src/metaserver/space/inode_volume_space_deallocate.h"
+#include "curvefs/src/metaserver/inode_storage.h"
+#include "curvefs/src/metaserver/storage/storage.h"
+#include "curvefs/src/metaserver/storage/rocksdb_storage.h"
+#include "curvefs/src/metaserver/storage/converter.h"
+#include "src/fs/ext4_filesystem_impl.h"
+#include "curvefs/test/metaserver/storage/utils.h"
+#include "curvefs/test/metaserver/space/mock_volume_space_manager.h"
+#include "curvefs/test/client/mock_metaserver_client.h"
+#include "curvefs/test/metaserver/space/utils.h"
+#include "curvefs/test/metaserver/copyset/mock/mock_copyset_node.h"
+#include "curvefs/src/client/rpcclient/task_excutor.h"
+
+namespace curvefs {
+namespace metaserver {
+
+using ::testing::_;
+using ::testing::Invoke;
+using ::testing::Return;
+
+using ::curvefs::client::rpcclient::MetaServerClientDone;
+
+namespace {
+auto localfs = curve::fs::Ext4FileSystemImpl::getInstance();
+}
+
+using curvefs::client::rpcclient::MockMetaServerClient;
+using curvefs::common::EmptyMsg;
+using curvefs::metaserver::NameGenerator;
+using curvefs::metaserver::copyset::MockCopysetNode;
+using curvefs::metaserver::storage::Key4VolumeExtentSlice;
+using curvefs::metaserver::storage::KVStorage;
+using curvefs::metaserver::storage::RandomStoragePath;
+using curvefs::metaserver::storage::RocksDBStorage;
+using curvefs::metaserver::storage::StorageOptions;
+
+
+class InodeVolumeSpaceDeallocateTest : public ::testing::Test {
+ protected:
+    void SetUp() override {
+        dataDir_ = RandomStoragePath();
+        StorageOptions options;
+        options.dataDir = dataDir_;
+        options.localFileSystem = localfs.get();
+        kvStorage_ = std::make_shared<RocksDBStorage>(options);
+        ASSERT_TRUE(kvStorage_->Open());
+
+        nameGen_ = std::make_shared<NameGenerator>(1);
+        inodeStorage_ = std::make_shared<InodeStorage>(kvStorage_, nameGen_, 0);
+
+        volumeSpaceManager_ = std::make_shared<MockVolumeSpaceManager>();
+        metaClient_ = std::make_shared<MockMetaServerClient>();
+
+        VolumeDeallocateCalOption calOpt;
+        calOpt.kvStorage = kvStorage_;
+        calOpt.inodeStorage = inodeStorage_;
+        calOpt.nameGen = nameGen_;
+
+        VolumeDeallocateExecuteOption executeOpt;
+        executeOpt.volumeSpaceManager = volumeSpaceManager_;
+        executeOpt.metaClient = metaClient_;
+        executeOpt.batchClean = 10;
+
+        EXPECT_CALL(*volumeSpaceManager_, GetBlockGroupSize(_))
+            .WillRepeatedly(Return(blockGroupSize_));
+        inodeVolumeSpaceDeallocate_ =
+            std::make_shared<InodeVolumeSpaceDeallocate>(fsId_, partitionId_,
+                                                         copysetNode_);
+        inodeVolumeSpaceDeallocate_->Init(calOpt);
+        inodeVolumeSpaceDeallocate_->Init(executeOpt);
+    }
+
+    void TearDown() override {
+        ASSERT_TRUE(kvStorage_->Close());
+        auto output = execShell("rm -rf " + dataDir_);
+        ASSERT_EQ(output.size(), 0);
+    }
+
+    void PrepareOneDeallocatableInode(Inode *inode, VolumeExtentSlice *slice) {
+        // inode
+        uint64_t len = 1024 * 1024 * 1024 + 8 * 1024;
+        inode->set_inodeid(1);
+        inode->set_fsid(fsId_);
+        inode->set_length(len);
+        inode->set_ctime(1681884587);
+        inode->set_ctime_ns(0);
+        inode->set_mtime(1681884587);
+        inode->set_mtime_ns(0);
+        inode->set_atime(1681884587);
+        inode->set_atime_ns(0);
+        inode->set_uid(100);
+        inode->set_gid(100);
+        inode->set_mode(777);
+        inode->set_nlink(0);
+        inode->set_type(FsFileType::TYPE_FILE);
+
+        // extent
+        uint64_t validBlockLen = blockGroupSize_ - 4 * 1024;
+        slice->set_offset(0);
+        uint16_t blockNum = len / validBlockLen;
+        for (int i = 0; i <= blockNum; i++) {
+            auto extent = slice->add_extents();
+            extent->set_fsoffset(validBlockLen * i);
+            extent->set_volumeoffset(blockGroupSize_ * i + 4 * 1024);
+            if (i == blockNum) {
+                extent->set_length(len - validBlockLen * (i - 1));
+                extent->set_isused(false);
+            } else {
+                extent->set_length(validBlockLen);
+                extent->set_isused(true);
+            }
+        }
+
+        // persist extent
+        Converter conv;
+        auto sliceKey = conv.SerializeToString(
+            Key4VolumeExtentSlice(fsId_, 1, slice->offset()));
+        auto st = kvStorage_->SSet(nameGen_->GetVolumeExtentTableName(),
+                                   sliceKey, *slice);
+        ASSERT_TRUE(st.ok());
+
+        // persist inode
+        auto inodeKey =
+            conv.SerializeToString(Key4Inode(inode->fsid(), inode->inodeid()));
+        st = kvStorage_->HSet(nameGen_->GetInodeTableName(), inodeKey, *inode);
+        ASSERT_TRUE(st.ok());
+
+        // persist deallocatable inode
+        st = kvStorage_->HSet(nameGen_->GetDeallocatableInodeTableName(),
+                              inodeKey, EmptyMsg());
+        ASSERT_TRUE(st.ok());
+    }
+
+    void PrepareDeallocatableGroup(Inode *inode, VolumeExtentSlice *slice,
+                                   DeallocatableBlockGroupMap *demap) {
+        PrepareOneDeallocatableInode(inode, slice);
+
+        for (auto extent : slice->extents()) {
+            DeallocatableBlockGroup tmp;
+            uint64_t blockgroupOffset =
+                extent.volumeoffset() / blockGroupSize_ * blockGroupSize_;
+            tmp.set_blockgroupoffset(blockgroupOffset);
+            tmp.set_deallocatablesize(extent.length());
+            tmp.add_inodeidlist(inode->inodeid());
+            Key4DeallocatableBlockGroup key(fsId_, blockgroupOffset);
+
+            auto st = kvStorage_->HSet(
+                nameGen_->GetDeallocatableBlockGroupTableName(),
+                key.SerializeToString(), tmp);
+            ASSERT_TRUE(st.ok());
+
+            (*demap)[blockgroupOffset] = std::move(tmp);
+        }
+    }
+
+ protected:
+    std::string dataDir_;
+    std::shared_ptr<KVStorage> kvStorage_;
+    std::shared_ptr<InodeStorage> inodeStorage_;
+    std::shared_ptr<MockVolumeSpaceManager> volumeSpaceManager_;
+    std::shared_ptr<MockMetaServerClient> metaClient_;
+    std::shared_ptr<NameGenerator> nameGen_;
+    std::shared_ptr<MockCopysetNode> copysetNode_;
+
+    std::shared_ptr<InodeVolumeSpaceDeallocate> inodeVolumeSpaceDeallocate_;
+
+    uint64_t fsId_ = 1;
+    uint32_t partitionId_ = 1;
+    uint64_t blockGroupSize_ = 128 * 1024 * 1024;
+};
+
+TEST_F(InodeVolumeSpaceDeallocateTest, Test_Status) {
+    ASSERT_FALSE(inodeVolumeSpaceDeallocate_->IsCanceled());
+
+    inodeVolumeSpaceDeallocate_->SetCanceled();
+    ASSERT_TRUE(inodeVolumeSpaceDeallocate_->IsCanceled());
+
+    inodeVolumeSpaceDeallocate_->SetDeallocateTask(1024);
+    ASSERT_EQ(1024, inodeVolumeSpaceDeallocate_->GetDeallocateTask());
+
+    inodeVolumeSpaceDeallocate_->ResetDeallocateTask();
+    ASSERT_FALSE(inodeVolumeSpaceDeallocate_->HasDeallocateTask());
+}
+
+TEST_F(InodeVolumeSpaceDeallocateTest, Test_CalDeallocatableSpace) {
+    // test with empty kvstorage
+    inodeVolumeSpaceDeallocate_->CalDeallocatableSpace();
+
+    // test with one inode kvstorage
+    Inode inode;
+    VolumeExtentSlice slice;
+    PrepareOneDeallocatableInode(&inode, &slice);
+    EXPECT_CALL(*metaClient_, UpdateDeallocatableBlockGroup(_, _, _))
+        .WillOnce(Return(MetaStatusCode::OK));
+    inodeVolumeSpaceDeallocate_->CalDeallocatableSpace();
+}
+
+TEST_F(InodeVolumeSpaceDeallocateTest, Test_DeallocatableSapceForInode) {
+    Inode inode;
+    VolumeExtentSlice slice;
+    PrepareOneDeallocatableInode(&inode, &slice);
+    auto key = Key4Inode(inode.fsid(), inode.inodeid());
+
+    DeallocatableBlockGroupMap increaseMap;
+    inodeVolumeSpaceDeallocate_->DeallocatableSapceForInode(key, &increaseMap);
+
+    ASSERT_EQ(increaseMap.size(), slice.extents_size());
+    auto iter = slice.extents().begin();
+    for (auto &item : increaseMap) {
+        auto expectedOffset =
+            iter->volumeoffset() / blockGroupSize_ * blockGroupSize_;
+        ASSERT_EQ(item.first, expectedOffset);
+        ASSERT_EQ(item.second.blockgroupoffset(), expectedOffset);
+
+        auto &increase = item.second.increase();
+        ASSERT_EQ(increase.increasedeallocatablesize(), iter->length());
+        ASSERT_EQ(increase.inodeidlistadd()[0], inode.inodeid());
+        ASSERT_EQ(increase.inodeidlistadd().size(), 1);
+
+        iter++;
+    }
+}
+
+TEST_F(InodeVolumeSpaceDeallocateTest, Test_DeallocateOneBlockGroup) {
+    uint64_t blockGroupOffset = 0;
+
+    // no decallocatable block group
+    inodeVolumeSpaceDeallocate_->DeallocateOneBlockGroup(blockGroupOffset);
+
+    // one deallocatable block group
+    {
+        Inode inode;
+        VolumeExtentSlice slice;
+        DeallocatableBlockGroupMap demap;
+        PrepareDeallocatableGroup(&inode, &slice, &demap);
+
+        EXPECT_CALL(*metaClient_, UpdateDeallocatableBlockGroup(fsId_, _, _))
+            .Times(2)
+            .WillRepeatedly(Return(MetaStatusCode::OK));
+        EXPECT_CALL(*volumeSpaceManager_,
+                    DeallocVolumeSpace(fsId_, blockGroupOffset, _))
+            .WillOnce(Return(true));
+        EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(fsId_, _, _, _))
+            .WillOnce(
+                Invoke([](uint32_t, uint64_t, const VolumeExtentSliceList &,
+                          MetaServerClientDone *done) {
+                    done->SetMetaStatusCode(MetaStatusCode::OK);
+                    done->Run();
+                }));
+        inodeVolumeSpaceDeallocate_->DeallocateOneBlockGroup(blockGroupOffset);
+    }
+}
+
+TEST_F(InodeVolumeSpaceDeallocateTest, Test_ProcessSepcifyInodeList) {
+    Inode inode;
+    VolumeExtentSlice slice;
+    uint64_t blockGroupOffset = 0;
+    DeallocatableBlockGroupMap demap;
+    PrepareDeallocatableGroup(&inode, &slice, &demap);
+    demap[blockGroupOffset].mutable_mark()->add_inodeidunderdeallocate(
+        inode.inodeid());
+
+    EXPECT_CALL(*metaClient_, UpdateDeallocatableBlockGroup(fsId_, _, _))
+        .Times(2)
+        .WillRepeatedly(Return(MetaStatusCode::OK));
+    EXPECT_CALL(*volumeSpaceManager_,
+                DeallocVolumeSpace(fsId_, blockGroupOffset, _))
+        .WillOnce(Return(true));
+    EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(fsId_, _, _, _))
+        .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentSliceList &,
+                            MetaServerClientDone *done) {
+            done->SetMetaStatusCode(MetaStatusCode::OK);
+            done->Run();
+        }));
+
+    inodeVolumeSpaceDeallocate_->ProcessSepcifyInodeList(blockGroupOffset,
+                                                         &demap);
+    ASSERT_TRUE(
+        demap[blockGroupOffset].mark().inodeidunderdeallocate().empty());
+}
+
+TEST_F(InodeVolumeSpaceDeallocateTest, Test_DeallocateInode) {
+    Inode inode;
+    VolumeExtentSlice slice;
+    DeallocatableBlockGroupMap demap;
+    PrepareDeallocatableGroup(&inode, &slice, &demap);
+    uint64_t blockGroupOffset = 0;
+    uint64_t decrease = 0;
+    Uint64Vec inodelist;
+    inodelist.Add(inode.inodeid());
+
+    EXPECT_CALL(*volumeSpaceManager_,
+                DeallocVolumeSpace(fsId_, blockGroupOffset, _))
+        .WillOnce(Return(true));
+    EXPECT_CALL(*metaClient_, AsyncUpdateVolumeExtent(fsId_, _, _, _))
+        .WillOnce(Invoke([](uint32_t, uint64_t, const VolumeExtentSliceList &,
+                            MetaServerClientDone *done) {
+            done->SetMetaStatusCode(MetaStatusCode::OK);
+            done->Run();
+        }));
+    inodeVolumeSpaceDeallocate_->DeallocateInode(blockGroupOffset, inodelist,
+                                                 &decrease);
+    ASSERT_EQ(decrease, slice.extents(0).length());
+}
+
+TEST_F(InodeVolumeSpaceDeallocateTest, Test_UpdateDeallocateInodeExtentSlice) {
+    Inode inode;
+    VolumeExtentSlice slice;
+    PrepareOneDeallocatableInode(&inode, &slice);
+    uint64_t blockGroupOffset = 0;
+    uint64_t decrease = 0;
+    VolumeExtentSliceList slicelist;
+    std::vector<Extent> deallocatableVolumeSpace;
+
+    auto st =
+        inodeStorage_->GetAllVolumeExtent(fsId_, inode.inodeid(), &slicelist);
+    ASSERT_EQ(st, MetaStatusCode::OK);
+
+    // deallocate blockGroupOffset = 0
+    {
+        inodeVolumeSpaceDeallocate_->UpdateDeallocateInodeExtentSlice(
+            blockGroupOffset, inode.inodeid(), &decrease, &slicelist,
+            &deallocatableVolumeSpace);
+
+        ASSERT_EQ(slice.extents(0).length(), decrease);
+        ASSERT_EQ(deallocatableVolumeSpace.size(), 1);
+        ASSERT_EQ(deallocatableVolumeSpace[0].offset,
+                  slice.extents(0).volumeoffset());
+        ASSERT_EQ(deallocatableVolumeSpace[0].len, slice.extents(0).length());
+        ASSERT_EQ(slice.extents_size() - 1, slicelist.slices(0).extents_size());
+
+        deallocatableVolumeSpace.clear();
+        decrease = 0;
+    }
+
+    // deallocate blockGroupOffset = blockGroupSize
+    {
+        blockGroupOffset = blockGroupSize_;
+        inodeVolumeSpaceDeallocate_->UpdateDeallocateInodeExtentSlice(
+            blockGroupOffset, inode.inodeid(), &decrease, &slicelist,
+            &deallocatableVolumeSpace);
+
+        ASSERT_EQ(slice.extents(1).length(), decrease);
+        ASSERT_EQ(deallocatableVolumeSpace.size(), 1);
+        ASSERT_EQ(deallocatableVolumeSpace[0].offset,
+                  slice.extents(1).volumeoffset());
+        ASSERT_EQ(deallocatableVolumeSpace[0].len, slice.extents(1).length());
+        ASSERT_EQ(slice.extents_size() - 2, slicelist.slices(0).extents_size());
+    }
+}
+
+}  // namespace metaserver
+}  // namespace curvefs