curvefs/client: fix client release read data cache core dump

after restarting metaserver, all previous copysets will be created
and recover from raft snapshot, during recovery, it will also restart
partition clean tasks, and each task needs corresponding copyset node.

deadlock happens because creating and recovering a copyset is
protected by CopysetNodeManager's lock, but partition clean tasks
also acquire this lock to get corresponding copyset node.

another problem is when copyset needs purge, we forget to stop related
tasks like partition cleaning task, s3 compact tasks. so, when these
tasks start to execution, metaserver will end up with segment fault.

this patch passes CopysetNode when creating MetaStore, so partition
clean task no need to acquire the lock, and, also call
`MetaStore::Clear` when copyset stop.

curvefs/conf/mds.conf

@@ -15,7 +15,7 @@ space.rpcTimeoutMs=500
 # metaserver options
 #
 metaserver.addr=127.0.0.1:6701  # __CURVEADM_TEMPLATE__ ${cluster_mds_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.metaserver | join_peer(hostvars, "metaserver_listen_port") }} __ANSIBLE_TEMPLATE__
-metaserver.rpcTimeoutMs=500
+metaserver.rpcTimeoutMs=5000
 metaserver.rpcRertyTimes=3
 metaserver.rpcRetryIntervalUs=1000000
 

curvefs/conf/metaserver.conf

@@ -10,7 +10,7 @@ s3.chunksize=67108864
 # if s3.enableDeleteObjects set True, batch size limit the object num of delete count per delete request
 s3.batchsize=100
 # if s3 sdk support batch delete objects, set True; other set False
-s3.enableDeleteObjects=True
+s3.enableDeleteObjects=False
 s3.endpoint=
 s3.bucket_name=
 s3.ak=
@@ -49,6 +49,8 @@ s3compactwq.s3infocache_size=100
 global.ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_metaserver_listen_host }} __ANSIBLE_TEMPLATE__
 global.port=16701  # __CURVEADM_TEMPLATE__ ${service_port} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_metaserver_listen_port }} __ANSIBLE_TEMPLATE__
 global.external_ip=127.0.0.1  # __CURVEADM_TEMPLATE__ ${service_external_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ curvefs_metaserver_listen_host }} __ANSIBLE_TEMPLATE__
+global.external_port=16701 # __CURVEADM_TEMPLATE__ ${service_external_port} __CURVEADM_TEMPLATE__
+global.enable_external_server=false
 
 # metaserver log directory
 # this config item can be replaced by start up option `-log_dir`

curvefs/conf/tools.conf

@@ -6,7 +6,7 @@ rpcTimeoutMs=10000
 rpcRetryTimes=5
 # topo file path
 topoFilePath=curvefs/test/tools/topo_example.json  # __CURVEADM_TEMPLATE__ /curvefs/tools/conf/topology.json __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ project_root_dest }}/conf/topology.json __ANSIBLE_TEMPLATE__
-# metaserver
+# metaserver external address
 metaserverAddr=127.0.0.1:6701  # __CURVEADM_TEMPLATE__ ${cluster_metaserver_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.metaserver | join_peer(hostvars, "metaserver_listen_port") }} __ANSIBLE_TEMPLATE__
 # etcd
 etcdAddr=127.0.0.1:12379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEADM_TEMPLATE__  __ANSIBLE_TEMPLATE__ {{ groups.etcd | join_peer(hostvars, "etcd_listen_client_port") }} __ANSIBLE_TEMPLATE__
@@ -24,5 +24,5 @@ s3.ak=ak
 s3.sk=sk
 s3.endpoint=endpoint
 s3.bucket_name=bucket
-s3.blocksize=1048576
-s3.chunksize=4194304
+s3.blocksize=4194304
+s3.chunksize=67108864

curvefs/proto/topology.proto

@@ -38,7 +38,7 @@ enum TopoStatusCode {
     TOPO_IP_PORT_DUPLICATED = 14;
     TOPO_NAME_DUPLICATED = 15;
     TOPO_CREATE_COPYSET_ON_METASERVER_FAIL = 16;
-    TOPO_CANNOT_REMOVE_NOT_RETIRED = 17;
+    TOPO_CANNOT_REMOVE_NOT_OFFLINE = 17;
     TOPO_POOL_EXIST = 18;
     TOPO_LEADER_NOT_FOUND = 19;
     TOPO_PARTITION_NOT_FOUND = 20;
@@ -73,9 +73,9 @@ message ZoneData {
 message ServerData {
     required uint32 serverId = 1;
     required string hostName = 2;
-    required string internalHostIP = 3;
+    required string internalIP = 3;
     required uint32 internalPort = 4;
-    required string externalHostIP = 5;
+    required string externalIP = 5;
     required uint32 externalPort = 6;
     required uint32 zoneId = 7;
     required uint32 PoolId = 8;
@@ -85,9 +85,9 @@ message MetaServerData {
     required uint32 MetaServerId = 1;
     required string hostName = 2;
     required string token = 3;
-    required string internalHostIP = 4;
+    required string internalIP = 4;
     required uint32 internalPort = 5;
-    required string externalHostIP = 6;
+    required string externalIP = 6;
     required uint32 externalPort = 7;
     required uint32 serverId = 8;
     required uint64 diskCapacity = 9;
@@ -115,8 +115,8 @@ message Copyset {
 message MetaServerInfo {
     required uint32 metaServerID = 1;
     required string hostname = 2;
-    required string hostIp = 3;
-    required uint32 port = 4;
+    required string internalIp = 3;
+    required uint32 internalPort = 4;
     required string externalIp = 5;
     required uint32 externalPort = 6;
     required OnlineState onlineState = 7;
@@ -126,8 +126,8 @@ message MetaServerInfo {
 // MetaServer message
 message MetaServerRegistRequest {
     required string hostName = 1;
-    required string hostIp = 2;
-    required uint32 port = 3;
+    required string internalIp = 2;
+    required uint32 internalPort = 3;
     required string externalIp = 4;
     required uint32 externalPort = 5;
 };
@@ -325,9 +325,10 @@ message GetMetaServerListInCopySetsRequest {
 
 message MetaServerLocation {
     required uint32 metaServerID = 1;
-    required string hostIp = 2;
-    required uint32 port = 3;
+    required string internalIp = 2;
+    required uint32 internalport = 3;
     optional string externalIp = 4;
+    optional uint32 externalPort = 5;
 }
 
 message CopySetServerInfo {

curvefs/src/client/rpcclient/mds_client.cpp

@@ -256,6 +256,31 @@ MdsClientImpl::CommitTx(const std::vector<PartitionTxId> &txIds) {
     return static_cast<TopoStatusCode>(rc);
 }
 
+template<typename T>
+void GetEndPoint(const T &info, butil::EndPoint *internal,
+    butil::EndPoint *external) {
+    const std::string &internalIp = info.internalip();
+    const std::string &externalIp = [&info]() {
+        if (info.has_externalip()) {
+            return info.externalip();
+        } else {
+            return info.internalip();
+        }
+    }();
+
+    const uint32_t internalPort = info.internalport();
+    const uint32_t externalPort = [&info]() {
+        if (info.has_externalport()) {
+            return info.externalport();
+        } else {
+            return info.internalport();
+        }
+    }();
+
+    butil::str2endpoint(internalIp.c_str(), internalPort, internal);
+    butil::str2endpoint(externalIp.c_str(), externalPort, external);
+}
+
 bool MdsClientImpl::GetMetaServerInfo(
     const PeerAddr &addr, CopysetPeerInfo<MetaserverID> *metaserverInfo) {
     std::vector<std::string> strs;
@@ -283,16 +308,9 @@ bool MdsClientImpl::GetMetaServerInfo(
         } else {
             const auto &info = response.metaserverinfo();
             MetaserverID metaserverID = info.metaserverid();
-            std::string internalIp = info.hostip();
-            std::string externalIp = internalIp;
-            if (info.has_externalip()) {
-                externalIp = info.externalip();
-            }
-            uint32_t port = info.port();
             butil::EndPoint internal;
-            butil::str2endpoint(internalIp.c_str(), port, &internal);
             butil::EndPoint external;
-            butil::str2endpoint(externalIp.c_str(), port, &external);
+            GetEndPoint(info, &internal, &external);
             *metaserverInfo = CopysetPeerInfo<MetaserverID>(
                 metaserverID, PeerAddr(internal), PeerAddr(external));
         }
@@ -329,18 +347,10 @@ bool MdsClientImpl::GetMetaServerListInCopysets(
                 CopysetPeerInfo<MetaserverID> csinfo;
                 ::curvefs::mds::topology::MetaServerLocation csl =
                     info.cslocs(j);
-                uint16_t port = csl.port();
-                std::string internalIp = csl.hostip();
                 csinfo.peerID = csl.metaserverid();
-                std::string externalIp = internalIp;
-                if (csl.has_externalip()) {
-                    externalIp = csl.externalip();
-                }
-
                 butil::EndPoint internal;
-                butil::str2endpoint(internalIp.c_str(), port, &internal);
                 butil::EndPoint external;
-                butil::str2endpoint(externalIp.c_str(), port, &external);
+                GetEndPoint(csl, &internal, &external);
                 csinfo.internalAddr = PeerAddr(internal);
                 csinfo.externalAddr = PeerAddr(external);
                 copysetseverl.AddCopysetPeerInfo(csinfo);

curvefs/src/client/s3/client_s3_cache_manager.cpp

@@ -120,16 +120,17 @@ void FsCacheManager::Get(std::list<DataCachePtr>::iterator iter) {
                                  lruReadDataCacheList_, iter);
 }
 
-void FsCacheManager::Delete(std::list<DataCachePtr>::iterator iter) {
+bool FsCacheManager::Delete(std::list<DataCachePtr>::iterator iter) {
     std::lock_guard<std::mutex> lk(lruMtx_);
 
     if (!(*iter)->InReadCache()) {
-        return;
+        return false;
     }
 
     (*iter)->SetReadCacheState(false);
     lruByte_ -= (*iter)->GetActualLen();
     lruReadDataCacheList_.erase(iter);
+    return true;
 }
 
 CURVEFS_ERROR FsCacheManager::FsSync(bool force) {
@@ -1249,8 +1250,9 @@ void ChunkCacheManager::AddReadDataCache(DataCachePtr dataCache) {
     for (auto key : deleteKeyVec) {
         auto iter = dataRCacheMap_.find(key);
         std::list<DataCachePtr>::iterator dcpIter = iter->second;
-        s3ClientAdaptor_->GetFsCacheManager()->Delete(dcpIter);
-        dataRCacheMap_.erase(iter);
+        if (s3ClientAdaptor_->GetFsCacheManager()->Delete(dcpIter)) {
+            dataRCacheMap_.erase(iter);
+        }
     }
     std::list<DataCachePtr>::iterator outIter;
     bool ret =
@@ -1287,9 +1289,10 @@ void ChunkCacheManager::ReleaseCache() {
     WriteLockGuard writeLockGuard(rwLockRead_);
     auto iter = dataRCacheMap_.begin();
     for (; iter != dataRCacheMap_.end(); iter++) {
-        s3ClientAdaptor_->GetFsCacheManager()->Delete(iter->second);
+        if (s3ClientAdaptor_->GetFsCacheManager()->Delete(iter->second)) {
+            dataRCacheMap_.erase(iter);
+        }
     }
-    dataRCacheMap_.clear();
 }
 
 void ChunkCacheManager::TruncateCache(uint64_t chunkPos) {
@@ -1331,8 +1334,9 @@ void ChunkCacheManager::TruncateReadCache(uint64_t chunkPos) {
         uint64_t dcLen = (*rIter->second)->GetLen();
         uint64_t dcActualLen = (*rIter->second)->GetActualLen();
         if ((dcChunkPos + dcLen) > chunkPos) {
-            s3ClientAdaptor_->GetFsCacheManager()->Delete(rIter->second);
-            dataRCacheMap_.erase(next(rIter).base());
+            if (s3ClientAdaptor_->GetFsCacheManager()->Delete(rIter->second)) {
+                dataRCacheMap_.erase(next(rIter).base());
+            }
         } else {
             break;
         }

curvefs/src/client/s3/client_s3_cache_manager.h

@@ -324,7 +324,7 @@ class FsCacheManager {
 
     bool Set(DataCachePtr dataCache,
              std::list<DataCachePtr>::iterator *outIter);
-    void Delete(std::list<DataCachePtr>::iterator iter);
+    bool Delete(std::list<DataCachePtr>::iterator iter);
     void Get(std::list<DataCachePtr>::iterator iter);
 
     CURVEFS_ERROR FsSync(bool force);

curvefs/src/client/s3/disk_cache_manager.cpp

@@ -248,12 +248,18 @@ bool DiskCacheManager::IsDiskCacheFull() {
 }
 
 bool DiskCacheManager::IsDiskCacheSafe() {
+    int64_t ratio = diskFsUsedRatio_.load(std::memory_order_seq_cst);
     uint64_t usedBytes = GetDiskUsedbytes();
-    if (usedBytes <= (safeRatio_ * maxUsableSpaceBytes_ / 100)) {
+    if ((usedBytes < (safeRatio_ * maxUsableSpaceBytes_ / 100))
+      && (ratio < safeRatio_)) {
         VLOG(3) << "disk cache is safe"
-                << ", usedBytes is: " << usedBytes;
+                << ", usedBytes is: " << usedBytes
+                << ", use ratio is: " << ratio;
         return true;
     }
+    VLOG(3) << "disk cache is not safe"
+                << ", usedBytes is: " << usedBytes
+                << ", use ratio is: " << ratio;
     return false;
 }
 

curvefs/src/mds/heartbeat/copyset_conf_generator.cpp

@@ -37,6 +37,11 @@ bool CopysetConfGenerator::GenCopysetConf(
     const ::curvefs::mds::topology::CopySetInfo &reportCopySetInfo,
     const ::curvefs::mds::heartbeat::ConfigChangeInfo &configChInfo,
     ::curvefs::mds::heartbeat::CopySetConf *copysetConf) {
+    // if copyset is creating return false directly
+    if (topo_->IsCopysetCreating(reportCopySetInfo.GetCopySetKey())) {
+        return false;
+    }
+
     // reported copyset not exist in topology
     // in this case an empty configuration will be sent to metaserver
     // to delete it
@@ -165,7 +170,7 @@ std::string CopysetConfGenerator::BuildPeerByMetaserverId(
     }
 
     return ::curvefs::mds::topology::BuildPeerIdWithIpPort(
-        metaServer.GetInternalHostIp(), metaServer.GetInternalPort(), 0);
+        metaServer.GetInternalIp(), metaServer.GetInternalPort(), 0);
 }
 }  // namespace heartbeat
 }  // namespace mds

curvefs/src/mds/heartbeat/heartbeat_manager.cpp

@@ -181,14 +181,14 @@ HeartbeatStatusCode HeartbeatManager::CheckRequest(
     }
 
     // mismatch ip address reported by metaserver and mds record
-    if (request.ip() != metaServer.GetInternalHostIp() ||
+    if (request.ip() != metaServer.GetInternalIp() ||
         request.port() != metaServer.GetInternalPort()) {
         LOG(ERROR) << "heartbeatManager receive heartbeat from metaServer: "
                    << request.metaserverid()
                    << ", but find report ip:" << request.ip()
                    << ", report port:" << request.port()
                    << " do not consistent with topo record ip:"
-                   << metaServer.GetInternalHostIp()
+                   << metaServer.GetInternalIp()
                    << ", record port:" << metaServer.GetInternalPort();
         return HeartbeatStatusCode::hbMetaServerIpPortNotMatch;
     }

curvefs/src/mds/metaserverclient/metaserver_client.cpp

@@ -349,6 +349,12 @@ FSStatusCode MetaserverClient::CreateCopySet(
 
         uint32_t maxRetry = options_.rpcRetryTimes;
         while (cntl.Failed() && maxRetry > 0) {
+            LOG(WARNING) << "Create copyset failed"
+                         << " from " << cntl.remote_side() << " to "
+                         << cntl.local_side()
+                         << " errCode = " << cntl.ErrorCode()
+                         << " errorText = " << cntl.ErrorText()
+                         << ", then will retry " << maxRetry << " times.";
             maxRetry--;
             bthread_usleep(options_.rpcRetryIntervalUs);
             cntl.Reset();

curvefs/src/mds/schedule/topoAdapter.cpp

@@ -227,7 +227,7 @@ bool TopoAdapterImpl::GetPeerInfo(MetaServerIdType id, PeerInfo *peerInfo) {
     if ((canGetMetaServer = topo_->GetMetaServer(id, &ms)) &&
         (canGetServer = topo_->GetServer(ms.GetServerId(), &server))) {
         *peerInfo = PeerInfo(ms.GetId(), server.GetZoneId(), server.GetId(),
-                             ms.GetInternalHostIp(), ms.GetInternalPort());
+                             ms.GetInternalIp(), ms.GetInternalPort());
     } else {
         LOG(ERROR) << "topoAdapter can not find metaServer(" << id
                    << ", res:" << canGetMetaServer
@@ -276,10 +276,10 @@ bool TopoAdapterImpl::MetaServerFromTopoToSchedule(
     if (topo_->GetServer(origin.GetServerId(), &server)) {
         out->info =
             PeerInfo{origin.GetId(), server.GetZoneId(), server.GetId(),
-                     origin.GetInternalHostIp(), origin.GetInternalPort()};
+                     origin.GetInternalIp(), origin.GetInternalPort()};
     } else {
         LOG(ERROR) << "can not get server:" << origin.GetId()
-                   << ", ip:" << origin.GetInternalHostIp()
+                   << ", ip:" << origin.GetInternalIp()
                    << ", port:" << origin.GetInternalPort() << " from topology";
 
         return false;

curvefs/src/mds/topology/topology.cpp

@@ -222,6 +222,8 @@ TopoStatusCode TopologyImpl::RemoveMetaServer(MetaServerIdType id) {
     WriteLockGuard wlockMetaServer(metaServerMutex_);
     auto it = metaServerMap_.find(id);
     if (it != metaServerMap_.end()) {
+        uint64_t metaserverCapacity =
+            it->second.GetMetaServerSpace().GetDiskCapacity();
         if (!storage_->DeleteMetaServer(id)) {
             return TopoStatusCode::TOPO_STORGE_FAIL;
         }
@@ -230,6 +232,17 @@ TopoStatusCode TopologyImpl::RemoveMetaServer(MetaServerIdType id) {
             ix->second.RemoveMetaServer(id);
         }
         metaServerMap_.erase(it);
+
+        // update pool
+        WriteLockGuard wlockPool(poolMutex_);
+        PoolIdType poolId = ix->second.GetPoolId();
+        auto it = poolMap_.find(poolId);
+        if (it != poolMap_.end()) {
+            it->second.SetDiskCapacity(it->second.GetDiskCapacity() -
+                metaserverCapacity);
+        } else {
+            return TopoStatusCode::TOPO_POOL_NOT_FOUND;
+        }
         return TopoStatusCode::TOPO_OK;
     } else {
         return TopoStatusCode::TOPO_METASERVER_NOT_FOUND;
@@ -432,13 +445,13 @@ ServerIdType TopologyImpl::FindServerByHostIpPort(const std::string &hostIp,
                                                   uint32_t port) const {
     ReadLockGuard rlockServer(serverMutex_);
     for (auto it = serverMap_.begin(); it != serverMap_.end(); it++) {
-        if (it->second.GetInternalHostIp() == hostIp) {
+        if (it->second.GetInternalIp() == hostIp) {
             if (0 == it->second.GetInternalPort()) {
                 return it->first;
             } else if (port == it->second.GetInternalPort()) {
                 return it->first;
             }
-        } else if (it->second.GetExternalHostIp() == hostIp) {
+        } else if (it->second.GetExternalIp() == hostIp) {
             if (0 == it->second.GetExternalPort()) {
                 return it->first;
             } else if (port == it->second.GetExternalPort()) {
@@ -496,7 +509,7 @@ bool TopologyImpl::GetMetaServer(const std::string &hostIp, uint32_t port,
     ReadLockGuard rlockMetaServerMap(metaServerMutex_);
     for (auto it = metaServerMap_.begin(); it != metaServerMap_.end(); it++) {
         ReadLockGuard rlockMetaServer(it->second.GetRWLockRef());
-        if (it->second.GetInternalHostIp() == hostIp &&
+        if (it->second.GetInternalIp() == hostIp &&
             it->second.GetInternalPort() == port) {
             *out = it->second;
             return true;
@@ -978,6 +991,13 @@ TopoStatusCode TopologyImpl::AddCopySet(const CopySetInfo &data) {
     }
 }
 
+TopoStatusCode TopologyImpl::AddCopySetCreating(const CopySetKey &key) {
+    WriteLockGuard wlockCopySetCreating(copySetCreatingMutex_);
+    auto iter = copySetCreating_.insert(key);
+    return iter.second ? TopoStatusCode::TOPO_OK :
+        TopoStatusCode::TOPO_ID_DUPLICATED;
+}
+
 TopoStatusCode TopologyImpl::RemoveCopySet(CopySetKey key) {
     WriteLockGuard wlockCopySetMap(copySetMutex_);
     auto it = copySetMap_.find(key);
@@ -992,6 +1012,11 @@ TopoStatusCode TopologyImpl::RemoveCopySet(CopySetKey key) {
     }
 }
 
+void TopologyImpl::RemoveCopySetCreating(CopySetKey key) {
+    WriteLockGuard wlockCopySetCreating(copySetCreatingMutex_);
+    copySetCreating_.erase(key);
+}
+
 TopoStatusCode TopologyImpl::UpdateCopySetTopo(const CopySetInfo &data) {
     ReadLockGuard rlockCopySetMap(copySetMutex_);
     CopySetKey key(data.GetPoolId(), data.GetId());
@@ -1318,7 +1343,7 @@ void TopologyImpl::GetMetaServersSpace(
         ReadLockGuard rlockMetaServer(i.second.GetRWLockRef());
         auto metaServerUsage = new curvefs::mds::topology::MetadataUsage();
         metaServerUsage->set_metaserveraddr(
-            i.second.GetInternalHostIp() + ":" +
+            i.second.GetInternalIp() + ":" +
             std::to_string(i.second.GetInternalPort()));
         auto const& space = i.second.GetMetaServerSpace();
         metaServerUsage->set_total(space.GetDiskCapacity());
@@ -1345,6 +1370,13 @@ std::string TopologyImpl::GetHostNameAndPortById(MetaServerIdType msId) {
     // get hostName of the metaserver
     return server.GetHostName() + ":" + std::to_string(ms.GetInternalPort());
 }
+
+bool TopologyImpl::IsCopysetCreating(const CopySetKey &key) const {
+    ReadLockGuard rlockCopySetCreating(copySetCreatingMutex_);
+    return copySetCreating_.count(key) != 0;
+}
+
+
 }  // namespace topology
 }  // namespace mds
 }  // namespace curvefs

curvefs/src/mds/topology/topology.h

@@ -79,6 +79,7 @@ class Topology {
     virtual TopoStatusCode AddServer(const Server &data) = 0;
     virtual TopoStatusCode AddMetaServer(const MetaServer &data) = 0;
     virtual TopoStatusCode AddCopySet(const CopySetInfo &data) = 0;
+    virtual TopoStatusCode AddCopySetCreating(const CopySetKey &key) = 0;
     virtual TopoStatusCode AddPartition(const Partition &data) = 0;
 
     virtual TopoStatusCode RemovePool(PoolIdType id) = 0;
@@ -86,6 +87,7 @@ class Topology {
     virtual TopoStatusCode RemoveServer(ServerIdType id) = 0;
     virtual TopoStatusCode RemoveMetaServer(MetaServerIdType id) = 0;
     virtual TopoStatusCode RemoveCopySet(CopySetKey key) = 0;
+    virtual void RemoveCopySetCreating(CopySetKey key) = 0;
     virtual TopoStatusCode RemovePartition(PartitionIdType id) = 0;
 
     virtual TopoStatusCode UpdatePool(const Pool &data) = 0;
@@ -231,6 +233,8 @@ class Topology {
             curvefs::mds::topology::MetadataUsage>* spaces) = 0;
 
     virtual std::string GetHostNameAndPortById(MetaServerIdType msId) = 0;
+
+    virtual bool IsCopysetCreating(const CopySetKey &key) const = 0;
 };
 
 class TopologyImpl : public Topology {
@@ -266,6 +270,7 @@ class TopologyImpl : public Topology {
     TopoStatusCode AddServer(const Server &data) override;
     TopoStatusCode AddMetaServer(const MetaServer &data) override;
     TopoStatusCode AddCopySet(const CopySetInfo &data) override;
+    TopoStatusCode AddCopySetCreating(const CopySetKey &key) override;
     TopoStatusCode AddPartition(const Partition &data) override;
 
     TopoStatusCode RemovePool(PoolIdType id) override;
@@ -273,6 +278,7 @@ class TopologyImpl : public Topology {
     TopoStatusCode RemoveServer(ServerIdType id) override;
     TopoStatusCode RemoveMetaServer(MetaServerIdType id) override;
     TopoStatusCode RemoveCopySet(CopySetKey key) override;
+    void RemoveCopySetCreating(CopySetKey key) override;
     TopoStatusCode RemovePartition(PartitionIdType id) override;
 
     TopoStatusCode UpdatePool(const Pool &data) override;
@@ -445,6 +451,8 @@ class TopologyImpl : public Topology {
 
     std::string GetHostNameAndPortById(MetaServerIdType msId) override;
 
+    bool IsCopysetCreating(const CopySetKey &key) const override;
+
  private:
     TopoStatusCode LoadClusterInfo();
 
@@ -461,6 +469,7 @@ class TopologyImpl : public Topology {
     std::unordered_map<MetaServerIdType, MetaServer> metaServerMap_;
     std::map<CopySetKey, CopySetInfo> copySetMap_;
     std::unordered_map<PartitionIdType, Partition> partitionMap_;
+    std::set<CopySetKey> copySetCreating_;
 
     // cluster info
     ClusterInformation clusterInfo_;
@@ -476,6 +485,7 @@ class TopologyImpl : public Topology {
     mutable RWLock metaServerMutex_;
     mutable RWLock copySetMutex_;
     mutable RWLock partitionMutex_;
+    mutable RWLock copySetCreatingMutex_;
 
     TopologyOption option_;
     curve::common::Thread backEndThread_;

curvefs/src/mds/topology/topology_item.cpp

@@ -130,9 +130,9 @@ bool Server::SerializeToString(std::string *value) const {
     ServerData data;
     data.set_serverid(id_);
     data.set_hostname(hostName_);
-    data.set_internalhostip(internalHostIp_);
+    data.set_internalip(internalIp_);
     data.set_internalport(internalPort_);
-    data.set_externalhostip(externalHostIp_);
+    data.set_externalip(externalIp_);
     data.set_externalport(externalPort_);
     data.set_zoneid(zoneId_);
     data.set_poolid(poolId_);
@@ -144,9 +144,9 @@ bool Server::ParseFromString(const std::string &value) {
     bool ret = data.ParseFromString(value);
     id_ = data.serverid();
     hostName_ = data.hostname();
-    internalHostIp_ = data.internalhostip();
+    internalIp_ = data.internalip();
     internalPort_ = data.internalport();
-    externalHostIp_ = data.externalhostip();
+    externalIp_ = data.externalip();
     externalPort_ = data.externalport();
     zoneId_ = data.zoneid();
     poolId_ = data.poolid();
@@ -158,9 +158,9 @@ bool MetaServer::SerializeToString(std::string *value) const {
     data.set_metaserverid(id_);
     data.set_hostname(hostName_);
     data.set_token(token_);
-    data.set_internalhostip(internalHostIp_);
+    data.set_internalip(internalIp_);
     data.set_internalport(internalPort_);
-    data.set_externalhostip(externalHostIp_);
+    data.set_externalip(externalIp_);
     data.set_externalport(externalPort_);
     data.set_serverid(serverId_);
     data.set_diskcapacity(space_.GetDiskCapacity());
@@ -176,9 +176,9 @@ bool MetaServer::ParseFromString(const std::string &value) {
     hostName_ = data.hostname();
     token_ = data.token();
     serverId_ = data.serverid();
-    internalHostIp_ = data.internalhostip();
+    internalIp_ = data.internalip();
     internalPort_ = data.internalport();
-    externalHostIp_ = data.externalhostip();
+    externalIp_ = data.externalip();
     externalPort_ = data.externalport();
     onlineState_ = OnlineState::UNSTABLE;
     space_.SetDiskCapacity(data.diskcapacity());

curvefs/src/mds/topology/topology_item.h

@@ -171,21 +171,21 @@ class Server {
     Server()
         : id_(UNINITIALIZE_ID),
           hostName_(""),
-          internalHostIp_(""),
+          internalIp_(""),
           internalPort_(0),
-          externalHostIp_(""),
+          externalIp_(""),
           externalPort_(0),
           zoneId_(UNINITIALIZE_ID),
           poolId_(UNINITIALIZE_ID) {}
     Server(ServerIdType id, const std::string &hostName,
-           const std::string &internalHostIp, uint32_t internalPort,
-           const std::string &externalHostIp, uint32_t externalPort,
+           const std::string &internalIp, uint32_t internalPort,
+           const std::string &externalIp, uint32_t externalPort,
            ZoneIdType zoneId, PoolIdType poolId)
         : id_(id),
           hostName_(hostName),
-          internalHostIp_(internalHostIp),
+          internalIp_(internalIp),
           internalPort_(internalPort),
-          externalHostIp_(externalHostIp),
+          externalIp_(externalIp),
           externalPort_(externalPort),
           zoneId_(zoneId),
           poolId_(poolId) {}
@@ -194,11 +194,11 @@ class Server {
 
     std::string GetHostName() const { return hostName_; }
 
-    std::string GetInternalHostIp() const { return internalHostIp_; }
+    std::string GetInternalIp() const { return internalIp_; }
 
     uint32_t GetInternalPort() const { return internalPort_; }
 
-    std::string GetExternalHostIp() const { return externalHostIp_; }
+    std::string GetExternalIp() const { return externalIp_; }
 
     uint32_t GetExternalPort() const { return externalPort_; }
 
@@ -221,9 +221,9 @@ class Server {
  private:
     ServerIdType id_;
     std::string hostName_;
-    std::string internalHostIp_;
+    std::string internalIp_;
     uint32_t internalPort_;
-    std::string externalHostIp_;
+    std::string externalIp_;
     uint32_t externalPort_;
     ZoneIdType zoneId_;
     PoolIdType poolId_;
@@ -259,9 +259,9 @@ class MetaServer {
           hostName_(""),
           token_(""),
           serverId_(UNINITIALIZE_ID),
-          internalHostIp_(""),
+          internalIp_(""),
           internalPort_(0),
-          externalHostIp_(""),
+          externalIp_(""),
           externalPort_(0),
           startUpTime_(0),
           onlineState_(OFFLINE),
@@ -269,16 +269,16 @@ class MetaServer {
 
     MetaServer(MetaServerIdType id, const std::string &hostName,
                const std::string &token, ServerIdType serverId,
-               const std::string &hostIp, uint32_t port,
-               const std::string &externalHostIp, uint32_t externalPort,
+               const std::string &internalIp, uint32_t internalPort,
+               const std::string &externalIp, uint32_t externalPort,
                OnlineState onlineState = OnlineState::OFFLINE)
         : id_(id),
           hostName_(hostName),
           token_(token),
           serverId_(serverId),
-          internalHostIp_(hostIp),
-          internalPort_(port),
-          externalHostIp_(externalHostIp),
+          internalIp_(internalIp),
+          internalPort_(internalPort),
+          externalIp_(externalIp),
           externalPort_(externalPort),
           startUpTime_(0),
           onlineState_(onlineState),
@@ -289,9 +289,9 @@ class MetaServer {
           hostName_(v.hostName_),
           token_(v.token_),
           serverId_(v.serverId_),
-          internalHostIp_(v.internalHostIp_),
+          internalIp_(v.internalIp_),
           internalPort_(v.internalPort_),
-          externalHostIp_(v.externalHostIp_),
+          externalIp_(v.externalIp_),
           externalPort_(v.externalPort_),
           startUpTime_(v.startUpTime_),
           onlineState_(v.onlineState_),
@@ -306,9 +306,9 @@ class MetaServer {
         hostName_ = v.hostName_;
         token_ = v.token_;
         serverId_ = v.serverId_;
-        internalHostIp_ = v.internalHostIp_;
+        internalIp_ = v.internalIp_;
         internalPort_ = v.internalPort_;
-        externalHostIp_ = v.externalHostIp_;
+        externalIp_ = v.externalIp_;
         externalPort_ = v.externalPort_;
         startUpTime_ = v.startUpTime_;
         onlineState_ = v.onlineState_;
@@ -329,19 +329,19 @@ class MetaServer {
 
     ServerIdType GetServerId() const { return serverId_; }
 
-    std::string GetInternalHostIp() const { return internalHostIp_; }
+    std::string GetInternalIp() const { return internalIp_; }
 
     uint32_t GetInternalPort() const { return internalPort_; }
 
-    void SetInternalHostIp(std::string internalHostIp) {
-        internalHostIp_ = internalHostIp;
+    void SetInternalIp(std::string internalIp) {
+        internalIp_ = internalIp;
     }
 
     void SetInternalPort(uint32_t internalPort) {
         internalPort_ = internalPort;
     }
 
-    std::string GetExternalHostIp() const { return externalHostIp_; }
+    std::string GetExternalIp() const { return externalIp_; }
 
     uint32_t GetExternalPort() const { return externalPort_; }
 
@@ -372,9 +372,9 @@ class MetaServer {
     std::string hostName_;
     std::string token_;
     ServerIdType serverId_;
-    std::string internalHostIp_;
+    std::string internalIp_;
     uint32_t internalPort_;
-    std::string externalHostIp_;
+    std::string externalIp_;
     uint32_t externalPort_;
     uint64_t startUpTime_;
     OnlineState onlineState_;  // 0:online、1: offline

curvefs/src/mds/topology/topology_manager.cpp

@@ -47,8 +47,8 @@ void TopologyManager::Init(const TopologyOption &option) { option_ = option; }
 
 void TopologyManager::RegistMetaServer(const MetaServerRegistRequest *request,
                                        MetaServerRegistResponse *response) {
-    std::string hostIp = request->hostip();
-    uint32_t port = request->port();
+    std::string hostIp = request->internalip();
+    uint32_t port = request->internalport();
     NameLockGuard lock(registMsMutex, hostIp + ":" + std::to_string(port));
 
     // here we get metaserver already registered in the cluster that have
@@ -56,7 +56,7 @@ void TopologyManager::RegistMetaServer(const MetaServerRegistRequest *request,
     // normally
     std::vector<MetaServerIdType> list = topology_->GetMetaServerInCluster(
         [&hostIp, &port](const MetaServer &ms) {
-            return (ms.GetInternalHostIp() == hostIp) &&
+            return (ms.GetInternalIp() == hostIp) &&
                    (ms.GetInternalPort() == port);
         });
     if (1 == list.size()) {
@@ -79,7 +79,8 @@ void TopologyManager::RegistMetaServer(const MetaServerRegistRequest *request,
     }
 
     ServerIdType serverId =
-        topology_->FindServerByHostIpPort(request->hostip(), request->port());
+        topology_->FindServerByHostIpPort(request->internalip(),
+            request->internalport());
     if (serverId == static_cast<ServerIdType>(UNINITIALIZE_ID)) {
         response->set_statuscode(TopoStatusCode::TOPO_SERVER_NOT_FOUND);
         return;
@@ -100,9 +101,9 @@ void TopologyManager::RegistMetaServer(const MetaServerRegistRequest *request,
         return;
     }
     if (request->has_externalip()) {
-        if (request->externalip() != server.GetExternalHostIp()) {
+        if (request->externalip() != server.GetExternalIp()) {
             LOG(ERROR) << "External ip of metaserver not match server's"
-                       << ", server external ip: " << server.GetExternalHostIp()
+                       << ", server external ip: " << server.GetExternalIp()
                        << ", request external ip: " << request->externalip();
             response->set_statuscode(TopoStatusCode::TOPO_INTERNAL_ERROR);
             return;
@@ -110,7 +111,7 @@ void TopologyManager::RegistMetaServer(const MetaServerRegistRequest *request,
     }
 
     MetaServer metaserver(metaServerId, request->hostname(), token, serverId,
-                          request->hostip(), request->port(),
+                          request->internalip(), request->internalport(),
                           request->externalip(), request->externalport(),
                           ONLINE);
 
@@ -141,9 +142,9 @@ void TopologyManager::ListMetaServer(const ListMetaServerRequest *request,
             MetaServerInfo *msInfo = response->add_metaserverinfos();
             msInfo->set_metaserverid(ms.GetId());
             msInfo->set_hostname(ms.GetHostName());
-            msInfo->set_hostip(ms.GetInternalHostIp());
-            msInfo->set_port(ms.GetInternalPort());
-            msInfo->set_externalip(ms.GetExternalHostIp());
+            msInfo->set_internalip(ms.GetInternalIp());
+            msInfo->set_internalport(ms.GetInternalPort());
+            msInfo->set_externalip(ms.GetExternalIp());
             msInfo->set_externalport(ms.GetExternalPort());
             msInfo->set_onlinestate(ms.GetOnlineState());
         } else {
@@ -178,9 +179,9 @@ void TopologyManager::GetMetaServer(const GetMetaServerInfoRequest *request,
     MetaServerInfo *msInfo = response->mutable_metaserverinfo();
     msInfo->set_metaserverid(ms.GetId());
     msInfo->set_hostname(ms.GetHostName());
-    msInfo->set_hostip(ms.GetInternalHostIp());
-    msInfo->set_port(ms.GetInternalPort());
-    msInfo->set_externalip(ms.GetExternalHostIp());
+    msInfo->set_internalip(ms.GetInternalIp());
+    msInfo->set_internalport(ms.GetInternalPort());
+    msInfo->set_externalip(ms.GetExternalIp());
     msInfo->set_externalport(ms.GetExternalPort());
     msInfo->set_onlinestate(ms.GetOnlineState());
 }
@@ -289,9 +290,9 @@ void TopologyManager::GetServer(const GetServerRequest *request,
     ServerInfo *info = new ServerInfo();
     info->set_serverid(sv.GetId());
     info->set_hostname(sv.GetHostName());
-    info->set_internalip(sv.GetInternalHostIp());
+    info->set_internalip(sv.GetInternalIp());
     info->set_internalport(sv.GetInternalPort());
-    info->set_externalip(sv.GetExternalHostIp());
+    info->set_externalip(sv.GetExternalIp());
     info->set_externalport(sv.GetExternalPort());
     info->set_zoneid(sv.GetZoneId());
     info->set_zonename(zone.GetName());
@@ -317,6 +318,12 @@ void TopologyManager::DeleteServer(const DeleteServerRequest *request,
                        << ", serverId = " << request->serverid();
             response->set_statuscode(TopoStatusCode::TOPO_INTERNAL_ERROR);
             return;
+        } else if (OnlineState::OFFLINE != ms.GetOnlineState()) {
+            LOG(ERROR) << "Can not delete server which have "
+                       << "metaserver not offline.";
+            response->set_statuscode(
+                TopoStatusCode::TOPO_CANNOT_REMOVE_NOT_OFFLINE);
+            return;
         } else {
             errcode = topology_->RemoveMetaServer(msId);
             if (errcode != TopoStatusCode::TOPO_OK) {
@@ -371,9 +378,9 @@ void TopologyManager::ListZoneServer(const ListZoneServerRequest *request,
             ServerInfo *info = response->add_serverinfo();
             info->set_serverid(sv.GetId());
             info->set_hostname(sv.GetHostName());
-            info->set_internalip(sv.GetInternalHostIp());
+            info->set_internalip(sv.GetInternalIp());
             info->set_internalport(sv.GetInternalPort());
-            info->set_externalip(sv.GetExternalHostIp());
+            info->set_externalip(sv.GetExternalIp());
             info->set_externalport(sv.GetExternalPort());
             info->set_zoneid(sv.GetZoneId());
             info->set_zonename(zone.GetName());
@@ -635,7 +642,7 @@ void TopologyManager::CreatePartitions(const CreatePartitionRequest *request,
         for (auto item : copysetMembers) {
             MetaServer metaserver;
             if (topology_->GetMetaServer(item, &metaserver)) {
-                std::string addr = metaserver.GetInternalHostIp() + ":" +
+                std::string addr = metaserver.GetInternalIp() + ":" +
                                    std::to_string(metaserver.GetInternalPort());
                 copysetMemberAddr.emplace(addr);
             } else {
@@ -696,7 +703,7 @@ bool TopologyManager::CreateCopysetNodeOnMetaServer(
     MetaServer metaserver;
     std::string addr;
     if (topology_->GetMetaServer(metaServerId, &metaserver)) {
-        addr = metaserver.GetInternalHostIp() + ":" +
+        addr = metaserver.GetInternalIp() + ":" +
                std::to_string(metaserver.GetInternalPort());
     } else {
         LOG(ERROR) << "Get metaserver info failed.";
@@ -715,6 +722,13 @@ bool TopologyManager::CreateCopysetNodeOnMetaServer(
     return true;
 }
 
+void TopologyManager::ClearCopysetCreating(PoolIdType poolId,
+    const std::set<CopySetIdType> &copysets) {
+    for (const auto &id : copysets) {
+        topology_->RemoveCopySetCreating(CopySetKey(poolId, id));
+    }
+}
+
 TopoStatusCode TopologyManager::CreateCopyset() {
     PoolIdType poolId;
     std::set<MetaServerIdType> metaServerIds;
@@ -731,7 +745,7 @@ TopoStatusCode TopologyManager::CreateCopyset() {
         MetaServer metaServer;
         if (topology_->GetMetaServer(it, &metaServer)) {
             metaServerAddrs.emplace(
-                metaServer.GetInternalHostIp() + ":" +
+                metaServer.GetInternalIp() + ":" +
                 std::to_string(metaServer.GetInternalPort()));
         } else {
             LOG(ERROR) << "get metaserver failed, metaserverId = " << it;
@@ -746,6 +760,12 @@ TopoStatusCode TopologyManager::CreateCopyset() {
         if (copysetId == static_cast<ServerIdType>(UNINITIALIZE_ID)) {
             return TopoStatusCode::TOPO_ALLOCATE_ID_FAIL;
         }
+        if (TopoStatusCode::TOPO_OK !=
+            topology_->AddCopySetCreating(CopySetKey(poolId, copysetId))) {
+            LOG(WARNING) << "the copyset key = (" << poolId
+                         << ", " << copysetId << ") is already creating.";
+            continue;
+        }
         copysetIds.emplace(copysetId);
     }
 
@@ -755,18 +775,20 @@ TopoStatusCode TopologyManager::CreateCopyset() {
         for (auto id : copysetIds) {
             CopySetInfo copyset(poolId, id);
             copyset.SetCopySetMembers(metaServerIds);
-            TopoStatusCode ret = topology_->AddCopySet(copyset);
-            // TODO(wanghai): delete copyset on metaserver
+            ret = topology_->AddCopySet(copyset);
             if (TopoStatusCode::TOPO_OK != ret) {
                 LOG(ERROR) << "Add copyset failed after create copyset."
                            << " poolId = " << poolId << ", copysetId = " << id
                            << ", error msg = " << TopoStatusCode_Name(ret);
+                ClearCopysetCreating(poolId, copysetIds);
                 return ret;
             }
         }
     } else {
+        ClearCopysetCreating(poolId, copysetIds);
         return TopoStatusCode::TOPO_CREATE_COPYSET_ON_METASERVER_FAIL;
     }
+    ClearCopysetCreating(poolId, copysetIds);
     return TopoStatusCode::TOPO_OK;
 }
 
@@ -801,9 +823,10 @@ void TopologyManager::GetMetaServerListInCopysets(
                 if (topology_->GetMetaServer(metaserverId, &metaserver)) {
                     MetaServerLocation *location = serverInfo->add_cslocs();
                     location->set_metaserverid(metaserver.GetId());
-                    location->set_hostip(metaserver.GetInternalHostIp());
-                    location->set_port(metaserver.GetInternalPort());
-                    location->set_externalip(metaserver.GetExternalHostIp());
+                    location->set_internalip(metaserver.GetInternalIp());
+                    location->set_internalport(metaserver.GetInternalPort());
+                    location->set_externalip(metaserver.GetExternalIp());
+                    location->set_externalport(metaserver.GetExternalPort());
                 } else {
                     LOG(INFO) << "GetMetaserver failed"
                               << " when GetMetaServerListInCopysets.";
@@ -873,7 +896,7 @@ void TopologyManager::GetCopysetOfPartition(
                     common::Peer *peer = cs.add_peers();
                     peer->set_id(ms.GetId());
                     peer->set_address(BuildPeerIdWithIpPort(
-                        ms.GetInternalHostIp(), ms.GetInternalPort()));
+                        ms.GetInternalIp(), ms.GetInternalPort()));
                 } else {
                     LOG(ERROR) << "GetMetaServer failed, id = " << msId;
                     response->set_statuscode(
@@ -902,7 +925,7 @@ TopoStatusCode TopologyManager::GetCopysetMembers(
         for (auto metaserverId : info.GetCopySetMembers()) {
             MetaServer server;
             if (topology_->GetMetaServer(metaserverId, &server)) {
-                std::string addr = server.GetExternalHostIp() + ":" +
+                std::string addr = server.GetExternalIp() + ":" +
                                    std::to_string(server.GetExternalPort());
                 addrs->emplace(addr);
             } else {
@@ -936,7 +959,7 @@ void TopologyManager::GetCopysetInfo(const uint32_t& poolId,
             if (topology_->GetMetaServer(msId, &ms)) {
                 common::Peer* peer = valueCopysetInfo->add_peers();
                 peer->set_id(ms.GetId());
-                peer->set_address(BuildPeerIdWithIpPort(ms.GetInternalHostIp(),
+                peer->set_address(BuildPeerIdWithIpPort(ms.GetInternalIp(),
                                                         ms.GetInternalPort()));
             } else {
                 LOG(ERROR) << "perrs: poolId=" << poolId
@@ -954,7 +977,7 @@ void TopologyManager::GetCopysetInfo(const uint32_t& poolId,
         auto peer = new common::Peer();
         if (topology_->GetMetaServer(msId, &ms)) {
             peer->set_id(ms.GetId());
-            peer->set_address(BuildPeerIdWithIpPort(ms.GetInternalHostIp(),
+            peer->set_address(BuildPeerIdWithIpPort(ms.GetInternalIp(),
                                                     ms.GetInternalPort()));
         } else {
             LOG(WARNING) << "leaderpeer: poolId=" << poolId
@@ -1019,7 +1042,7 @@ void TopologyManager::ListCopysetsInfo(ListCopysetInfoResponse* response) {
             if (topology_->GetMetaServer(msId, &ms)) {
                 common::Peer* peer = valueCopysetInfo->add_peers();
                 peer->set_id(ms.GetId());
-                peer->set_address(BuildPeerIdWithIpPort(ms.GetInternalHostIp(),
+                peer->set_address(BuildPeerIdWithIpPort(ms.GetInternalIp(),
                                                         ms.GetInternalPort()));
             } else {
                 LOG(ERROR) << "perrs: poolId=" << i.GetPoolId()
@@ -1037,7 +1060,7 @@ void TopologyManager::ListCopysetsInfo(ListCopysetInfoResponse* response) {
         auto peer = new common::Peer();
         if (topology_->GetMetaServer(msId, &ms)) {
             peer->set_id(ms.GetId());
-            peer->set_address(BuildPeerIdWithIpPort(ms.GetInternalHostIp(),
+            peer->set_address(BuildPeerIdWithIpPort(ms.GetInternalIp(),
                                                     ms.GetInternalPort()));
         } else {
             LOG(WARNING) << "leaderpeer: poolId=" << i.GetPoolId()
@@ -1116,9 +1139,9 @@ void TopologyManager::ListServer(ListServerResponse* response) {
             auto serverInfo = response->add_serverinfos();
             serverInfo->set_serverid(server.GetId());
             serverInfo->set_hostname(server.GetHostName());
-            serverInfo->set_internalip(server.GetInternalHostIp());
+            serverInfo->set_internalip(server.GetInternalIp());
             serverInfo->set_internalport(server.GetInternalPort());
-            serverInfo->set_externalip(server.GetExternalHostIp());
+            serverInfo->set_externalip(server.GetExternalIp());
             serverInfo->set_externalport(server.GetExternalPort());
             serverInfo->set_zoneid(server.GetZoneId());
             serverInfo->set_poolid(server.GetPoolId());
@@ -1141,9 +1164,9 @@ void TopologyManager::ListMetaserverOfCluster(
             MetaServerInfo* msInfo = response->add_metaserverinfos();
             msInfo->set_metaserverid(ms.GetId());
             msInfo->set_hostname(ms.GetHostName());
-            msInfo->set_hostip(ms.GetInternalHostIp());
-            msInfo->set_port(ms.GetInternalPort());
-            msInfo->set_externalip(ms.GetExternalHostIp());
+            msInfo->set_internalip(ms.GetInternalIp());
+            msInfo->set_internalport(ms.GetInternalPort());
+            msInfo->set_externalip(ms.GetExternalIp());
             msInfo->set_externalport(ms.GetExternalPort());
             msInfo->set_onlinestate(ms.GetOnlineState());
             msInfo->set_serverid(ms.GetServerId());

curvefs/src/mds/topology/topology_manager.h

@@ -164,6 +164,10 @@ class TopologyManager {
                                 const uint32_t& copysetId,
                                 CopysetValue* copysetValue);
 
+
+    virtual void ClearCopysetCreating(PoolIdType poolId,
+        const std::set<CopySetIdType> &copysets);
+
  private:
     std::shared_ptr<Topology> topology_;
     std::shared_ptr<MetaserverClient> metaserverClient_;

curvefs/src/metaserver/copyset/copyset_node.cpp

@@ -108,7 +108,7 @@ bool CopysetNode::Init(const CopysetNodeOptions& options) {
     }
 
     // create metastore
-    metaStore_ = absl::make_unique<MetaStoreImpl>();
+    metaStore_ = absl::make_unique<MetaStoreImpl>(this);
 
     InitRaftNodeOptions();
 
@@ -149,6 +149,11 @@ void CopysetNode::Stop() {
         applyQueue_->Stop();
         applyQueue_.reset();
     }
+
+    if (metaStore_) {
+        metaStore_->Clear();
+        metaStore_.reset();
+    }
 }
 
 int CopysetNode::LoadConfEpoch(const std::string& file) {

curvefs/src/metaserver/copyset/copyset_node_manager.cpp

@@ -164,10 +164,33 @@ CopysetNode* CopysetNodeManager::GetCopysetNode(PoolId poolId,
     return nullptr;
 }
 
-bool CopysetNodeManager::IsCopysetNodeExist(PoolId poolId,
-                                            CopysetId copysetId) {
+int CopysetNodeManager::IsCopysetNodeExist(
+    const CreateCopysetRequest::Copyset& copyset) {
     ReadLockGuard lock(lock_);
-    return copysets_.count(ToGroupId(poolId, copysetId)) != 0;
+    auto iter = copysets_.find(ToGroupId(copyset.poolid(),
+                                         copyset.copysetid()));
+    if (iter == copysets_.end()) {
+        return 0;
+    } else {
+        auto copysetNode = iter->second.get();
+        std::vector<Peer> peers;
+        copysetNode->ListPeers(&peers);
+        if (peers.size() != copyset.peers_size()) {
+            return -1;
+        }
+
+        for (int i = 0; i < copyset.peers_size(); i++) {
+            auto cspeer = copyset.peers(i);
+            auto iter = std::find_if(
+            peers.begin(), peers.end(),
+            [&cspeer](const Peer& p) { return
+                cspeer.address() == p.address();});
+            if (iter == peers.end()) {
+                return -1;
+            }
+        }
+    }
+    return 1;
 }
 
 bool CopysetNodeManager::CreateCopysetNode(PoolId poolId, CopysetId copysetId,

curvefs/src/metaserver/copyset/copyset_node_manager.h

@@ -57,7 +57,11 @@ class CopysetNodeManager {
 
     virtual CopysetNode* GetCopysetNode(PoolId poolId, CopysetId copysetId);
 
-    bool IsCopysetNodeExist(PoolId poolId, CopysetId copysetId);
+    /**
+     * @return 0: not exist; 1: key exist and peers are exactly same;
+     * -1: key exist but peers are not exactly same
+     */
+    int IsCopysetNodeExist(const CreateCopysetRequest::Copyset& copyset);
 
     bool CreateCopysetNode(PoolId poolId, CopysetId copysetId,
                            const braft::Configuration& conf,

curvefs/src/metaserver/copyset/copyset_service.cpp

@@ -79,13 +79,17 @@ void CopysetServiceImpl::GetCopysetsStatus(
 
 COPYSET_OP_STATUS CopysetServiceImpl::CreateOneCopyset(
     const CreateCopysetRequest::Copyset& copyset) {
-    bool exists =
-        manager_->IsCopysetNodeExist(copyset.poolid(), copyset.copysetid());
-    if (exists) {
+    int exists = manager_->IsCopysetNodeExist(copyset);
+    if (-1 == exists) {
+        LOG(ERROR) << "Copyset "
+                   << ToGroupIdString(copyset.poolid(), copyset.copysetid())
+                   << " already exists, but peers not exactly the same.";
+        return COPYSET_OP_STATUS_EXIST;
+    } else if (1 == exists) {
         LOG(WARNING) << "Copyset "
                      << ToGroupIdString(copyset.poolid(), copyset.copysetid())
-                     << " already exists";
-        return COPYSET_OP_STATUS_EXIST;
+                     << " already exists.";
+        return COPYSET_OP_STATUS_SUCCESS;
     }
 
     braft::Configuration conf;

curvefs/src/metaserver/metaserver.cpp

@@ -26,6 +26,7 @@
 #include <brpc/server.h>
 #include <gflags/gflags.h>
 #include <glog/logging.h>
+#include <braft/builtin_service_impl.h>
 
 #include "absl/memory/memory.h"
 #include "curvefs/src/metaserver/copyset/copyset_service.h"
@@ -64,6 +65,8 @@ void Metaserver::InitOptions(std::shared_ptr<Configuration> conf) {
     conf_ = conf;
     conf_->GetValueFatalIfFail("global.ip", &options_.ip);
     conf_->GetValueFatalIfFail("global.port", &options_.port);
+    conf_->GetBoolValue("global.enable_external_server",
+        &options_.enableExternalServer);
 
     std::string value;
     conf_->GetValueFatalIfFail("bthread.worker_count", &value);
@@ -86,7 +89,10 @@ void Metaserver::InitRegisterOptions() {
                                &registerOptions_.metaserverInternalIp);
     conf_->GetValueFatalIfFail("global.external_ip",
                                &registerOptions_.metaserverExternalIp);
-    conf_->GetValueFatalIfFail("global.port", &registerOptions_.metaserverPort);
+    conf_->GetValueFatalIfFail("global.port",
+                               &registerOptions_.metaserverInternalPort);
+    conf_->GetValueFatalIfFail("global.external_port",
+                               &registerOptions_.metaserverExternalPort);
     conf_->GetValueFatalIfFail("mds.register_retries",
                                &registerOptions_.registerRetries);
     conf_->GetValueFatalIfFail("mds.register_timeoutMs",
@@ -170,37 +176,64 @@ void Metaserver::Run() {
 
     PartitionCleanManager::GetInstance().Run();
 
-    brpc::Server server;
-    butil::ip_t ip;
-    LOG_IF(FATAL, 0 != butil::str2ip(options_.ip.c_str(), &ip))
-        << "convert " << options_.ip << " to ip failed";
-    butil::EndPoint listenAddr(ip, options_.port);
-
+    // add internal server
     server_ = absl::make_unique<brpc::Server>();
     metaService_ = absl::make_unique<MetaServerServiceImpl>(
         copysetNodeManager_, inflightThrottle_.get());
     copysetService_ =
         absl::make_unique<CopysetServiceImpl>(copysetNodeManager_);
+    raftCliService2_ = absl::make_unique<RaftCliService2>(copysetNodeManager_);
 
     // add metaserver service
     LOG_IF(FATAL, server_->AddService(metaService_.get(),
                                       brpc::SERVER_DOESNT_OWN_SERVICE) != 0)
         << "add metaserverService error";
-
     LOG_IF(FATAL, server_->AddService(copysetService_.get(),
                                       brpc::SERVER_DOESNT_OWN_SERVICE) != 0)
         << "add copysetservice error";
 
+    butil::ip_t ip;
+    LOG_IF(FATAL, 0 != butil::str2ip(options_.ip.c_str(), &ip))
+        << "convert " << options_.ip << " to ip failed";
+    butil::EndPoint listenAddr(ip, options_.port);
+
     // add raft-related service
     copysetNodeManager_->AddService(server_.get(), listenAddr);
 
-    // start rpc server
+    // start internal rpc server
     brpc::ServerOptions option;
     if (options_.bthreadWorkerCount != -1) {
         option.num_threads = options_.bthreadWorkerCount;
     }
     LOG_IF(FATAL, server_->Start(listenAddr, &option) != 0)
-        << "start brpc server error";
+        << "start internal brpc server error";
+
+    // add external server
+    if (options_.enableExternalServer) {
+        externalServer_ = absl::make_unique<brpc::Server>();
+        LOG_IF(FATAL, externalServer_->AddService(metaService_.get(),
+            brpc::SERVER_DOESNT_OWN_SERVICE) != 0)
+            << "add metaserverService error";
+        LOG_IF(FATAL, externalServer_->AddService(copysetService_.get(),
+            brpc::SERVER_DOESNT_OWN_SERVICE) != 0)
+            << "add copysetService error";
+        LOG_IF(FATAL, externalServer_->AddService(raftCliService2_.get(),
+            brpc::SERVER_DOESNT_OWN_SERVICE) != 0)
+            << "add raftCliService2 error";
+        LOG_IF(FATAL, externalServer_->AddService(new braft::RaftStatImpl{},
+            brpc::SERVER_OWNS_SERVICE) != 0)
+            << "add raftStatService error";
+
+        butil::ip_t ip;
+        LOG_IF(FATAL, 0 != butil::str2ip(
+            registerOptions_.metaserverExternalIp.c_str(), &ip))
+            << "convert " << registerOptions_.metaserverExternalIp
+            << " to ip failed";
+        butil::EndPoint listenAddr(ip, registerOptions_.metaserverExternalPort);
+        // start external rpc server
+        LOG_IF(FATAL, externalServer_->Start(listenAddr, &option) != 0)
+            << "start external brpc server error";
+    }
 
     // try start s3compact wq
     LOG_IF(FATAL, S3CompactManager::GetInstance().Run() != 0);
@@ -221,7 +254,10 @@ void Metaserver::Stop() {
     }
 
     LOG(INFO) << "MetaServer is going to quit";
-
+    if (options_.enableExternalServer) {
+        externalServer_->Stop(0);
+        externalServer_->Join();
+    }
     server_->Stop(0);
     server_->Join();
 

curvefs/src/metaserver/metaserver.h

@@ -31,6 +31,7 @@
 #include "curvefs/src/metaserver/copyset/apply_queue.h"
 #include "curvefs/src/metaserver/copyset/config.h"
 #include "curvefs/src/metaserver/copyset/copyset_node_manager.h"
+#include "curvefs/src/metaserver/copyset/raft_cli_service2.h"
 #include "curvefs/src/metaserver/copyset/copyset_service.h"
 #include "curvefs/src/metaserver/register.h"
 #include "curvefs/src/metaserver/heartbeat.h"
@@ -48,11 +49,13 @@ using ::curvefs::metaserver::copyset::ApplyQueue;
 using ::curvefs::metaserver::copyset::CopysetNodeManager;
 using ::curvefs::metaserver::copyset::CopysetNodeOptions;
 using ::curvefs::metaserver::copyset::CopysetServiceImpl;
+using ::curvefs::metaserver::copyset::RaftCliService2;
 
 struct MetaserverOptions {
     std::string ip;
     int port;
     int bthreadWorkerCount = -1;
+    bool enableExternalServer;
 };
 
 class Metaserver {
@@ -87,8 +90,11 @@ class Metaserver {
     MetaServerMetadata metadate_;
 
     std::unique_ptr<brpc::Server> server_;
+    std::unique_ptr<brpc::Server> externalServer_;
+
     std::unique_ptr<MetaServerServiceImpl> metaService_;
     std::unique_ptr<CopysetServiceImpl> copysetService_;
+    std::unique_ptr<RaftCliService2> raftCliService2_;
 
     HeartbeatOptions heartbeatOptions_;
     Heartbeat heartbeat_;

curvefs/src/metaserver/metastore.cpp

@@ -28,10 +28,10 @@
 #include <vector>
 #include "curvefs/src/metaserver/partition_clean_manager.h"
 #include "curvefs/src/metaserver/storage.h"
+#include "curvefs/src/metaserver/copyset/copyset_node.h"
 
 namespace curvefs {
 namespace metaserver {
-MetaStoreImpl::MetaStoreImpl() {}
 
 // NOTE: if we use set we need define hash function, it's complicate
 using PartitionContainerType = std::unordered_map<uint32_t, PartitionInfo>;
@@ -44,20 +44,13 @@ using InodeIteratorType = MapContainerIterator<InodeContainerType>;
 using DentryIteratorType = SetContainerIterator<DentryContainerType>;
 using PendingTxIteratorType = MapContainerIterator<PendingTxContainerType>;
 
+MetaStoreImpl::MetaStoreImpl(copyset::CopysetNode* node) : copysetNode_(node) {}
+
 bool MetaStoreImpl::LoadPartition(uint32_t partitionId, void* entry) {
     auto partitionInfo = reinterpret_cast<PartitionInfo*>(entry);
     partitionId = partitionInfo->partitionid();
     auto partition = std::make_shared<Partition>(*partitionInfo);
     partitionMap_.emplace(partitionId, partition);
-    if (partitionInfo->status() == PartitionStatus::DELETING) {
-        std::shared_ptr<PartitionCleaner> partitionCleaner =
-            std::make_shared<PartitionCleaner>(GetPartition(partitionId));
-        copyset::CopysetNode *copysetNode =
-            copyset::CopysetNodeManager::GetInstance().GetCopysetNode(
-                partition->GetPoolId(), partition->GetCopySetId());
-        PartitionCleanManager::GetInstance().Add(partitionId, partitionCleaner,
-                                                 copysetNode);
-    }
     return true;
 }
 
@@ -69,9 +62,10 @@ bool MetaStoreImpl::LoadInode(uint32_t partitionId, void* entry) {
     }
 
     auto inode = reinterpret_cast<Inode*>(entry);
-    auto rc = partition->InsertInode(*inode);
+    MetaStatusCode rc = partition->InsertInode(*inode);
     if (rc != MetaStatusCode::OK) {
-        LOG(ERROR) << "InsertInode failed, retCode = " << rc;
+        LOG(ERROR) << "InsertInode failed, retCode = "
+                   << MetaStatusCode_Name(rc);
         return false;
     }
     return true;
@@ -85,9 +79,10 @@ bool MetaStoreImpl::LoadDentry(uint32_t partitionId, void* entry) {
     }
 
     auto dentry = reinterpret_cast<Dentry*>(entry);
-    auto rc = partition->CreateDentry(*dentry, true);
+    MetaStatusCode rc = partition->CreateDentry(*dentry, true);
     if (rc != MetaStatusCode::OK) {
-        LOG(ERROR) << "CreateDentry failed, retCode = " << rc;
+        LOG(ERROR) << "CreateDentry failed, retCode = "
+                   << MetaStatusCode_Name(rc);
         return false;
     }
     return true;
@@ -137,6 +132,17 @@ bool MetaStoreImpl::Load(const std::string& pathname) {
         partitionMap_.clear();
         LOG(ERROR) << "Load metadata failed.";
     }
+
+    for (auto it = partitionMap_.begin(); it != partitionMap_.end(); it++) {
+        if (it->second->GetStatus() == PartitionStatus::DELETING) {
+            uint32_t partitionId = it->second->GetPartitionId();
+            std::shared_ptr<PartitionCleaner> partitionCleaner =
+                std::make_shared<PartitionCleaner>(GetPartition(partitionId));
+            PartitionCleanManager::GetInstance().Add(
+                partitionId, partitionCleaner, copysetNode_);
+        }
+    }
+
     return succ;
 }
 
@@ -268,14 +274,14 @@ MetaStatusCode MetaStoreImpl::DeletePartition(
     auto it = partitionMap_.find(partitionId);
     if (it == partitionMap_.end()) {
         LOG(WARNING) << "DeletePartition, partition is not found"
-                     << ", partitionId = " <<  partitionId;
+                     << ", partitionId = " << partitionId;
         response->set_statuscode(MetaStatusCode::PARTITION_NOT_FOUND);
         return MetaStatusCode::PARTITION_NOT_FOUND;
     }
 
     if (it->second->IsDeletable()) {
         LOG(INFO) << "DeletePartition, partition is deletable, delete it"
-                  << ", partitionId = " <<  partitionId;
+                  << ", partitionId = " << partitionId;
         TrashManager::GetInstance().Remove(partitionId);
         it->second->ClearS3Compact();
         PartitionCleanManager::GetInstance().Remove(partitionId);
@@ -286,21 +292,18 @@ MetaStatusCode MetaStoreImpl::DeletePartition(
 
     if (it->second->GetStatus() != PartitionStatus::DELETING) {
         LOG(INFO) << "DeletePartition, set partition to deleting"
-                  << ", partitionId = " <<  partitionId;
+                  << ", partitionId = " << partitionId;
         it->second->ClearDentry();
         std::shared_ptr<PartitionCleaner> partitionCleaner =
             std::make_shared<PartitionCleaner>(GetPartition(partitionId));
-        copyset::CopysetNode *copysetNode =
-            copyset::CopysetNodeManager::GetInstance().GetCopysetNode(
-                it->second->GetPoolId(), it->second->GetCopySetId());
         PartitionCleanManager::GetInstance().Add(partitionId, partitionCleaner,
-                                                 copysetNode);
+                                                 copysetNode_);
         it->second->SetStatus(PartitionStatus::DELETING);
         TrashManager::GetInstance().Remove(partitionId);
         it->second->ClearS3Compact();
     } else {
         LOG(INFO) << "DeletePartition, partition is already deleting"
-                  << ", partitionId = " <<  partitionId;
+                  << ", partitionId = " << partitionId;
     }
 
     response->set_statuscode(MetaStatusCode::PARTITION_DELETING);
@@ -580,8 +583,8 @@ MetaStatusCode MetaStoreImpl::GetOrModifyS3ChunkInfo(
         response->set_statuscode(status);
         return status;
     }
-    MetaStatusCode status = partition->GetOrModifyS3ChunkInfo(fsId, inodeId,
-        request->s3chunkinfoadd(), request->s3chunkinforemove(),
+    MetaStatusCode status = partition->GetOrModifyS3ChunkInfo(
+        fsId, inodeId, request->s3chunkinfoadd(), request->s3chunkinforemove(),
         request->returns3chunkinfomap(), response->mutable_s3chunkinfomap());
     response->set_statuscode(status);
     return status;

curvefs/src/metaserver/metastore.h

@@ -33,6 +33,11 @@
 
 namespace curvefs {
 namespace metaserver {
+
+namespace copyset {
+class CopysetNode;
+}  // namespace copyset
+
 // dentry
 using curvefs::metaserver::GetDentryRequest;
 using curvefs::metaserver::GetDentryResponse;
@@ -122,7 +127,7 @@ class MetaStore {
 
 class MetaStoreImpl : public MetaStore {
  public:
-    MetaStoreImpl();
+    explicit MetaStoreImpl(copyset::CopysetNode* node);
 
     bool Load(const std::string& pathname) override;
     bool Save(const std::string& path,
@@ -202,6 +207,8 @@ class MetaStoreImpl : public MetaStore {
     RWLock rwLock_;  // protect partitionMap_
     std::map<uint32_t, std::shared_ptr<Partition>> partitionMap_;
     std::list<uint32_t> partitionIds_;
+
+    copyset::CopysetNode* copysetNode_;
 };
 }  // namespace metaserver
 }  // namespace curvefs

curvefs/src/metaserver/partition.cpp

@@ -269,10 +269,6 @@ MetaStatusCode Partition::InsertInode(const Inode& inode) {
         return MetaStatusCode::PARTITION_ID_MISSMATCH;
     }
 
-    if (GetStatus() == PartitionStatus::DELETING) {
-        return MetaStatusCode::PARTITION_DELETING;
-    }
-
     return inodeManager_->InsertInode(inode);
 }
 

curvefs/src/metaserver/register.cpp

@@ -67,16 +67,17 @@ int Register::RegisterToMDS(MetaServerMetadata *metadata) {
     }
 
     req.set_hostname(hostname);
-    req.set_hostip(ops_.metaserverInternalIp);
-    req.set_port(ops_.metaserverPort);
+    req.set_internalip(ops_.metaserverInternalIp);
+    req.set_internalport(ops_.metaserverInternalPort);
     req.set_externalip(ops_.metaserverExternalIp);
-    req.set_externalport(ops_.metaserverPort);
+    req.set_externalport(ops_.metaserverExternalPort);
 
     LOG(INFO) << " Registering to MDS " << mdsEps_[inServiceIndex_]
               << ". hostname: " << hostname
               << ", internal ip: " << ops_.metaserverInternalIp
-              << ", port: " << ops_.metaserverPort
-              << ", external ip: " << ops_.metaserverExternalIp;
+              << ", internal port: " << ops_.metaserverInternalPort
+              << ", external ip: " << ops_.metaserverExternalIp
+              << ", external port: " << ops_.metaserverExternalPort;
 
     int retries = ops_.registerRetries;
     while (retries >= 0) {
@@ -87,7 +88,7 @@ int Register::RegisterToMDS(MetaServerMetadata *metadata) {
 
         if (channel.Init(mdsEps_[inServiceIndex_].c_str(), NULL) != 0) {
             LOG(ERROR) << ops_.metaserverInternalIp << ":"
-                       << ops_.metaserverPort
+                       << ops_.metaserverInternalPort
                        << " Fail to init channel to MDS "
                        << mdsEps_[inServiceIndex_];
             return -1;
@@ -99,12 +100,13 @@ int Register::RegisterToMDS(MetaServerMetadata *metadata) {
             break;
         } else {
             LOG(INFO) << ops_.metaserverInternalIp << ":"
-                       << ops_.metaserverPort << " Fail to register to MDS "
-                       << mdsEps_[inServiceIndex_]
-                       << ", cntl errorCode: " << cntl.ErrorCode() << ","
-                       << " cntl error: " << cntl.ErrorText() << ","
-                       << " statusCode: " << resp.statuscode() << ","
-                       << " going to sleep and try again.";
+                      << ops_.metaserverInternalPort
+                      << " Fail to register to MDS "
+                      << mdsEps_[inServiceIndex_]
+                      << ", cntl errorCode: " << cntl.ErrorCode() << ","
+                      << " cntl error: " << cntl.ErrorText() << ","
+                      << " statusCode: " << resp.statuscode() << ","
+                      << " going to sleep and try again.";
             if (cntl.ErrorCode() == EHOSTDOWN ||
                 cntl.ErrorCode() == brpc::ELOGOFF) {
                 inServiceIndex_ = (inServiceIndex_ + 1) % mdsEps_.size();
@@ -115,7 +117,8 @@ int Register::RegisterToMDS(MetaServerMetadata *metadata) {
     }
 
     if (retries <= 0) {
-        LOG(ERROR) << ops_.metaserverInternalIp << ":" << ops_.metaserverPort
+        LOG(ERROR) << ops_.metaserverInternalIp << ":"
+                   << ops_.metaserverInternalPort
                    << " Fail to register to MDS for " << ops_.registerRetries
                    << " times.";
         return -1;
@@ -125,7 +128,7 @@ int Register::RegisterToMDS(MetaServerMetadata *metadata) {
     metadata->set_id(resp.metaserverid());
     metadata->set_token(resp.token());
 
-    LOG(INFO) << ops_.metaserverInternalIp << ":" << ops_.metaserverPort
+    LOG(INFO) << ops_.metaserverInternalIp << ":" << ops_.metaserverInternalPort
               << " Successfully registered to MDS: " << mdsEps_[inServiceIndex_]
               << ", metaserver id: " << metadata->id() << ","
               << " token: " << metadata->token();

curvefs/src/metaserver/register.h

@@ -36,7 +36,8 @@ struct RegisterOptions {
     std::string mdsListenAddr;
     std::string metaserverInternalIp;
     std::string metaserverExternalIp;
-    int metaserverPort;
+    uint32_t metaserverInternalPort;
+    uint32_t metaserverExternalPort;
     int registerRetries;
     int registerTimeout;
 };

curvefs/src/tools/create/curvefs_create_topology_tool.cpp

@@ -124,6 +124,21 @@ int CurvefsBuildTopologyTool::HandleBuildCluster() {
         return DealFailedRet(ret, "scan cluster");
     }
 
+    ret = RemoveServersNotInNewTopo();
+    if (ret != 0) {
+        return DealFailedRet(ret, "remove server");
+    }
+
+    ret = RemoveZonesNotInNewTopo();
+    if (ret != 0) {
+        return DealFailedRet(ret, "remove zone");
+    }
+
+    ret = RemovePoolsNotInNewTopo();
+    if (ret != 0) {
+        return DealFailedRet(ret, "remove pool");
+    }
+
     ret = CreatePool();
     if (ret != 0) {
         return DealFailedRet(ret, "create pool");
@@ -270,6 +285,8 @@ int CurvefsBuildTopologyTool::ScanCluster() {
             [it](Pool& data) { return data.name == it->poolname(); });
         if (ix != poolDatas.end()) {
             poolDatas.erase(ix);
+        } else {
+            poolToDel.emplace_back(it->poolid());
         }
     }
 
@@ -291,6 +308,8 @@ int CurvefsBuildTopologyTool::ScanCluster() {
             });
         if (ix != zoneDatas.end()) {
             zoneDatas.erase(ix);
+        } else {
+            zoneToDel.emplace_back(it->zoneid());
         }
     }
 
@@ -313,6 +332,8 @@ int CurvefsBuildTopologyTool::ScanCluster() {
                                });
         if (ix != serverDatas.end()) {
             serverDatas.erase(ix);
+        } else {
+            serverToDel.emplace_back(it->serverid());
         }
     }
 
@@ -325,7 +346,6 @@ int CurvefsBuildTopologyTool::ListPool(std::list<PoolInfo>* poolInfos) {
     ListPoolResponse response;
     brpc::Controller cntl;
     cntl.set_timeout_ms(FLAGS_rpcTimeoutMs);
-    cntl.set_log_id(1);
 
     LOG(INFO) << "ListPool send request: " << request.DebugString();
     stub.ListPool(&cntl, &request, &response, nullptr);
@@ -357,7 +377,6 @@ int CurvefsBuildTopologyTool::GetZonesInPool(PoolIdType poolid,
 
     brpc::Controller cntl;
     cntl.set_timeout_ms(FLAGS_rpcTimeoutMs);
-    cntl.set_log_id(1);
 
     LOG(INFO) << "ListZoneInPool, send request: " << request.DebugString();
 
@@ -390,7 +409,6 @@ int CurvefsBuildTopologyTool::GetServersInZone(
     request.set_zoneid(zoneid);
     brpc::Controller cntl;
     cntl.set_timeout_ms(FLAGS_rpcTimeoutMs);
-    cntl.set_log_id(1);
 
     LOG(INFO) << "ListZoneServer, send request: " << request.DebugString();
 
@@ -415,6 +433,115 @@ int CurvefsBuildTopologyTool::GetServersInZone(
     return 0;
 }
 
+int CurvefsBuildTopologyTool::RemovePoolsNotInNewTopo() {
+    TopologyService_Stub stub(&channel_);
+    for (auto it : poolToDel) {
+        DeletePoolRequest request;
+        DeletePoolResponse response;
+        request.set_poolid(it);
+
+        brpc::Controller cntl;
+        cntl.set_timeout_ms(FLAGS_rpcTimeoutMs);
+
+        LOG(INFO) << "ClearPool, send request: " << request.DebugString();
+
+        stub.DeletePool(&cntl, &request, &response, nullptr);
+
+        if (cntl.ErrorCode() == EHOSTDOWN ||
+            cntl.ErrorCode() == brpc::ELOGOFF) {
+            return kRetCodeRedirectMds;
+        } else if (cntl.Failed()) {
+            LOG(ERROR) << "ClearPool errcorde = " << response.statuscode()
+                       << ", error content:" << cntl.ErrorText()
+                       << " , poolId = " << it;
+            return kRetCodeCommonErr;
+        }
+
+        if (response.statuscode() != TopoStatusCode::TOPO_OK) {
+            LOG(ERROR) << "ClearPool rpc response fail. "
+                       << "Message is :" << response.DebugString()
+                       << " , poolId =" << it;
+            return response.statuscode();
+        } else {
+            LOG(INFO) << "Received ClearPool response success, "
+                      << response.DebugString();
+        }
+    }
+    return 0;
+}
+
+int CurvefsBuildTopologyTool::RemoveZonesNotInNewTopo() {
+    TopologyService_Stub stub(&channel_);
+    for (auto it : zoneToDel) {
+        DeleteZoneRequest request;
+        DeleteZoneResponse response;
+        request.set_zoneid(it);
+
+        brpc::Controller cntl;
+        cntl.set_timeout_ms(FLAGS_rpcTimeoutMs);
+
+        LOG(INFO) << "ClearZone, send request: " << request.DebugString();
+
+        stub.DeleteZone(&cntl, &request, &response, nullptr);
+
+        if (cntl.ErrorCode() == EHOSTDOWN ||
+            cntl.ErrorCode() == brpc::ELOGOFF) {
+            return kRetCodeRedirectMds;
+        } else if (cntl.Failed()) {
+            LOG(ERROR) << "ClearZone, errcorde = " << response.statuscode()
+                       << ", error content:" << cntl.ErrorText()
+                       << " , zoneId = " << it;
+            return kRetCodeCommonErr;
+        }
+        if (response.statuscode() != TopoStatusCode::TOPO_OK) {
+            LOG(ERROR) << "ClearZone Rpc response fail. "
+                       << "Message is :" << response.DebugString()
+                       << " , zoneId = " << it;
+            return response.statuscode();
+        } else {
+            LOG(INFO) << "Received ClearZone Rpc success, "
+                      << response.DebugString();
+        }
+    }
+    return 0;
+}
+
+int CurvefsBuildTopologyTool::RemoveServersNotInNewTopo() {
+    TopologyService_Stub stub(&channel_);
+    for (auto it : serverToDel) {
+        DeleteServerRequest request;
+        DeleteServerResponse response;
+        request.set_serverid(it);
+
+        brpc::Controller cntl;
+        cntl.set_timeout_ms(FLAGS_rpcTimeoutMs);
+
+        LOG(INFO) << "ClearServer, send request: " << request.DebugString();
+
+        stub.DeleteServer(&cntl, &request, &response, nullptr);
+
+        if (cntl.ErrorCode() == EHOSTDOWN ||
+            cntl.ErrorCode() == brpc::ELOGOFF) {
+            return kRetCodeRedirectMds;
+        } else if (cntl.Failed()) {
+            LOG(ERROR) << "ClearServer, errcorde = " << response.statuscode()
+                       << ", error content : " << cntl.ErrorText()
+                       << " , serverId = " << it;
+            return kRetCodeCommonErr;
+        }
+        if (response.statuscode() != TopoStatusCode::TOPO_OK) {
+            LOG(ERROR) << "ClearServer Rpc response fail. "
+                       << "Message is :" << response.DebugString()
+                       << " , serverId = " << it;
+            return response.statuscode();
+        } else {
+            LOG(INFO) << "Received ClearServer Rpc success, "
+                      << response.DebugString();
+        }
+    }
+    return 0;
+}
+
 int CurvefsBuildTopologyTool::CreatePool() {
     TopologyService_Stub stub(&channel_);
     for (auto it : poolDatas) {
@@ -431,7 +558,6 @@ int CurvefsBuildTopologyTool::CreatePool() {
 
         brpc::Controller cntl;
         cntl.set_timeout_ms(FLAGS_rpcTimeoutMs);
-        cntl.set_log_id(1);
 
         LOG(INFO) << "CreatePool, send request: " << request.DebugString();
 
@@ -470,7 +596,6 @@ int CurvefsBuildTopologyTool::CreateZone() {
 
         brpc::Controller cntl;
         cntl.set_timeout_ms(FLAGS_rpcTimeoutMs);
-        cntl.set_log_id(1);
 
         LOG(INFO) << "CreateZone, send request: " << request.DebugString();
 
@@ -485,7 +610,7 @@ int CurvefsBuildTopologyTool::CreateZone() {
                        << " , zoneName = " << it.name;
             return kRetCodeCommonErr;
         }
-        if (response.statuscode() != 0) {
+        if (response.statuscode() != TopoStatusCode::TOPO_OK) {
             LOG(ERROR) << "CreateZone Rpc response fail. "
                        << "Message is :" << response.DebugString()
                        << " , zoneName = " << it.name;
@@ -513,7 +638,6 @@ int CurvefsBuildTopologyTool::CreateServer() {
 
         brpc::Controller cntl;
         cntl.set_timeout_ms(FLAGS_rpcTimeoutMs);
-        cntl.set_log_id(1);
 
         LOG(INFO) << "CreateServer, send request: " << request.DebugString();
 

curvefs/src/tools/create/curvefs_create_topology_tool.h

@@ -133,6 +133,9 @@ class CurvefsBuildTopologyTool : public curvefs::tools::CurvefsTool {
     int InitPoolData();
     int ScanCluster();
     int ScanPool();
+    int RemovePoolsNotInNewTopo();
+    int RemoveZonesNotInNewTopo();
+    int RemoveServersNotInNewTopo();
     int CreatePool();
     int CreateZone();
     int CreateServer();
@@ -150,6 +153,10 @@ class CurvefsBuildTopologyTool : public curvefs::tools::CurvefsTool {
     std::list<Zone> zoneDatas;
     std::list<Pool> poolDatas;
 
+    std::list<ServerIdType> serverToDel;
+    std::list<ZoneIdType> zoneToDel;
+    std::list<PoolIdType> poolToDel;
+
     std::vector<std::string> mdsAddressStr_;
     int mdsAddressIndex_;
     brpc::Channel channel_;

curvefs/src/tools/curvefs_tool_define.cpp

@@ -339,7 +339,8 @@ std::string MetaserverInfo2Str(
     std::stringstream ret;
     ret << "metaserverId:" << metaserver.metaserverid()
         << ", hostname:" << metaserver.hostname()
-        << ", hostIp:" << metaserver.hostip() << ", port:" << metaserver.port()
+        << ", InternalIp:" << metaserver.internalip()
+        << ", internalPort:" << metaserver.internalport()
         << ", externalIp:" << metaserver.externalip()
         << ", externalPort:" << metaserver.externalport() << ", onlineState:"
         << mds::topology::OnlineState_Name(metaserver.onlinestate())

curvefs/test/client/rpcclient/mds_client_test.cpp

@@ -515,8 +515,8 @@ TEST_F(MdsClientImplTest, test_GetMetaServerInfo) {
     auto metaserverInfo = new curvefs::mds::topology::MetaServerInfo();
     metaserverInfo->set_metaserverid(1);
     metaserverInfo->set_hostname("hangzhou");
-    metaserverInfo->set_hostip("127.0.0.1");
-    metaserverInfo->set_port(5000);
+    metaserverInfo->set_internalip("127.0.0.1");
+    metaserverInfo->set_internalport(5000);
     metaserverInfo->set_externalip("127.0.0.1");
     metaserverInfo->set_externalport(5000);
     metaserverInfo->set_onlinestate(::curvefs::mds::topology::ONLINE);
@@ -559,8 +559,8 @@ TEST_F(MdsClientImplTest, GetMetaServerListInCopysets) {
     auto l2 = copysetInfo->add_cslocs();
     auto l3 = copysetInfo->add_cslocs();
     l1->set_metaserverid(1);
-    l1->set_hostip("127.0.0.1");
-    l1->set_port(9000);
+    l1->set_internalip("127.0.0.1");
+    l1->set_internalport(9000);
     l1->set_externalip("127.0.0.1");
     l2->CopyFrom(*l1);
     l2->set_metaserverid(2);

curvefs/test/client/test_disk_cache_manager.cpp

@@ -292,7 +292,25 @@ TEST_F(TestDiskCacheManager, IsDiskCacheFull) {
 }
 
 TEST_F(TestDiskCacheManager, IsDiskCacheSafe) {
-    int ret = diskCacheManager_->IsDiskCacheSafe();
+    S3ClientAdaptorOption option;
+    option.diskCacheOpt.diskCacheType = (DiskCacheType)2;
+    option.diskCacheOpt.cacheDir = "/mnt/test_unit";
+    option.diskCacheOpt.trimCheckIntervalSec = 1;
+    option.diskCacheOpt.fullRatio = 0;
+    option.diskCacheOpt.safeRatio = 0;
+    option.diskCacheOpt.maxUsableSpaceBytes = 0;
+    option.diskCacheOpt.cmdTimeoutSec = 5;
+    option.diskCacheOpt.asyncLoadPeriodMs = 10;
+    S3Client *client = nullptr;
+    diskCacheManager_->Init(client, option);
+    bool ret = diskCacheManager_->IsDiskCacheSafe();
+    ASSERT_EQ(false, ret);
+
+    option.diskCacheOpt.fullRatio = 100;
+    option.diskCacheOpt.safeRatio = 99;
+    option.diskCacheOpt.maxUsableSpaceBytes = 100000000;
+    diskCacheManager_->Init(client, option);
+    ret = diskCacheManager_->IsDiskCacheSafe();
     ASSERT_EQ(true, ret);
 }
 

curvefs/test/mds/heartbeat/heartbeat_manager_test.cpp

@@ -150,7 +150,7 @@ TEST_F(TestHeartbeatManager, test_checkReuqest_abnormal) {
               response.statuscode());
 
     // 4. port not same
-    metaServer.SetInternalHostIp("192.168.10.1");
+    metaServer.SetInternalIp("192.168.10.1");
     metaServer.SetInternalPort(11000);
     EXPECT_CALL(*topology_, GetMetaServer(_, _))
         .WillOnce(DoAll(SetArgPointee<1>(metaServer), Return(true)));

curvefs/test/mds/topology/test_topology.cpp

@@ -1782,6 +1782,21 @@ TEST_F(TestTopology, AddCopySet_StorageFail) {
     ASSERT_EQ(TopoStatusCode::TOPO_STORGE_FAIL, ret);
 }
 
+TEST_F(TestTopology, CopySetCreating) {
+    PoolIdType poolId = 0x11;
+    CopySetIdType copysetId = 0x51;
+
+    ASSERT_EQ(TopoStatusCode::TOPO_OK,
+        topology_->AddCopySetCreating(CopySetKey(poolId, copysetId)));
+    ASSERT_EQ(TopoStatusCode::TOPO_ID_DUPLICATED,
+        topology_->AddCopySetCreating(CopySetKey(poolId, copysetId)));
+
+    ASSERT_TRUE(topology_->IsCopysetCreating(CopySetKey(poolId, copysetId)));
+
+    topology_->RemoveCopySetCreating(CopySetKey(poolId, copysetId));
+    ASSERT_FALSE(topology_->IsCopysetCreating(CopySetKey(poolId, copysetId)));
+}
+
 TEST_F(TestTopology, RemoveCopySet_success) {
     PoolIdType poolId = 0x11;
     CopySetIdType copysetId = 0x51;

curvefs/test/mds/topology/test_topology_helper.cpp

@@ -44,9 +44,9 @@ bool CompareZone(const Zone &lh, const Zone &rh) {
 bool CompareServer(const Server &lh, const Server &rh) {
     return lh.GetId() == rh.GetId() &&
            lh.GetHostName() == rh.GetHostName() &&
-           lh.GetInternalHostIp() == rh.GetInternalHostIp() &&
+           lh.GetInternalIp() == rh.GetInternalIp() &&
            lh.GetInternalPort() == rh.GetInternalPort() &&
-           lh.GetExternalHostIp() == rh.GetExternalHostIp() &&
+           lh.GetExternalIp() == rh.GetExternalIp() &&
            lh.GetExternalPort() == rh.GetExternalPort() &&
            lh.GetZoneId() == rh.GetZoneId() &&
            lh.GetPoolId() == rh.GetPoolId();
@@ -57,9 +57,9 @@ bool CompareMetaServer(const MetaServer &lh, const MetaServer &rh) {
            lh.GetHostName() == rh.GetHostName() &&
            lh.GetToken() == rh.GetToken() &&
            lh.GetServerId() == rh.GetServerId() &&
-           lh.GetInternalHostIp() == rh.GetInternalHostIp() &&
+           lh.GetInternalIp() == rh.GetInternalIp() &&
            lh.GetInternalPort() == rh.GetInternalPort() &&
-           lh.GetExternalHostIp() == rh.GetExternalHostIp() &&
+           lh.GetExternalIp() == rh.GetExternalIp() &&
            lh.GetExternalPort() == rh.GetExternalPort() &&
            lh.GetStartUpTime() == rh.GetStartUpTime() &&
            lh.GetMetaServerSpace().GetDiskCapacity() ==

curvefs/test/mds/topology/test_topology_manager.cpp

@@ -199,8 +199,8 @@ TEST_F(TestTopologyManager, test_RegistMetaServer_SuccessWithExIp) {
 
     MetaServerRegistRequest request;
     request.set_hostname("metaserver");
-    request.set_hostip("testInternalIp");
-    request.set_port(0);
+    request.set_internalip("testInternalIp");
+    request.set_internalport(0);
     request.set_externalip("externalIp1");
     request.set_externalport(0);
 
@@ -219,7 +219,7 @@ TEST_F(TestTopologyManager, test_RegistMetaServer_SuccessWithExIp) {
     ASSERT_EQ(token, response.token());
     MetaServer metaserver;
     ASSERT_TRUE(topology_->GetMetaServer(csId, &metaserver));
-    ASSERT_EQ("externalIp1", metaserver.GetExternalHostIp());
+    ASSERT_EQ("externalIp1", metaserver.GetExternalIp());
 }
 
 TEST_F(TestTopologyManager, test_RegistMetaServer_ExIpNotMatch) {
@@ -234,8 +234,8 @@ TEST_F(TestTopologyManager, test_RegistMetaServer_ExIpNotMatch) {
 
     MetaServerRegistRequest request;
     request.set_hostname("metaserver");
-    request.set_hostip("testInternalIp");
-    request.set_port(0);
+    request.set_internalip("testInternalIp");
+    request.set_internalport(0);
     request.set_externalip("externalIp2");
     request.set_externalport(0);
 
@@ -260,8 +260,8 @@ TEST_F(TestTopologyManager, test_RegistMetaServer_ServerNotFound) {
 
     MetaServerRegistRequest request;
     request.set_hostname("metaserver");
-    request.set_hostip("unExistIp");
-    request.set_port(100);
+    request.set_internalip("unExistIp");
+    request.set_internalport(100);
     request.set_externalip("externalIp1");
     request.set_externalport(0);
 
@@ -283,8 +283,8 @@ TEST_F(TestTopologyManager, test_RegistMetaServer_AllocateIdFail) {
 
     MetaServerRegistRequest request;
     request.set_hostname("metaserver");
-    request.set_hostip("testInternalIp");
-    request.set_port(100);
+    request.set_internalip("testInternalIp");
+    request.set_internalport(100);
     request.set_externalip("externalIp1");
     request.set_externalport(0);
 
@@ -310,8 +310,8 @@ TEST_F(TestTopologyManager, test_RegistMetaServer_AddMetaServerFail) {
 
     MetaServerRegistRequest request;
     request.set_hostname("metaserver");
-    request.set_hostip("testInternalIp");
-    request.set_port(100);
+    request.set_internalip("testInternalIp");
+    request.set_internalport(100);
     request.set_externalip("externalIp1");
     request.set_externalport(0);
 
@@ -354,15 +354,15 @@ TEST_F(TestTopologyManager, test_ListMetaServer_ByIdSuccess) {
 
     ASSERT_THAT(response.metaserverinfos(0).metaserverid(),
                 AnyOf(csId1, csId2));
-    ASSERT_EQ("ip1", response.metaserverinfos(0).hostip());
+    ASSERT_EQ("ip1", response.metaserverinfos(0).internalip());
     ASSERT_EQ("ip2", response.metaserverinfos(0).externalip());
-    ASSERT_THAT(response.metaserverinfos(0).port(), AnyOf(100, 200));
+    ASSERT_THAT(response.metaserverinfos(0).internalport(), AnyOf(100, 200));
 
     ASSERT_THAT(response.metaserverinfos(1).metaserverid(),
                 AnyOf(csId1, csId2));
-    ASSERT_EQ("ip1", response.metaserverinfos(1).hostip());
+    ASSERT_EQ("ip1", response.metaserverinfos(1).internalip());
     ASSERT_EQ("ip2", response.metaserverinfos(1).externalip());
-    ASSERT_THAT(response.metaserverinfos(1).port(), AnyOf(100, 200));
+    ASSERT_THAT(response.metaserverinfos(1).internalport(), AnyOf(100, 200));
 }
 
 TEST_F(TestTopologyManager, test_ListMetaServer_ServerNotFound) {
@@ -410,8 +410,8 @@ TEST_F(TestTopologyManager, test_GetMetaServer_ByIdSuccess) {
     ASSERT_TRUE(response.has_metaserverinfo());
 
     ASSERT_EQ(csId1, response.metaserverinfo().metaserverid());
-    ASSERT_EQ("ip1", response.metaserverinfo().hostip());
-    ASSERT_EQ(100, response.metaserverinfo().port());
+    ASSERT_EQ("ip1", response.metaserverinfo().internalip());
+    ASSERT_EQ(100, response.metaserverinfo().internalport());
     ASSERT_EQ("ip2", response.metaserverinfo().externalip());
     ASSERT_EQ(100, response.metaserverinfo().externalport());
 }
@@ -777,6 +777,48 @@ TEST_F(TestTopologyManager, test_DeleteServer_success) {
     ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
 }
 
+TEST_F(TestTopologyManager, test_DeleteServerHaveMetaserver_success) {
+    PoolIdType poolId = 0x11;
+    ZoneIdType zoneId = 0x21;
+    ServerIdType serverId = 0x31;
+    PrepareAddPool(poolId);
+    PrepareAddZone(zoneId);
+    PrepareAddServer(serverId, "hostname1", "ip1", 0, "ip2", 0, zoneId, poolId);
+    PrepareAddMetaServer(0x41, "ms1", "token1", 0x31, "ip1", 0, "ip2", 8888,
+        OnlineState::OFFLINE);
+
+    DeleteServerRequest request;
+    request.set_serverid(serverId);
+
+    DeleteServerResponse response;
+
+    EXPECT_CALL(*storage_, DeleteMetaServer(_)).WillOnce(Return(true));
+    EXPECT_CALL(*storage_, DeleteServer(_)).WillOnce(Return(true));
+
+    serviceManager_->DeleteServer(&request, &response);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_OK, response.statuscode());
+}
+
+TEST_F(TestTopologyManager, test_DeleteServerHaveMetaserver_fail) {
+    PoolIdType poolId = 0x11;
+    ZoneIdType zoneId = 0x21;
+    ServerIdType serverId = 0x31;
+    PrepareAddPool(poolId);
+    PrepareAddZone(zoneId);
+    PrepareAddServer(serverId, "hostname1", "ip1", 0, "ip2", 0, zoneId, poolId);
+    PrepareAddMetaServer(0x41, "ms1", "token1", 0x31, "ip1", 0, "ip2", 8888);
+    DeleteServerRequest request;
+    request.set_serverid(serverId);
+
+    DeleteServerResponse response;
+
+    serviceManager_->DeleteServer(&request, &response);
+
+    ASSERT_EQ(TopoStatusCode::TOPO_CANNOT_REMOVE_NOT_OFFLINE,
+        response.statuscode());
+}
+
 TEST_F(TestTopologyManager, test_ListZoneServer_ByIdSuccess) {
     PoolIdType poolId = 0x11;
     ZoneIdType zoneId = 0x21;
@@ -1282,9 +1324,9 @@ TEST_F(TestTopologyManager, test_GetMetaServerListInCopySets_success) {
 
     ASSERT_THAT(response.csinfo(0).cslocs(0).metaserverid(),
                 AnyOf(0x41, 0x42, 0x43));
-    ASSERT_EQ("ip1", response.csinfo(0).cslocs(0).hostip());
+    ASSERT_EQ("ip1", response.csinfo(0).cslocs(0).internalip());
     ASSERT_EQ("ip2", response.csinfo(0).cslocs(0).externalip());
-    ASSERT_EQ(0, response.csinfo(0).cslocs(0).port());
+    ASSERT_EQ(0, response.csinfo(0).cslocs(0).internalport());
 }
 
 TEST_F(TestTopologyManager, test_GetMetaServerListInCopySets_CopysetNotFound) {
@@ -2219,8 +2261,9 @@ TEST_F(TestTopologyManager, test_GetMetaServerListInCopysets_Success) {
 
     ASSERT_THAT(response.csinfo(0).cslocs(0).metaserverid(),
                 AnyOf(0x41, 0x42, 0x43));
-    ASSERT_EQ(response.csinfo(0).cslocs(0).hostip(), "127.0.0.1");
-    ASSERT_THAT(response.csinfo(0).cslocs(0).port(), AnyOf(7777, 7778, 7779));
+    ASSERT_EQ(response.csinfo(0).cslocs(0).internalip(), "127.0.0.1");
+    ASSERT_THAT(response.csinfo(0).cslocs(0).internalport(),
+        AnyOf(7777, 7778, 7779));
 }
 
 TEST_F(TestTopologyManager, test_GetMetaServerListInCopysets_Fail) {

curvefs/test/mds/topology/test_topology_service.cpp

@@ -107,8 +107,8 @@ TEST_F(TestTopologyService, test_RegistMetaServer_success) {
     brpc::Controller cntl;
     MetaServerRegistRequest request;
     request.set_hostname("metaserver");
-    request.set_hostip("127.0.0.1");
-    request.set_port(8888);
+    request.set_internalip("127.0.0.1");
+    request.set_internalport(8888);
     request.set_externalip("127.0.0.1");
     request.set_externalport(9999);
 
@@ -134,8 +134,8 @@ TEST_F(TestTopologyService, test_RegistMetaServer_fail) {
     brpc::Controller cntl;
     MetaServerRegistRequest request;
     request.set_hostname("metaserver");
-    request.set_hostip("127.0.0.1");
-    request.set_port(8888);
+    request.set_internalip("127.0.0.1");
+    request.set_internalport(8888);
     request.set_externalip("127.0.0.1");
     request.set_externalport(9999);
 

curvefs/test/metaserver/copyset/copyset_node_manager_test.cpp

@@ -149,7 +149,17 @@ TEST_F(CopysetNodeManagerTest, CreateCopysetTest_Common) {
         nodeManager_->AddService(&server, butil::EndPoint(ip, kPort)));
 
     EXPECT_TRUE(nodeManager_->CreateCopysetNode(kPoolId, kCopysetId, conf));
-    EXPECT_TRUE(nodeManager_->IsCopysetNodeExist(kPoolId, kCopysetId));
+
+    CreateCopysetRequest::Copyset copyset;
+    copyset.set_poolid(kPoolId);
+    copyset.set_copysetid(kCopysetId);
+    copyset.add_peers()->set_address("127.0.0.1:29920:0");
+    copyset.add_peers()->set_address("127.0.0.1:29921:0");
+    copyset.add_peers()->set_address("127.0.0.1:29922:0");
+    EXPECT_EQ(1, nodeManager_->IsCopysetNodeExist(copyset));
+
+    copyset.mutable_peers(0)->set_address("127.0.0.1:29923:0");
+    EXPECT_EQ(-1, nodeManager_->IsCopysetNodeExist(copyset));
 
     // create same copyset will failed
     EXPECT_FALSE(nodeManager_->CreateCopysetNode(kPoolId, kCopysetId, conf));
@@ -180,7 +190,6 @@ TEST_F(CopysetNodeManagerTest, DeleteCopysetNodeTest_Success) {
         nodeManager_->AddService(&server, butil::EndPoint(ip, kPort)));
 
     EXPECT_TRUE(nodeManager_->CreateCopysetNode(kPoolId, kCopysetId, conf));
-    EXPECT_TRUE(nodeManager_->IsCopysetNodeExist(kPoolId, kCopysetId));
 
     // create same copyset will failed
     EXPECT_FALSE(nodeManager_->CreateCopysetNode(kPoolId, kCopysetId, conf));
@@ -218,7 +227,6 @@ TEST_F(CopysetNodeManagerTest,
         nodeManager_->AddService(&server, butil::EndPoint(ip, kPort)));
 
     EXPECT_TRUE(nodeManager_->CreateCopysetNode(kPoolId, kCopysetId, conf));
-    EXPECT_TRUE(nodeManager_->IsCopysetNodeExist(kPoolId, kCopysetId));
 
     // create same copyset will failed
     EXPECT_FALSE(nodeManager_->CreateCopysetNode(kPoolId, kCopysetId, conf));

curvefs/test/metaserver/copyset/copyset_node_snapshot_test.cpp

@@ -345,6 +345,7 @@ TEST_F(CopysetNodeRaftSnapshotTest,
     node->ListPeers(&peers);
     EXPECT_EQ(3, peers.size());
     EXPECT_EQ(epochBefore, node->GetConfEpoch());
+    node->SetMetaStore(nullptr);
 }
 
 TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadFailed) {
@@ -372,6 +373,7 @@ TEST_F(CopysetNodeRaftSnapshotTest, SnapshotLoadTest_MetaStoreLoadFailed) {
         .Times(0);
 
     EXPECT_NE(0, node->on_snapshot_load(&reader));
+    node->SetMetaStore(nullptr);
 }
 
 }  // namespace copyset

curvefs/test/metaserver/copyset/copyset_service_test.cpp

@@ -141,6 +141,28 @@ TEST_F(CopysetServiceTest, CreateCopysetTest) {
 
         stub.CreateCopysetNode(&cntl, &request, &response, nullptr);
 
+        ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText();
+        ASSERT_EQ(COPYSET_OP_STATUS::COPYSET_OP_STATUS_SUCCESS,
+                  response.status());
+    }
+
+    // create copyset exist
+    {
+        CopysetService_Stub stub(&channel_);
+        brpc::Controller cntl;
+
+        CreateCopysetRequest request;
+        CreateCopysetResponse response;
+
+        auto* copyset = request.add_copysets();
+        copyset->set_poolid(poolId_);
+        copyset->set_copysetid(copysetId_);
+        copyset->add_peers()->set_address("127.0.0.1:29960:0");
+        copyset->add_peers()->set_address("127.0.0.1:29961:0");
+        copyset->add_peers()->set_address("127.0.0.1:29963:0");
+
+        stub.CreateCopysetNode(&cntl, &request, &response, nullptr);
+
         ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText();
         ASSERT_EQ(COPYSET_OP_STATUS::COPYSET_OP_STATUS_EXIST,
                   response.status());

curvefs/test/metaserver/copyset/meta_operator_test.cpp

@@ -154,6 +154,9 @@ TEST_F(MetaOperatorTest, OnApplyErrorTest) {
     mock::MockMetaStore* mockMetaStore = new mock::MockMetaStore();
     node.SetMetaStore(mockMetaStore);
 
+    ON_CALL(*mockMetaStore, Clear())
+        .WillByDefault(Return(true));
+
     brpc::Controller cntl;
 
 #define OPERATOR_ON_APPLY_TEST(TYPE)                                           \
@@ -257,6 +260,9 @@ TEST_F(MetaOperatorTest, OnApplyFromLogErrorTest) {
     mock::MockMetaStore* mockMetaStore = new mock::MockMetaStore();
     node.SetMetaStore(mockMetaStore);
 
+    ON_CALL(*mockMetaStore, Clear())
+        .WillByDefault(Return(true));
+
     brpc::Controller cntl;
 
 #define OPERATOR_ON_APPLY_FROM_LOG_TEST(TYPE)                                \
@@ -378,6 +384,8 @@ TEST_F(MetaOperatorTest, PropostTest_RequestCanBypassProcess) {
     auto* mockRaftNode = new MockRaftNode();
     node.SetRaftNode(mockRaftNode);
 
+    ON_CALL(*mockMetaStore, Clear())
+        .WillByDefault(Return(true));
     EXPECT_CALL(*mockRaftNode, apply(_))
         .Times(0);
     EXPECT_CALL(*mockRaftNode, shutdown(_))

curvefs/test/metaserver/metastore_test.cpp

@@ -94,7 +94,10 @@ class MetastoreTest : public ::testing::Test {
                    first.partitionid() == second.partitionid() &&
                    first.start() == second.start() &&
                    first.end() == second.end() &&
-                   first.nextid() == second.nextid();
+                   first.nextid() == second.nextid() &&
+                   first.status() == second.status() &&
+                   first.inodenum() == second.inodenum() &&
+                   first.dentrynum() == second.dentrynum();
         if (!ret) {
             LOG(INFO) << "first partition :" << first.ShortDebugString()
                       << ", second partiton : " << second.ShortDebugString();
@@ -132,7 +135,7 @@ class MetastoreTest : public ::testing::Test {
 };
 
 TEST_F(MetastoreTest, partition) {
-    MetaStoreImpl metastore;
+    MetaStoreImpl metastore(nullptr);
     CreatePartitionRequest createPartitionRequest;
     CreatePartitionResponse createPartitionResponse;
     PartitionInfo partitionInfo;
@@ -227,7 +230,7 @@ TEST_F(MetastoreTest, partition) {
 }
 
 TEST_F(MetastoreTest, test_inode) {
-    MetaStoreImpl metastore;
+    MetaStoreImpl metastore(nullptr);
 
     // create partition1 partition2
     CreatePartitionRequest createPartitionRequest;
@@ -461,7 +464,7 @@ TEST_F(MetastoreTest, test_inode) {
 }
 
 TEST_F(MetastoreTest, test_dentry) {
-    MetaStoreImpl metastore;
+    MetaStoreImpl metastore(nullptr);
 
     // create partition1 partition2
     CreatePartitionRequest createPartitionRequest;
@@ -686,7 +689,7 @@ TEST_F(MetastoreTest, test_dentry) {
 }
 
 TEST_F(MetastoreTest, persist_success) {
-    MetaStoreImpl metastore;
+    MetaStoreImpl metastore(nullptr);
     uint32_t partitionId = 4;
     uint32_t partitionId2 = 2;
     // create partition1
@@ -800,7 +803,159 @@ TEST_F(MetastoreTest, persist_success) {
     ASSERT_TRUE(done.IsSuccess());
 
     // load MetaStoreImpl to new meta
-    MetaStoreImpl metastoreNew;
+    MetaStoreImpl metastoreNew(nullptr);
+    LOG(INFO) << "MetastoreTest test Load";
+    ASSERT_TRUE(metastoreNew.Load("./metastore_test"));
+
+    // compare two meta
+    ASSERT_TRUE(ComparePartition(
+        metastoreNew.GetPartition(partitionId)->GetPartitionInfo(),
+        metastore.GetPartition(partitionId)->GetPartitionInfo()));
+
+    ASSERT_TRUE(ComparePartition(
+        metastoreNew.GetPartition(partitionId2)->GetPartitionInfo(),
+        metastore.GetPartition(partitionId2)->GetPartitionInfo()));
+
+    metastoreNew.GetPartition(partitionId)
+        ->GetInode(fsId, createInodeResponse1.inode().inodeid(), &tempInode);
+    ASSERT_TRUE(CompareInode(tempInode, createInodeResponse1.inode()));
+    ASSERT_EQ(tempInode.nlink(), createInodeResponse1.inode().nlink() + 2);
+
+    // clear meta
+    LOG(INFO) << "MetastoreTest test Clear";
+    ASSERT_TRUE(metastore.Clear());
+}
+
+TEST_F(MetastoreTest, persist_deleting_partition_success) {
+    MetaStoreImpl metastore(nullptr);
+    uint32_t partitionId = 4;
+    uint32_t partitionId2 = 2;
+    // create partition1
+    CreatePartitionRequest createPartitionRequest;
+    CreatePartitionResponse createPartitionResponse;
+    PartitionInfo partitionInfo;
+    partitionInfo.set_fsid(1);
+    partitionInfo.set_poolid(2);
+    partitionInfo.set_copysetid(3);
+    partitionInfo.set_partitionid(partitionId);
+    partitionInfo.set_start(100);
+    partitionInfo.set_end(1000);
+    partitionInfo.set_txid(100);
+    partitionInfo.set_status(PartitionStatus::READWRITE);
+    createPartitionRequest.mutable_partition()->CopyFrom(partitionInfo);
+    MetaStatusCode ret = metastore.CreatePartition(&createPartitionRequest,
+                                                   &createPartitionResponse);
+    ASSERT_EQ(ret, MetaStatusCode::OK);
+    ASSERT_EQ(createPartitionResponse.statuscode(), ret);
+
+    ret = metastore.CreatePartition(&createPartitionRequest,
+                                    &createPartitionResponse);
+    ASSERT_EQ(ret, MetaStatusCode::OK);
+    ASSERT_EQ(createPartitionResponse.statuscode(), ret);
+
+    // create partition2
+    PartitionInfo partitionInfo2 = partitionInfo;
+    partitionInfo2.set_partitionid(partitionId2);
+    createPartitionRequest.mutable_partition()->CopyFrom(partitionInfo2);
+    ret = metastore.CreatePartition(&createPartitionRequest,
+                                    &createPartitionResponse);
+    ASSERT_EQ(ret, MetaStatusCode::OK);
+    ASSERT_EQ(createPartitionResponse.statuscode(), ret);
+
+    // add 2 inode to partion1
+    CreateInodeRequest createInodeRequest;
+    CreateInodeResponse createInodeResponse1;
+    CreateInodeResponse createInodeResponse2;
+
+    uint32_t poolId = 2;
+    uint32_t copysetId = 3;
+    uint32_t fsId = 1;
+    uint64_t length = 2;
+    uint32_t uid = 100;
+    uint32_t gid = 200;
+    uint32_t mode = 777;
+    FsFileType type = FsFileType::TYPE_DIRECTORY;
+
+    createInodeRequest.set_poolid(poolId);
+    createInodeRequest.set_copysetid(copysetId);
+    createInodeRequest.set_partitionid(partitionId);
+    createInodeRequest.set_fsid(fsId);
+    createInodeRequest.set_length(length);
+    createInodeRequest.set_uid(uid);
+    createInodeRequest.set_gid(gid);
+    createInodeRequest.set_mode(mode);
+    createInodeRequest.set_type(type);
+
+    ret = metastore.CreateInode(&createInodeRequest, &createInodeResponse1);
+    ASSERT_EQ(createInodeResponse1.statuscode(), ret);
+    ASSERT_EQ(createInodeResponse1.statuscode(), MetaStatusCode::OK);
+    ASSERT_EQ(createInodeResponse1.inode().inodeid(), 100);
+
+    createInodeRequest.set_partitionid(partitionId);
+    ret = metastore.CreateInode(&createInodeRequest, &createInodeResponse2);
+    ASSERT_EQ(createInodeResponse2.statuscode(), ret);
+    ASSERT_EQ(createInodeResponse2.statuscode(), MetaStatusCode::OK);
+
+    // add 2 dentry to partiton1
+    CreateDentryRequest createDentryRequest;
+    CreateDentryResponse createDentryResponse1;
+    CreateDentryResponse createDentryResponse2;
+    Dentry dentry1;
+    dentry1.set_fsid(fsId);
+    dentry1.set_inodeid(2000);
+    dentry1.set_parentinodeid(100);
+    dentry1.set_name("dentry1");
+    dentry1.set_txid(1);
+
+
+    createDentryRequest.set_poolid(poolId);
+    createDentryRequest.set_copysetid(copysetId);
+    createDentryRequest.set_partitionid(partitionId);
+    createDentryRequest.mutable_dentry()->CopyFrom(dentry1);
+
+    ret = metastore.CreateDentry(&createDentryRequest, &createDentryResponse1);
+    ASSERT_EQ(createDentryResponse1.statuscode(), MetaStatusCode::OK);
+    ASSERT_EQ(createDentryResponse1.statuscode(), ret);
+
+    Dentry dentry2 = dentry1;
+    dentry2.set_inodeid(2);
+    dentry2.set_name("dentry2");
+    createDentryRequest.mutable_dentry()->CopyFrom(dentry2);
+    ret = metastore.CreateDentry(&createDentryRequest, &createDentryResponse2);
+    ASSERT_EQ(createDentryResponse2.statuscode(), MetaStatusCode::OK);
+    ASSERT_EQ(createDentryResponse2.statuscode(), ret);
+
+    Inode tempInode;
+    metastore.GetPartition(partitionId)
+        ->GetInode(fsId, createInodeResponse1.inode().inodeid(), &tempInode);
+    LOG(INFO) << "tempInode = " << tempInode.DebugString();
+    ASSERT_EQ(tempInode.nlink(), createInodeResponse1.inode().nlink() + 2);
+
+    DeletePartitionRequest deletePartitionRequest;
+    DeletePartitionResponse deletePartitionResponse;
+    deletePartitionRequest.set_poolid(poolId);
+    deletePartitionRequest.set_copysetid(copysetId);
+    deletePartitionRequest.set_partitionid(partitionId);
+
+    ret = metastore.DeletePartition(&deletePartitionRequest,
+                                    &deletePartitionResponse);
+    ASSERT_EQ(deletePartitionResponse.statuscode(),
+              MetaStatusCode::PARTITION_DELETING);
+    ASSERT_EQ(deletePartitionResponse.statuscode(), ret);
+    ASSERT_EQ(metastore.GetPartition(partitionId)->GetPartitionInfo().status(),
+              PartitionStatus::DELETING);
+
+    // dump MetaStoreImpl to file
+    OnSnapshotSaveDoneImpl done;
+    LOG(INFO) << "MetastoreTest test Save";
+    ASSERT_TRUE(metastore.Save("./metastore_test", &done));
+
+    // wait meta save to file
+    done.Wait();
+    ASSERT_TRUE(done.IsSuccess());
+
+    // load MetaStoreImpl to new meta
+    MetaStoreImpl metastoreNew(nullptr);
     LOG(INFO) << "MetastoreTest test Load";
     ASSERT_TRUE(metastoreNew.Load("./metastore_test"));
 
@@ -824,7 +979,7 @@ TEST_F(MetastoreTest, persist_success) {
 }
 
 TEST_F(MetastoreTest, persist_partition_fail) {
-    MetaStoreImpl metastore;
+    MetaStoreImpl metastore(nullptr);
     uint32_t partitionId = 4;
     // create partition1
     CreatePartitionRequest createPartitionRequest;
@@ -848,7 +1003,7 @@ TEST_F(MetastoreTest, persist_partition_fail) {
 }
 
 TEST_F(MetastoreTest, persist_dentry_fail) {
-    MetaStoreImpl metastore;
+    MetaStoreImpl metastore(nullptr);
     uint32_t partitionId = 4;
 
     // create partition1

robot/Resources/keywords/deploy.py

@@ -315,9 +315,9 @@ def destroy_curvefs():
         for mountpoint in config.fs_mount_dir:
             cmd = "sudo /home/nbs/.curveadm/bin/curveadm umount %s%s"%(config.fs_mount_path,mountpoint)
             shell_operator.ssh_exec(ssh, cmd)
-        cmd = "/home/nbs/.curveadm/bin/curveadm stop"
+        cmd = "echo 'yes'| /home/nbs/.curveadm/bin/curveadm stop"
         ret = shell_operator.run_exec(cmd)
-        cmd = "echo 'y' | /home/nbs/.curveadm/bin/curveadm clean"
+        cmd = "echo 'yes' | /home/nbs/.curveadm/bin/curveadm clean"
         ret = shell_operator.run_exec(cmd)
     except Exception:
         logger.error("destroy curvefs fail.")

src/common/s3_adapter.cpp

@@ -523,7 +523,8 @@ int S3Adapter::DeleteObjects(const std::list<Aws::String>& keyList) {
 
         return 0;
     } else {
-        LOG(ERROR) << response.GetError().GetMessage() << "failed";
+        LOG(ERROR) << response.GetError().GetMessage() << " failed, "
+                   << deleteObjectsRequest.SerializePayload();
         return -1;
     }
     return 0;