docs/source/design/transfer-engine/index.md

@@ -291,6 +291,7 @@ For advanced users, TransferEngine provides the following advanced runtime optio
 - `MC_WORKERS_PER_CTX` The number of asynchronous worker threads corresponding to each device instance
 - `MC_SLICE_SIZE` The segmentation granularity of user requests in Transfer Engine
 - `MC_RETRY_CNT` The maximum number of retries in Transfer Engine
+- `MC_AUTO_GID_MAX_RETRIES` The maximum number of automatic local GID reprobe retries during classic RDMA handshake recovery. Default value 2. Set to 0 to disable automatic GID retry.
 - `MC_LOG_LEVEL` This option can be set as `TRACE`/`INFO`/`WARNING`/`ERROR` (see [glog doc](https://github.com/google/glog/blob/master/docs/logging.md)), and more detailed logs will be output during runtime
 - `MC_DISABLE_METACACHE` Disable local meta cache to prevent transfer failure due to dynamic memory registrations, which may downgrades the performance
 - `MC_HANDSHAKE_LISTEN_BACKLOG` The backlog size of socket listening for handshaking, default value is 128

docs/source/zh_archive/transfer-engine.md

@@ -407,6 +407,7 @@ int init(const std::string &metadata_conn_string,
 - `MC_WORKERS_PER_CTX` 每个设备实例对应的异步工作线程数量
 - `MC_SLICE_SIZE` Transfer Engine 中用户请求的切分粒度
 - `MC_RETRY_CNT` Transfer Engine 中最大重试次数
+- `MC_AUTO_GID_MAX_RETRIES` classic RDMA 握手恢复过程中自动重探测本地 GID 的最大重试次数，默认值 2。设置为 0 可以关闭自动 GID 重试。
 - `MC_LOG_LEVEL` 该选项可以设置成`TRACE`/`INFO`/`WARNING`/`ERROR`（详情见 [glog doc](https://github.com/google/glog/blob/master/docs/logging.md)），则在运行时会输出更详细的日志
 - `MC_HANDSHAKE_LISTEN_BACKLOG` 监听握手连接的 backlog 大小, 默认值 128
 - `MC_HANDSHAKE_MAX_LENGTH` P2P 模式下握手消息的最大长度（字节）。有效范围：1MB 到 128MB。默认值为 1MB (1048576 字节)。当单个 RDMA 实例注册大量内存缓冲区（>10,000）时，需要增大此值以避免握手失败。示例：设置为 10485760 表示 10MB

mooncake-transfer-engine/include/config.h

@@ -50,6 +50,7 @@ struct GlobalConfig {
     int workers_per_ctx = 2;
     size_t slice_size = 65536;
     int retry_cnt = 9;
+    int auto_gid_max_retries = 2;
     int handshake_listen_backlog = 128;
     bool metacache = true;
     int log_level = google::INFO;

mooncake-transfer-engine/include/transport/rdma_transport/rdma_context.h

@@ -29,11 +29,13 @@
 #include <list>
 #include <map>
 #include <memory>
+#include <mutex>
 #include <string>
 #include <thread>
 #include <unordered_map>
 
 #include "common.h"
+#include "rdma_gid_probe.h"
 #include "rdma_transport.h"
 #include "transport/transport.h"
 
@@ -41,6 +43,7 @@ namespace mooncake {
 
 class RdmaEndPoint;
 class RdmaTransport;
+class RdmaContextTestPeer;
 class WorkerPool;
 class EndpointStore;
 
@@ -51,6 +54,11 @@ enum class GidNetworkState {
     GID_NOT_FOUND = 2         // No suitable GID found
 };
 
+struct GidSelectionSnapshot {
+    std::string gid;
+    int gid_index = -1;
+};
+
 struct RdmaCq {
     RdmaCq() : native(nullptr), outstanding(0) {}
     ibv_cq *native;
@@ -68,6 +76,8 @@ struct MemoryRegionMeta {
 // including Memory Region, CQ, EndPoint (QPs), etc.
 class RdmaContext {
    public:
+    friend class RdmaContextTestPeer;
+
     RdmaContext(RdmaTransport &engine, const std::string &device_name);
 
     ~RdmaContext();
@@ -148,7 +158,16 @@ class RdmaContext {
 
     std::string gid() const;
 
-    int gidIndex() const { return gid_index_; }
+    GidSelectionSnapshot gidSelection() const;
+
+    int gidIndex() const;
+
+    bool autoGidSelectionEnabled() const { return auto_gid_selection_enabled_; }
+
+    bool reprobeAutoGid(
+        const GidSelectionSnapshot &expected_selection,
+        const std::vector<AutoGidSelectionIdentity> &tried_selections = {},
+        std::string *previous_gid = nullptr, std::string *next_gid = nullptr);
 
     ibv_context *context() const { return context_; }
 
@@ -215,6 +234,9 @@ class RdmaContext {
     ibv_mtu active_mtu_;
     uint8_t num_lag_ports_ = 0;  // 0/1 = not in LAG; ≥2 = LAG active
     ibv_gid gid_;
+    mutable std::mutex gid_lock_;
+    mutable std::mutex gid_reprobe_lock_;
+    bool auto_gid_selection_enabled_ = false;
 
     RWSpinlock memory_regions_lock_;
     MemoryRegionMap memory_region_map_;

mooncake-transfer-engine/include/transport/rdma_transport/rdma_endpoint.h

@@ -148,15 +148,31 @@ class RdmaEndPoint {
     size_t getQPNumber() const;
 
    private:
+    enum class SetupConnectionFailureStage {
+        kNone,
+        kPeerValidation,
+        kReset,
+        kInit,
+        kRtr,
+        kRts,
+    };
+
+    struct SetupConnectionFailureInfo {
+        SetupConnectionFailureStage stage = SetupConnectionFailureStage::kNone;
+        int sys_errno = 0;
+    };
+
     std::vector<uint32_t> qpNum() const;
 
     int doSetupConnection(const std::string &peer_gid, uint16_t peer_lid,
                           std::vector<uint32_t> peer_qp_num_list,
-                          std::string *reply_msg = nullptr);
+                          std::string *reply_msg = nullptr,
+                          SetupConnectionFailureInfo *failure_info = nullptr);
 
     int doSetupConnection(int qp_index, const ibv_gid &peer_gid,
                           uint16_t peer_lid, uint32_t peer_qp_num,
-                          std::string *reply_msg = nullptr);
+                          int local_gid_index, std::string *reply_msg = nullptr,
+                          SetupConnectionFailureInfo *failure_info = nullptr);
 
    private:
     static constexpr uint64_t kWaitExistingHandshakeTimeoutNano =

mooncake-transfer-engine/include/transport/rdma_transport/rdma_gid_probe.h

@@ -0,0 +1,247 @@
+// Copyright 2026 KVCache.AI
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef RDMA_GID_PROBE_H
+#define RDMA_GID_PROBE_H
+
+#include <cerrno>
+#include <infiniband/verbs.h>
+
+#include <algorithm>
+#include <optional>
+#include <string>
+#include <string_view>
+#include <vector>
+
+namespace mooncake {
+
+enum class AutoGidCandidateClass {
+    kNetworkRoutable = 0,
+    kNoNetworkRoutable = 1,
+    kNetworkDegraded = 2,
+    kNoNetworkDegraded = 3,
+    kFallbackNonzero = 4,
+};
+
+enum class AutoGidRetryAction {
+    kDoNotRetry = 0,
+    kRetryWithReprobedGid = 1,
+    kRetryWithObservedChange = 2,
+};
+
+struct AutoGidCandidate {
+    int gid_index = -1;
+    std::string gid;
+    uint32_t gid_type = 0;
+    bool has_network_device = false;
+    bool is_ipv4_mapped = false;
+    bool is_link_local_ipv6 = false;
+    bool is_overlay_network = false;
+    bool is_overlay_ipv4 = false;
+    bool is_null_gid = false;
+    bool query_succeeded = true;
+};
+
+struct AutoGidSelection {
+    int gid_index = -1;
+    std::string gid;
+    AutoGidCandidateClass candidate_class =
+        AutoGidCandidateClass::kFallbackNonzero;
+};
+
+struct AutoGidSelectionIdentity {
+    int gid_index = -1;
+    std::string gid;
+};
+
+inline const char* autoGidCandidateClassToString(
+    AutoGidCandidateClass candidate_class) {
+    switch (candidate_class) {
+        case AutoGidCandidateClass::kNetworkRoutable:
+            return "network-routable";
+        case AutoGidCandidateClass::kNoNetworkRoutable:
+            return "no-network-routable";
+        case AutoGidCandidateClass::kNetworkDegraded:
+            return "network-degraded";
+        case AutoGidCandidateClass::kNoNetworkDegraded:
+            return "no-network-degraded";
+        case AutoGidCandidateClass::kFallbackNonzero:
+            return "fallback-nonzero";
+    }
+    return "unknown";
+}
+
+inline std::optional<AutoGidCandidateClass> classifyAutoGidCandidate(
+    const AutoGidCandidate& candidate) {
+    if (!candidate.query_succeeded || candidate.gid_index < 0 ||
+        candidate.is_null_gid) {
+        return std::nullopt;
+    }
+
+    if (candidate.gid_type != IBV_GID_TYPE_ROCE_V2 &&
+        candidate.gid_type != IBV_GID_TYPE_IB) {
+        return std::nullopt;
+    }
+
+    const bool is_roce_v2 = candidate.gid_type == IBV_GID_TYPE_ROCE_V2;
+    const bool is_overlay =
+        is_roce_v2 && (candidate.is_overlay_network ||
+                       (candidate.is_ipv4_mapped && candidate.is_overlay_ipv4));
+    const bool is_link_local =
+        is_roce_v2 && !candidate.is_ipv4_mapped && candidate.is_link_local_ipv6;
+    const bool is_degraded = is_overlay || is_link_local;
+
+    if (candidate.has_network_device) {
+        return is_degraded ? AutoGidCandidateClass::kNetworkDegraded
+                           : AutoGidCandidateClass::kNetworkRoutable;
+    }
+
+    return is_degraded ? AutoGidCandidateClass::kNoNetworkDegraded
+                       : AutoGidCandidateClass::kNoNetworkRoutable;
+}
+
+inline int autoGidCandidateClassPriority(
+    AutoGidCandidateClass candidate_class) {
+    switch (candidate_class) {
+        case AutoGidCandidateClass::kNetworkRoutable:
+            return 0;
+        case AutoGidCandidateClass::kNoNetworkRoutable:
+            return 1;
+        case AutoGidCandidateClass::kNetworkDegraded:
+            return 2;
+        case AutoGidCandidateClass::kNoNetworkDegraded:
+            return 3;
+        case AutoGidCandidateClass::kFallbackNonzero:
+            return 4;
+    }
+    return 5;
+}
+
+inline std::vector<AutoGidSelection> rankAutoGidCandidates(
+    const std::vector<AutoGidCandidate>& candidates) {
+    std::vector<AutoGidSelection> ranked;
+    int first_query_success_fallback = -1;
+    std::string first_query_success_fallback_gid;
+
+    for (const auto& candidate : candidates) {
+        auto candidate_class = classifyAutoGidCandidate(candidate);
+        if (candidate_class.has_value()) {
+            ranked.push_back(
+                {candidate.gid_index, candidate.gid, *candidate_class});
+            continue;
+        }
+
+        if (candidate.query_succeeded && !candidate.is_null_gid &&
+            first_query_success_fallback < 0) {
+            first_query_success_fallback = candidate.gid_index;
+            first_query_success_fallback_gid = candidate.gid;
+        }
+    }
+
+    std::stable_sort(
+        ranked.begin(), ranked.end(),
+        [](const AutoGidSelection& lhs, const AutoGidSelection& rhs) {
+            int lhs_priority =
+                autoGidCandidateClassPriority(lhs.candidate_class);
+            int rhs_priority =
+                autoGidCandidateClassPriority(rhs.candidate_class);
+            if (lhs_priority != rhs_priority) {
+                return lhs_priority < rhs_priority;
+            }
+            return lhs.gid_index < rhs.gid_index;
+        });
+
+    if (first_query_success_fallback >= 0) {
+        ranked.push_back({first_query_success_fallback,
+                          first_query_success_fallback_gid,
+                          AutoGidCandidateClass::kFallbackNonzero});
+    }
+
+    return ranked;
+}
+
+inline std::optional<AutoGidSelection> selectBestAutoGidCandidate(
+    const std::vector<AutoGidCandidate>& candidates) {
+    auto ranked = rankAutoGidCandidates(candidates);
+    if (ranked.empty()) {
+        return std::nullopt;
+    }
+    return ranked.front();
+}
+
+inline bool shouldAttemptAutoGidHandshakeRetry(bool auto_gid_selection_enabled,
+                                               int retry_count, int max_retries,
+                                               bool failure_happened_at_rtr,
+                                               int sys_errno) {
+    return auto_gid_selection_enabled && retry_count < max_retries &&
+           failure_happened_at_rtr && sys_errno == EINVAL;
+}
+
+inline bool didAutoGidSelectionChange(int previous_gid_index,
+                                      std::string_view previous_gid,
+                                      int current_gid_index,
+                                      std::string_view current_gid) {
+    return previous_gid_index != current_gid_index ||
+           previous_gid != current_gid;
+}
+
+inline bool matchesAutoGidSelection(const AutoGidSelectionIdentity& identity,
+                                    const AutoGidSelection& selection) {
+    return identity.gid_index == selection.gid_index &&
+           identity.gid == selection.gid;
+}
+
+inline bool hasTriedAutoGidSelection(
+    const std::vector<AutoGidSelectionIdentity>& tried_selections,
+    const AutoGidSelection& selection) {
+    return std::any_of(tried_selections.begin(), tried_selections.end(),
+                       [&](const AutoGidSelectionIdentity& identity) {
+                           return matchesAutoGidSelection(identity, selection);
+                       });
+}
+
+inline std::optional<AutoGidSelection> reselectAutoGidCandidate(
+    const std::vector<AutoGidCandidate>& candidates, int current_gid_index,
+    std::string_view current_gid,
+    const std::vector<AutoGidSelectionIdentity>& tried_selections = {}) {
+    auto ranked = rankAutoGidCandidates(candidates);
+    for (const auto& selection : ranked) {
+        if (!didAutoGidSelectionChange(current_gid_index, current_gid,
+                                       selection.gid_index, selection.gid)) {
+            continue;
+        }
+        if (hasTriedAutoGidSelection(tried_selections, selection)) {
+            continue;
+        }
+        return selection;
+    }
+    return std::nullopt;
+}
+
+inline AutoGidRetryAction decideAutoGidRetryAction(
+    bool reprobe_changed, int previous_gid_index, std::string_view previous_gid,
+    int current_gid_index, std::string_view current_gid) {
+    if (reprobe_changed) {
+        return AutoGidRetryAction::kRetryWithReprobedGid;
+    }
+    if (didAutoGidSelectionChange(previous_gid_index, previous_gid,
+                                  current_gid_index, current_gid)) {
+        return AutoGidRetryAction::kRetryWithObservedChange;
+    }
+    return AutoGidRetryAction::kDoNotRetry;
+}
+
+}  // namespace mooncake
+
+#endif  // RDMA_GID_PROBE_H

mooncake-transfer-engine/include/transport/rdma_transport/rdma_transport.h

@@ -36,11 +36,13 @@ namespace mooncake {
 class RdmaContext;
 class RdmaEndPoint;
 class TransferMetadata;
+class RdmaTransportTestPeer;
 class WorkerPool;
 
 class RdmaTransport : public Transport {
     friend class RdmaContext;
     friend class RdmaEndPoint;
+    friend class RdmaTransportTestPeer;
     friend class WorkerPool;
 
    public:
@@ -101,6 +103,9 @@ class RdmaTransport : public Transport {
    private:
     int allocateLocalSegmentID();
 
+    int refreshLocalDeviceDesc(const std::string &device_name, uint16_t lid,
+                               const std::string &gid);
+
     int preTouchMemory(void *addr, size_t length);
 
    public:
@@ -134,6 +139,7 @@ class RdmaTransport : public Transport {
     // "192.168.0.y:port") for NIC path construction, while
     // local_server_name_ keeps the TCP-reachable address for P2P routing.
     std::string rdma_server_name_;
+    std::mutex local_desc_lock_;
 };
 
 using TransferRequest = Transport::TransferRequest;
@@ -144,4 +150,4 @@ using BatchID = Transport::BatchID;
 
 }  // namespace mooncake
 
-#endif  // RDMA_TRANSPORT_H_
+#endif  // RDMA_TRANSPORT_H_

mooncake-transfer-engine/src/config.cpp

@@ -224,6 +224,18 @@ void loadGlobalConfig(GlobalConfig& config) {
                 << "Ignore value from environment variable MC_RETRY_CNT";
     }
 
+    const char* auto_gid_max_retries_env =
+        std::getenv("MC_AUTO_GID_MAX_RETRIES");
+    if (auto_gid_max_retries_env) {
+        int val = atoi(auto_gid_max_retries_env);
+        if (val >= 0 && val <= 16) {
+            config.auto_gid_max_retries = val;
+        } else {
+            LOG(WARNING) << "Ignore value from environment variable "
+                            "MC_AUTO_GID_MAX_RETRIES";
+        }
+    }
+
     const char* disable_metacache = std::getenv("MC_DISABLE_METACACHE");
     if (disable_metacache) {
         config.metacache = false;

mooncake-transfer-engine/src/transport/rdma_transport/rdma_context.cpp

@@ -16,7 +16,9 @@
 
 #include <algorithm>
 #include <cerrno>
+#include <cstdio>
 #include <fcntl.h>
+#include <netinet/in.h>
 #include <sys/epoll.h>
 #include <unistd.h>
 
@@ -39,6 +41,7 @@
 #include <hsa/hsa_ext_amd.h>
 #endif
 #include "transport/rdma_transport/endpoint_store.h"
+#include "transport/rdma_transport/rdma_gid_probe.h"
 #include "transport/rdma_transport/rdma_endpoint.h"
 #include "transport/rdma_transport/rdma_transport.h"
 #include "transport/rdma_transport/worker_pool.h"
@@ -132,7 +135,7 @@ bool isKernelDmabufSupported() {
             LOG(WARNING)
                 << "Kernel lacks CONFIG_PCI_P2PDMA / CONFIG_DMABUF_MOVE_NOTIFY "
                 << "(p2pdma=" << found[0] << " move_notify=" << found[1]
-                << "); HIP dmabuf MR registration disabled, falling back to "
+                << "), HIP dmabuf MR registration disabled, falling back to "
                 << "ibv_reg_mr() (which requires an amdgpu peermem driver). "
                 << "Rebuild kernel with both options for GPU-direct RDMA.";
         }
@@ -141,6 +144,17 @@ bool isKernelDmabufSupported() {
     return supported;
 }
 #endif  // USE_HIP_DMABUF
+
+std::string gidBytesToString(const uint8_t *raw) {
+    std::string gid_str;
+    char buf[16] = {0};
+    const static size_t kGidLength = 16;
+    for (size_t i = 0; i < kGidLength; ++i) {
+        snprintf(buf, sizeof(buf), "%02x", raw[i]);
+        gid_str += i == 0 ? buf : std::string(":") + buf;
+    }
+    return gid_str;
+}
 }  // namespace
 
 RdmaContext::RdmaContext(RdmaTransport &engine, const std::string &device_name)
@@ -661,16 +675,16 @@ std::string RdmaContext::nicPath() const {
     return MakeNicPath(engine_.rdma_server_name_, device_name_);
 }
 
-std::string RdmaContext::gid() const {
-    std::string gid_str;
-    char buf[16] = {0};
-    const static size_t kGidLength = 16;
-    for (size_t i = 0; i < kGidLength; ++i) {
-        sprintf(buf, "%02x", gid_.raw[i]);
-        gid_str += i == 0 ? buf : std::string(":") + buf;
-    }
+std::string RdmaContext::gid() const { return gidSelection().gid; }
 
-    return gid_str;
+GidSelectionSnapshot RdmaContext::gidSelection() const {
+    std::lock_guard<std::mutex> guard(gid_lock_);
+    return {gidBytesToString(gid_.raw), gid_index_};
+}
+
+int RdmaContext::gidIndex() const {
+    std::lock_guard<std::mutex> guard(gid_lock_);
+    return gid_index_;
 }
 
 ibv_cq *RdmaContext::cq() {
@@ -711,16 +725,53 @@ static std::string readGidNdev(const std::string &device_name, uint8_t port,
     return ndev;
 }
 
-// Returns 1 if the GID has an associated network device, 0 otherwise.
-static int hasNetworkDevice(const std::string &device_name, uint8_t port,
-                            int gid_index) {
-    return !readGidNdev(device_name, port, gid_index).empty() ? 1 : 0;
+static inline bool isOverlayNetwork(const std::string &ndev) {
+    return ndev.find("flannel") == 0 || ndev.find("cni") == 0 ||
+           ndev.find("calico") == 0 || ndev.find("vxlan") == 0 ||
+           ndev.find("docker") == 0 || ndev == "tunl0";
+}
+
+static inline bool isOverlayIPv4(const struct in6_addr *addr) {
+    if (!ipv6_addr_v4mapped(addr)) return false;
+
+    uint32_t ipv4 = ntohl(addr->s6_addr32[3]);
+    uint8_t octet1 = (ipv4 >> 24) & 0xFF;
+    uint8_t octet2 = (ipv4 >> 16) & 0xFF;
+
+    if (octet1 == 10) return true;
+    if (octet1 == 172 && octet2 >= 16 && octet2 <= 31) return true;
+    if (octet1 == 100 && octet2 >= 64 && octet2 <= 127) return true;
+    return false;
+}
+
+static inline bool isLinkLocalIpv6(const struct in6_addr *addr) {
+    return IN6_IS_ADDR_LINKLOCAL(addr);
 }
 
 static const char *GidNetworkStateToString(GidNetworkState state) {
-    return (state == GidNetworkState::GID_WITH_NETWORK)
-               ? "with network device"
-               : "without network device";
+    switch (state) {
+        case GidNetworkState::GID_WITH_NETWORK:
+            return "with network device";
+        case GidNetworkState::GID_WITHOUT_NETWORK:
+            return "without network device";
+        case GidNetworkState::GID_NOT_FOUND:
+            return "not found";
+    }
+    return "unknown";
+}
+
+static GidNetworkState autoGidStateFromSelection(
+    const AutoGidSelection &selection) {
+    switch (selection.candidate_class) {
+        case AutoGidCandidateClass::kNetworkRoutable:
+        case AutoGidCandidateClass::kNetworkDegraded:
+            return GidNetworkState::GID_WITH_NETWORK;
+        case AutoGidCandidateClass::kNoNetworkRoutable:
+        case AutoGidCandidateClass::kNoNetworkDegraded:
+        case AutoGidCandidateClass::kFallbackNonzero:
+            return GidNetworkState::GID_WITHOUT_NETWORK;
+    }
+    return GidNetworkState::GID_NOT_FOUND;
 }
 
 GidNetworkState RdmaContext::findBestGidIndex(const std::string &device_name,
@@ -728,67 +779,167 @@ GidNetworkState RdmaContext::findBestGidIndex(const std::string &device_name,
                                               ibv_port_attr &port_attr,
                                               uint8_t port, int &gid_index) {
     gid_index = -1;
-    int i;
-    struct ibv_gid_entry gid_entry;
-    int fallback_ipv4_gid_without_network = -1;
-    int fallback_ipv6_gid_with_network = -1;
-    int fallback_ipv6_gid_without_network = -1;
-    GidNetworkState state = GidNetworkState::GID_NOT_FOUND;
+    std::vector<AutoGidCandidate> candidates;
+    candidates.reserve(port_attr.gid_tbl_len);
 
-    for (i = 0; i < port_attr.gid_tbl_len; i++) {
+    for (int i = 0; i < port_attr.gid_tbl_len; i++) {
+        AutoGidCandidate candidate;
+        candidate.gid_index = i;
+
+        struct ibv_gid_entry gid_entry;
         if (ibv_query_gid_ex(context, port, i, &gid_entry, 0)) {
-            // Reached end of valid GID indices
-            break;
-        }
-
-        if (gid_entry.gid_type != IBV_GID_TYPE_ROCE_V2 &&
-            gid_entry.gid_type != IBV_GID_TYPE_IB) {
+            candidate.query_succeeded = false;
+            candidates.push_back(candidate);
             continue;
         }
 
-        const bool is_ipv4_gid =
-            gid_entry.gid_type == IBV_GID_TYPE_ROCE_V2 &&
-            ipv6_addr_v4mapped((struct in6_addr *)gid_entry.gid.raw);
-        const bool has_network_device = hasNetworkDevice(device_name, port, i);
+        const auto *gid_addr =
+            reinterpret_cast<const struct in6_addr *>(gid_entry.gid.raw);
+        std::string ndev = readGidNdev(device_name, port, i);
+        candidate.gid = gidBytesToString(gid_entry.gid.raw);
+        candidate.gid_type = gid_entry.gid_type;
+        candidate.has_network_device = !ndev.empty();
+        candidate.is_ipv4_mapped = ipv6_addr_v4mapped(gid_addr);
+        candidate.is_link_local_ipv6 = isLinkLocalIpv6(gid_addr);
+        candidate.is_overlay_network =
+            candidate.has_network_device && isOverlayNetwork(ndev);
+        candidate.is_overlay_ipv4 =
+            candidate.is_ipv4_mapped && isOverlayIPv4(gid_addr);
+        candidate.is_null_gid = isNullGid(&gid_entry.gid);
+        candidates.push_back(candidate);
+    }
 
-        if (is_ipv4_gid) {
-            if (has_network_device) {
-                gid_index = i;
-                return GidNetworkState::GID_WITH_NETWORK;
-            }
-            if (fallback_ipv4_gid_without_network < 0) {
-                gid_index = i;
-                fallback_ipv4_gid_without_network = i;
-                state = GidNetworkState::GID_WITHOUT_NETWORK;
-            }
+    auto selection = selectBestAutoGidCandidate(candidates);
+    if (!selection.has_value()) {
+        return GidNetworkState::GID_NOT_FOUND;
+    }
+
+    gid_index = selection->gid_index;
+    VLOG(1) << "Selected auto GID[" << gid_index << "] on " << device_name
+            << " with class "
+            << autoGidCandidateClassToString(selection->candidate_class);
+    return autoGidStateFromSelection(*selection);
+}
+
+bool RdmaContext::reprobeAutoGid(
+    const GidSelectionSnapshot &expected_selection,
+    const std::vector<AutoGidSelectionIdentity> &tried_selections,
+    std::string *previous_gid, std::string *next_gid) {
+    std::lock_guard<std::mutex> reprobe_guard(gid_reprobe_lock_);
+    std::string current_gid_string;
+    std::string next_gid_string;
+    int current_gid_index = -1;
+    int next_gid_index = -1;
+    uint16_t current_lid = 0;
+    ibv_context *current_context = nullptr;
+    uint8_t current_port = 0;
+    AutoGidCandidateClass next_candidate_class =
+        AutoGidCandidateClass::kFallbackNonzero;
+    {
+        std::lock_guard<std::mutex> guard(gid_lock_);
+        if (!auto_gid_selection_enabled_ || !context_) {
+            return false;
+        }
+
+        current_gid_index = gid_index_;
+        current_gid_string = gidBytesToString(gid_.raw);
+        current_lid = lid_;
+        current_context = context_;
+        current_port = port_;
+        if (current_gid_index != expected_selection.gid_index ||
+            current_gid_string != expected_selection.gid) {
+            return false;
+        }
+    }
+
+    ibv_port_attr port_attr;
+    if (ibv_query_port(current_context, current_port, &port_attr)) {
+        PLOG(WARNING) << "Failed to reprobe port attributes on " << device_name_
+                      << "/" << static_cast<int>(current_port);
+        return false;
+    }
+
+    std::vector<AutoGidCandidate> candidates;
+    candidates.reserve(port_attr.gid_tbl_len);
+    for (int i = 0; i < port_attr.gid_tbl_len; ++i) {
+        AutoGidCandidate candidate;
+        candidate.gid_index = i;
+
+        struct ibv_gid_entry gid_entry;
+        if (ibv_query_gid_ex(current_context, current_port, i, &gid_entry, 0)) {
+            candidate.query_succeeded = false;
+            candidates.push_back(candidate);
             continue;
         }
 
-        if (has_network_device && fallback_ipv6_gid_with_network < 0) {
-            fallback_ipv6_gid_with_network = i;
-        }
+        const auto *gid_addr =
+            reinterpret_cast<const struct in6_addr *>(gid_entry.gid.raw);
+        std::string ndev = readGidNdev(device_name_, current_port, i);
+        candidate.gid = gidBytesToString(gid_entry.gid.raw);
+        candidate.gid_type = gid_entry.gid_type;
+        candidate.has_network_device = !ndev.empty();
+        candidate.is_ipv4_mapped = ipv6_addr_v4mapped(gid_addr);
+        candidate.is_link_local_ipv6 = isLinkLocalIpv6(gid_addr);
+        candidate.is_overlay_network =
+            candidate.has_network_device && isOverlayNetwork(ndev);
+        candidate.is_overlay_ipv4 =
+            candidate.is_ipv4_mapped && isOverlayIPv4(gid_addr);
+        candidate.is_null_gid = isNullGid(&gid_entry.gid);
+        candidates.push_back(candidate);
+    }
 
-        if (!has_network_device && fallback_ipv6_gid_without_network < 0) {
-            fallback_ipv6_gid_without_network = i;
+    auto selection = reselectAutoGidCandidate(
+        candidates, current_gid_index, current_gid_string, tried_selections);
+    if (!selection.has_value()) {
+        if (next_gid) {
+            *next_gid = current_gid_string;
+        }
+        return false;
+    }
+
+    ibv_gid new_gid = {};
+    if (ibv_query_gid(current_context, current_port, selection->gid_index,
+                      &new_gid)) {
+        return false;
+    }
+    if (isNullGid(&new_gid)) {
+        return false;
+    }
+
+    if (previous_gid) {
+        *previous_gid = current_gid_string;
+    }
+    next_gid_string = gidBytesToString(new_gid.raw);
+
+    int publish_ret = engine_.refreshLocalDeviceDesc(device_name_, current_lid,
+                                                     next_gid_string);
+    if (publish_ret) {
+        LOG(ERROR) << "Failed to refresh local device descriptor for "
+                   << device_name_ << ": " << publish_ret;
+        return false;
+    }
+
+    {
+        std::lock_guard<std::mutex> guard(gid_lock_);
+        gid_ = new_gid;
+        gid_index_ = selection->gid_index;
+        next_gid_index = selection->gid_index;
+        next_candidate_class = selection->candidate_class;
+        if (next_gid) {
+            *next_gid = next_gid_string;
         }
     }
 
-    if (fallback_ipv4_gid_without_network >= 0) {
-        gid_index = fallback_ipv4_gid_without_network;
-        return GidNetworkState::GID_WITHOUT_NETWORK;
+    if (next_gid_string.empty()) {
+        return false;
     }
 
-    if (fallback_ipv6_gid_with_network >= 0) {
-        gid_index = fallback_ipv6_gid_with_network;
-        return GidNetworkState::GID_WITH_NETWORK;
-    }
-
-    if (fallback_ipv6_gid_without_network >= 0) {
-        gid_index = fallback_ipv6_gid_without_network;
-        return GidNetworkState::GID_WITHOUT_NETWORK;
-    }
-
-    return state;
+    LOG(WARNING) << "Auto GID reprobe switched " << device_name_ << "/"
+                 << static_cast<int>(port_) << " from index "
+                 << current_gid_index << " (" << current_gid_string << ") to "
+                 << next_gid_index << " (" << next_gid_string << "), class "
+                 << autoGidCandidateClassToString(next_candidate_class);
+    return true;
 }
 
 int RdmaContext::openRdmaDevice(const std::string &device_name, uint8_t port,
@@ -950,6 +1101,7 @@ int RdmaContext::openRdmaDevice(const std::string &device_name, uint8_t port,
 
         updateGlobalConfig(device_attr);
         GidNetworkState gid_state;
+        auto_gid_selection_enabled_ = gid_index < 0;
         if (gid_index < 0) {
             int found_gid_index = -1;
             gid_state = findBestGidIndex(device_name, context, port_attr, port,
@@ -967,7 +1119,7 @@ int RdmaContext::openRdmaDevice(const std::string &device_name, uint8_t port,
             }
         } else {
             // Also check network state for user-specified GID
-            bool has_ndev = hasNetworkDevice(device_name, port, gid_index);
+            bool has_ndev = !readGidNdev(device_name, port, gid_index).empty();
             if (!has_ndev) {
                 LOG(WARNING) << "User-specified GID index " << gid_index
                              << " on " << device_name << "/" << port
@@ -1001,7 +1153,10 @@ int RdmaContext::openRdmaDevice(const std::string &device_name, uint8_t port,
         lid_ = attr.lid;
         active_mtu_ = attr.active_mtu;
         active_speed_ = attr.active_speed;
-        gid_index_ = gid_index;
+        {
+            std::lock_guard<std::mutex> guard(gid_lock_);
+            gid_index_ = gid_index;
+        }
 
         ibv_free_device_list(devices);
         return 0;

mooncake-transfer-engine/src/transport/rdma_transport/rdma_endpoint.cpp

@@ -17,6 +17,7 @@
 #include <glog/logging.h>
 
 #include <cassert>
+#include <cerrno>
 #include <cstddef>
 #include <chrono>
 #include <thread>
@@ -28,11 +29,40 @@
 
 #include "common.h"
 #include "config.h"
+#include "transport/rdma_transport/rdma_gid_probe.h"
 
 namespace mooncake {
-const static uint8_t MAX_HOP_LIMIT = 16;
-const static uint8_t TIMEOUT = 14;
-const static uint8_t RETRY_CNT = 7;
+constexpr uint8_t kMaxHopLimit = 16;
+constexpr uint8_t kTimeout = 14;
+constexpr uint8_t kRetryCount = 7;
+
+static GidSelectionSnapshot fillLocalHandshakeDesc(
+    RdmaContext &context, const std::string &peer_nic,
+    const std::vector<uint32_t> &qp_num,
+    RdmaEndPoint::HandShakeDesc &local_desc) {
+    auto gid_selection = context.gidSelection();
+    local_desc.local_nic_path = context.nicPath();
+    local_desc.local_lid = context.lid();
+    local_desc.local_gid = gid_selection.gid;
+    local_desc.peer_nic_path = peer_nic;
+    local_desc.qp_num = qp_num;
+    local_desc.reply_msg.clear();
+    return gid_selection;
+}
+
+static void rememberAutoGidSelection(
+    std::vector<AutoGidSelectionIdentity> &attempted_selections,
+    const GidSelectionSnapshot &selection) {
+    auto already_attempted =
+        std::any_of(attempted_selections.begin(), attempted_selections.end(),
+                    [&](const AutoGidSelectionIdentity &attempted) {
+                        return attempted.gid_index == selection.gid_index &&
+                               attempted.gid == selection.gid;
+                    });
+    if (!already_attempted) {
+        attempted_selections.push_back({selection.gid_index, selection.gid});
+    }
+}
 
 RdmaEndPoint::RdmaEndPoint(RdmaContext &context)
     : context_(context),
@@ -262,6 +292,8 @@ int RdmaEndPoint::setupConnectionsByActive() {
     HandShakeDesc local_desc, peer_desc;
     std::string peer_server_name, peer_nic_name;
     bool do_rpc = false;
+    int auto_gid_retry_count = 0;
+    std::vector<AutoGidSelectionIdentity> attempted_auto_gid_selections;
 
     {
         RWSpinlock::WriteGuard guard(lock_);
@@ -290,12 +322,6 @@ int RdmaEndPoint::setupConnectionsByActive() {
                 disconnectUnlocked();
                 return ERR_INVALID_ARGUMENT;
             }
-
-            local_desc.local_nic_path = context_.nicPath();
-            local_desc.local_lid = context_.lid();
-            local_desc.local_gid = context_.gid();
-            local_desc.peer_nic_path = peer_nic_path_;
-            local_desc.qp_num = qpNum();
         }
     }
 
@@ -332,96 +358,162 @@ int RdmaEndPoint::setupConnectionsByActive() {
         return connected() ? 0 : ERR_ENDPOINT;
     }
 
-    // Perform the RPC without holding the lock to avoid deadlock and allow
-    // "simultaneous open" handshake handling.
-    int rc = context_.engine().sendHandshake(peer_server_name, local_desc,
-                                             peer_desc);
+    for (;;) {
+        std::vector<uint32_t> local_qp_num;
+        {
+            RWSpinlock::ReadGuard guard(lock_);
+            local_qp_num = qpNum();
+        }
+        auto local_gid_selection = fillLocalHandshakeDesc(
+            context_, peer_nic_path_, local_qp_num, local_desc);
+        rememberAutoGidSelection(attempted_auto_gid_selections,
+                                 local_gid_selection);
+        peer_desc = HandShakeDesc();
 
-    // We should check the RPC return code before comparing `peer_qp_num_list_`
-    // with `peer_desc.qp_num`, since a failed RPC may result in an
-    // invalid `peer_desc.qp_num`.
-    //
-    // If the RPC is failed, even if the state is CONNECTED, (which means
-    // it is handled by setupConnectionsByPassive in another thread during the
-    // RPC, or "simultaneous open"), we should resetConnection to be safe.
-    // Because we're not sure whether the peer needs a connection
-    // re-establishment. (We don't know `peer_desc.qp_num`)
-    if (rc) {
-        RWSpinlock::WriteGuard write_guard(lock_);
-        resetConnection("handshake RPC failure");
-        return rc;
-    }
+        // Perform the RPC without holding the lock to avoid deadlock and allow
+        // "simultaneous open" handshake handling.
+        int rc = context_.engine().sendHandshake(peer_server_name, local_desc,
+                                                 peer_desc);
 
-    // Re-acquire lock after RPC to finalize state transition
-    RWSpinlock::WriteGuard guard(lock_);
-
-    // Handle simultaneous open: if the peer initiates a connection during our
-    // RPC and it is passively established in setupConnectionsByPassive, simply
-    // reuse the existing endpoint.
-    if (connected()) {
-        if (peer_qp_num_list_ == peer_desc.qp_num) {
-            LOG(INFO) << "Received same peer QP numbers, reusing connection.";
-            return 0;
+        // We should check the RPC return code before comparing
+        // `peer_qp_num_list_` with `peer_desc.qp_num`, since a failed RPC may
+        // result in an invalid `peer_desc.qp_num`.
+        //
+        // If the RPC is failed, even if the state is CONNECTED, (which means
+        // it is handled by setupConnectionsByPassive in another thread during
+        // the RPC, or "simultaneous open"), we should resetConnection to be
+        // safe. Because we're not sure whether the peer needs a connection
+        // re-establishment. (We don't know `peer_desc.qp_num`)
+        if (rc) {
+            RWSpinlock::WriteGuard write_guard(lock_);
+            resetConnection("handshake RPC failure");
+            return rc;
         }
 
-        // This mismatch scenario should be rare. It may occur when a peer
-        // first sends us an Active RPC and establishes a connection,
-        // then restarts, and eventually accepts and responds to our
-        // Active RPC.
-        LOG(WARNING) << "Peer QP list mismatch on connected endpoint, "
-                        "re-establishing connection: "
-                     << toString();
+        bool retry_with_new_gid = false;
+        {
+            // Re-acquire lock after RPC to finalize state transition
+            RWSpinlock::WriteGuard guard(lock_);
 
-        int ret = resetConnection("re-establishing connection (active)");
-        if (ret) return ret;
-    }
-
-    if (!peer_desc.reply_msg.empty()) {
-        LOG(ERROR) << "Rejected handshake request by peer "
-                   << local_desc.peer_nic_path;
-        disconnectUnlocked();
-        return ERR_REJECT_HANDSHAKE;
-    }
-
-    if (peer_desc.local_nic_path != peer_nic_path_ ||
-        peer_desc.peer_nic_path != local_desc.local_nic_path) {
-        LOG(ERROR) << "Invalid argument: received packet mismatch, "
-                      "local.local_nic_path: "
-                   << local_desc.local_nic_path
-                   << ", local.peer_nic_path: " << local_desc.peer_nic_path
-                   << ", peer.local_nic_path: " << peer_desc.local_nic_path
-                   << ", peer.peer_nic_path: " << peer_desc.peer_nic_path;
-        disconnectUnlocked();
-        return ERR_REJECT_HANDSHAKE;
-    }
-
-    if (!peer_desc.local_gid.empty()) {
-        int ret = doSetupConnection(peer_desc.local_gid, peer_desc.local_lid,
-                                    peer_desc.qp_num);
-        if (ret != 0) {
-            resetConnection("failed connection setup (active)");
-        }
-        return ret;
-    } else {
-        auto segment_desc =
-            context_.engine().meta()->getSegmentDescByName(peer_server_name);
-        if (segment_desc) {
-            for (auto &nic : segment_desc->devices) {
-                if (nic.name == peer_nic_name) {
-                    int ret =
-                        doSetupConnection(nic.gid, nic.lid, peer_desc.qp_num);
-                    if (ret != 0) {
-                        resetConnection("failed connection setup (active)");
-                    }
-                    return ret;
+            // Handle simultaneous open: if the peer initiates a connection
+            // during our RPC and it is passively established in
+            // setupConnectionsByPassive, simply reuse the existing endpoint.
+            if (connected()) {
+                if (peer_qp_num_list_ == peer_desc.qp_num) {
+                    LOG(INFO)
+                        << "Received same peer QP numbers, reusing connection.";
+                    return 0;
                 }
+
+                // This mismatch scenario should be rare. It may occur when a
+                // peer first sends us an Active RPC and establishes a
+                // connection, then restarts, and eventually accepts and
+                // responds to our Active RPC.
+                LOG(WARNING) << "Peer QP list mismatch on connected endpoint, "
+                                "re-establishing connection: "
+                             << toString();
+
+                int ret =
+                    resetConnection("re-establishing connection (active)");
+                if (ret) return ret;
+            }
+
+            if (!peer_desc.reply_msg.empty()) {
+                LOG(ERROR) << "Rejected handshake request by peer "
+                           << local_desc.peer_nic_path;
+                disconnectUnlocked();
+                return ERR_REJECT_HANDSHAKE;
+            }
+
+            if (peer_desc.local_nic_path != peer_nic_path_ ||
+                peer_desc.peer_nic_path != local_desc.local_nic_path) {
+                LOG(ERROR) << "Invalid argument: received packet mismatch, "
+                              "local.local_nic_path: "
+                           << local_desc.local_nic_path
+                           << ", local.peer_nic_path: "
+                           << local_desc.peer_nic_path
+                           << ", peer.local_nic_path: "
+                           << peer_desc.local_nic_path
+                           << ", peer.peer_nic_path: "
+                           << peer_desc.peer_nic_path;
+                disconnectUnlocked();
+                return ERR_REJECT_HANDSHAKE;
+            }
+
+            int ret = ERR_DEVICE_NOT_FOUND;
+            std::string failure_message;
+            SetupConnectionFailureInfo failure_info;
+            if (!peer_desc.local_gid.empty()) {
+                ret = doSetupConnection(peer_desc.local_gid,
+                                        peer_desc.local_lid, peer_desc.qp_num,
+                                        &failure_message, &failure_info);
+            } else {
+                auto segment_desc =
+                    context_.engine().meta()->getSegmentDescByName(
+                        peer_server_name);
+                if (segment_desc) {
+                    for (auto &nic : segment_desc->devices) {
+                        if (nic.name == peer_nic_name) {
+                            ret = doSetupConnection(
+                                nic.gid, nic.lid, peer_desc.qp_num,
+                                &failure_message, &failure_info);
+                            break;
+                        }
+                    }
+                }
+            }
+
+            if (ret == 0) {
+                return 0;
+            }
+
+            if (shouldAttemptAutoGidHandshakeRetry(
+                    context_.autoGidSelectionEnabled(), auto_gid_retry_count,
+                    globalConfig().auto_gid_max_retries,
+                    failure_info.stage == SetupConnectionFailureStage::kRtr,
+                    failure_info.sys_errno)) {
+                std::string previous_gid;
+                std::string next_gid;
+                bool reprobe_changed = context_.reprobeAutoGid(
+                    local_gid_selection, attempted_auto_gid_selections,
+                    &previous_gid, &next_gid);
+                auto current_gid_selection = context_.gidSelection();
+                auto retry_action = decideAutoGidRetryAction(
+                    reprobe_changed, local_gid_selection.gid_index,
+                    local_gid_selection.gid, current_gid_selection.gid_index,
+                    current_gid_selection.gid);
+                if (retry_action != AutoGidRetryAction::kDoNotRetry) {
+                    int reset_ret = resetConnection(
+                        retry_action ==
+                                AutoGidRetryAction::kRetryWithReprobedGid
+                            ? "retry after auto GID reprobe (active)"
+                            : "retry with externally reprobed GID (active)");
+                    if (reset_ret) return reset_ret;
+                    status_.store(CONNECTING, std::memory_order_relaxed);
+                    ++auto_gid_retry_count;
+                    retry_with_new_gid = true;
+                    LOG(WARNING)
+                        << "Retry active handshake with updated local GID on "
+                        << context_.deviceName() << ": "
+                        << local_gid_selection.gid << " -> "
+                        << current_gid_selection.gid << " (attempt "
+                        << auto_gid_retry_count << "/"
+                        << globalConfig().auto_gid_max_retries << ")";
+                }
+            }
+
+            if (!retry_with_new_gid) {
+                if (ret == ERR_DEVICE_NOT_FOUND) {
+                    LOG(ERROR) << "Peer NIC " << peer_nic_name
+                               << " not found in " << peer_server_name;
+                    disconnectUnlocked();
+                } else {
+                    resetConnection("failed connection setup (active)");
+                }
+                return ret;
             }
         }
     }
-    LOG(ERROR) << "Peer NIC " << peer_nic_name << " not found in "
-               << peer_server_name;
-    disconnectUnlocked();
-    return ERR_DEVICE_NOT_FOUND;
 }
 
 int RdmaEndPoint::setupConnectionsByPassive(const HandShakeDesc &peer_desc,
@@ -430,11 +522,8 @@ int RdmaEndPoint::setupConnectionsByPassive(const HandShakeDesc &peer_desc,
     if (connected()) {
         // If already connected with the same peer QP info, return success
         if (peer_qp_num_list_ == peer_desc.qp_num) {
-            local_desc.local_nic_path = context_.nicPath();
-            local_desc.local_lid = context_.lid();
-            local_desc.local_gid = context_.gid();
-            local_desc.peer_nic_path = peer_nic_path_;
-            local_desc.qp_num = qpNum();
+            fillLocalHandshakeDesc(context_, peer_nic_path_, qpNum(),
+                                   local_desc);
             LOG(INFO) << "Received same peer QP numbers, reusing connection.";
             return 0;
         }
@@ -473,38 +562,82 @@ int RdmaEndPoint::setupConnectionsByPassive(const HandShakeDesc &peer_desc,
         return ERR_INVALID_ARGUMENT;
     }
 
-    local_desc.local_nic_path = context_.nicPath();
-    local_desc.local_lid = context_.lid();
-    local_desc.local_gid = context_.gid();
-    local_desc.peer_nic_path = peer_nic_path_;
-    local_desc.qp_num = qpNum();
+    status_.store(CONNECTING, std::memory_order_relaxed);
+
+    auto attempt_setup_with_peer = [&](const std::string &peer_gid,
+                                       uint16_t peer_lid) -> int {
+        int auto_gid_retry_count = 0;
+        std::vector<AutoGidSelectionIdentity> attempted_auto_gid_selections;
+        for (;;) {
+            auto local_gid_selection = fillLocalHandshakeDesc(
+                context_, peer_nic_path_, qpNum(), local_desc);
+            rememberAutoGidSelection(attempted_auto_gid_selections,
+                                     local_gid_selection);
+
+            SetupConnectionFailureInfo failure_info;
+            int ret = doSetupConnection(peer_gid, peer_lid, peer_desc.qp_num,
+                                        &local_desc.reply_msg, &failure_info);
+            if (ret == 0) {
+                return 0;
+            }
+
+            if (!shouldAttemptAutoGidHandshakeRetry(
+                    context_.autoGidSelectionEnabled(), auto_gid_retry_count,
+                    globalConfig().auto_gid_max_retries,
+                    failure_info.stage == SetupConnectionFailureStage::kRtr,
+                    failure_info.sys_errno)) {
+                resetConnection("failed connection setup (passive)");
+                return ret;
+            }
+
+            std::string previous_gid;
+            std::string next_gid;
+            bool reprobe_changed = context_.reprobeAutoGid(
+                local_gid_selection, attempted_auto_gid_selections,
+                &previous_gid, &next_gid);
+            auto current_gid_selection = context_.gidSelection();
+            auto retry_action = decideAutoGidRetryAction(
+                reprobe_changed, local_gid_selection.gid_index,
+                local_gid_selection.gid, current_gid_selection.gid_index,
+                current_gid_selection.gid);
+            if (retry_action == AutoGidRetryAction::kDoNotRetry) {
+                resetConnection("failed connection setup (passive)");
+                return ret;
+            }
+
+            int reset_ret = resetConnection(
+                retry_action == AutoGidRetryAction::kRetryWithReprobedGid
+                    ? "retry after auto GID reprobe (passive)"
+                    : "retry with externally reprobed GID (passive)");
+            if (reset_ret) return reset_ret;
+            status_.store(CONNECTING, std::memory_order_relaxed);
+            ++auto_gid_retry_count;
+            LOG(WARNING) << "Retry passive handshake with updated local GID on "
+                         << context_.deviceName() << ": "
+                         << local_gid_selection.gid << " -> "
+                         << current_gid_selection.gid << " (attempt "
+                         << auto_gid_retry_count << "/"
+                         << globalConfig().auto_gid_max_retries << ")";
+        }
+    };
 
     if (!peer_desc.local_gid.empty()) {
-        int ret = doSetupConnection(peer_desc.local_gid, peer_desc.local_lid,
-                                    peer_desc.qp_num, &local_desc.reply_msg);
-        if (ret != 0) {
-            resetConnection("failed connection setup (passive)");
-        }
-        return ret;
+        return attempt_setup_with_peer(peer_desc.local_gid,
+                                       peer_desc.local_lid);
     } else {
         auto segment_desc =
             context_.engine().meta()->getSegmentDescByName(peer_server_name);
         if (segment_desc) {
             for (auto &nic : segment_desc->devices) {
                 if (nic.name == peer_nic_name) {
-                    int ret =
-                        doSetupConnection(nic.gid, nic.lid, peer_desc.qp_num,
-                                          &local_desc.reply_msg);
-                    if (ret != 0) {
-                        resetConnection("failed connection setup (passive)");
-                    }
-                    return ret;
+                    return attempt_setup_with_peer(nic.gid, nic.lid);
                 }
             }
         }
     }
     local_desc.reply_msg =
         "Peer nic not found in that server: " + peer_nic_path_;
+    status_.store(UNCONNECTED, std::memory_order_relaxed);
     LOG(ERROR) << local_desc.reply_msg;
     return ERR_DEVICE_NOT_FOUND;
 }
@@ -714,13 +847,18 @@ static int parseGidString(const std::string &gid_str, ibv_gid &gid_out) {
 int RdmaEndPoint::doSetupConnection(const std::string &peer_gid,
                                     uint16_t peer_lid,
                                     std::vector<uint32_t> peer_qp_num_list,
-                                    std::string *reply_msg) {
+                                    std::string *reply_msg,
+                                    SetupConnectionFailureInfo *failure_info) {
     if (qp_list_.size() != peer_qp_num_list.size()) {
         std::string message =
             "QP count mismatch in peer and local endpoints, check "
             "MC_MAX_EP_PER_CTX";
         LOG(ERROR) << "[Handshake] " << message;
         if (reply_msg) *reply_msg = message;
+        if (failure_info) {
+            failure_info->stage = SetupConnectionFailureStage::kPeerValidation;
+            failure_info->sys_errno = 0;
+        }
         return ERR_INVALID_ARGUMENT;
     }
 
@@ -731,12 +869,18 @@ int RdmaEndPoint::doSetupConnection(const std::string &peer_gid,
         std::string message = "Invalid peer GID: " + peer_gid;
         LOG(ERROR) << "[Handshake] " << message;
         if (reply_msg) *reply_msg = message;
+        if (failure_info) {
+            failure_info->stage = SetupConnectionFailureStage::kPeerValidation;
+            failure_info->sys_errno = 0;
+        }
         return ret;
     }
 
+    int local_gid_index = context_.gidIndex();
     for (int qp_index = 0; qp_index < (int)qp_list_.size(); ++qp_index) {
         int ret = doSetupConnection(qp_index, peer_gid_raw, peer_lid,
-                                    peer_qp_num_list[qp_index], reply_msg);
+                                    peer_qp_num_list[qp_index], local_gid_index,
+                                    reply_msg, failure_info);
         if (ret) return ret;
     }
 
@@ -747,7 +891,8 @@ int RdmaEndPoint::doSetupConnection(const std::string &peer_gid,
 
 int RdmaEndPoint::doSetupConnection(int qp_index, const ibv_gid &peer_gid,
                                     uint16_t peer_lid, uint32_t peer_qp_num,
-                                    std::string *reply_msg) {
+                                    int local_gid_index, std::string *reply_msg,
+                                    SetupConnectionFailureInfo *failure_info) {
     if (qp_index < 0 || qp_index > (int)qp_list_.size())
         return ERR_INVALID_ARGUMENT;
     auto &qp = qp_list_[qp_index];
@@ -761,6 +906,10 @@ int RdmaEndPoint::doSetupConnection(int qp_index, const ibv_gid &peer_gid,
         std::string message = "Failed to modify QP to RESET";
         PLOG(ERROR) << "[Handshake] " << message;
         if (reply_msg) *reply_msg = message + ": " + strerror(errno);
+        if (failure_info) {
+            failure_info->stage = SetupConnectionFailureStage::kReset;
+            failure_info->sys_errno = errno;
+        }
         return ERR_ENDPOINT;
     }
 
@@ -779,6 +928,10 @@ int RdmaEndPoint::doSetupConnection(int qp_index, const ibv_gid &peer_gid,
             "Failed to modify QP to INIT, check local context port num";
         PLOG(ERROR) << "[Handshake] " << message;
         if (reply_msg) *reply_msg = message + ": " + strerror(errno);
+        if (failure_info) {
+            failure_info->stage = SetupConnectionFailureStage::kInit;
+            failure_info->sys_errno = errno;
+        }
         return ERR_ENDPOINT;
     }
 
@@ -790,8 +943,8 @@ int RdmaEndPoint::doSetupConnection(int qp_index, const ibv_gid &peer_gid,
         attr.path_mtu = globalConfig().mtu_length;
     attr.ah_attr.grh.dgid = peer_gid;
     // TODO gidIndex and portNum must fetch from REMOTE
-    attr.ah_attr.grh.sgid_index = context_.gidIndex();
-    attr.ah_attr.grh.hop_limit = MAX_HOP_LIMIT;
+    attr.ah_attr.grh.sgid_index = local_gid_index;
+    attr.ah_attr.grh.hop_limit = kMaxHopLimit;
     // Set traffic class if configured (-1 means use default)
     if (globalConfig().ib_traffic_class >= 0) {
         attr.ah_attr.grh.traffic_class =
@@ -816,14 +969,18 @@ int RdmaEndPoint::doSetupConnection(int qp_index, const ibv_gid &peer_gid,
             "Failed to modify QP to RTR, check mtu, gid, peer lid, peer qp num";
         PLOG(ERROR) << "[Handshake] " << message;
         if (reply_msg) *reply_msg = message + ": " + strerror(errno);
+        if (failure_info) {
+            failure_info->stage = SetupConnectionFailureStage::kRtr;
+            failure_info->sys_errno = errno;
+        }
         return ERR_ENDPOINT;
     }
 
     // RTR -> RTS
     memset(&attr, 0, sizeof(attr));
     attr.qp_state = IBV_QPS_RTS;
-    attr.timeout = TIMEOUT;
-    attr.retry_cnt = RETRY_CNT;
+    attr.timeout = kTimeout;
+    attr.retry_cnt = kRetryCount;
     attr.rnr_retry = 7;  // or 7,RNR error
     attr.sq_psn = 0;
     attr.max_rd_atomic = 16;
@@ -835,6 +992,10 @@ int RdmaEndPoint::doSetupConnection(int qp_index, const ibv_gid &peer_gid,
         std::string message = "Failed to modify QP to RTS";
         PLOG(ERROR) << "[Handshake] " << message;
         if (reply_msg) *reply_msg = message + ": " + strerror(errno);
+        if (failure_info) {
+            failure_info->stage = SetupConnectionFailureStage::kRts;
+            failure_info->sys_errno = errno;
+        }
         return ERR_ENDPOINT;
     }
 
@@ -900,4 +1061,5 @@ int RdmaEndPoint::doSetupConnection(int qp_index, const ibv_gid &peer_gid,
 
     return 0;
 }
+
 }  // namespace mooncake

mooncake-transfer-engine/src/transport/rdma_transport/rdma_transport.cpp

@@ -375,7 +375,7 @@ int RdmaTransport::unregisterLocalMemoryInternal(void *addr,
 }
 
 int RdmaTransport::allocateLocalSegmentID() {
-    auto desc = metadata_->getSegmentDesc(local_server_name_);
+    auto desc = metadata_->getSegmentDescByID(LOCAL_SEGMENT_ID);
     if (!desc) desc = std::make_shared<SegmentDesc>();
     desc->name = local_server_name_;
     // Store RDMA server name for dual-NIC setups; when it differs from
@@ -402,6 +402,34 @@ int RdmaTransport::allocateLocalSegmentID() {
     return 0;
 }
 
+int RdmaTransport::refreshLocalDeviceDesc(const std::string &device_name,
+                                          uint16_t lid,
+                                          const std::string &gid) {
+    std::lock_guard<std::mutex> guard(local_desc_lock_);
+    auto original_desc = metadata_->getSegmentDescByID(LOCAL_SEGMENT_ID);
+    if (!original_desc) {
+        return ERR_ADDRESS_NOT_REGISTERED;
+    }
+
+    auto updated_desc = std::make_shared<SegmentDesc>(*original_desc);
+    for (auto &device : updated_desc->devices) {
+        if (device.name != device_name) continue;
+        device.lid = lid;
+        device.gid = gid;
+        metadata_->addLocalSegment(LOCAL_SEGMENT_ID, local_server_name_,
+                                   std::move(updated_desc));
+        int ret = metadata_->updateLocalSegmentDesc();
+        if (ret) {
+            auto rollback_desc = original_desc;
+            metadata_->addLocalSegment(LOCAL_SEGMENT_ID, local_server_name_,
+                                       std::move(rollback_desc));
+        }
+        return ret;
+    }
+
+    return ERR_DEVICE_NOT_FOUND;
+}
+
 int RdmaTransport::registerLocalMemoryBatch(
     const std::vector<RdmaTransport::BufferEntry> &buffer_list,
     const std::string &location) {

mooncake-transfer-engine/tent/src/transport/rdma/rdma_transport.cpp

@@ -21,9 +21,11 @@
 #include <sys/time.h>
 
 #include <cassert>
+#include <cerrno>
 #include <cstddef>
 #include <cstdlib>
 #include <future>
+#include <limits>
 #include <set>
 #include <sstream>
 
@@ -47,6 +49,39 @@
 
 namespace mooncake {
 namespace tent {
+
+namespace {
+
+uint16_t getRdmaBindDefaultPort(const Config& config) {
+    constexpr const char* kKey = "rpc_server_port";
+    if (!config.contains(kKey)) return 0;
+
+    json raw_value = config.get<json>(kKey, json());
+    if (raw_value.is_number_integer() || raw_value.is_number_unsigned()) {
+        long long value = raw_value.get<long long>();
+        if (value >= 0 && value <= static_cast<long long>(
+                                       std::numeric_limits<uint16_t>::max())) {
+            return static_cast<uint16_t>(value);
+        }
+        return 0;
+    }
+
+    if (raw_value.is_string()) {
+        const std::string string_value = raw_value.get<std::string>();
+        char* end = nullptr;
+        errno = 0;
+        unsigned long value = std::strtoul(string_value.c_str(), &end, 10);
+        if (errno == 0 && end != string_value.c_str() && *end == '\0' &&
+            value <= std::numeric_limits<uint16_t>::max()) {
+            return static_cast<uint16_t>(value);
+        }
+    }
+
+    return 0;
+}
+
+}  // namespace
+
 static Status configureLaneCount(std::shared_ptr<Config> conf,
                                  std::shared_ptr<RdmaParams> params) {
     constexpr int kUnset = -1;
@@ -217,12 +252,12 @@ Status RdmaTransport::install(std::string& local_segment_name,
     local_topology_ = local_topology;
 
     // In dual-NIC environments (e.g. separate TCP and RDMA interfaces),
-    // transports/rdma/bind_address allows NIC paths to use an RDMA-reachable
-    // IP while local_segment_name_ keeps the TCP-reachable address for P2P.
+    // transports/rdma/bind_address allows NIC paths to use an
+    // RDMA-reachable IP while local_segment_name_ keeps the
+    // TCP-reachable address for P2P.
     const auto rdma_bind_addr = conf_->get("transports/rdma/bind_address", "");
     if (!rdma_bind_addr.empty()) {
-        const auto default_port =
-            conf_->get("rpc_server_port", static_cast<uint16_t>(0));
+        const uint16_t default_port = getRdmaBindDefaultPort(*conf_);
         auto [host_name, port] =
             parseHostNameWithPort(local_segment_name, default_port);
         rdma_server_name_ = rdma_bind_addr + ":" + std::to_string(port);

mooncake-transfer-engine/tests/CMakeLists.txt

@@ -36,9 +36,16 @@ target_link_libraries(rdma_loopback_test PUBLIC transfer_engine gtest gtest_main
 # add_test(NAME rdma_loopback_test COMMAND rdma_loopback_test)
 
 # This test verifies endpoint re-establishment in RDMATransport.
-# Intended for manual testing only.
 add_executable(rdma_endpoint_reestablish_test ${WORKSPACE}/rdma_endpoint_reestablish_test.cpp)
 target_link_libraries(rdma_endpoint_reestablish_test PUBLIC transfer_engine gtest gtest_main )
+if (UNIX AND NOT APPLE)
+    target_link_options(rdma_endpoint_reestablish_test PRIVATE
+        "-Wl,--wrap=ibv_modify_qp"
+        "-Wl,--wrap=ibv_query_gid"
+        "-Wl,--wrap=_ibv_query_gid_ex")
+endif()
+add_test(NAME rdma_endpoint_reestablish_test COMMAND rdma_endpoint_reestablish_test)
+set_tests_properties(rdma_endpoint_reestablish_test PROPERTIES LABELS "rdma")
 
 if (USE_CXL)
     add_executable(cxl_transport_test ${WORKSPACE}/cxl_transport_test.cpp)
@@ -98,6 +105,18 @@ add_executable(transfer_metadata_test ${WORKSPACE}/transfer_metadata_test.cpp)
 target_link_libraries(transfer_metadata_test PUBLIC transfer_engine gtest gtest_main)
 add_test(NAME transfer_metadata_test COMMAND transfer_metadata_test)
 
+add_executable(config_test ${WORKSPACE}/config_test.cpp)
+target_link_libraries(config_test PUBLIC transfer_engine gtest gtest_main)
+add_test(NAME config_test COMMAND config_test)
+
+add_executable(rdma_gid_probe_test ${WORKSPACE}/rdma_gid_probe_test.cpp)
+target_link_libraries(rdma_gid_probe_test PUBLIC transfer_engine gtest gtest_main)
+add_test(NAME rdma_gid_probe_test COMMAND rdma_gid_probe_test)
+
+add_executable(rdma_context_reprobe_test ${WORKSPACE}/rdma_context_reprobe_test.cpp)
+target_link_libraries(rdma_context_reprobe_test PUBLIC transfer_engine gtest gtest_main)
+add_test(NAME rdma_context_reprobe_test COMMAND rdma_context_reprobe_test)
+
 add_executable(topology_test ${WORKSPACE}/topology_test.cpp)
 target_link_libraries(topology_test PUBLIC transfer_engine gtest gtest_main)
 add_test(NAME topology_test COMMAND topology_test)

mooncake-transfer-engine/tests/config_test.cpp

@@ -23,7 +23,10 @@ namespace {
 
 class PkeyIndexEnvTest : public ::testing::Test {
    protected:
-    void TearDown() override { ::unsetenv("MC_PKEY_INDEX"); }
+    void TearDown() override {
+        ::unsetenv("MC_PKEY_INDEX");
+        ::unsetenv("MC_AUTO_GID_MAX_RETRIES");
+    }
 };
 
 TEST_F(PkeyIndexEnvTest, DefaultIsZeroWhenUnset) {
@@ -79,5 +82,28 @@ TEST_F(PkeyIndexEnvTest, EmptyStringKeepsDefault) {
     EXPECT_EQ(config.pkey_index, 4);
 }
 
+TEST_F(PkeyIndexEnvTest, AutoGidRetriesDefaultsToTwoWhenUnset) {
+    ::unsetenv("MC_AUTO_GID_MAX_RETRIES");
+    GlobalConfig config;
+    config.auto_gid_max_retries = 2;
+    loadGlobalConfig(config);
+    EXPECT_EQ(config.auto_gid_max_retries, 2);
+}
+
+TEST_F(PkeyIndexEnvTest, AutoGidRetriesAcceptsValidOverride) {
+    ASSERT_EQ(::setenv("MC_AUTO_GID_MAX_RETRIES", "0", 1), 0);
+    GlobalConfig config;
+    loadGlobalConfig(config);
+    EXPECT_EQ(config.auto_gid_max_retries, 0);
+}
+
+TEST_F(PkeyIndexEnvTest, AutoGidRetriesRejectsOutOfRangeOverride) {
+    ASSERT_EQ(::setenv("MC_AUTO_GID_MAX_RETRIES", "99", 1), 0);
+    GlobalConfig config;
+    config.auto_gid_max_retries = 5;
+    loadGlobalConfig(config);
+    EXPECT_EQ(config.auto_gid_max_retries, 5);
+}
+
 }  // namespace
 }  // namespace mooncake

mooncake-transfer-engine/tests/rdma_context_reprobe_test.cpp

@@ -0,0 +1,144 @@
+// Copyright 2026 KVCache.AI
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <array>
+#include <cstring>
+#include <memory>
+#include <string>
+
+#include "common.h"
+#include "error.h"
+#include "transfer_metadata.h"
+#include "transport/rdma_transport/rdma_context.h"
+#include "transport/rdma_transport/rdma_transport.h"
+
+#if defined(__has_feature)
+#define MC_HAS_FEATURE(x) __has_feature(x)
+#else
+#define MC_HAS_FEATURE(x) 0
+#endif
+#if defined(__SANITIZE_ADDRESS__) || MC_HAS_FEATURE(address_sanitizer)
+#include <sanitizer/lsan_interface.h>
+#define MC_LSAN_IGNORE_OBJECT(p) __lsan_ignore_object(p)
+#else
+#define MC_LSAN_IGNORE_OBJECT(p) ((void)(p))
+#endif
+
+using namespace mooncake;
+
+namespace mooncake {
+
+class RdmaTransportTestPeer {
+   public:
+    static void bindMetadata(RdmaTransport &transport,
+                             std::shared_ptr<TransferMetadata> metadata,
+                             std::string local_server_name) {
+        transport.metadata_ = std::move(metadata);
+        transport.local_server_name_ = std::move(local_server_name);
+    }
+};
+
+class RdmaContextTestPeer {
+   public:
+    static void seedAutoGidState(RdmaContext &context, ibv_context *verbs_ctx,
+                                 uint8_t port, uint16_t lid, const ibv_gid &gid,
+                                 int gid_index) {
+        context.context_ = verbs_ctx;
+        context.port_ = port;
+        context.lid_ = lid;
+        context.gid_ = gid;
+        context.gid_index_ = gid_index;
+        context.auto_gid_selection_enabled_ = true;
+    }
+
+    static void disableContextForTeardown(RdmaContext &context) {
+        context.context_ = nullptr;
+    }
+};
+
+}  // namespace mooncake
+
+namespace {
+
+ibv_gid makeGid(const std::array<uint8_t, 16> &bytes) {
+    ibv_gid gid = {};
+    std::memcpy(gid.raw, bytes.data(), bytes.size());
+    return gid;
+}
+
+std::string formatGid(const std::array<uint8_t, 16> &bytes) {
+    std::string gid;
+    char buf[4] = {0};
+    for (size_t i = 0; i < bytes.size(); ++i) {
+        std::snprintf(buf, sizeof(buf), "%02x", bytes[i]);
+        gid += i == 0 ? buf : std::string(":") + buf;
+    }
+    return gid;
+}
+
+constexpr std::array<uint8_t, 16> kCurrentGid = {
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11};
+class RdmaContextReprobeTest : public ::testing::Test {
+   protected:
+    void SetUp() override {
+        transport_ = new RdmaTransport();
+        MC_LSAN_IGNORE_OBJECT(transport_);
+        metadata_ = std::make_shared<TransferMetadata>(P2PHANDSHAKE);
+        RdmaTransportTestPeer::bindMetadata(*transport_, metadata_,
+                                            "local-rdma-segment");
+
+        auto local_desc = std::make_shared<TransferMetadata::SegmentDesc>();
+        local_desc->name = "local-rdma-segment";
+        local_desc->protocol = "rdma";
+        local_desc->devices.push_back(
+            {"synthetic0", 23, formatGid(kCurrentGid), ""});
+        ASSERT_EQ(
+            metadata_->addLocalSegment(LOCAL_SEGMENT_ID, "local-rdma-segment",
+                                       std::move(local_desc)),
+            0);
+
+        context_ = new RdmaContext(*transport_, "synthetic0");
+        MC_LSAN_IGNORE_OBJECT(context_);
+        RdmaContextTestPeer::seedAutoGidState(
+            *context_, reinterpret_cast<ibv_context *>(0x1), /*port=*/1,
+            /*lid=*/23, makeGid(kCurrentGid), /*gid_index=*/0);
+    }
+
+    std::shared_ptr<TransferMetadata::SegmentDesc> localDesc() const {
+        return metadata_->getSegmentDescByID(LOCAL_SEGMENT_ID);
+    }
+
+    RdmaTransport *transport_ = nullptr;
+    std::shared_ptr<TransferMetadata> metadata_;
+    RdmaContext *context_ = nullptr;
+};
+
+TEST_F(RdmaContextReprobeTest,
+       ReprobeStopsWhenExpectedSelectionDoesNotMatchCurrentState) {
+    auto before_desc = localDesc();
+    ASSERT_TRUE(before_desc);
+
+    bool changed = context_->reprobeAutoGid({formatGid(kCurrentGid), 9}, {});
+
+    EXPECT_FALSE(changed);
+    EXPECT_EQ(context_->gidIndex(), 0);
+    EXPECT_EQ(context_->gid(), formatGid(kCurrentGid));
+    auto after_desc = localDesc();
+    EXPECT_EQ(after_desc.get(), before_desc.get());
+}
+
+}  // namespace

mooncake-transfer-engine/tests/rdma_endpoint_reestablish_test.cpp

@@ -16,49 +16,169 @@
  * RDMA Endpoint Re-establishment Test
  *
  * Purpose:
- * This test verifies that TE correctly handles endpoint re-establish
- * during simulated Initiator restarts.
+ * This test verifies that TE correctly handles endpoint re-establish during
+ * simulated initiator restarts, and that classic RDMA can recover from a first
+ * RTR/EINVAL by reprobeing the next auto-selected local GID.
  *
  * How to run:
- * 1. Start etcd:
- *    etcd --listen-client-urls http://127.0.0.1:18222 \
- *      --advertise-client-urls http://127.0.0.1:18222
- *
- * 2. Run test
- *    sudo env MC_METADATA_SERVER=127.0.0.1:18222 \
- *      MC_TARGET_SERVER_NAME=127.0.0.1:12345 \
- *      MC_INITIATOR_SERVER_NAME=127.0.0.1:12346 \
- *      MC_TARGET_DEVICE_NAME=erdma_0 MC_INITIATOR_DEVICE_NAME=erdma_1 \
- *      ./build/mooncake-transfer-engine/tests/rdma_endpoint_reestablish_test
+ *   sudo env MC_METADATA_SERVER=P2PHANDSHAKE \
+ *     MC_TARGET_SERVER_NAME=127.0.0.1:12345 \
+ *     MC_INITIATOR_SERVER_NAME=127.0.0.1:12346 \
+ *     MC_TARGET_DEVICE_NAME=erdma_0 MC_INITIATOR_DEVICE_NAME=erdma_1 \
+ *     ./build/mooncake-transfer-engine/tests/rdma_endpoint_reestablish_test
  */
 
-#include <gflags/gflags.h>
-#include <glog/logging.h>
-#include <gtest/gtest.h>
-#include <numa.h>
-#include <sys/time.h>
-#include <unistd.h>
-
+#include <algorithm>
+#include <cerrno>
 #include <cstdlib>
 #include <fstream>
 #include <iomanip>
 #include <memory>
+#include <mutex>
 #include <sstream>
+#include <string>
+#include <unordered_map>
 #include <vector>
 
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <infiniband/verbs.h>
+#include <numa.h>
+#include <sys/time.h>
+#include <unistd.h>
+
+#include "common.h"
 #include "transfer_engine.h"
 #include "transport/transport.h"
-#include "common.h"
 
 using namespace mooncake;
 
-// Size of the pre-registered memory.
-constexpr size_t kRAMBufSize = 256ull << 24;  // 256 MB
+namespace {
 
-// Actual data payload size for RDMA Read/Write.
-constexpr size_t kDataLength = 16ull << 24;  // 16MB
+constexpr size_t kRAMBufSize = 256ull << 24;
+constexpr size_t kDataLength = 16ull << 24;
 
-std::string formatDeviceNames(const std::string &device_names) {
+bool usesP2PHandshake(const std::string& metadata_server) {
+    return metadata_server == P2PHANDSHAKE;
+}
+
+std::vector<std::string> getAvailableRdmaDevices() {
+    int num_devices = 0;
+    ibv_device** device_list = ibv_get_device_list(&num_devices);
+    std::vector<std::string> devices;
+    if (device_list == nullptr) {
+        return devices;
+    }
+    devices.reserve(num_devices);
+    for (int i = 0; i < num_devices; ++i) {
+        devices.emplace_back(ibv_get_device_name(device_list[i]));
+    }
+    ibv_free_device_list(device_list);
+    return devices;
+}
+
+struct RtrFaultInjectionState {
+    std::mutex mu;
+    bool synthetic_gid_swap_enabled = false;
+    std::string synthetic_gid_device;
+    bool fail_first_rtr_einval = false;
+    std::string fail_rtr_device;
+    int injected_failures = 0;
+    std::unordered_map<std::string, std::vector<int>> rtr_sgid_history;
+    std::unordered_map<std::string, std::vector<std::string>> rtr_gid_history;
+} g_rtr_fault_injection_state;
+
+std::string formatGidBytes(const uint8_t* raw) {
+    std::ostringstream oss;
+    oss << std::hex << std::setfill('0');
+    for (size_t i = 0; i < 16; ++i) {
+        if (i != 0) {
+            oss << ":";
+        }
+        oss << std::setw(2) << static_cast<int>(raw[i]);
+    }
+    return oss.str();
+}
+
+void resetRtrFaultInjectionState() {
+    std::lock_guard<std::mutex> guard(g_rtr_fault_injection_state.mu);
+    g_rtr_fault_injection_state.synthetic_gid_swap_enabled = false;
+    g_rtr_fault_injection_state.synthetic_gid_device.clear();
+    g_rtr_fault_injection_state.fail_first_rtr_einval = false;
+    g_rtr_fault_injection_state.fail_rtr_device.clear();
+    g_rtr_fault_injection_state.injected_failures = 0;
+    g_rtr_fault_injection_state.rtr_sgid_history.clear();
+    g_rtr_fault_injection_state.rtr_gid_history.clear();
+}
+
+void configureRtrFaultInjection(const std::string& device_name) {
+    std::lock_guard<std::mutex> guard(g_rtr_fault_injection_state.mu);
+    g_rtr_fault_injection_state.synthetic_gid_swap_enabled = true;
+    g_rtr_fault_injection_state.synthetic_gid_device = device_name;
+    g_rtr_fault_injection_state.fail_first_rtr_einval = true;
+    g_rtr_fault_injection_state.fail_rtr_device = device_name;
+    g_rtr_fault_injection_state.injected_failures = 0;
+    g_rtr_fault_injection_state.rtr_sgid_history.clear();
+    g_rtr_fault_injection_state.rtr_gid_history.clear();
+}
+
+std::vector<int> getRtrSgidHistory(const std::string& device_name) {
+    std::lock_guard<std::mutex> guard(g_rtr_fault_injection_state.mu);
+    auto iter = g_rtr_fault_injection_state.rtr_sgid_history.find(device_name);
+    if (iter == g_rtr_fault_injection_state.rtr_sgid_history.end()) {
+        return {};
+    }
+    return iter->second;
+}
+
+int getInjectedFailureCount() {
+    std::lock_guard<std::mutex> guard(g_rtr_fault_injection_state.mu);
+    return g_rtr_fault_injection_state.injected_failures;
+}
+
+std::vector<std::string> getRtrGidHistory(const std::string& device_name) {
+    std::lock_guard<std::mutex> guard(g_rtr_fault_injection_state.mu);
+    auto iter = g_rtr_fault_injection_state.rtr_gid_history.find(device_name);
+    if (iter == g_rtr_fault_injection_state.rtr_gid_history.end()) {
+        return {};
+    }
+    return iter->second;
+}
+
+void recordRtrAttempt(const std::string& device_name, int sgid_index,
+                      const std::string& gid) {
+    std::lock_guard<std::mutex> guard(g_rtr_fault_injection_state.mu);
+    g_rtr_fault_injection_state.rtr_sgid_history[device_name].push_back(
+        sgid_index);
+    g_rtr_fault_injection_state.rtr_gid_history[device_name].push_back(gid);
+}
+
+int maybeSwapSyntheticGidIndex(const std::string& device_name, int gid_index) {
+    std::lock_guard<std::mutex> guard(g_rtr_fault_injection_state.mu);
+    if (!g_rtr_fault_injection_state.synthetic_gid_swap_enabled ||
+        g_rtr_fault_injection_state.synthetic_gid_device != device_name) {
+        return gid_index;
+    }
+    if (gid_index == 0) return 1;
+    if (gid_index == 1) return 0;
+    return gid_index;
+}
+
+bool shouldInjectRtrEinval(const std::string& device_name, int sgid_index) {
+    std::lock_guard<std::mutex> guard(g_rtr_fault_injection_state.mu);
+    if (!g_rtr_fault_injection_state.fail_first_rtr_einval ||
+        g_rtr_fault_injection_state.fail_rtr_device != device_name ||
+        sgid_index != 0) {
+        return false;
+    }
+    g_rtr_fault_injection_state.fail_first_rtr_einval = false;
+    g_rtr_fault_injection_state.synthetic_gid_swap_enabled = false;
+    ++g_rtr_fault_injection_state.injected_failures;
+    return true;
+}
+
+std::string formatDeviceNames(const std::string& device_names) {
     std::stringstream ss(device_names);
     std::string item;
     std::vector<std::string> tokens;
@@ -76,7 +196,7 @@ std::string formatDeviceNames(const std::string &device_names) {
     return formatted;
 }
 
-std::string makeNicPriorityMatrix(const std::string &device_name) {
+std::string makeNicPriorityMatrix(const std::string& device_name) {
     auto formatted_devices = formatDeviceNames(device_name);
     return "{\"cpu:0\": [[" + formatted_devices +
            "],[]], "
@@ -84,8 +204,8 @@ std::string makeNicPriorityMatrix(const std::string &device_name) {
            formatted_devices + "],[]]}";
 }
 
-void wait_for_transfer(TransferEngine *engine, BatchID batch_id,
-                       const std::string &op_name) {
+void waitForTransfer(TransferEngine* engine, BatchID batch_id,
+                     const std::string& op_name) {
     bool completed = false;
     TransferStatus status;
     while (!completed) {
@@ -103,27 +223,26 @@ void wait_for_transfer(TransferEngine *engine, BatchID batch_id,
 
 struct TEContext {
     std::unique_ptr<TransferEngine> engine_{};
-    uint8_t *local_addr_{};
+    uint8_t* local_addr_{};
     bool segment_opened_{false};
     SegmentHandle segment_handle_{};
     uint64_t remote_base_{};
 
-    TEContext(const std::string &local_server_name,
-              const std::string &metadata_server, const std::string &segment_id,
-              const std::string &device_name) {
+    TEContext(const std::string& local_server_name,
+              const std::string& metadata_server, const std::string& segment_id,
+              const std::string& device_name) {
         engine_ = std::make_unique<TransferEngine>(false);
         auto hostname_port = parseHostNameWithPort(local_server_name);
         engine_->init(metadata_server, local_server_name, hostname_port.first,
                       hostname_port.second);
 
         auto nic_priority_matrix = makeNicPriorityMatrix(device_name);
-        void *args[2] = {const_cast<char *>(nic_priority_matrix.c_str()),
+        void* args[2] = {const_cast<char*>(nic_priority_matrix.c_str()),
                          nullptr};
-
-        Transport *xport = engine_->installTransport("rdma", args);
+        Transport* xport = engine_->installTransport("rdma", args);
         LOG_ASSERT(xport);
 
-        local_addr_ = static_cast<uint8_t *>(numa_alloc_onnode(kRAMBufSize, 0));
+        local_addr_ = static_cast<uint8_t*>(numa_alloc_onnode(kRAMBufSize, 0));
         memset(local_addr_, 0, kDataLength);
 
         int rc =
@@ -145,16 +264,21 @@ struct TEContext {
         numa_free(local_addr_, kRAMBufSize);
         if (segment_opened_) engine_->closeSegment(segment_handle_);
     }
+
+    std::string localSegmentName() const {
+        return engine_->getLocalIpAndPort();
+    }
 };
 
 class RDMAEndpointReestablishTest : public ::testing::Test {
    protected:
     void SetUp() override {
+        resetRtrFaultInjectionState();
         google::InitGoogleLogging("RDMAEndpointReestablishTest");
         FLAGS_logtostderr = true;
 
-        const char *env = std::getenv("MC_METADATA_SERVER");
-        metadata_server = env ? env : "127.0.0.1:18222";
+        const char* env = std::getenv("MC_METADATA_SERVER");
+        metadata_server = env ? env : P2PHANDSHAKE;
         LOG(INFO) << "metadata_server: " << metadata_server;
 
         env = std::getenv("MC_TARGET_SERVER_NAME");
@@ -165,16 +289,99 @@ class RDMAEndpointReestablishTest : public ::testing::Test {
         initiator_server_name = env ? env : "127.0.0.1:12346";
         LOG(INFO) << "initiator_server_name: " << initiator_server_name;
 
+        auto devices = getAvailableRdmaDevices();
+        if (devices.size() < 2) {
+            GTEST_SKIP() << "Need at least two RDMA devices, found "
+                         << devices.size();
+        }
+
         env = std::getenv("MC_TARGET_DEVICE_NAME");
-        target_device_name = env ? env : "erdma_0";
+        target_device_name = env ? env : devices[0];
         LOG(INFO) << "target_device_name: " << target_device_name;
 
         env = std::getenv("MC_INITIATOR_DEVICE_NAME");
-        initiator_device_name = env ? env : "erdma_1";
+        initiator_device_name = env ? env : devices[1];
         LOG(INFO) << "initiator_device_name: " << initiator_device_name;
     }
 
-    void TearDown() override { google::ShutdownGoogleLogging(); }
+    void TearDown() override {
+        google::ShutdownGoogleLogging();
+        resetRtrFaultInjectionState();
+    }
+
+    void runEndpointReestablishScenario(const std::string& target_device,
+                                        const std::string& initiator_device) {
+        LOG(INFO) << "========== Setting up Target ==========";
+        TEContext target_ctx(target_server_name, metadata_server, "",
+                             target_device);
+        const std::string target_segment_name =
+            usesP2PHandshake(metadata_server) ? target_ctx.localSegmentName()
+                                              : target_server_name;
+        LOG(INFO) << "Resolved target segment name: " << target_segment_name;
+        LOG(INFO)
+            << "Target is up. Waiting for RDMA connections and operations...";
+
+        LOG(INFO) << "========== Phase 1: Start, Connect & Write ==========";
+        {
+            TEContext init_ctx(initiator_server_name, metadata_server,
+                               target_segment_name, initiator_device);
+            for (size_t i = 0; i < kDataLength; ++i) {
+                init_ctx.local_addr_[i] = static_cast<uint8_t>(i % 256);
+            }
+
+            LOG(INFO) << "Writing " << kDataLength << " bytes to Target...";
+            auto batch_id = init_ctx.engine_->allocateBatchID(1);
+            TransferRequest entry;
+            entry.opcode = TransferRequest::WRITE;
+            entry.length = kDataLength;
+            entry.source = init_ctx.local_addr_;
+            entry.target_id = init_ctx.segment_handle_;
+            entry.target_offset = init_ctx.remote_base_;
+
+            Status s = init_ctx.engine_->submitTransfer(batch_id, {entry});
+            ASSERT_EQ(s, Status::OK());
+            waitForTransfer(init_ctx.engine_.get(), batch_id, "WRITE");
+            LOG(INFO) << "Phase 1: Write Completed. Tearing down connection...";
+        }
+
+        LOG(INFO) << "Simulating Initiator Crash/Restart... Waiting 2 seconds.";
+        sleep(2);
+
+        LOG(INFO)
+            << "========== Phase 2: Restart, Re-establish Endpoint & Read "
+               "==========";
+        {
+            TEContext init_ctx(initiator_server_name, metadata_server,
+                               target_segment_name, initiator_device);
+            LOG(INFO) << "Reading data back over new Endpoint...";
+            auto batch_id = init_ctx.engine_->allocateBatchID(1);
+            TransferRequest entry;
+            entry.opcode = TransferRequest::READ;
+            entry.length = kDataLength;
+            entry.source = init_ctx.local_addr_;
+            entry.target_id = init_ctx.segment_handle_;
+            entry.target_offset = init_ctx.remote_base_;
+
+            Status s = init_ctx.engine_->submitTransfer(batch_id, {entry});
+            ASSERT_EQ(s, Status::OK());
+            waitForTransfer(init_ctx.engine_.get(), batch_id, "READ");
+
+            bool ok = true;
+            for (size_t i = 0; i < kDataLength; ++i) {
+                if (init_ctx.local_addr_[i] != static_cast<uint8_t>(i % 256)) {
+                    ok = false;
+                    LOG(ERROR) << "Data mismatch at offset " << i
+                               << ", expected " << (i % 256) << ", got "
+                               << (int)init_ctx.local_addr_[i];
+                    break;
+                }
+            }
+
+            ASSERT_TRUE(ok) << "Endpoint Reconstruction Verification Failed!";
+            LOG(INFO) << ">>> ENDPOINT RECONSTRUCTION VERIFICATION: "
+                         "\033[32mSUCCESS\033";
+        }
+    }
 
     std::string metadata_server;
     std::string target_server_name;
@@ -184,77 +391,94 @@ class RDMAEndpointReestablishTest : public ::testing::Test {
 };
 
 TEST_F(RDMAEndpointReestablishTest, EndpointReestablish) {
-    // 1. Setup Target (will stay alive until test function exits)
-    LOG(INFO) << "========== Setting up Target ==========";
-    TEContext target_ctx(target_server_name, metadata_server, "",
-                         target_device_name);
-    LOG(INFO) << "Target is up. Waiting for RDMA connections and operations...";
-
-    // 2. Phase 1: Initiator Start, Connect & Write
-    LOG(INFO) << "========== Phase 1: Start, Connect & Write ==========";
-    {
-        TEContext init_ctx(initiator_server_name, metadata_server,
-                           target_server_name, initiator_device_name);
-
-        // Fill buffer with test pattern
-        for (size_t i = 0; i < kDataLength; ++i) {
-            init_ctx.local_addr_[i] = static_cast<uint8_t>(i % 256);
-        }
-
-        LOG(INFO) << "Writing " << kDataLength << " bytes to Target...";
-        auto batch_id = init_ctx.engine_->allocateBatchID(1);
-        TransferRequest entry;
-        entry.opcode = TransferRequest::WRITE;
-        entry.length = kDataLength;
-        entry.source = init_ctx.local_addr_;
-        entry.target_id = init_ctx.segment_handle_;
-        entry.target_offset = init_ctx.remote_base_;
-
-        Status s = init_ctx.engine_->submitTransfer(batch_id, {entry});
-        ASSERT_EQ(s, Status::OK());
-
-        wait_for_transfer(init_ctx.engine_.get(), batch_id, "WRITE");
-
-        LOG(INFO) << "Phase 1: Write Completed. Tearing down connection...";
-    }
-
-    LOG(INFO) << "Simulating Initiator Crash/Restart... Waiting 2 seconds.";
-    sleep(2);
-
-    // 3. Phase 2: Restart, Re-establish Endpoint & Read
-    LOG(INFO) << "========== Phase 2: Restart, Re-establish Endpoint & Read "
-                 "==========";
-    {
-        TEContext init_ctx(initiator_server_name, metadata_server,
-                           target_server_name, initiator_device_name);
-
-        LOG(INFO) << "Reading data back over new Endpoint...";
-        auto batch_id = init_ctx.engine_->allocateBatchID(1);
-        TransferRequest entry;
-        entry.opcode = TransferRequest::READ;
-        entry.length = kDataLength;
-        entry.source = init_ctx.local_addr_;
-        entry.target_id = init_ctx.segment_handle_;
-        entry.target_offset = init_ctx.remote_base_;
-
-        Status s = init_ctx.engine_->submitTransfer(batch_id, {entry});
-        ASSERT_EQ(s, Status::OK());
-
-        wait_for_transfer(init_ctx.engine_.get(), batch_id, "READ");
-
-        bool ok = true;
-        for (size_t i = 0; i < kDataLength; ++i) {
-            if (init_ctx.local_addr_[i] != static_cast<uint8_t>(i % 256)) {
-                ok = false;
-                LOG(ERROR) << "Data mismatch at offset " << i << ", expected "
-                           << (i % 256) << ", got "
-                           << (int)init_ctx.local_addr_[i];
-                break;
-            }
-        }
-
-        ASSERT_TRUE(ok) << "Endpoint Reconstruction Verification Failed!";
-        LOG(INFO)
-            << ">>> ENDPOINT RECONSTRUCTION VERIFICATION: \033[32mSUCCESS\033";
-    }
+    runEndpointReestablishScenario(target_device_name, initiator_device_name);
+}
+
+TEST_F(RDMAEndpointReestablishTest, EndpointReestablishReverseDevices) {
+    runEndpointReestablishScenario(initiator_device_name, target_device_name);
+}
+
+TEST_F(RDMAEndpointReestablishTest, ActiveHandshakeRetriesAfterAutoGidReprobe) {
+    configureRtrFaultInjection(initiator_device_name);
+    runEndpointReestablishScenario(target_device_name, initiator_device_name);
+
+    EXPECT_EQ(getInjectedFailureCount(), 1);
+    auto sgid_history = getRtrSgidHistory(initiator_device_name);
+    auto gid_history = getRtrGidHistory(initiator_device_name);
+    ASSERT_EQ(sgid_history.size(), gid_history.size());
+    ASSERT_GE(sgid_history.size(), 2u);
+    EXPECT_EQ(sgid_history.front(), 0);
+    EXPECT_FALSE(gid_history.front().empty());
+    EXPECT_TRUE(std::any_of(
+        gid_history.begin() + 1, gid_history.end(),
+        [&](const std::string& gid) { return gid != gid_history.front(); }));
+}
+
+TEST_F(RDMAEndpointReestablishTest,
+       PassiveHandshakeRetriesAfterAutoGidReprobe) {
+    configureRtrFaultInjection(target_device_name);
+    runEndpointReestablishScenario(target_device_name, initiator_device_name);
+
+    EXPECT_EQ(getInjectedFailureCount(), 1);
+    auto sgid_history = getRtrSgidHistory(target_device_name);
+    auto gid_history = getRtrGidHistory(target_device_name);
+    ASSERT_EQ(sgid_history.size(), gid_history.size());
+    ASSERT_GE(sgid_history.size(), 2u);
+    EXPECT_EQ(sgid_history.front(), 0);
+    EXPECT_FALSE(gid_history.front().empty());
+    EXPECT_TRUE(std::any_of(
+        gid_history.begin() + 1, gid_history.end(),
+        [&](const std::string& gid) { return gid != gid_history.front(); }));
+}
+
+}  // namespace
+
+extern "C" int __real__ibv_query_gid_ex(ibv_context* context, uint8_t port_num,
+                                        int gid_index,
+                                        struct ibv_gid_entry* entry,
+                                        uint32_t flags, size_t entry_size);
+
+extern "C" int __wrap__ibv_query_gid_ex(ibv_context* context, uint8_t port_num,
+                                        int gid_index,
+                                        struct ibv_gid_entry* entry,
+                                        uint32_t flags, size_t entry_size) {
+    const std::string device_name = ibv_get_device_name(context->device);
+    int wrapped_gid_index = maybeSwapSyntheticGidIndex(device_name, gid_index);
+    return __real__ibv_query_gid_ex(context, port_num, wrapped_gid_index, entry,
+                                    flags, entry_size);
+}
+
+extern "C" int __real_ibv_query_gid(ibv_context* context, uint8_t port_num,
+                                    int gid_index, union ibv_gid* gid);
+
+extern "C" int __wrap_ibv_query_gid(ibv_context* context, uint8_t port_num,
+                                    int gid_index, union ibv_gid* gid) {
+    const std::string device_name = ibv_get_device_name(context->device);
+    int wrapped_gid_index = maybeSwapSyntheticGidIndex(device_name, gid_index);
+    return __real_ibv_query_gid(context, port_num, wrapped_gid_index, gid);
+}
+
+extern "C" int __real_ibv_modify_qp(ibv_qp* qp, ibv_qp_attr* attr,
+                                    int attr_mask);
+
+extern "C" int __wrap_ibv_modify_qp(ibv_qp* qp, ibv_qp_attr* attr,
+                                    int attr_mask) {
+    if (qp != nullptr && attr != nullptr && attr->qp_state == IBV_QPS_RTR &&
+        (attr_mask & IBV_QP_AV)) {
+        const std::string device_name =
+            ibv_get_device_name(qp->context->device);
+        int sgid_index = attr->ah_attr.grh.sgid_index;
+        union ibv_gid actual_gid = {};
+        std::string gid_string;
+        if (__real_ibv_query_gid(qp->context, attr->ah_attr.port_num,
+                                 sgid_index, &actual_gid) == 0) {
+            gid_string = formatGidBytes(actual_gid.raw);
+        }
+        recordRtrAttempt(device_name, sgid_index, gid_string);
+        if (shouldInjectRtrEinval(device_name, sgid_index)) {
+            errno = EINVAL;
+            return -1;
+        }
+    }
+    return __real_ibv_modify_qp(qp, attr, attr_mask);
 }

mooncake-transfer-engine/tests/rdma_gid_probe_test.cpp

@@ -0,0 +1,402 @@
+// Copyright 2026 KVCache.AI
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <gtest/gtest.h>
+
+#include <vector>
+
+#include "transport/rdma_transport/rdma_gid_probe.h"
+
+using namespace mooncake;
+
+namespace {
+
+AutoGidCandidate makeCandidate(int gid_index, uint32_t gid_type,
+                               bool has_network_device, bool is_ipv4_mapped,
+                               bool is_link_local_ipv6,
+                               bool is_overlay_network = false,
+                               bool is_overlay_ipv4 = false,
+                               bool is_null_gid = false,
+                               bool query_succeeded = true,
+                               std::string gid = "") {
+    AutoGidCandidate candidate;
+    candidate.gid_index = gid_index;
+    candidate.gid =
+        gid.empty() ? "gid-" + std::to_string(gid_index) : std::move(gid);
+    candidate.gid_type = gid_type;
+    candidate.has_network_device = has_network_device;
+    candidate.is_ipv4_mapped = is_ipv4_mapped;
+    candidate.is_link_local_ipv6 = is_link_local_ipv6;
+    candidate.is_overlay_network = is_overlay_network;
+    candidate.is_overlay_ipv4 = is_overlay_ipv4;
+    candidate.is_null_gid = is_null_gid;
+    candidate.query_succeeded = query_succeeded;
+    return candidate;
+}
+
+TEST(RdmaGidProbeTest, PrefersNetworkBackedRoutableCandidate) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/0, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/false,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto selection = selectBestAutoGidCandidate(candidates);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 1);
+    EXPECT_EQ(selection->candidate_class,
+              AutoGidCandidateClass::kNetworkRoutable);
+}
+
+TEST(RdmaGidProbeTest, DemotesLinkLocalBehindRoutableNetworkCandidate) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/0, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/false,
+                      /*is_link_local_ipv6=*/true),
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto selection = selectBestAutoGidCandidate(candidates);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 1);
+    EXPECT_EQ(selection->candidate_class,
+              AutoGidCandidateClass::kNetworkRoutable);
+}
+
+TEST(RdmaGidProbeTest, DemotesOverlayCandidateBehindNormalNetworkCandidate) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/0, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false,
+                      /*is_overlay_network=*/true),
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto selection = selectBestAutoGidCandidate(candidates);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 1);
+}
+
+TEST(RdmaGidProbeTest,
+     PrefersNoNetworkRoutableOverDegradedNetworkBackedCandidate) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/0, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/false,
+                      /*is_link_local_ipv6=*/true),
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/false,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto selection = selectBestAutoGidCandidate(candidates);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 1);
+    EXPECT_EQ(selection->candidate_class,
+              AutoGidCandidateClass::kNoNetworkRoutable);
+}
+
+TEST(RdmaGidProbeTest, KeepsNoNetworkFallbackAsLastResort) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/3, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/false,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto selection = selectBestAutoGidCandidate(candidates);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 3);
+    EXPECT_EQ(selection->candidate_class,
+              AutoGidCandidateClass::kNoNetworkRoutable);
+}
+
+TEST(RdmaGidProbeTest, FallsBackToFirstNonzeroCandidateWhenNeeded) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/0, IBV_GID_TYPE_ROCE_V1,
+                      /*has_network_device=*/false,
+                      /*is_ipv4_mapped=*/false,
+                      /*is_link_local_ipv6=*/false,
+                      /*is_overlay_network=*/false,
+                      /*is_overlay_ipv4=*/false,
+                      /*is_null_gid=*/false),
+        makeCandidate(/*gid_index=*/2, IBV_GID_TYPE_ROCE_V1,
+                      /*has_network_device=*/false,
+                      /*is_ipv4_mapped=*/false,
+                      /*is_link_local_ipv6=*/false,
+                      /*is_overlay_network=*/false,
+                      /*is_overlay_ipv4=*/false,
+                      /*is_null_gid=*/false),
+    };
+
+    auto selection = selectBestAutoGidCandidate(candidates);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 0);
+    EXPECT_EQ(selection->candidate_class,
+              AutoGidCandidateClass::kFallbackNonzero);
+}
+
+TEST(RdmaGidProbeTest, DoesNotTreatIbCandidateAsLinkLocalIpv6Penalty) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/0, IBV_GID_TYPE_IB,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/false,
+                      /*is_link_local_ipv6=*/true),
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/false,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto selection = selectBestAutoGidCandidate(candidates);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 0);
+    EXPECT_EQ(selection->candidate_class,
+              AutoGidCandidateClass::kNetworkRoutable);
+}
+
+TEST(RdmaGidProbeTest, SkipsInvalidAndNullCandidates) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/0, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false,
+                      /*is_overlay_network=*/false,
+                      /*is_overlay_ipv4=*/false,
+                      /*is_null_gid=*/false,
+                      /*query_succeeded=*/false),
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false,
+                      /*is_overlay_network=*/false,
+                      /*is_overlay_ipv4=*/false,
+                      /*is_null_gid=*/true),
+        makeCandidate(/*gid_index=*/2, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto selection = selectBestAutoGidCandidate(candidates);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 2);
+}
+
+TEST(RdmaGidProbeTest, KeepsStableOrderingWithinSameCandidateClass) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/3, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto ranked = rankAutoGidCandidates(candidates);
+    ASSERT_EQ(ranked.size(), 2u);
+    EXPECT_EQ(ranked[0].gid_index, 1);
+    EXPECT_EQ(ranked[1].gid_index, 3);
+}
+
+TEST(RdmaGidProbeTest, ReprobeStillPicksBestCandidateFromFreshSnapshot) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+        makeCandidate(/*gid_index=*/3, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/false,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto selection = selectBestAutoGidCandidate(candidates);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 1);
+    EXPECT_EQ(selection->candidate_class,
+              AutoGidCandidateClass::kNetworkRoutable);
+}
+
+TEST(RdmaGidProbeTest, ReprobeDetectsSameIndexGidRefresh) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false,
+                      /*is_overlay_network=*/false,
+                      /*is_overlay_ipv4=*/false,
+                      /*is_null_gid=*/false,
+                      /*query_succeeded=*/true,
+                      /*gid=*/"00:11:22"),
+        makeCandidate(/*gid_index=*/3, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/false,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false),
+    };
+
+    auto selection = reselectAutoGidCandidate(
+        candidates, /*current_gid_index=*/1, /*current_gid=*/"00:11:21");
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 1);
+    EXPECT_EQ(selection->gid, "00:11:22");
+    EXPECT_EQ(selection->candidate_class,
+              AutoGidCandidateClass::kNetworkRoutable);
+}
+
+TEST(RdmaGidProbeTest, ReprobeSkipsRetryWhenBestSelectionDidNotChange) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false,
+                      /*is_overlay_network=*/false,
+                      /*is_overlay_ipv4=*/false,
+                      /*is_null_gid=*/false,
+                      /*query_succeeded=*/true,
+                      /*gid=*/"00:11:22"),
+    };
+
+    auto selection = reselectAutoGidCandidate(
+        candidates, /*current_gid_index=*/1, /*current_gid=*/"00:11:22");
+    EXPECT_FALSE(selection.has_value());
+}
+
+TEST(RdmaGidProbeTest, ReprobeSkipsAlreadyTriedCandidates) {
+    std::vector<AutoGidCandidate> candidates = {
+        makeCandidate(/*gid_index=*/1, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/true,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false,
+                      /*is_overlay_network=*/false,
+                      /*is_overlay_ipv4=*/false,
+                      /*is_null_gid=*/false,
+                      /*query_succeeded=*/true,
+                      /*gid=*/"00:11:22"),
+        makeCandidate(/*gid_index=*/3, IBV_GID_TYPE_ROCE_V2,
+                      /*has_network_device=*/false,
+                      /*is_ipv4_mapped=*/true,
+                      /*is_link_local_ipv6=*/false,
+                      /*is_overlay_network=*/false,
+                      /*is_overlay_ipv4=*/false,
+                      /*is_null_gid=*/false,
+                      /*query_succeeded=*/true,
+                      /*gid=*/"00:11:33"),
+    };
+
+    std::vector<AutoGidSelectionIdentity> tried = {
+        {1, "00:11:22"},
+    };
+    auto selection = reselectAutoGidCandidate(
+        candidates, /*current_gid_index=*/1, /*current_gid=*/"00:11:21", tried);
+    ASSERT_TRUE(selection.has_value());
+    EXPECT_EQ(selection->gid_index, 3);
+    EXPECT_EQ(selection->gid, "00:11:33");
+}
+
+TEST(RdmaGidProbeTest, DetectsSameIndexGidByteChangesAsSelectionChanges) {
+    EXPECT_TRUE(didAutoGidSelectionChange(/*previous_gid_index=*/1,
+                                          /*previous_gid=*/"00:11:22",
+                                          /*current_gid_index=*/1,
+                                          /*current_gid=*/"00:11:23"));
+
+    EXPECT_FALSE(didAutoGidSelectionChange(/*previous_gid_index=*/1,
+                                           /*previous_gid=*/"00:11:22",
+                                           /*current_gid_index=*/1,
+                                           /*current_gid=*/"00:11:22"));
+}
+
+TEST(RdmaGidProbeTest, HandshakeRetryRespectsConfiguredRetryBudget) {
+    EXPECT_TRUE(shouldAttemptAutoGidHandshakeRetry(
+        /*auto_gid_selection_enabled=*/true,
+        /*retry_count=*/0,
+        /*max_retries=*/2,
+        /*failure_happened_at_rtr=*/true, EINVAL));
+
+    EXPECT_TRUE(shouldAttemptAutoGidHandshakeRetry(
+        /*auto_gid_selection_enabled=*/true,
+        /*retry_count=*/1,
+        /*max_retries=*/2,
+        /*failure_happened_at_rtr=*/true, EINVAL));
+
+    EXPECT_FALSE(shouldAttemptAutoGidHandshakeRetry(
+        /*auto_gid_selection_enabled=*/false,
+        /*retry_count=*/0,
+        /*max_retries=*/2,
+        /*failure_happened_at_rtr=*/true, EINVAL));
+
+    EXPECT_FALSE(shouldAttemptAutoGidHandshakeRetry(
+        /*auto_gid_selection_enabled=*/true,
+        /*retry_count=*/2,
+        /*max_retries=*/2,
+        /*failure_happened_at_rtr=*/true, EINVAL));
+
+    EXPECT_FALSE(shouldAttemptAutoGidHandshakeRetry(
+        /*auto_gid_selection_enabled=*/true,
+        /*retry_count=*/0,
+        /*max_retries=*/0,
+        /*failure_happened_at_rtr=*/true, EINVAL));
+}
+
+TEST(RdmaGidProbeTest, HandshakeRetryOnlyTriggersForRtrEinval) {
+    EXPECT_FALSE(shouldAttemptAutoGidHandshakeRetry(
+        /*auto_gid_selection_enabled=*/true,
+        /*retry_count=*/0,
+        /*max_retries=*/2,
+        /*failure_happened_at_rtr=*/false, EINVAL));
+
+    EXPECT_FALSE(shouldAttemptAutoGidHandshakeRetry(
+        /*auto_gid_selection_enabled=*/true,
+        /*retry_count=*/0,
+        /*max_retries=*/2,
+        /*failure_happened_at_rtr=*/true, ENOENT));
+}
+
+TEST(RdmaGidProbeTest, RetryActionRequiresObservedOrReprobedChange) {
+    EXPECT_EQ(decideAutoGidRetryAction(
+                  /*reprobe_changed=*/false, /*previous_gid_index=*/1,
+                  /*previous_gid=*/"00:11:22", /*current_gid_index=*/1,
+                  /*current_gid=*/"00:11:22"),
+              AutoGidRetryAction::kDoNotRetry);
+
+    EXPECT_EQ(decideAutoGidRetryAction(
+                  /*reprobe_changed=*/true, /*previous_gid_index=*/1,
+                  /*previous_gid=*/"00:11:22", /*current_gid_index=*/1,
+                  /*current_gid=*/"00:11:23"),
+              AutoGidRetryAction::kRetryWithReprobedGid);
+
+    EXPECT_EQ(decideAutoGidRetryAction(
+                  /*reprobe_changed=*/false, /*previous_gid_index=*/1,
+                  /*previous_gid=*/"00:11:22", /*current_gid_index=*/1,
+                  /*current_gid=*/"00:11:23"),
+              AutoGidRetryAction::kRetryWithObservedChange);
+}
+
+}  // namespace