From 4136d2b73b916ecf9367cee4575a80f075afb5df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E9=B9=A4=E7=94=B7?= Date: Sun, 8 Feb 2026 11:41:17 +0800 Subject: [PATCH] [TE] Add AWS EFA transport using libfabric (#1509) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * [TE] Add AWS EFA transport using libfabric Add EfaTransport as a new transport backend for AWS Elastic Fabric Adapter (EFA) devices. EFA exposes RDMA-like NICs but does not support the full ibverbs QP API, so this transport uses libfabric's FI_EP_RDM (Reliable Datagram Message) endpoint type instead. Architecture (per EFA device): EfaTransport → EfaContext → EfaEndPoint - EfaContext: owns fabric/domain/AV/CQ/MR resources - EfaEndPoint: one RDM endpoint per peer, with address-vector addressing - Dedicated CQ poller thread per device for responsive completion draining Key design decisions: - FI_THREAD_SAFE requested from provider; per-endpoint spinlock on fi_write as safety net for concurrent submission threads - Atomic CAS reservation of CQ and WR capacity before posting fi_write to prevent CQ overflow under high concurrency - CQ error path drains all queued errors (fi_cq_readerr loop) before returning, per libfabric semantics - Retry-with-backoff on CQ/WR full instead of immediate slice failure - Thread-safe endpoint creation via atomic getOrInsert to prevent duplicate endpoints for the same peer - Handshake exchanges EFA endpoint addresses via dedicated efa_addr field in HandShakeDesc Build: cmake -DUSE_EFA=ON (requires libfabric from AWS EFA installer) Tested on p6-b200.48xlarge (8 EFA devices, 8×400 Gbps): 59.72 GB/s Co-Authored-By: Claude Opus 4.6 * [TE] Add EFA unit tests and bench tool support Add efa_transport_test with 5 test cases: - InstallTransport: verify EFA transport installation - LoopbackWrite: basic loopback write operation - WriteAndRead: write then read with data integrity check - MultiWrite: batch write (16 requests) - StressMultipleBatches: stress test (20 batches × 8 requests) Add --protocol=efa support to transfer_engine_bench with manual topology discovery (EFA needs explicit discover() since TransferEngine(false) skips auto-discovery). Co-Authored-By: Claude Opus 4.6 * [Docs] Add EFA transport documentation Add comprehensive EFA transport documentation covering: - Prerequisites and build instructions - Usage with vLLM (prefill/decode disaggregation) - Unit test descriptions and environment variables - Benchmark results on p6-b200.48xlarge: 59.72 GB/s (EFA) vs 9.5 GB/s (TCP iperf3) vs 0.11 GB/s (Mooncake TCP transport) - EFA vs RoCE RDMA comparison table - Thread safety design notes - Troubleshooting guide Add EfaTransport to the transfer-engine index toctree and supported transport lists. Co-Authored-By: Claude Opus 4.6 * [TE] Address PR review: ifdef EFA fields, use find_package for libfabric - Wrap efa_addr in HandShakeDesc with #ifdef USE_EFA in transfer_metadata.h - Wrap efa_addr serialization/deserialization with #ifdef USE_EFA in transfer_metadata.cpp - Replace hardcoded /opt/amazon/efa paths with find_path/find_library in common.cmake - Remove redundant hardcoded EFA paths from all CMakeLists.txt files - Fix git clone URL in efa-transport.md to use official kvcache-ai/Mooncake repo Co-Authored-By: Claude Opus 4.6 * [Docs] Update EFA benchmark results with tuned parameters (170 GB/s) Update benchmark documentation with comprehensive parameter tuning results from cross-machine testing on p6-b200.48xlarge instances. Key finding: MC_SLICE_SIZE=262144 nearly doubles EFA throughput from ~70 to ~170 GB/s, reaching 88% of RoCE RDMA performance. Co-Authored-By: Claude Opus 4.6 * Fix clang-format violation in transfer_metadata.h Remove extra space before comment on efa_addr field to satisfy clang-format-20 style check. Co-Authored-By: Claude Opus 4.6 * [Docs] Add EFA latency benchmark script, rename efa doc to underscore - Add efa_latency_bench.py: automated benchmark script that measures EFA throughput for tuned/default configs via SSH and plots Latency vs Cache Size chart - Add efa_latency_bench.png: benchmark results chart - Rename efa-transport.md -> efa_transport.md to match naming convention of other transport docs (ascend_transport.md, etc.) - Update toctree reference in index.md Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Ubuntu Co-authored-by: Claude Opus 4.6 Co-authored-by: Ubuntu --- .../design/transfer-engine/efa_transport.md | 353 +++++++++ docs/source/design/transfer-engine/index.md | 12 +- mooncake-common/common.cmake | 23 + mooncake-integration/CMakeLists.txt | 5 + .../transfer_engine/transfer_engine_py.cpp | 34 +- .../example/efa_latency_bench.png | Bin 0 -> 64738 bytes .../example/efa_latency_bench.py | 454 +++++++++++ .../example/transfer_engine_bench.cpp | 7 +- .../include/transfer_metadata.h | 3 + .../transport/efa_transport/efa_context.h | 177 +++++ .../transport/efa_transport/efa_endpoint.h | 164 ++++ .../transport/efa_transport/efa_transport.h | 149 ++++ mooncake-transfer-engine/src/CMakeLists.txt | 7 + .../src/multi_transport.cpp | 8 + .../src/transfer_metadata.cpp | 12 +- .../src/transport/CMakeLists.txt | 6 + .../transport/efa_transport/CMakeLists.txt | 14 + .../transport/efa_transport/efa_context.cpp | 562 ++++++++++++++ .../transport/efa_transport/efa_endpoint.cpp | 441 +++++++++++ .../transport/efa_transport/efa_transport.cpp | 732 ++++++++++++++++++ mooncake-transfer-engine/tests/CMakeLists.txt | 6 + .../tests/efa_transport_test.cpp | 338 ++++++++ 22 files changed, 3501 insertions(+), 6 deletions(-) create mode 100644 docs/source/design/transfer-engine/efa_transport.md create mode 100644 mooncake-transfer-engine/example/efa_latency_bench.png create mode 100755 mooncake-transfer-engine/example/efa_latency_bench.py create mode 100644 mooncake-transfer-engine/include/transport/efa_transport/efa_context.h create mode 100644 mooncake-transfer-engine/include/transport/efa_transport/efa_endpoint.h create mode 100644 mooncake-transfer-engine/include/transport/efa_transport/efa_transport.h create mode 100644 mooncake-transfer-engine/src/transport/efa_transport/CMakeLists.txt create mode 100644 mooncake-transfer-engine/src/transport/efa_transport/efa_context.cpp create mode 100644 mooncake-transfer-engine/src/transport/efa_transport/efa_endpoint.cpp create mode 100644 mooncake-transfer-engine/src/transport/efa_transport/efa_transport.cpp create mode 100644 mooncake-transfer-engine/tests/efa_transport_test.cpp diff --git a/docs/source/design/transfer-engine/efa_transport.md b/docs/source/design/transfer-engine/efa_transport.md new file mode 100644 index 00000000..b6fee161 --- /dev/null +++ b/docs/source/design/transfer-engine/efa_transport.md @@ -0,0 +1,353 @@ +# AWS EFA Transport for Mooncake + +This document describes how to build and use Mooncake with AWS Elastic Fabric Adapter (EFA) support using libfabric. + +## Prerequisites + +### 1. AWS EFA Driver and libfabric + +EFA driver and libfabric should be pre-installed on AWS instances with EFA support (e.g., p6-b200.48xlarge, p5e.48xlarge, p4d.24xlarge). + +Verify installation: +```bash +# Check EFA devices +fi_info -p efa + +# Verify libfabric location +ls /opt/amazon/efa/lib/libfabric.so +ls /opt/amazon/efa/include/rdma/fabric.h +``` + +If not installed, follow [AWS EFA documentation](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html). + +### 2. Build Dependencies + +```bash +# Ubuntu/Debian +sudo apt-get update +sudo apt-get install -y \ + build-essential \ + cmake \ + git \ + libgflags-dev \ + libgoogle-glog-dev \ + libjsoncpp-dev \ + libnuma-dev \ + libibverbs-dev \ + libboost-all-dev \ + libcurl4-openssl-dev \ + libyaml-cpp-dev \ + libgtest-dev \ + pybind11-dev \ + python3-dev + +# Install yalantinglibs (required) +cd /tmp +git clone https://github.com/alibaba/yalantinglibs.git +cd yalantinglibs +mkdir build && cd build +cmake .. -DCMAKE_INSTALL_PREFIX=/usr/local +make -j$(nproc) +sudo make install +``` + +## Building Mooncake with EFA Support + +### 1. Clone the Repository + +```bash +git clone https://github.com/kvcache-ai/Mooncake.git +cd Mooncake +git submodule update --init --recursive +``` + +### 2. Build with EFA Enabled + +```bash +mkdir build && cd build + +cmake .. \ + -DUSE_EFA=ON \ + -DWITH_TE=ON \ + -DWITH_STORE=ON \ + -DBUILD_UNIT_TESTS=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo + +make -j$(nproc) +``` + +### 3. Install Python Package + +```bash +# Copy built modules to wheel directory +cp mooncake-integration/engine.cpython-*.so ../mooncake-wheel/mooncake/ +cp mooncake-asio/libasio.so ../mooncake-wheel/mooncake/ + +# Install with pip +pip install -e ../mooncake-wheel --no-build-isolation +``` + +## Verification + +Test EFA transport initialization: + +```python +from mooncake.engine import TransferEngine + +te = TransferEngine() +result = te.initialize('127.0.0.1', 'P2PHANDSHAKE', 'efa', '') +print(f'Initialize result: {result}') # Should be 0 + +# You should see logs like: +# EFA device (libfabric): rdmap79s0, domain: rdmap79s0-rdm, provider: efa +``` + +## Usage with vLLM + +### Prefill Instance + +```bash +VLLM_MOONCAKE_BOOTSTRAP_PORT=8998 \ +vllm serve -tp 8 \ + --port 8010 \ + --trust-remote-code \ + --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_producer","kv_connector_extra_config":{"mooncake_protocol":"efa"}}' +``` + +### Decode Instance + +```bash +vllm serve -tp 8 \ + --port 8020 \ + --trust-remote-code \ + --kv-transfer-config '{"kv_connector":"MooncakeConnector","kv_role":"kv_consumer","kv_connector_extra_config":{"mooncake_protocol":"efa"}}' +``` + +## Unit Tests + +Run the EFA transport unit tests (requires EFA hardware): + +```bash +./build/mooncake-transfer-engine/tests/efa_transport_test +``` + +The test suite includes: + +| Test | Description | +|------|-------------| +| `InstallTransport` | Verify EFA transport installation | +| `LoopbackWrite` | Loopback write operation | +| `WriteAndRead` | Write then read with data integrity check | +| `MultiWrite` | Batch write (16 requests) | +| `StressMultipleBatches` | Stress test (20 batches x 8 requests) | + +You can also run all unit tests via CTest: + +```bash +cd build && ctest --output-on-failure +``` + +Environment variables for test configuration: + +```bash +export MC_METADATA_SERVER=P2PHANDSHAKE # default +export MC_LOCAL_SERVER_NAME=127.0.0.1:12345 # default +``` + +## Performance Benchmark + +Use `transfer_engine_bench` to measure EFA transport throughput between two nodes. + +### Target Node (receiver) + +```bash +./build/mooncake-transfer-engine/example/transfer_engine_bench \ + --mode=target \ + --protocol=efa \ + --metadata_server=P2PHANDSHAKE +``` + +### Initiator Node (sender) + +```bash +./build/mooncake-transfer-engine/example/transfer_engine_bench \ + --mode=initiator \ + --protocol=efa \ + --metadata_server=P2PHANDSHAKE \ + --segment_id=: \ + --operation=write \ + --duration=10 \ + --threads=8 \ + --block_size=65536 \ + --batch_size=128 \ + --buffer_size=1073741824 \ + --report_unit=GB +``` + +Replace `:` with the target node's address shown in the target's startup log (e.g., `ip-172-31-29-226:12345`). + +### Key Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--block_size` | 65536 | Bytes per transfer request | +| `--batch_size` | 128 | Requests per batch | +| `--threads` | 12 | Concurrent submission threads | +| `--buffer_size` | 1 GB | Total buffer size | +| `--duration` | 10 | Test duration in seconds | +| `--operation` | read | `read` or `write` | +| `--report_unit` | GB | `GB\|GiB\|Gb\|MB\|MiB\|Mb` | + +### Benchmark Results + +Tested on two p6-b200.48xlarge instances (8 EFA devices each, 8×400 Gbps) in the same AWS placement group. + +#### Optimized Results + +With tuned parameters (`MC_SLICE_SIZE=262144`): + +| Operation | Throughput | Configuration | +|-----------|-----------|---------------| +| **Write** | **167.63 GB/s** | threads=48, block_size=128KB, batch_size=128, MC_SLICE_SIZE=256KB | +| **Read** | **171.89 GB/s** | threads=48, block_size=128KB, batch_size=128, MC_SLICE_SIZE=256KB | + +#### Parameter Tuning Results + +The following table shows how different parameters affect write throughput: + +| block_size | threads | batch_size | MC_SLICE_SIZE | Throughput | +|-----------|---------|------------|---------------|-----------| +| 64KB | 8 | 128 | default (64KB) | 69.47 GB/s | +| 256KB | 8 | 128 | default | 70.09 GB/s | +| 64KB | 16 | 128 | default | 78.80 GB/s | +| 64KB | 32 | 256 | default | 87.65 GB/s | +| 64KB | 64 | 256 | default | 85.72 GB/s | +| 128KB | 32 | 128 | default | 92.33 GB/s | +| 128KB | 32 | 128 | 128KB | 152.26 GB/s | +| 128KB | 32 | 128 | 256KB | 156.18 GB/s | +| 128KB | 48 | 128 | 256KB | **160.34 GB/s** | +| 128KB | 64 | 128 | 256KB | 158.82 GB/s | + +Key findings: +- **MC_SLICE_SIZE** is the most impactful tuning parameter — increasing from default 64KB to 256KB nearly **doubles** throughput (92→160 GB/s) +- **block_size=128KB** outperforms 64KB by ~10-15% +- **threads=48** is optimal for 8 EFA devices; 64 threads shows slight diminishing returns +- **batch_size=128** is sufficient; increasing to 256+ causes "Cannot select device" errors at higher thread counts + +#### Cross-Transport Comparison + +| Transport | Throughput | Per-NIC Bandwidth | Notes | +|-----------|-----------|-------------------|-------| +| **EFA (tuned)** | **168-172 GB/s** | ~207-214 Gbps × 8 NICs | MC_SLICE_SIZE=256KB, threads=48 | +| **EFA (default)** | **69.47 GB/s** | ~86 Gbps × 8 NICs | Default parameters | +| TCP (iperf3 baseline) | 9.5 GB/s | 76 Gbps total | Kernel TCP stack, 8 parallel streams | +| TCP (Mooncake) | 0.11 GB/s | — | Mooncake TCP transport, unoptimized for throughput | + +**EFA (tuned) vs TCP**: EFA delivers **17.7x** the raw TCP bandwidth by bypassing the kernel network stack. + +**EFA vs RoCE RDMA**: On comparable 8×400 Gbps RoCE networks, Mooncake's RDMA transport achieves ~190 GB/s. Tuned EFA reaches **~88%** of RoCE performance, demonstrating that proper parameter tuning can largely close the gap between SRD-based EFA and hardware-offloaded RDMA. + +### Tuning Tips + +- **Set `MC_SLICE_SIZE=262144` (256KB)** — this is the single most important tuning knob, nearly doubling throughput from defaults +- Increase `--threads` to 32-48 to saturate multiple EFA devices (6 threads per device is a good starting point) +- Use `--block_size=131072` (128KB) for optimal per-request efficiency +- Keep `--batch_size=128`; higher values may cause device selection failures with many threads +- Allocate buffers on both NUMA nodes for balanced NIC utilization (the bench tool does this by default) +- Avoid `--block_size=256KB` or larger with many threads — this can trigger "Cannot select device" errors due to buffer boundary alignment across 8 EFA devices + +## Technical Details + +### Why libfabric instead of ibverbs? + +AWS EFA exposes RDMA-like devices through the ibverbs interface, but does not support the full ibverbs API. Specifically: +- Queue Pair (QP) creation fails with "Operation not supported" (error 95) +- EFA requires using libfabric's `FI_EP_RDM` (Reliable Datagram Message) endpoint type + +### EFA Transport Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ EfaTransport │ +├─────────────────────────────────────────────────────┤ +│ EfaContext (per device) │ +│ ├── fi_info (fabric info) │ +│ ├── fid_fabric (fabric handle) │ +│ ├── fid_domain (protection domain) │ +│ ├── fid_av (address vector for peer lookup) │ +│ ├── fid_cq (completion queues) │ +│ └── fid_mr (memory regions) │ +├─────────────────────────────────────────────────────┤ +│ EfaEndpoint (per connection) │ +│ ├── fid_ep (RDM endpoint) │ +│ ├── fi_addr_t (peer address) │ +│ └── local_addr (local endpoint address) │ +└─────────────────────────────────────────────────────┘ +``` + +### Thread Safety + +The EFA transport requests `FI_THREAD_SAFE` from the libfabric provider and adds per-endpoint spinlocks to serialize `fi_write` calls. This is necessary because: + +- Multiple submission threads may route slices to the same endpoint concurrently +- libfabric RDM endpoints default to `FI_THREAD_UNSPEC` (no thread safety guarantees) +- Concurrent `fi_write` without serialization corrupts provider internals, causing completions to silently vanish + +CQ completion queues are polled by dedicated worker threads (one per EFA device) that run independently of submission threads. + +### EFA vs RoCE RDMA + +| Feature | EFA (libfabric SRD) | RoCE (ibverbs) | +|---------|--------------------|--------------------| +| Protocol | Scalable Reliable Datagram | RDMA over Converged Ethernet | +| Endpoint type | `FI_EP_RDM` (message-based) | Queue Pairs (true RDMA) | +| Write operation | Software-emulated via messages + ACKs | Hardware-offloaded one-sided RDMA | +| CPU overhead | Moderate (provider processes ACKs) | Minimal (NIC handles everything) | +| Throughput (8×400G) | ~170 GB/s (tuned) | ~190 GB/s | +| AWS availability | All EFA-enabled instances | Not available on AWS | + +### Supported AWS Instance Types + +- p6-b200.48xlarge (8 EFA devices, `rdmap*` naming) +- p5e.48xlarge (16 EFA devices, `rdmap*` naming) +- p4d.24xlarge (4 EFA devices) +- Other EFA-enabled instances + +Use `fi_info -p efa` to list available EFA devices on your instance. + +## Troubleshooting + +### No EFA devices found + +``` +EfaTransport: No EFA devices found +``` + +Solution: Verify EFA is available with `fi_info -p efa` + +### Permission denied + +``` +fi_fabric failed: Permission denied +``` + +Solution: Ensure proper permissions or run with sudo for testing + +### libfabric not found + +``` +cannot find -lfabric +``` + +Solution: Verify `/opt/amazon/efa/lib` is in the library path: +```bash +export LD_LIBRARY_PATH=/opt/amazon/efa/lib:$LD_LIBRARY_PATH +``` + +### Workers hang under high concurrency + +If `transfer_engine_bench` hangs with some workers never completing: + +1. **Ensure both nodes are running the same build** — the CQ backpressure and thread-safety fixes must be present on both sides +2. **Reduce concurrency** to verify basic connectivity: `--threads=1 --batch_size=16` +3. **Check CQ poller threads**: logs should show "Started N CQ polling worker threads" where N matches the number of EFA devices diff --git a/docs/source/design/transfer-engine/index.md b/docs/source/design/transfer-engine/index.md index 98146661..119b3775 100644 --- a/docs/source/design/transfer-engine/index.md +++ b/docs/source/design/transfer-engine/index.md @@ -11,7 +11,7 @@ Mooncake Transfer Engine is a high-performance, zero-copy data transfer library As shown in the diagram, each specific client corresponds to a `TransferEngine`, which not only includes a RAM Segment but also integrates management for high-speed transfers across multiple threads and network cards. The RAM Segment, in principle, corresponds to the entire virtual address space of this `TransferEngine`, but in reality, only parts of it (known as a `Buffer`) are registered for (GPUDirect) RDMA Read/Write. Each Buffer can have separate permissions (corresponding to RDMA `rkey`, etc.) and network card affinity (e.g., preferred NICs for different types of memory). -Mooncake Transfer Engine provides interfaces through the `TransferEngine` class (located in `mooncake-transfer-engine/include/transfer_engine.h`), where the specific data transfer functions for different backends are implemented by the `Transport` class, currently supporting `TcpTransport`, `RdmaTransport`, `NVMeoFTransport`, `NvlinkTransport`, `IntraNodeNvlinkTransport`, and `HipTransport`. +Mooncake Transfer Engine provides interfaces through the `TransferEngine` class (located in `mooncake-transfer-engine/include/transfer_engine.h`), where the specific data transfer functions for different backends are implemented by the `Transport` class, currently supporting `TcpTransport`, `RdmaTransport`, `EfaTransport`, `NVMeoFTransport`, `NvlinkTransport`, `IntraNodeNvlinkTransport`, and `HipTransport`. ### Segment Segment represents a collection of source address ranges and target address ranges available during the data transfer process in Transfer Engine. That is, all local and remote addresses involved in `BatchTransfer` requests must be within the valid segment range. Transfer Engine supports the following two types of Segments. @@ -155,7 +155,7 @@ The following video shows a normal run as described above, with the Target on th ![transfer-engine-running](../../image/transfer-engine-running.gif) ## Transfer Engine C/C++ API -Transfer Engine provides interfaces through the `TransferEngine` class (located in `mooncake-transfer-engine/include/transfer_engine.h`), where the specific data transfer functions for different backends are implemented by the `Transport` class, currently supporting `TcpTransport`, `RdmaTransport`, `NVMeoFTransport`, `NvlinkTransport` (for NVIDIA GPUs), `IntraNodeNvlinkTransport` (for NVIDIA GPUs), and `HipTransport` (for AMD GPUs). +Transfer Engine provides interfaces through the `TransferEngine` class (located in `mooncake-transfer-engine/include/transfer_engine.h`), where the specific data transfer functions for different backends are implemented by the `Transport` class, currently supporting `TcpTransport`, `RdmaTransport`, `EfaTransport` (for AWS EFA), `NVMeoFTransport`, `NvlinkTransport` (for NVIDIA GPUs), `IntraNodeNvlinkTransport` (for NVIDIA GPUs), and `HipTransport` (for AMD GPUs). For a complete C++ API reference, see [Transfer Engine C++ API Reference](cpp-api.md). @@ -311,6 +311,14 @@ For advanced users, TransferEngine provides the following advanced runtime optio cpp-api :::: +## EFA Transport (AWS) + +:::{toctree} +:maxdepth: 1 + +efa_transport +::: + ## Ascend Transport Component :::{toctree} diff --git a/mooncake-common/common.cmake b/mooncake-common/common.cmake index 4855061c..318b2464 100644 --- a/mooncake-common/common.cmake +++ b/mooncake-common/common.cmake @@ -68,6 +68,29 @@ option(USE_UBSHMEM "option for using ascend npu with shmem" OFF) option(USE_ASCEND_HETEROGENEOUS "option for transferring between ascend npu and gpu" OFF) option(USE_MNNVL "option for using Multi-Node NVLink transport" OFF) option(USE_CXL "option for using CXL protocol" OFF) +option(USE_EFA "option for using AWS EFA transport" OFF) + +if (USE_EFA) + # Find libfabric headers and library; default to AWS EFA installer path + find_path(LIBFABRIC_INCLUDE_DIR rdma/fabric.h + HINTS /opt/amazon/efa/include + PATH_SUFFIXES include) + find_library(LIBFABRIC_LIBRARY fabric + HINTS /opt/amazon/efa/lib + PATH_SUFFIXES lib lib64) + + if (NOT LIBFABRIC_INCLUDE_DIR OR NOT LIBFABRIC_LIBRARY) + message(FATAL_ERROR "libfabric not found. Install AWS EFA or set LIBFABRIC_INCLUDE_DIR/LIBFABRIC_LIBRARY.") + endif() + + get_filename_component(LIBFABRIC_LIB_DIR ${LIBFABRIC_LIBRARY} DIRECTORY) + include_directories(${LIBFABRIC_INCLUDE_DIR}) + link_directories(${LIBFABRIC_LIB_DIR}) + add_compile_definitions(USE_EFA) + message(STATUS "AWS EFA (libfabric) transport is enabled") + message(STATUS " libfabric include: ${LIBFABRIC_INCLUDE_DIR}") + message(STATUS " libfabric library: ${LIBFABRIC_LIBRARY}") +endif() option(USE_ETCD "option for enable etcd as metadata server" OFF) option(USE_ETCD_LEGACY "option for enable etcd based on etcd-cpp-api-v3" OFF) option(USE_REDIS "option for enable redis as metadata server" OFF) diff --git a/mooncake-integration/CMakeLists.txt b/mooncake-integration/CMakeLists.txt index 10c9284e..c677e2ff 100644 --- a/mooncake-integration/CMakeLists.txt +++ b/mooncake-integration/CMakeLists.txt @@ -39,6 +39,11 @@ if (WITH_TE) INSTALL_RPATH "$ORIGIN" ) + # Propagate EFA compile definition to engine target + if(USE_EFA) + target_compile_definitions(engine PRIVATE USE_EFA) + endif() + target_link_libraries(engine PRIVATE $ ) diff --git a/mooncake-integration/transfer_engine/transfer_engine_py.cpp b/mooncake-integration/transfer_engine/transfer_engine_py.cpp index b193302e..da45791d 100644 --- a/mooncake-integration/transfer_engine/transfer_engine_py.cpp +++ b/mooncake-integration/transfer_engine/transfer_engine_py.cpp @@ -158,12 +158,30 @@ int TransferEnginePy::initializeExt(const char *local_hostname, const char *protocol, const char *device_name, const char *metadata_type) { - (void)(protocol); + std::string proto = protocol ? std::string(protocol) : ""; std::string conn_string = buildConnString(metadata_type, metadata_server); auto device_name_safe = device_name ? std::string(device_name) : ""; auto device_filter = buildDeviceFilter(device_name_safe); + +#ifdef USE_EFA + // When using EFA protocol, we still need topology discovery but won't + // auto-install RDMA + bool use_efa = (proto == "efa"); + // Disable auto_discover to prevent RDMA transport installation, we'll + // install EFA manually + engine_ = std::make_unique(false, device_filter); + // Manually discover topology for EFA to populate device list + if (use_efa) { + engine_->getLocalTopology()->discover(device_filter); + LOG(INFO) << "Topology discovery complete for EFA. Found " + << engine_->getLocalTopology()->getHcaList().size() + << " devices."; + } +#else engine_ = std::make_unique(true, device_filter); +#endif + if (getenv("MC_LEGACY_RPC_PORT_BINDING")) { auto hostname_port = parseHostNameWithPort(local_hostname); int ret = @@ -176,6 +194,20 @@ int TransferEnginePy::initializeExt(const char *local_hostname, if (ret) return -1; } +#ifdef USE_EFA + // Install EFA transport when protocol is "efa" + if (use_efa) { + LOG(INFO) + << "Installing EFA transport as requested by protocol parameter"; + auto transport = engine_->installTransport("efa", nullptr); + if (!transport) { + LOG(ERROR) << "Failed to install EFA transport"; + return -1; + } + LOG(INFO) << "EFA transport installed successfully"; + } +#endif + free_list_.resize(kSlabSizeKBTabLen); return 0; } diff --git a/mooncake-transfer-engine/example/efa_latency_bench.png b/mooncake-transfer-engine/example/efa_latency_bench.png new file mode 100644 index 0000000000000000000000000000000000000000..34693c0710e20d6ea46a2fdce1137fe938cf7af6 GIT binary patch literal 64738 zcmce8bx>7*^z8)%1TG*TT}nzwh?JCpiUNY5gtQVO-Q7rt($b*^P4ws=Kb+z9y5Nw#J%U9Pwl<-T5F#`#e262@oDf;C=}rx8EGXH>J%D+PEe54nCj91GBF!$Xn*(zxpc#$E51XC}h@D`yg1me^T~+SzB^gIoB}SR{YG!ZI97uAu(@(Bcxw zcjoW!(f_}{wG8V>6zeLrF;ewE&vf$TB(3d|{PYsK&eHA-_Wi3@Q7tVk4?aC;>3$n< z@+HGyxiWgKqM{-oD9Fa{0?M)Mil?_+DUB67MG!VB|JTB}jC2C~`STb#m06V)6`FLf znlrmPpIHez%LkRWm@JdL=Ccg5p8Bens|MGzw7bA`kA$2Y`>KpgKoB#DRd8hF*(e?@ ze9z=$#<8(6N9VcT_mdWT>KB)8WgA-w>*?t^abmou!TEEuOyORn|IgnFMN*KESk;&c zDW~6~M|u1Btp7}EMy_9a4G&o{Tsbx}Nl8i6&fZ>`2o3xqY%pDeRQk^s3l=N-e|$UG zW>J0Eh(#=3vsgS53}evN*7o)Fwd~b$SZ!fXKEuekJa9j`-{Z_Prscbq7$(kGNBi-^t|SSgZy%0eqPqnk`G>~ zU+asfmf54>%;2)UH1%3l`z>s{NN#mf3k!>bgM$k?RpN{Ng%gPm!+J9viPRnCdrri3 zA}djkEnhR=4pyviw0c3wYrgGDCsDOCPEc;c_27eLoehjttK7cba;Vg1#?V+(g^*6< zn|YR}C#oY^Dp^l6LnU*eQOm6R(?j(mVF{aQ^m64kL*}DGrpuSlGBPqA93J8k5Txi- zySK#(_(w)Y#=A?NtW@!JLG)lV0dGa+u{r&GPv(<@oz>6k`F=`iccGitrtmrkqqHRjO6A>e7!=_bc+hV!ojZNUV@&)&f zoSdAAtel*Fdz>J0A!-4`9LurmFadew$FI3m9k+fN?H=tf^P}ssctu5HpELzqE%fn6 zUDRkSU#@C}-Tx8BTaJKYt874E;M=F)-xZ5{xVnlbyYF$pbX821W^Mj5 zN?x6gw+Ign)$L7+jI6J6+wUv3V1#XBu<`r5`2Irvz3Qq#wYzukJ__#~8A8OVey4oxw4R;D#6r z9Cn#&slHzEYWaq*hE_E!DjQbD>M#Z_9pvQjaZ?an^%E}6dQ63*AijDkFfi~G4$k=4 z*q7%7l$YqMUAE2Htn{9D|1fGFp>}%seyh^Z%xh4GH-tg~tlL{i|YTO?2NY+z7 zfZd{&ql=O3wxu35-M;+n=Lfy*!_MH(Nx}|Gn&k>J5Y6D=;+S#~C%onh2nm_V@04s+ zc|7kt-H~V#&Aqa*F}dC;WwbK-#nCZ!gw=ku219c1H*>iV65PWEQ>&$ie50eIhgQ3j z8&$G2W0`KgNTtWc!7()MSgt-EUeaDL8Cj@WZKSn7T%T@lRcRY>+eXd^)@U8E6w8GR zzsn}lsZuMk)O$2CAZ>3IP|2{ zaa}FzZE34MK58ua5;&! z=FMlKKWJB$-Oqd^nBKk(>qpDLz<{HRYJ{j>{x73}9 zMqJll4OK`uZ#4I8pX=_n8FUzNyEXOo^90$zGunsCN!^Th(iM`0+6?oD<}Kiu*{{1oY*PC%C;gpPl~7t>9@`E+l5tLyf@6hmbM^AUMv<@ku9~WL}6i0-sc~ zm*)6>{Ej*$d_UzDq2$rd6K(fh19Y7(d?rlTu=&gMboLv5=&{|Iih^3 zB;De;RI^rgCPLQtm#YOVM{5ccGPxdps{j2xC7*n`OF>HB)RY@?tXvG=`5S3Yw_%OC zs)&e)8V3djvX1+&2k&n$r<)BF+0S@oR@HR7y1HI?RB++@_wU9t1`QCT7J~8Q;(g;2~)hB77P}#t?nXs~A zc{hqXcB|sOLUNMcri^{%W{-yL?{63x-CtF8+!|zgE2VQEH<&wDndk_Z^%t~QlT@ar zvPkYuMifjyGK~{5djT`m6v8+&`o%}H)Ozau6EY-Bltz)_m`|ofYrlhEZY+;T-I%h= zA&+obDxJxDSiuD`n$dm7i-`UTswi#PVaSfM++i8Vv~>jn!}YP+=lS9@3%~N!A&Tj` z+_1N^s+}(_t3r~)r45~ZwvF+IAix2`r62Kpey*EO!b|O&U`OVQB?0(=z$@*y{;0^b z<((~+B)W8SsvAn&JB=wdwV|}D7py8)ONn-)pFH7ktdwM zJdb1dPb!(t`3gRemcExeSipqdgQN&Sm>O2&NWRly(d519uRL{0VKz3u)@a^z`s1wu zi$SNEMB^qrJUqyCOd7|$FA!|18S;#(BrICnIS`k@@!UmqMZk0YurG=#2LXi`+sN6%-M%9emg>>|bbI{ns1GrmA`)C;hWclP zN*|Td3Ov#)roX+tVW0fdVQEm*tgxjpgN)9sJVwi+a#Q7zn^$(#9_PheTY)UMcKsQ_ z2k)fuFflQI1dq5ZOTg}HgRNcaxXM&jRRx>Ni{j#=L0i`oJf*5!TO;2FkPOaY8hr>A zzc;;@{mOtvbuP^4-Dn=dK1m6WgVm<|%7+8&$*$pgVikLv^F|j&?U@11Om56}IXXJ> z>DBt88)&j;=H3MaM``N}L|r-RfBZuD%J!`WO6Aww>OYotc2+0so5;CTQg4Sa#F~@+ z4oLA1=q=K6NsJeD7_IR#&JHA?r_@V9A!p)J$YSP|Er9=Z%{>WT>BSeidn0hk29Grjs1VS(VH`hk3-kRhwNubvtb!j~^c5=$L~0*lz&3OlP030-5JL6qONoWoALUV&EE6rJAOC>&l+S2KhA!k@r?M|1?XvR|k2%;sR;2wn~l)imCU#||Q{2_(S zOhy*F$V#3U zS1s)%d`8^&Ujcp{lxzOgQ*5DAKA|9WczN;FQIXkz8UT(*IslJzDABgJtrr24@$Wk{zTmH_j!dm8i1uwV1137Aa<1xWnw&p z%K5ThXrfItuTBR9p}>$3TvNO#IH-x*mCpCyNe33#1T#oDcXhI|54Oc0#|ju3?o-g+ z&eArpcwvbj?M%F+7Zb`fgYnq6ym)sr=tqIE@>1otcDc=Z zf3=3Y!KGLvH{J}QU}a%pk)BYF5Q6 z8>QbDQfWR^)>yecY;yMEqn}xo2CY%*Ne-|#{i35)`%91aM~)G^1NWL*NY447xi8`Tm9IRGcAJ(ekpHf1Z50LWR_!k# zNWxqg%EZGw^7X0K=vPn@8SD(J!fgHS%T4+I_6BYWMAhEIBORBl7X3@esqppprv<=f z?}h*?<&Hx@r|6oM)m46gsxM*PXcDj3U4prP;?NO#{zl%0n~3X9&X5xX8vue5bdo1n z+$Q_Tp?Wxk1pwZ5B^n?95>yZL!ESRO(-)>SVH8nGO$eIm5QFz>c>q2ZaP7=~Lph<&EMA!VoHJV2&&t*dC?R9Ks9-a<9be)LH*5~^bj;jS#5Zr0++__VH z)#?NMkFIAi5Yd!m8X6w41p4l2LAplSPU-lyDPnuuzLGr2z!!a z?b1QU{-a{^#4&l7xfD2DehrUXHU{^X$~!;6Qbcvg%TsACBM5b(IgD9-k3ln@uVgqy zHk!wN#!yf%$9eNvxs}`AMvLtN)SY_5Te(dPtv}XF?$@4UVyazPSt*%>ScaINSD%0uY&((01GG=9W-xp*cLy`G@5Js%W53q=U~7d%D!u%lwWdGoUj=z?cxs z{qMf!y#GP*ytsq}hstNF_ITkLA7aT-01NMdL{N`gK0aDLM#^$sjktY4_34YN+Q4;` z4=}(UY0S~9OE^x=8e3lOgq=$*X3q^^{Va#lDIiTaAAG2R7~helRsMUpvb@d)uvN#> zP`MN2(T|F6F`*uTSNa0>YYe7ug}1M6Z+F_u0ctSpFEBP8D0*d-HRiC z0r}{SZHRRh zHkWSKdwDeVC0~G8I@^N~DprAf_?{+931}&f!a`T~930;#kRBHLa+x)XvnC5ng6Ori zn%5se3VNrO``Eb2X1c9KkzmyDLAEv$Ylhaub@z|wp{PJJN+tl5Pz106G~UYkdJ0Ub z_`ynDTav^;mgg)~c?fG_JXpe?CL7(@)PxI@gCs4e31WpU7yv-n-aUaK;j-amx{K}0 z^{*r3{KBx}TUrUbKitO(YN#EqbkU-JG5JJwL|H<&2E}$S;dh>z5`nR{Mx_pmRM-5U z17N-pM^T*PD)RXd?MXAz_w41(gu;?b?=B|^Ga#JFidv>h`9)PI(WfJ8qmi7imT74z z`uP~2DBIrda;#GARcYyZz}$EB^;uBv?(PsGxb$GVF%`4TURv10JKCdow^e>6j`rl0 zNFZckZ%Fi$pU`V1wq)(^{;nSj^xx}*QH5kH6J2-9F25Ek6%%AZ`rHccAuU9SlwY|N) z95%Oa-BN;tD-%_60NPg)ooC4x8ER^0AlB4s${Mi7S;^kEV}O3B&5Hv&4b?DzY%dwvEX`^|BR`5 z1Ya{=r{q4jYL@00q`;{Wk7GMKkv}&uj*pj{*Pi5eiBOA7hv$B6jhFC< zYXrobJOE*kzATKhU;;%_^Ot8#Ma z_a+1sfs+FN_-alQ{9#;=lyAX|t5@=wpA(`dA<)n#Ojl;r|*cW$-SJ8bK>S+NUjXkfNl zrFC2-UHZ3jk_NpFW3|{QA+vsIAk;2smhe>{?PUx`Tp2vV6K#!PuMJ+h1{W>$YW<$GdAsj2!}vXe1>!7I4SB_x^DA4OhdXN}nDKm6 z0*2@(jW6Fqk@%8M^hfs?(3K2NsBi#9dtP)3Rllwdn`iCK2dtG|;T#(r7M+eho!m4t zjry;PHFeC}6RqrnYcyJc5;lxi4Ty?fT zV{FkuV-aQN4%wgCb!l|Z7ydbCpmAUElgd(4G;yhXEdq8;2TQ(Zwez9Niy$9Q*{l6I zDd7=HqSN=j;Fwj6)sem8cA;<0CsXoET0ZrDkpH!=FN*sZE1Z%0 zt3$p(&_X(cyUrrQwoD=aE4V135{67PVG}l84UO*%7hnF z%69oMZ)f}Ss?+Wb))M0PPtwnM;F_QHe8zj~ zVF5Fg{z3vTzOgQf@!Z(;#-nd#MSG{Z%e~VbcyQUFWd9Shq^*Zr7K_YCn5Vr3r{eCB z`Pvn*;(rxXcJmg$ej+%*by}!Z2o^LRbLl&7biOFNTu#S6l2j{*9*N zH5IYgdq`Q=+s0lVUNbP;#GBjpzkU;Lu7&1Am!jiC`v6447@eOFs;-t|yBGq;v$Tq! zNv}(-?+P_3F9gk@Z;*`W6Tf^ePH{>+dJdze%TeQo|DlS8NP!Jq-T0X;PeQ`s3CVuK zp(s24h*;e1Qt++=0VKpDb(9~HFZ5*3Ty~-PD%7#-*kHU79r9=0^v(CrstL7i6&6cf zcCXzkWX0Y(f^co=`lNB)71OMk?y@`Si>S^@c+bE6K+=R@iVqtTg(+{GDf>1BOz8+H zLkz^zWt;}?`j!#C zGD-DMpJPceQKd{RyP`fih9_9|ixfL;en#6goa^5QhrP_w(y|m((0RHRnJIZqadz;P ztTh1gKZouyFDr?PN2HU-+jrmg?3%>b0tQwr?NEPDWYgyZ6i;Gnh161{8Mb$cU?yL) zKf11l4-K;F;ac2?TVetWl1JT}=$7M_*8{wTkeNIG#DqsDfGlB{7Cciz4=_hNh#f{8m zH_brPNF+_@1)BVh@)hrRR%kuMbqNr2`L2?>ePkWGq6NI(PS zA05P;%i7PQRaOrbE08Ip2fOQ$7w*?~>)nx@7#q9R@PZrzDMJ|;zUBr61kmmqVYIoU zKezl6Lk1g1isPgS*%Adv7lzZ<@I64EXS4l*wxjYCyjfB43<@Lz79O5^cvvk$#Kgp) zR1%#i#3{(xxF%*)dvNTvI`T;_yLM&t!G&HClbQIn4o+n&9#-@kUmc%}T*1XxA z;Kd8fQK&2)WNH3tAOVJFkg3jdMEm)xk+foLRj(t0a~}%k$HT>e^+85LLgHXlSd-mZ zSJX0V#`^5)@vfp;`JKQgStG}ClWZU_(@NqH(LHsz2f9a=SQY5X)L9WRF*4uZ-Q@DB&h7R?wR~xjfSA~!Z#d;PF+L?v!rP(mlJh_cPS!prKmjk30Qx8i z4UMyME*lHWN{@!ccOb^lE&llAoR%BE@0Ta?5B|!?uKHSHHQ~74E*vZFwDw8?F+_yL zM}(iwXMA2+{l56M367gPj*i(7PcNMOzB@8SIZt~zq83GOm|nl+Ri!=XcKgni9-@X` zu4VH5I1#JIMV-4FT?+JBY?k^^x}bdh{E*XN*cS68h_@ol-_RwWwix7p~tD^c9C%6j76mWvthJaPQBc z9o$R2qYW4JTSn4%*K3@j4&374$H@+2)trT7fh;0^)kIJifMOYinzJwWiPy%d+V&k# zv`*vUfdom^fzPi4G&!%IUaFe~a;ED8Nhfsou(n#fKW%@xA@G_zlp8V!N+WNyg+n8t zDsnPMYEOw&;odK$H{m!^hZ|&I(6C+Xe+@hp!YlK%$O9E43^6>?2FgLOO#hXi^)J>)9if=Zts%F2U7GxcE% zESyvc=dA@Ng74`XC6?zwZaIJDN*u_~WHdBiA&MD)e}@fBa0lNJqN@OsLVN-sUjE2^ zavLOVN3bv;tTe)|(0YJy4P;X-j9w0G79a{rsbstc(JC3#deN>DxEsWz0aC=~e6QG4 zDWW@^b&P(e`{QtBd3vL$jQB>#@|T6zyG~3*7xY3qsr|k*aJ3X~#8hsM$f;6ja>3H& z7azIE_O=o~KLnZUSBVufY>reQPf;85y~LgXiH-I}KWmw^E(tyR*_`uq10cjV}V_`?(0uwh4xAzuUZbZof+O*Mp0Y=dF;eN^o)occrtpz=x z3lK|2bF)m6>)IWV9lyeg0aXU_lc>uUw}QmxS)e-%L8CnYvJt!p!C++BD$5l#AEc5w zK7wHeG+zvmz$O!`ogJDrd&S`2@uY@%xtlqnLg6OydTF2U*}01H%c|MC(+AZ~Hb*1E zZbuz@+h-m$i~eeFi<-r10~HRK6~m|5YPs*~RzVm_2g32{9;Y{O1y6(q+)H2&M)Vh& zvPdmTzsP&ZB?oS(S3xCqp4fP&J%#Q@-*lsIIAZW3X75GTV>Hx`(R zPz!^lUDOH>;pC)_mRX|N?EGTtX3ZQVu@-OD*c};ClrpfQS#-*Wi(UX1Xw#SUtLO#f zIN5z?pnk6Iee!;AxVJem6|G}9_w%FrJ*xVqCL3^T+>NwsXDHqtv{CRY0`&;f68>6H z1nZS&GjngLWP%8O88t=+dOI-h&DqsQ2$KQD>IAIbsW%WEjEf&?W;!en_sUbHx_5Sh z7EsB~8ZN6u=9iSTXUD1@wp?cOVQ`;x%280WZch#OOI<0|btlQmoZ)&noEqDl|t>f20Cf$8Z+{hgja2mD~}))Mkvbgd}^ zsy~T1m5+Tp5bbSXF~5Q5BVTR zxy93<=K_Oc^m}`4jY&u-O2>WoIS6^9e(TuS*cfMMY3B#89zQ93ls({)B7FRjn?i%X z75psz8KhYhbad!YmDckay(KEo-ksZYVL^b|FH38GGhtt@@3Us~lFu6zO1N39c%AkW z;|D`q^vkB4Q4+L~dOsQW^B?Tw#8eBzPhWalC4%0=@xElz!(CG(eGF1hY;fWwgS(*1 zB77d?#QOUBL0e5A13eeF{7fv;ML^<^?{q2r`TBI)-Pd2&zP=hT&u0x*Qf1eihtt_B zA+_nE1SBo@Em$rlCKTPsDP@=I;~rds@(e}eewz&TW}4$F1+w0w^9Ius427#DU;OJC z*mg>8n-$Abc$)3kfd~Z@gs&bI5o2y{E(GE=2uNh)EkrSvogf?7E08@J=tc zm$(y_mOHm-26H>44F_9pW?}Z?pFWLjS{8QpkrhI#PvuKx1fKHoLZS-_hpFx(Rc=K4 z8bzjfHNYQtnbV_UuGl?`deZy?_!hFDhU-Am@9ZzXEdEF*?idOx_xZ&VU{4Fo2BKj6 z!;TY#W?=_8CT=WA;oj6E@xV_!Zh=KGsW zkQS$Ekg~MHt>;zUrKMPwY+yW2I4^WQ_NoqV|3lt8{*T@|SsQ!-VMSmO913(WKb>ou zun)P_^G*>HhtLi``v#&k7<^hGol#%XZzh~s0>(Ajbqx<*p9&#~dQZ$@sSPwMW01Fj z2W$l67qBb{or_8W>j|hGV!-RlLjVS+nGl%04uF*whnV0Bf|$B{Qg0VzIs4_|cCgRL zfq?lDA~p!FEN}u477FZO5H#viLL}qtJ3u6q1s!e|Txe=VrqqNqmq&rn1)3l%J<0h6 z2-_WFh2X}3&qA1L;QvHf-ARn{)VP72CPYa|NkKVRsNo$DP|_n+U9)ieOPs0L7WXM) zdH=g+nzYd+7Z0bH?OiL|Y_oOzmNo@cdOCv;PB#+i! z*%L9lIcczng3m(#_~_7h;H=?$_}hcbN3S6`uRtP6AMy^J1L~R;GA3B__&|mTkBH#V zsd9Z(`$oSd97)B|I_|OE8LFTP;?Z1wS_A5VSBt*&qgvD`4Eh~ljF;2`U|JE%ftdM% z;^N)dO9qa+Q_-_S<;6ogoDV;(Kr%ouTNFra;0pZkFoOaL9CkiFzAoP{Wp?v$Kt(#m z*X5$@;}KZN@y-4ay`F?QQQiH17-MCnQ}O6tZzHRDyZ+NxX|mvQ_{(1l&_k^0iu$0M zp+ZPZJajEQ`4xji9LPUOFt%^N^s4$#*tWD3$rVSd_PnqpqPZNfI?ULCyoY?0oKoO9`2O~w1-d%n**v5qV4)-PfnPG6tJiA`qSRZ>%< zg3J~Wa9Y(L^Mjw4OcwR!r)MDBOu#f)+g;#s000Z70692FHr(n6%{@*5tOr71Uv!9t1IK&>CWI| zQQr6%2UQFHS>&DfVHx?+J}%3Zz1?q1ej*?P*wzcGSqT|8PTuK)#)_&W;JTFx1q^No zx7T=Yb|UMzTp6`8gn!`Iv-!{ipS)kGk_QuIp$|R0-po@@Q&GqBy=7>eZY zS#aIQ+UxYMWz{5qeOC8hJyjCNiajlbU2c?CJfuRr*MVLAI4ejN|MI{0YjUuX7)s&0 zc*t9D4V+~-qIUJsq>Mi6_}}~Qu93(5q9o#$wJ5J7-$)`FKH?#eJ#|v!TQ%3!0at6* zkOpRym6dOe*I@?r1FiTb&Wq#;Rc6jrelsC;+%4nb2|GR^jK*ua@qFMcQ{IL;*?i_J zc1#c1(NMy$N+s7u49bvK^(O1%+kY#uutD2dak*LibceL!?pdtFYRvrMcXx*0s-_^& zDL}?NP>e95yr#5r){KzRhbYR;#J zZi$v->mprUIM+uM;?2JX)<3;x6MwwA6zPmNaDpa!Wvq z{y12FOQ+_s+G6;9a0qEeXJ(>j*E{a~2l_vmCia4&1me$4FjRG%VzUO{+gHGvB39$4 zZU@tSc~)3<1?;#jogIw6j7#|n6a}siTQFO?U^}lB0_@iBr(|GQ%Bck02_=BYjq;Rt zhX0qsPdS~1(Yz77GP6+ffURZFrc(k!6-(z?n+%ZH;7SeP8aV{h7=l<_L3snUN<2Cx z!ux`t9_kDk*$p`T1@238t{HP#3ICVhXijhy_O@BLlQF<}G*Ja1QW_=;DG=Z`el&i3p`2ji2cOa-} z+}gjlUxk82yMW%s@p=fJVs^G{9cc>@E4qGoi{p&Lr9+V=eJ?G8(6YEU79&-#>-RQh zpY$ftgXF-40+!#pui9$j78G@j;D{&zH}&kdw^&g0A?fI|PWAAZD3o{OP<2m(2V@2S z2msA4C?&yPf1>#wA8l6`CJvhFD#Tz6#2F0GW@@q};^(J_ zK~R=h511A6TMUcG30ri40+fGPk^c_H4kQA~!^6X~pagj<0O-X=3aF1Efg!0JoO_YX zt4Kr`%xgB_%VcNK$)k_29P{G%gI8MEy7`)y(O9HBSB$bEPpPdNH*N_g*U~2@s$@I(w1zw;kuqUl6J!8Ipf zJt>3e>;+ZD!_GmVMs|)R@KzsOGU?4q0~>}b5HH5h>7c%70&s%R6s5;U`?G*_Kpuyw z0?X!A#Mc8nP272>?@-~}FAd&-_JYNJ;Ef!v00M$SGVtZgwx0El7oZnF1ZsvwN9k3`9+rIr=PbNWXHq-B+a$&=K zYERffO6B%uaH9=y`%eLmMNJGAnovR;2T^t<5G7knL$XkyQD3qBpfv(+f*5h9OW=yi zH<5xWnv6~E4bY`88KoE)lQ;D5fp-!)>~`}#2B6a;HcDB3ATV0>TtEw&?arWt1U&&q ztHV~)9Qr1(bU|U~d36KHqdr>sy2I!AxQm`Fqa1gPE_kE^nk~Hp66#-ZxMa#n@^C|W z*yGp@DqN(@N5~4e08%^Z!-bJ54_G~VKz;1e7mMed2RNg+R6F!s3K& zqv#>F>O8o`kj(%@UAuMdIZ4SRz_y4z!2b}jpT7YHiJJ5rcyBfqeno)c@K<5Bc4ge; zr2dKRNdMX=2sNO8h8iz=wL$H)hS(5|yV$_B1|~03)QLrvi&ry|Pv&H+B*;!6I4n2c zj+EP#XeT>3(&EfUBCY{V$z|>^jj{ieaQ#!hT^YRX+8&2?IK&Kc5NW#lnVFeO!JUFM zk^r({0206}|9Z4WDA@pKmPTPQt)M4u0b@%tfm~W#7R9=BS%>0pN^2o*Df8mOMc zn+~>g%-IoD1Q12<&ui>lJSvrB z#R8;a=4BSNtBbm>FztWDE>RZ%2U7 z=<8E+-<_g_O4y6GgYvAtOBGasMElAP0?qf#P(ckvZJ7|$sh<%If80sEenPX`Y1-@AGN`{*EtHSpvQ1yx4l>~3UPZZl%igWYxhpFcqViaA$ z0M~T8?E}q4{PgDs5{k*G#FGz8I@jzm0y$%$hf${La>e}G9$xxXfvs87lOS*5V2i+0Q0Of1&o4ep4hGDx&vMCy2OYaVwUKt4 zwRpdI$ygOIX;j8~J_xCQ!E|;jjOpg3Oq)@eS2TijMbO$53cfb&ZL(+J-T?vw3Ll6k z;^Nlls+BW@fBt@5X(Qeiny_%7W_}$?5~$8Kiny06*6;ak!Rx~5>lx|k+o4wrd}VJy zvL|Xvo%@x~4%{w~9UN$y%fJ1siY!Kwz~$l#T3T0zYUA=Yxc;94mkri#d@!M=10@21 zHV68r7~D4QyYGzQa#I9p$sx{j@DuAJOemZf*#7ma{~0y2hD5)GM8R*bzc z>+0RRFNQRM_nr#7%`gV64A@4m6qpau0HcRAAVFUfViRARY@vRPK+Tf~b8u_|%O5*`M4P7F z%7=yzxPSk?==yPJi6D9JS;Wdw60hkYuc3UQVu9RsJAJt$O0$A3cUz%eX2sL@bEYJD z%S$l!DQD2U=4@Gm2J5cq* zFzvrJ z+o&KLHv(PT42c^-tMEF+#&#!yO;kJ`unxCYStR1y0n}*&-GWQ7QxHT1DJb97%=7#W zKPVmQ5K0^n0MdQwy1QnN=L3C+5uu?qPeLT3GD{Kd#XOcZ&~qfYY3AFqv3$#0!hppM z3BQnup&aQov52t5yqR67t7@B?{Gq2)BhIKdORK>2S2&=O6`+3C+5|d9SgQay>*lKU zA-xPwpvR|wZ2{J7w(GmAInpEuE>VPs2U{VSe9~nP3n0RCHD!GV2Md7errak@r-_KF z#vBi&Sc*CA$kqO~#S&G%Zy1{FH@UKl-?o5@oAqh_K`yKKWvq%ro2`DOH{hh#6N(3& z4*F?5pHXS{!d^vm8%iF{QK$l~9_3xde~;MDvou8!PF&SLoEQAvz;{HiK{uHJhx$3| z3^0iFmDzE@aWVkhBh1|l$lRQhpW*#xo&v=KV`4XV6^K|#q&Tq%5sdVf0_s9)AvCQ} zTle`pk~0un!K07t4A2hZGwNC#{D>Lv1(LENd=2;k&z^%G;h=_h`d-irXg7r^H;Z$- z=Y8?RWqjg$*x3heCfPD7z$l2tDSFB4V4uA6{dZ+yy`@vnL0wKJ*2hi&79*`H2+8>OA0!NK-+GE78iJzLEtE!+@ipS#x0w-BP*tdedqe#w&_)wUQkB>uDLP`KwWIZ;+3y+-oi=c{Gr{KoFP z!e5ywZTDN45%X>tCuf-JKg!ICG3u-uAC-N(oCQH9!Qr%*4uAA>)^NuE26vtnZ(1D3 z#r+O&eWw_b3>m(IKSmosbe2q(KxVD@=&Ug#LtkKBEUI83&y>47OmrIDwCUT^({ z@Xmi1+U?xp*vUTb+p-%w*LQp(LhGbSi1Xo9;Fq#Vk_i7XdcbV+w&3O1KiFGcbEpYV zaM!FPuuVoK4VN;+1&Dn=Qam}=;nk+em{#mvA=&pRel8J8xGUyet;hT8AqqQ~= z&FEz;b0;vom`Y)V^LH*-j@4F^4(Gfxp7-DoS2mDXQ5=q(XjDv2BeADrwos-!oa_Aa zlKzqPgF$!d+l@1WVfnED7C|~eK_~6YU#9j}(o9U)4>t7Fb}<%`u&H_(Xqy2u2Tw$pIA9~s zATNEm*LoXDV>0Nav9~+T#I)4Jr@PFgw#tw&@ML6WmOvio+WO3wY&GZd=r6N8d!{FqzZZGEbnl26y!B0t6MTLsEQ{d+-&-5Ay9 zy)*V5Ig6H-g==Hqw#+W{)(hssOaAT14iXQd9(Q~Vca&_no6b{ujdyR9H27jQ$&v%~ zgZ^61tWAK(*vrEA#aCHyI}ji+}gm|2CK~$(3VF_j#W#tG;g= zJIYTb`+B0JZqEQw*n!OP%t>FQ%$8cX9vJZs`Rfz&me&dV%C)`yLcF50;YuXw|3;qI z`0RN=Y0jEkfTaG&3_Z$$$m_k&7vXP$tAPQ^CpSLs*`m*M?i|V!I#}g6zp*l5$Ld|3 zR8o_~>{uvT7TCW!!G7-G<_fXByem^U^B>L~9;;!~MDKt)LrOY6{#068^Y?FOwJjdH zHItQspoQ1YdU6g=9mKxd#gH&PY-S~U3SK- zIMgIf3Ye7m2MGi+$hWvX$(QSPW%ea`J)R0tExF==00O&aQvKhHnCm=!TJ?4Cc2fkV9kl2%G7?&G^GhsG)epKB|!otIj*M` zbi#9&gZJNV#w#*$Z%>ehx!(B`f5Q%V_Rf5!!z(w!@N*kk%zU#rpjhU`#Y7h7D);Xd zoJ*HKA5j&_W175h)0xHjW=}OZ`~z9)s=w|{8Fw2|nj<sZyH(?0$x0+!cfvt!0D6?VTa6xM+9ukg~ zzXy&F4U7j09#sJSx0JV8n9SLyFCgLQ@RVP7Rko|hpalA>kiEPq6Vu;-HLeM1Z&hrd zSN=HKDx+>EKWAFGZw*T=VsZca{MCWtw@7HRX4fVOll%7$xqwO~VDW8Js$&ge6j_%M7k5R;@u;O~*Nikdv7CdPT_2zJ+BWJU7ccF8wtvG-TK z4Bm8SfDfNMPozx4svGd)r%CoN+M>Ptl0Hz&17B{GR{z<9S#{UyPmCZwGqBW z62|>ER@762XE$_iPT$J`HMonuOpBJpD5Z=KXqNxG3@0zJ6pUcM?1o>fbLw~$Ouf>R zY=2i0Y93O(Zrs@Ieg8gtsS^99QzT*HZFSukBCSx2?XJ`j&kk3LL6HRX+1R|x`nHc& ziKQ1J0fR?#VqyYm<8^}8{1VGC?{#l@iV#WILFY5CWCp+1@3A3)Qf3-n0Fy#W+VOoz zm~Z9ie!)y{vpYI)0+#s}7<~|K4rCWrCMIv7U&9-MDAz&CCZYSBrjK9^Z0$knn8VX_ zhwF_`dV{^44-39tm0lw(R@eLXSzQ2W1O{QD8T?|A!0xvI& zP!(Ve8Ev4{u~qsB_S5Lyot@E%2^(3i9_jj5^x6bZmcKsAL!I05g|shI_4vrAl8Jp{tzLYxZk-<5p-4_Et`7EOn#T+md zfwJ;H1gdKWd4d7$Nv}9|eQvdG5A&7=JI-sLr6=bL?)A*@$0jny2gnd9){fEugOBw_ zF7^_zssqYRC!N*t0%)~ka&8w~w5+p>8vE}x!Lyv>pPHe+B$5=T7oIPIThrOusaFkO zLnX8y@2B|HZTa95E5QRh3ih+4*UI6c2xV7Jk)&TjGcOw1K7PyumLU+XC~vb`n{4D? z*_XqkNmYjhE z0P>#X-wkrlfGTI|c8p~)RKQy*D?hKW5=f`kaOU9oK-c#f3W`RP>`KF41~7<1^O`iM z6tHw4e6nUP!-KTOpfW-PCFG%0_RYvq95)SI<=eiqv+0z>a@70OTP6F(NU35;30Ddk zi4{k2DXSjjotISnwOJRzUJs8fcu!xAJP~A}j|8!QgO=C?k4L!Fwh1Tk#I6E97-(C5 z+Lv=4#)}w&KywTPeoEXZ`6V)MlNI?cqccZm3Z*LBt3@_qbTzcjF)7vjpZbCfjI+R_ zAY?JzhR9J;9+F@XN&#yP(mo20oQZzi@PJwsqAH8d06)FE8;Md@YVm6eB8e*h8pYlQ zR;F7}nlmSv9rY1|A@mcDrtqrXRH@HUXMP_1K|4siMb^whZ7vb~W{BU6T! z)%IIdUL2^?@$IqjR6kChVMn$Cs94Z|2u=B8bBZWoaEK$bXRUQVY-JrK;?m7H{`|%D z?N!CJ#QAYjl9rUe>0!IrMF^yz@bRvJub2I_rLHSd|GBfjuoIRWG+3-RGGR=Pj%WK` zJ(pf9o;E@Du-f~+^cgy+?^hfRfc^(`QJl!JI>}`h2lbdTFf8XFUIhW8M zJ3LKxN~_vS>`Ey&Hw5ax^@|lWZetQ8`PKJr6hTL2cA#Q`cg8!oIn{qR7Z!f1)eo9% z`ih0!Ntc|U`9+f~IzUG9Z_i_iKS7gO@wtS)@Zo8CRARMn+z=SV17zIOP9QoKblriE z3lrflI8{CeAa&dWrtpu4)pZruELqtqGxgTE_l2Mtd z>`D}K(nFPWIjj#g_{r4T(on1t#T^YLSzz#~HVp6+)Is@z+$Upv*KK&6*+RzYu z{r>0;!xOeCU)GH*v3RV$fb%*q`hWh$b7bP};dU3b-J*YRHG5M8Z)p@ksVDPaQd!|1 zV(t6mj?&Q0$xPl~j~kVDT=;2fcoRV2|9k~Q&QwDNgze;=zlRNDA5}G;d=Z$$%gVgd z?!O!Hq4PLHwfCkek>j=#W4=x$kUF+q$1z>C{O9(CU%!R}pIiN3!9sV9wdG<1MYf%A zo=EDuVRqlN;N_=#;6HJ2b_hb_c`QH}2Bc~>N;=WxVdU#axEKpHBkTiEvRq?>N%>CY4HFV>F$M^WbcuM7_Z=L#v) z$CPXQyYuplRQq>R(mk-dfCTP8_7M$1ZUMUfZ)qR#oXg_dy@fq*ZLXRAcM`7-qAD*L zO8bnr-42o;*_JY+LtAECDU^p*Z71m8zTWlXTQf*U00aNKQ85|D{??`k{`{KEgdnth zdOha=`C2{i|L4y`J=U-nFtj8#Wd#(hef^UneG$E&imJ&oIwXsW5gVXD5I0{0)$1(V z``{a|rn9lcnJXk#Fp<)n#kkc4^7O~Xe_NYwI#9>+SVfzxSl%CD#3pF}g z)iE;R#OH%>3jsEkb4*Sq00Sq8y@KbzlS!H)=6=iZ=?#k~r`bB+GjjDUNcjgF?aCM4 z+&YdoDiRT70Ar(+(BRMrB2;OALp^5?2L}nbfDwDC9?;02qmRz8MLrU5dQl?EW*@Y> zxJqkx_Tu@(ZR196x|c5W?_<7o?#j8dh=oBmvWBD?0p8{;dLwA8ta=0r*1Jnzu7hfM z5~gm*oOZz3fHY5`W9aX{#K@ySoPdZ_AW1954CE-n(I@06@&ly2-9kg1+;WGL$62uS z7ymWq=Y3)Qyy!VLX-}eJ7q?WbI9wbg0IUPN8$)7CNT*<1-u*qp!hf7VfH;n_K_I1o zt*7zS&<`X-e&{I@Ck|o{8Ryy?6}@g5GIuEP>lUA5o^f_`i*`2bpXdKJ)JfJwsRgdA z1D<_o8-gjj7R({3eMf=+-jV;ami$0pLMUmFG}B9Qh2aT)MLL9pl2+;D$-6eutU1k# z&%{TMWwV60Yd)ct7vH!N6PfK_MXPgi%Q*BrXI&FQs3b^~9e>yJC)OYna!7|X6)isC zM0};tk@YYGRGi(CJlC3^=jEL}YCKl2jyo`d7gN`&vGHjWmjej&?Tvo0Jq& z?3bn+Q0Q6Sa^=$2x`)a0vp++lRTv}K;c(VXS7o2eMV5QnCCj@br1HCO5!yG};NJqC z!_j^Q{G(R*q2iaxauD_Z&O@nX9eZK!KuhJS_+nA|6L>UnXIQC)G?P1!zC6W(@qme= zlP6^PT}U{J#H|XfP>OQyyL`_{c|D4d3aIz~OeX#mia3dj~@J6$f)6Baj|jB#*GyqJiOI;P2445L4fRB?8De9 zs4kq%5Kej34Ks9PoQ61&_;!Q+45J0D0KPEykmMVqBk1<9B~fVz2T-pDrq8@{Ys^FO z^9AjiRDI~>rd#qTddcHVLr)ieUx%UM+wVRFKEa6cuD3D?{ zk_Jy;>+Py9;tQ1L6}WFNX?}BYIS`+5;+qT6okmO z3MfDLXi*LNFHiHsZDbb2nAnwfGie+@AwI|)m^Uz*#{270BOi_r@>By=Wv=@9fHlh@ zrmrSswKlVbjiKZbx{Dic9VTWsQ4ct9U7^){f+sJvoTszqqThKlQ?lgRU*nGQxuNhi z-(`y*?K{qnhLoN92~Mc@HiAhx@cs812T;d@fSSq6x7t`C&BXgBAUCUJAQ~-Y?jH(k zmj3c|7c3e#Np5^oTByNPQZH5b(1@$MlXm(8Q(oeM$4}k_BYV@UeVN6FbTV|fC2OL| z_|A9{4G+X5JHfMBs|2a{aGA}|{LSd4!&~IPJEte~A2)De+x#rQo_s_*`}s|7i)(*9 zgxDGPPEcl_;TY^NB`oUs5Mk8dxI_W9Fb$8#UAR~!4TzpkE$pmZ)*0HT4WHRMV)AeN z*^arz%Hc;X=f{|8+sF~6;dr^2q-EgP#Tl0PzlAl^Ew-Hp(FP|izu_{Z)Z^g3T;!w% zHQs4?>G3*%oXF9%`#?+9qgncn&Tn$(4nt=x4Swgz_g5?g)FSQLcB57T>H&s1xlKLn z{ADF6VofQc**aenJKj8Skb;K3Lh9aeQgYi1`cod2Dx6R34BwSH*H>M>IHCG4gcggu z(e57|`Touq!NdU7g9670_#6O599F*-H0)z&Qdswd$!t7^HS-^J@RqtCO(aWx|9b;9 zR;ZI1Hw6?|*Fij&_SDD2A3{fHX1RR6frUb(yda$33tP)PVR%_%<^yz{aeu*u%6Ll3 zbY<@MzkQ3~1FVErDltv*H*7t<9&TgipYCqM`iKIDq&uRQR6F+WIR3*Pk7@6V+(*GU zPwD^n6{W!1H(?n$r?oa>2k3PpbP?qa9y1|efonKHY*($o`lzb70nWoX>IV?MxR73c zLKFNDoYO46zxrO|PdW@;`JzWl%IwG5E{t7}hPwt};yXJ}Z_&wPJd~+a|17!g^*KnK z0qth{{i1|CqZ1-IBQ$p{RQD|`ED!?A)q!u1!5U!`(qw?eFTkZh6#j@VR?Y8jlFSwl z*T=`_!@q%~4`apjxJA-^QNQeADq4t|9o!zR0NyUPQ>QZb=K)dpA51_L!qvz3VfXpUgY>1co|6N&J(^cb4XvT4f<6Q*a zeO;(LGjHz6X`C3S`VPVdPWdl5-=IeINHXjZLTZ@%Ma;@x3S6IL{k%=7{3TqHR)L9$ zm)EF@%6PALQp?(~`}Y1$7qzG(C#MZx{MhVqV`wgnwJ_C0ntx#4mEz#_8+&TaC2l+z zeSJhUENOB&AUpa)gt%R$fx?N0dmkfC5c3YaHR4u7Du}4+aq<-~P=+uX^5#hJm(Jw5 zUDbJa=>VKE(3N&9`vtavgR;>R>zkQQDtGk=!l`{x_k`V~M!E64s!}Y?V;h4V_OS&| zCEUA5P3JDs*e>Y?L7hfNmFRUAuB%GeVg27;hVMz-K$eh(KV`X8a~Ku##MYe(-JDDs zFWgeB_Z_(JbaAS3LHaN-$NFZ{7!GY^cc*h&Fi~ZTQS5M08yDBTaJ`x7U?)osS3KBB zD>-ye$UVNM0;)_6$JBLQ-X5zSYihKJ&^#MFB(U!>cB8|Mm9$mn_vFu1<4r>kX&=e#Emm@+;?bq(fr-wFUxe`%b%Q0acDei zu2dgg44U-ZI;L=w=+jE&Gvn_5P49dq^Si(cI}R6@9=AxWW9^l4>eK5$1uD7cB_SKv z&!u$le#X@lMz!z#p~t+-@As)@1sLSZ4HGR=ENIeX<<=f_xoxoa+iz=CwtLyzox4$+ z+OJy4K+}rFJAT%rdeY7BXA0|*2-zL=__Dih*rUq*|J^p_n9|oaF{&ym7n@QIp=#Lx zxWy)SSdtKi78j(#fB7_ygY0MzA~!P&%XK)8LeTyQG)iIzN5jj(9Qpys;VIZb5#0*> z2?eZewd~+Pbk(4+F}QQiDb?Bm20$wNE=jG@Z(Yq)b1gN5oFXduecv`vqs>(Yh$-rf znKn{j&^1hMe zr6gMggut_mijaWV^QZq%yT9uk1u!0a`@La&>6kG^7}Nl8$&|DtoP)XquTfz z>-uDzN4Qh+rZyp%6CF5`n0NU^I|CPIeYfBI@QaIfT1S>ItNG$vep;s*->Op~>x@o9U+I!_`OCp3Ku5IZy4xMq>6 zhY}GfgrN|*x|QTe+O(?R;DsvqttvYOC^$Fp$BytRBzA;-3bf?&gM~OQPEc_+(&bxI zdwa(2)?R;>U5C-H&#Ac3;BoB@@<;+(aMBUt2og}v67gmKE?6iL)i_droXT<&r=19c z2V_e;jj)pY>yQ^wT|~Ak6;oASE~wfl3}f!8>+s(Q{ZMk*?}m`!l<}Cwx$~1OF&}zu zH>5Lt8=_t4h3nX(zk|Ragq~n$+tVfrFl=!j+AV3-R0z&F@hzpLqvI774FOZAKGpCL z#L8jYlNdqArd7i7*bE>oF(fyRSd6o=m4YpG9+qwwege_@0UEP&yQ?5b?)e{I97g*A z^l*q7iKqhZG@41@aiHxf<@UMHQ`p}u{!D0oKNs3z#x_Ybn-y;?wb1M@2sr+_sO0s4 zGB^w-$3o_(lC;wW$jwJEGOABFMod{b{fn^n98JC#;R{m0+W2OiH_gxik(^ILA`DQXF71tZVR(7LS6Z&9 zUpDpM_H36{FKp<&o|_i4w#4bJffD)P;R`Hpuxs=?zdZ?P{4+lO7?gI1sqa9qK~@d; zatCpRaIYjZ?7f#h6Epqs&xe@WCK>6$tD}gD8yy|Bl^Qt$8Tm2z_7Y?HVa<(%V-FTO zy6-W+nusGn>H=8qFvyO0@j{Jm7kr+O&Rsz#TKztlF8I}KkB^Cqllfe z{QmUuoAVp)B;591%_*>_;+Jli-BFFOUGJ#VvtE}3drJ+PXJm8z57ujzo>Za|ZaFwe z*lyS+Y0`bCM6RG8uUF+=VU?XaH&vbU4^9?d zEp^(Yar)WR?}}B}634~A|$ zd>f+v`>#{`Idhx<>tYU%Mcm`~?<D1hz?0AwvD)^$a4L{)(4%gVNd|-O!J!m{c^DA!G$N4ro@Cw#af1~F zSpHW^a)4E=DMazdJ!RYeVdFKpc$f$sAwd)#tdd^lbMJpI6@C@$=1kfUlI%15^uh=yx zW$!-IrFqjkJX`}yPYO;6lfVliH=sax3y%OuSuLQ})S3G~n10boXbVuNMN0-F3^)H3 zIiE`W6o?^BmQ7pQ2BO82#(6KyW#(=bZS|AGkxet?owc8I4*k%gjB(o5ccNEj*cNSh zCg&-4Wt~eS_JYS^-WPXdw%ByTQ>}yi;NNp!%DibW6PIiEm^gu3Q3m%-m_5@GKW-@4 zOmRX%9yS4Js5$t~ty@N53~toiLqt#D>F*(%E*8wzGe|oiK`I0za|iPJh+bH}5J(Kc_5m_hatn!92zc4B1XxX4g^&f= zGAeT6fLe8NjxA6w%8T2XMIotoCSr9s%~>0gy1knANz6QeGpxcwVEfg5mt`8>{Hjpq8_ zgJJ%!zJXwhIFT?w|IZI(rw>slcch7gIaWb+R2X6=vtPYQE(;KWsAe94#_nok(Bm;m z$X5VHhH0#BUQatwXj0jRicXoz&f!Z5&<4V@PfNShiPjfvY#*W`lwHOv5+O~)j$+~D zL?L@v{P&5Zi3GmlThkJw(?ZlZ`W#N&2Moe&zYS+`gs}E16YDwRZwYfa9Ays6HEq}f z?bqT@rTFwdd8oDSY@s7Ci2Y9dedKA^5ih{?mg3fZYjhZ36;{aAl^1EDtkv-B=(xpo z=Ym{iCJVs?^hl}&%n!jS?Mi$iOyW^n#mG6=Ms365RnfoFDLro|23khO_*=o{O0)DM zw)c+H32Si}!7^#Y-?$qTM?jT$;WExpx|8%~;Q<0M8deMB&0Pa`|D=Ed;=v`IF6Z87 zai16+sbVz|_>rOHBAhs+xaHuGk1i~HA6BkW-Ei3r51E)fk{aO+lN)H9YJG2Rh7IOW zd=e=lc}0K0c+MR$4NR|Ep_C5?$s)gyn^|HHqN@p(82~@XSSZc?3GV9Z^PLVvhWDp_ zTMKApj~;r_A~-k^`7ZVuzDyF}0CC9gVBHJwj0$w`U)XyjwZ@o%EfBgwsDo)2+n~7G z3Y%`6Cu2ZGWR{<&eMYHEq}1A}#fR;hFF56;H|*N$d_fN%7$yR62Kth5#Fx(?Todsh zf(NYs)co!J4Nt;_6&-cdg=Y>Cfs)pHJ8m<16N8jTE}Zg{tZcdCqQ}X*I7?^vQ#NI) zwt_5e`(d0KIUuLM;WRm)fo4g9Km82i#@c*>2xeCTaLUi{OhZvI*2J@%N%kC)`7`bX zV3bI-O9Po6x!@<1ucl@6v9gn+dP&Dz!8clfv* z_=4imAs|p8bQc-D&6Z7QHOwgHDY>|K=C|B7;GhpuI^(9_u>vGjq3rtRdjJhmAZ`Zd z;rayh#u)#JkFJP%KvqCZ7^iven$CgP_4C&a3l9$aV*BCr$Y?73-Xi!0865m3TF1EI zHwrz=o)YqsA(-GipY~5@hOyYI7MlYnS1z*x(P@>lcOsUyoZZ>h)PX!$m?%(^P9kDg zCH@rUg{-42woiqHoLOP9`Pab>pymj-E`$thz^j%4QnX;|qrL)mRZ)Mdb8$9ztgnAk zl1VJvd+#^*rj>^ghucD;QqO7#^rw$CWl87Nh;eR-q5c(}L%K-J6=xjw8UlJExhNt! zwuGpl7^xSvLoMt};vZddU2Z1vx`v#|I;WOl-69Ecj5@64;I`C*#!J@VotrleKR-G^ zgin})WGaCiG*J3Gh$4EHb0!DA&vm?gw!B|esMR$%Nk0U2?r!fz57=MrX4qabLQ8Z% zXyZWc8Al{xKepiwK4mzm1st0$`SoHlv$yWRs6{g^3G3WgTNe?3<;P1|q0gQeO0R?d zi5)cqcgXP10!4pA!VX%Ai(2VA9e1(}7nkat4~~sp7yYS}+qv>C62HD}qDIev7bF8R z;7KfhW)()Rjq~uLyrsN|X!~ElM=#+8l8cMF8~Zv#w>yvjN-mW~^B2cPdoPEP?zxk1 zUXlhp`prdnWO({$Tye@?>0ejgjvWS;Bbkx4{B63}K%GbLjeG0Y?+&gy)u$rIKkxgv z&a&{(p0g>$&oZqP?+`Ai(KR{>ZM|vQ-u#n~55hR`%kqKs>1{zrV!-g=hO>E0Znw4a zNNrIek@zNMa(sLn;J8DfhmO9{%Rx(s1G1i#gJwfCa-lDVQ0am%IC zkAnyriDDm3GEp*HVF512hA1BPS~cB;QiT;0z0PRt|5d#nHzlT6oZ-onC$RJD0+B@f zeOoQRQAqLhbknHsNwl_dPRlK?6%l)&=V%8YW7~|@2NBCaGA!f0C{F?Sl;ky}ip6zc z$(6vZfH(m!rw(h%zpN9_Ul+3=nD<(T#%0KP$LZSTeG{l__wRW<+**t<4~^?`~vdpdshCB&|DuDL-dg>zW(rLjrTLNulO|eMuiobC!<=UiD5K<7eRf6^V0~ zUXGN}TIS^K=VG&8PdN^BG*-GpP|Pxmp{ zz>@w^Wh3D!HlRiTN5{WOGs$4N_01{Ay1 z9lxW3X0L9LA}Eo3SH$8fStP;$jngbuj~}o691#UA*?6toXzR*Lia}O@wFhBWhskm5 ze@|(q!0@0hG3(sm+O~2Zjn3AaB>40wX5ABq#ohh3am>6^MfT?U``H7D{pIjOLFepp zL-H|vM7=^Du7Bs}ObtK3x)u&825-vd53iR`zb+PmlAA4TAk`4>XMo$klXtdhLdi*J zX#Yp@e>W(;^j6PxJ=Y87nysl5S|<~oYcFWg@hv1Tn@s;dR6k_p*f*9Rui+ffG{p<1 zuk2`93IBl|J-!D(u8ft5#ciqrVH-3g3d9SvZy>3e5MA#`h11E=y}o7TPXAf|z&eQ> z$`7=K*+D3y{%?L{ZHq`}zgy&w#{6tUkoY85q6A;vr_f!;k(2gdhDQSI1CaisnAqdl zGoVRU$D2%RI<)~S=ybiAw?sZ6=X7AtPId1r1t<~qJ;o0sp;k{$B%fEHEizW#l4_{t zwi2~ew)6tJCCmVnEhR>8#a+fO!n&M639u5eY$7E#G`VQYva+xgli={y$Zt6+Rk&jX zl^vR@ShduJ5t(6Y@I}JR(U&ZGVBiI_D~FSz|>)! z-hC(U7x z!O-gF%D0dUc@scuJsiHqFt+2%PtpL!N~}YS-(y}2>Zoy6-V@NVK(E+X<}}!_4q*!| z&cWqf_lNrE$91D_$5wX9PrCsTTA1=%xN9cD7z~+HL}{rqoM!5qF-Z?D%iWm%wQ^o% zaI;HI&ZcB{-dOp%e9`#2kKi2v8$DJouIL6$baE*`akG+|Lh6Xq<&@k`TC% zcG{fpD(LcGIjk2+GK8Q*i5Un#%jYeI ze0qi~$Fhi3=#ka-m8`)b98n()7FK^lagayeW#YKq1i%I!Q|dV7F13G<0V&oDdRLS* zP^xw<-vEc7p^AncT+7>+Ig5M|%pu~GwBm#|_LyZ7PGscyf?KyjVq;H_!J&1GH$`XzEtSOSK#2022}w6zq{Ri)O2R76AFKUmI*SI6z({I z$bD|$x>(VbLQA0^vwFa0c3_l8>$QRdYr57L#{-&LqrYEik-G0k8jMjtNSJecc@UO; z?-C{eIK{ZB9(2NE4F!_&6@!Qps~_CGtAsH`#O@ymEtY~G@})n3R8E853oDYy*Rwc> zj^atq06U>&WPFagiyUnTDSMoT&L@@%gAI6c(r_j}7S-3i<&;8YN(7FruueTV`HW91 zpzGwD*W%fl2rY6>Lxm;|wGKf{3NzP09sUY}HKx!W?<~%oJ90F56B+16P8am_lhXG# zf|UgKF)6qvf`7vUf0V?(+OIVcc6#qc0$68KRe$t3bBU$YnmA>5e3Fg)QJ1fO0P#nG z5$>_9Bzx{r{m=I_sdGR@qA$(YXAeK}Lii6+F8ikO(^HT@Lk!#Glj`0YfJu(fi!)3C(WGVWD58M;la;?0Jqv&CnLjRkk8x6Kf zW^0&{94Sk@z0~+pJ{K6K?z~^RO`EK!UN(UO;U1#zzZkdQeHDz4in(etFlegb%>`m~ z@*7RYuf{~;4}5R%{TJ@X$-{4dIVTt7-;EO%pHrG1|2!oEaxF={gj*cTzHe*2tD7v| zg!!ZtxRK6H@y&6vBW~C?j7q?z(Gk7aTO9=@`x;am*ru-Vs~~C3=^jKoLF@QcE7D#o9UHcdbdKOC>?#?+P?7P>UAGYsNUETV73$f1l!pgND`6Vm|?ai!Y%f|bK z-Z1w0xAGcdL@yH#kN6TV&`5Ia8(CRdR}-_<11OH8 zOPBiB^(FluByeq{e>TzqDqeQuU5B*-wsx`Q=RQ9vJ;Haf_A`gUGa=fF{73O^VH&?z z@PB?qM5)*1-I3`N3_{zDRVu2^F{X5~OX+X&ALfet5kmiRK^qg3DOyAsae_RH9mrh05fYlQAFndEyVh>J@(%Br@w~8u;{yP_f)=KBPDW_7+m-eRy_GSv46H( zG!CidjDC|Xw9evqAaH)y88iXOcIT-J_d)=G>ho-}FM5S{N+Cj3{e+Mg~%&8JSL@L)KA$1>_8 zRx0>h+yPG_0kh61_Cd-22Yor>gSO+wbHz3i6}UtGy_O?Xi~ zXuo00;RDb|qF4VRFq|VQbDWbtGrr?=82=&f)B-7{y}POo1y`ODZL+V)%T0P<;(1?E zxZU$s`E!?xoUVP6e!9mwiJo5DzBLR1julNjX!(Btt!j>5JPXK$nt)(ZF-hnN%*@S^ zLAxP5it50PhX0+hJ^aav{wTt(plEDBjz(e8uOEt9E8itq=oq_K+tn}&t~+nLXVwH5 zKWx{(9T0u5FK!e6`1ptMN)H3EJ#B*zI{#9{%cqC&wDK8ttibLrOAFJ4&j%FaM9W4) zK<>mz6ofQ@ZugvOR{%5y~;%B^jB=vRvhZkKaR@P zIE@;qZ$=<;-{(x74zo)HfK>5u?6BhgD!IkDw!!=636+Y^JpF^m>ZnbBe@zg+uS%7@ z^S2>!SbH@~cTNHd`+kZwWy3g>^F-n3%GfJXG+|7sDfsI;R8o zsLc)T;KHC8>pcA&UisDp`OTJor$5P!CNtVmXC32&iEEm}ujAx+MGhDR);?mDDZIxZ zBH%4~y+-PxCC*z9w0qTX4a7YhbH8E68bs%K{BA?o-^Hsn|i8>NgpxeQKa^s(c3hSQEjpz zA1?H9hvaPEdpG|?)t6^B7IbscOB2)oTN0!mKW7H{6jF&$2Eu~&|0s5Yu%J(K^(#=$ z->|*hJTv2hv8t@OnUFuR3!hhljKp@^MI0Fw)`_C}orx#PXDudn*Om&H^-qsab}Y(f zjoXDgd=t@4Q1X;smKbIyCa9+N^Eb(KzdFMD&TXB}JdaE3{CG7s#ZZR)=Zp?irKMaOXR$c86 zA7jOEpMXu*7|*I@hRlfRwAJ|;ZoZF+#0r1EH)i3a$)7;xf|Fp)VnNO7ViTNRX*kW9_at$(v7VDV<87_k9)G z&T{a?Xa!$X)ZCo$Cq84hwNdv*=l)h!Rm=3Xy9(%s<8bwj!MjI9oZq|6#u+b-(L1WVydK`R2qFF!>Ee#qZFDaRu2hdJT#(6 zlS*id-|LCC7O@W;0!uvS2Ndpqw#kyyuy(X5nyLD+1|5r@-ro0{pK6nC!ihtg2X+TJ z$8_Ate#(8UCgroOWYN9+FM8fy9A#61g;ttrOAnUcxL0gqhldkuOzWwJUanPId6b@; zY=m0>q|>8`(#k?j`u>O&*USi;&gGUN=a78FN1EvH^P9ZGc%z>dM>-XG&AXWhdhL6{ zx_g_o{Cd-g#{_%aYdcrqQ#VRssj017y*0a6aec$<1Mvgk>)LyI*X^GMNq{GN3l5+b zIO8Jhz$P*=k3R{_`6+Uo;`9(*kx{C|5q7qMsCKwL^g01$%ITB+8YQp_Mn-iW601W ztBp5sF)?zv8fFs*^jHi=u16});}n{Q=YbKv!mCv_#2+@enJ0^b`PFgdVNn!+T#ucn39|84+4Pp?WM*IY8W_qT$HDj|E|Y(IvQn_{bJ zqVKRgxO?h%nq_1vj9EBlW2@A*qu<@I*zrW}99BaA&h?7P_$^Mr08U*F7(;c^{W zDcNUs2-FB$?IpxKm9NG%K1&N*raVqHn57lo6YMPhDOJfrg*+N{-{+3S? zhtdQ3O(G01;s(!@Qtu|B6SIum%bdd=nG2jeW&HN?>8S?`vObo6?E7f9@S^ti0Q+;_ zynto;!-?_`zts5ysoGV|?=`MPE%_I!+*yVJ|C)1BN8d3SJJ5Pcb4cOD&AhHF*VPTJ zvu#W`-fV0S4OB8xgSr09rl1&kzNTs-#rT3J!Op&3x@4B`wA9@6PrL@<46mGY_ z&eRp+z_+j9V8IWnGyje_F3euwY}vjlT|1rj;j%SAX?i`Q%!927L!y=wOaW07kIPN! zJ`ynUsu?G>+#S>LqF%m;viO8gy)VR{Ur6OC*z)71L2$panhF=bMaD9}(4}^yuCwar zA$uEb-NaM1Zfmw}mgnH;u#;9fant&Mb-#&7HSss*e2sSosuL7WxoA&HWyma_ z5}V>*H|1RDipXkh5z8*mVZHuw;{M&uj09q#K&t@f{BksUw@YZ6Z;!LR{KnajHp2#^ zvNd0;rX_3}l$!l!#ycbU@>^Z6&l|b>Ir`Gf*bN7&&zmWm%LdFzY4+DEEwACS`&Sth za={Q0vYc4wFQk9N>w!CjaDe>M9W*2y*=>@p5psr#WJCeq^rT?0J@n`S}Uk~XS|`gB&IsE;dj$T#kx@&zCJkb;#Q>6 zNpu9sbrwpIuo5}Na67vuhdCYLaxZO81IuEr%uQ!W?rkC|Q&;ILqnpLv`#cSMzAQSl zkQU4=Nt8O^bXr^Yl6jx#+RgrM%TL1oL6kxDbsiQMTVq}s&1l{D2OEXW_8gWxy5Z;p zTKX}=f7c=%H#~P)9#rNqg?y+wV1qaH(VVNd^F)1;+OJ7k?%;L06o9>GZa_c#NiF1wE<502aeQI21{dy*JpYJ_Y|3ky>i4T(Zuln~FzeXo zd)|z3_*%+?MSazp<;~Zus>l#u>)#V!)!gh`R6YCI@z(OTvG@qO#jwv=eO=GM*Egd! z{Bu7{<1k&iMNf7swOG()`eVb(0X8GO&wK1w+sS)p8XD}W^d`3WUV8WhM}o1VGy?)_ zjin{y+d*Y@zr_Qa4`sU>F+HCQMmNEA>p_c=<&Piuam(-*YwGOP>8VrB#orPOtIGG- z#~MpU2g84WvUU?S$X;bts@{@NVoV$#VqBc=KBjNr7+CUhcEi8A(Jf_dtnT8^k5UPQ zv#dGo?bZjC&ABQC#`rdd@~nFqvs6VJQVuQ&?Pu3C-nnDP*cH;w5N%+Qi?>#+)wN`D zBuB>*qF(F%-4TWtisjeGzEfU4(ta_%@#W7tWOzhdchZ&Q3tI+r2w6ZUYMz&ir*yOF z;yjzO*e)C3~E^ckE_+oR+&-op=I4q`?bh<_ z4{>KK!_J+ocIRSs#a7Z@!*AoMZ^lYa_E=Vf^J>#Srwzwn-%ZU-u1Q*_ z_m7t$r`?h0f!^`%%+YNBaktQ0YBPTU*+Si>y^Y&tcbn@u%Vz$aKIvPpv-}3?N2O#3 zH}u5f5FPE*yS~2d=gR9@%JV-zGk<+~7pn-_N$+7u* z9~LDgF}`*4zU-GDV%=g_K6C!f#s+3Z+aQ`JtUF11rebN#nL50ArdQ*ln5gBbE3@8T z_8H4jGm;zO*yUn;Ha_s`P$|F3+5S7h3>|N1_h)<9xqlEbV^ZF6I+R3**T+t65nk9n zKd-GFUBM8(<;It3A8lLGKHC&lVmQ{;ZlZ7C85kd8uGK&KQz6n_VCtgIl2c@j4&9Cu z%U(&Z4_7MmvG~4QFtw&UMWTx}j-<;^!1qADNdcV9@brFIpub!l-BwgTbLK;S?|PcB ziCvR-`kNzi1*a~0q^K+ehYFCHYM~7si5f9`=#SG?How2^ll0T)k<%9Xq@C&Kg(rtw zH~_S+Wg^1>Y7U3i)OHOEx;PFShDp%jy#_Z_?8hESi;kc;{*~CTf8H6UJ{V`_BCeC45@PWZK9e8iuM-A4AfO=!*Qx^ko5hn<^q zjP-lH&S zn=Ng@nUu$CRjVF5E8sC4ySbsO{fd|OVr<8+!TH6pUFSWx*b*KD2M^FR<})$x=TcP1 zLS#4k`I{SyDDv63pnY~9ZQ$BTvJZ1Q=WD5EUEKwgLLF7e!|yfeJ9vK9cfviMg42sW z*?micJk33Kjy*GOk&CNlF0~~fu6PcqRrX|j570*PanN9Hz2XQbPgTm1E5&s!T|WKF zn*zv^t~q>jLqoyYDVH|-)M+7=^STYazxveJ81mkZ6a9%rFM|Hp&?_Y9^DsWcx@Ko4 zfhotqj>h9d^_IYs3HBuH*Xs@Gt<4dv+R_*6ijZ$~Hz`Dv#yHs#Q)p<*+NGxI8D9q6e zl2HuNN~_PWu2zGYBJ{%D#4`i9Ugwh_(aSw+4=d|X=ayWYtxI&l@j4Y-`}(tDg5!>9 zwV@)M0nIy)>hP1W_3(y4t>})FpBwCIyp?QCbc9jJVC345A3tn{wY9YgRTcbwxNera z#s93UyL0z$W%!A689{-2%|A-g7!1i5aal5j-;FSM?5%cZ?Z`vUsx@sT_jlaYJV9@Z zcbC;=PWE~~u-M&FsNd=`uJSZW!Bci=?&s2{s;ccmLWEpylif+3wHxHKdHipMq`ukj zDPAAJ7ZzG*F7RH`j@4C8*!ec1$#pbi5Ava~#c| ze#~5^rG*bd)veFQ&i+0)2a|o;%7|5DFnfHUimAhEeiE*X7xQlk7BrVsSS2OZSzWK9 z$>@t#3^84_4cL?5sLOO@SN*$7$F2veoVm|NlJn%J`|ey=VqwT({@_vPspn98740+7 ztIsy~{rdG0$~_s68M_<%e?kbe5tGKTbbus(#Y7v?_x-#!`Ln%rsw`%;F%_?~+R2o- z)rCNy2~*tlH@|ssFeteb=mX}IO%Hqz@?M;XU&o#lxK`SnPB=*MV;WERC!sXfEkkt# zsgV^gvyLu7m};Nf<})oR5N4c_6I;?^WYq9EWZuR@_`>#8o96ff_tKL9v3K7T%XvAb zh5`!zr%vVfY9%=BTXKR=JoGgXpr>Y##+*pd<}vo<%Kml`#v%9F2MmUrOkx7N-e_x)U)l7gViN!^Gww87$6oJp^6tAELh4~p9h zB3s&j`ak&LJIrz8x<}s3u-}Dj*DJ`s|7M*|kbGE{%Y);qR{Y|i+*m`?KYn_p42DgS z;2xtQY5rb>d%@SK)%j&Vz9>l zsKEH=cM9mpfAcZJvvtwYr}J=S;@0bIS3oqK=bLFTl+CI*J;7-ak-spdi`90_%S-$4 zDCWIgg9xBNzfTwRb-#h1pe^+Py~fPHobQ_nfx_SLEa!433gg+qOX_O?Kdl!0v2z#n zlsA1#n>kZ(-v4CYdPUoj*!u9Fw}*TS?%23150p&6LNM^1*<3`)AV;B>SqG6uP6e2ZR1K zVw{vKrpL_oKfMFp39qp5Z8D0K4DQ0b&4#%T#C06h5Ih4k^~xaR{d(Q=d|@cU}ysWDa8 z!mnOAuC|FPH{C6WyFrHeMsZD41k zF-BA<%yg{ADsa{8Ys>eP!@>0wWZ3&t-q^@CcWz)r4HHv}_z1KYjzSKXB%|%6xxu13 zA=L&#i9FB;^9@OX?-=|I?}q7?cAhoo-MURAe)~&TzJy&z7c;W zA;V%f?G5udSN~uxDTksp9Aoev0QY}wA?fi$v0W&V)mn_uIOx%iGc*A1Az;36!)|TZ zpHWv?hc&k*=Ui8_>&1I5T|3Rht7Di#^H6e}HQ}U=ii)Cu^KXtqr!50xzzuM%qRCVI z-To6m!&nX|I^$!QXl#}@_;C&YVbpiK~P*34<#(51rL3_-j$q5F`E|&7?h>=dd zyU|vWJHFpM{say0i+{vIslfjJD0P?V?zek(CN-3w7*71*#%rE*n2!Pe`!kINi)#wu ziPd|YouU=iar1qOONQv_S^dnF=%1u=`TM|&GospuwWYFI z&5YAg@wPYT2M>9U=)E$~R@nw)Yw8b*Ijs{{-u9((OX&|LD=y zw6wHcx9K54js8VJjQF#Rf{4s@aenrlWn%~?^WK8`o?_Ws9TV>~b^4T)ABrK1AFn>x znC7$nml-`@-#1&$Hy^E@I7TYA@viFM!mKp?N$LGzs`XqOl}zqJrcVo8iauWVTvBm> zA*F&E&-ybd=am!Gf>by$?#L%OS=h+f80}3JzeO_K2Hhmg15Sd4GTB>n^z^RX!BQ9? z=W@s&OWV(|Fl4$jTJgX!?IPKOxrNnPk0)jUkvvSU99YA~#�Fe!D0Zyx)*JZyLNR zw@u$7jkIe?AJ3*mlAVLYvF&PO&c{C~-V0UdddA=dVhjfIn^nO~@?#WadJJZST)lNl zSsvE@^%FrCRB0*&sZn6;c-v^(a8CIVQ``1czqf4cP-9eHZZ0Xsaa1ED%mx*9t zkv#9T^Q^^K!OxP#Un9deQUZeFR~KBYE9HOP4sK!%B#*@Fgt)iBDI_5=5yBTtOkEx< z1f3STfrX@8`7aF*74~OL z8}rNX)jx|LI3MyluB{uLDc)_`@9fvt6=r5w>`QBYZcitkLN|Vr(tesqkS(h7{m{%a z7%Dz48|L<&&i!ez@wy6YmmO!!r(IWs_PX?_g>@)v8s^nnNE`_GELpp8)w7#Efmn0Y z>sf~aet3+~&8Yw8s40sJKfgG!Gh#%IH|3MMORD0Txx;%rm2R7{2kw2u9x9GBloUM$ z^@}j&%d+3+I`p=jB=R7v$qs59`ZqmlU@-^Y8VES778VxfSG_`?^UBG!eBHKp9lsyd z{Ftz{lV?YRUaN($Yxq*gcJWP>kq!o({_W;@=eS@fM63hm1o~#e=tA)`h>Eb*62#i|Kvq2ba!!R1}OECa<== z!Y;#xI_>qR-j3Xb-^p_Kj_HTSDjCpsyS+2LJUUisn>zTSskcMfiTUUrx?|O;VfQS4 zT{7EtY@?`{l<4Cj9mKH_tp74ZH>-wS}k5P zFXeN}^TlG;oxO)8qifEoeMY5RR&*s)xP;hX(fDR_qn=~1u7SnheoO};BaT$#jXZQjGN$GlUrJ>NcC z$l|V+$@We)i4MEV`MSYA*Nxosy4Ftql|7^J$jY*~p3kR1Bgkb?j5qkj;rRwm%%P+} z%lgH;f_I+CP%>nHqDqqS{CDMN1(M{}e;gt@RE4GwE@%$Bz6YzrH17t;?BZa={ZG;) z?h~XkarQ2yK^wh$frBm2jAf(()BL@YRy&E9SWlHzS$WsQcMUV}Ihx@S3#D%a*=s}c)RFrYdy{gg1FI-z&II(&N!gEJuO zsUrlK-A2BHky-|x_sLJvoAy;RTrXulqgRSy?eDXIHN*My?T zYnSv6-nj3)EYuyXboNmASYhxhSbGxSJBDo8e_T(G8{^uLRTNr#=;&<1be2B2mTkFp zUisnuu!A!rv`v2^+}*){_{FupzGx5edb@%8_E_bPt-D2j-se=Qv|F53HZ#l%TDN$5 zI*?+~?U(D7I}BT8KQTG(5yNMTXa#|~GCNCrrU>(x47nxlV5oko;KeyRXs~n~UI6l& zp9JpJO7yz^G3!wCVwz+SlbS2bkv3Yh6Z9LHEkys^vU(G>aNYB8~eA8Z!Fi~$FZ+kA=AUS&$_ zQ%^;x0PTHKejc=Y4cpk zccP$^QXx41_NC(mmn5fRO8>-Ta8jHJ`gv4yu6r^J_I{-)y=3MSxK|DzqcR8tb~OHa zO-fFd^TyR$d9;t44G$|nR76H-I-Yob%9zFVuv77kg{XR|hpIG^)hDcGhu5lRZ4-{& zXP4Q+e1z&>{l3OVezK2k5x{+*T?Y~p$tpXC>INKV4>(ou~7v?m+TUkPm(oUBVV zVN4j>X}QO}q^CUg@QV4mzi8~ zd-Ai8xBj0rck1tR=l^P@K5$>X`3Hwt{<__( z$pTkdt&<~|(|phat>9|y>|oM*yXW;~L*{m2RffH-PQI}vl{IXEb_xw=J<(kyR_7KM z((cG*sjBIXwkr&%NB*I5$r@J!J91t1e}VbIYH5b-?t&KD$&9-fE+tJ*xJI40!YI@d zk~&}S6Jos})D{9r{vjFUoA7tWm038C=T=%(^L3++%EoO@VWgQQLlf~a{?$7jZ_^>h zr(w zujajldSTNh*D4Xb{Bh~XMN1>-9Qc=lPtE=i{-orLZ%qG;i^FFJpbt62*avtaI1t#?RgVNow$}oOf zjN-wxV=unCUOQ!%o{28r8$q&JKXy4CXp2i5S4)d(UE+Jlgv4@qwFj@LO(15*kq@sz z&$6`ai>jTaK>FsW+a)s|;eWgx*Y1A3_QkfL9!sl>_EO<MTFVQyRC?(%QAR9#8Thx@EtFKWKj*IVEcq=gvn#?w`p4FYH+-Z~Y zX;u-L)GB3LUmdF5OBxnQk|=f0RK|-@uM7V(aLF~ZXSEBR>B+>iJToB=edfR0uG{jz zHWt@UlXkgHVM=qFr{FuEUmYF&qS0e(sji{`!Vt!~g|W5SX-|k2Sy`a~KxcR>=H_*o zzwY#_^m-KCBzyCjNs)|PBTk0>FX{e+6P%tKzgjY9T%q#_@W%!ZE;yxTFr_KU2K^MT*gf%yDHskGu?1oq?+|0!{kl+ z6xZhjGiy$^V(hJNi)pixQtZZLD~{bNj-D@mXl%pPeFiv$22PD0yw#NE#DkY5WVfJm zFOurED|}u_JSwj1%`bHB;hDPe(8I!HrwTx+ z7hH*fcw$$({QZT(0n!>iunT3c$=Dxg4JXxC^6s%h!@TjN|F#z|tI{7Vb!hw=*4P+4 zv{g-&ZA2ybG!AGNDNr2MJR#`0xMpG2cxJle>%CTP8kNiBJ|w`c>}1OU6VWyAwbx!D z_D1`OsgwRjU#@5jZmzytc!F-AjWOs9^*Yd|{UN3O*vKTQSx2>2bbqi|Gq&Y2?h>zb)HnQEw@bhDmd>FZ(=mZ9; zxxsUHVJ7V7)OT|wM7eH^Y@^@p7MU5tFYGNC>~*}A`=+WtV3-9@FJN2ZULKG3r1J+S z>#%+<@)9O*I@eKrmCRL>hLZqw*(gs68Yu0DbWW>mTDM5r*2v6jE(9&r<3vJrfk0qh zo|Fxcr_%B`B^7h-<%N`2Q86NNT+wa&VsChH-JrGkdOZvSy5a)Fms|HfE{>WhF4_8f z_>gtk8|z{n{aZIw7rFa&iFtZ4&7IM8oV8)o;Rti{gA38$cHhL|%(?GDbEeZ6AG>^tuKB%rExGKJv!L};tR0Q?yFF-FfKQeZ3D>BT_{q)J%D(q+ZuTsyjoH=Y z!u7N&oo6%MAoC!(AE4isq7P_31J@fWM4F8&&)9`q8 z{C!N#O~h8|pvq3Yf%iJn2HLWOA$(=`U-BH;IMb&2-Pa zVxw6$s{h9Dp(>xn{}THZA+(O*Ws<%2Vi5zXuYKK{rBm?j{wZBHo|zj~*zGE=>&hC{ z)z_oNJxDB^boMVv-I=!&AJd{TvpKu@?U}!P#i_eqke4qyZg1>k7dvXy3v}S#95lT@$m5J`;n|zvv_K(MDMQ%U7L&^ zf&Q6r4yspTr@X#yD7z!=WTxM&8_rTLPtn6k6Xe!Y!OO*w{TB#UcH+HjAYzNQr6oxf zmks920|uh&jwJ8bxt++g`LgV0svZyJI?3YxZOQ_0r*q`6Sv&S0{716J*_)G{2yDr)*t`IAH3nzkLgS%!h(;?qe>~9 ztGOk7UPUZLUH`4XkH8H$(@NwFWt=t(m=cPRGn(1IZ6USW^@hKruX@6&&imydno6;E z5!CVWA9{8ps`=fs-=^gBpx<~gXpPk2FF-@667`!8&CR(RFK9-})u_8d&)#Mou&~}y z`@HSB>UN=^9QMYuJJe$ImDcK^i@&}fd+^q0NY!ug%~i;fB&8EKH#Gy8a?uu9mefn@ z{mfZ~co@c{uqO{35j=lbwCT=mMT3JhHtt%E?z;CVjDuEYI5~(^)3?shnRaWYyNx5f zIaXJ))VJ7B|Dq=+w#jqXy_~CbSv*#^wT?b0&4Zj6xH6o{)6S`Ntm>54b6Fk}0Jfr4 zL{UjEcvHT=;qKQr)IV$o=BRa(Sf#-0`b!2QKeu!Fq04pr^26R>0tM^NvSt}&+Fzt1Go8raRWN}s}-Oq{jS@vzF zT5MS@_S#PBQQkF?*LKY3R(YmHHsy}yYGv|z5@^>XrCuF40lW8-sAZVli25UkZ}84c zwxT6&<_nLkSX6_?nynKFQ3ASOjNXew#yUSWRaL)(Jopi6p$k2n%V)D%g&E%6A857yQYcC(k3Op$34JNyN1)UZUWPf z(z8#vDoe9K@QL37&=_W_o}?6sJwve2 zROA>&$o|>MU$EiWZzsnM160iwwmuwWHF_cLY6VhniUo8bxJ5a)Q%(Kv%?bo0^g zB_Zra5u)Qk&3A6s{19v1_FL|r{|PT|Utpoj@?@fzfb=ad4t+3tDTYm4P{DDX*fm#+ zAD=_lUj{2KgAFrr0d#sB|aAJ5bmR8K7GDj@_=4m#-O{>rAzORh0E`~ z`=i1_l!1LXeiJ9iGBHGuH1$`& z@^vG#*3?)>K+W0hNFFeb&%> z^>=VEu8S(5YTtcn7k}83XAAuPRq;d3An!wHaG))WPI9+YI2B-FcKZcIKQ!jPz3+iD z7Vu@K=*EmqR6xv~T#s#}C#ElA?@8Stw?WeGO#XW&*yl&f6dm3PL7GXlGejtUm#ufxPSJ0QL2V(2U3?SWCw9v^9Oy zuox$G&0bBnS#6$lY4< z`8=EMq}_quD0IvVTioMA=Ii!m$&SuU@zgI%i3P48h*8mr~8lcR7W?ZZ`RqDgvaH2&*zEQ6<>ro8$ z_~E)63JD?Arl;q($s1YGxUF@m=VWf--r~c#s0N0NuqVnR$CUJ9Q_M%F%F=vdJUq@a z_5WTi_Y}c(n3c(Y5DOIQ+>8H~C-QkFi{a(f&1mwES)4lLO@R(iS>79$n(aRs%kLk1 z;2^dYyGtpSi)^p89<=t}s~YyNdug4r_I$nO*PdIMMkszh@7|D-AgD+7&(2>*MPkg5E zL568n_;=19VV{*^gUnf|Q`+J_RTWgeW3POce!NR)>iXK=CtuoYPF%#Tb0q*esOlV^ z$BE*|jiL6|M{}}KoEPOgQGOFYPk*iCRz=3Fx1%WEOj4SH^h`?4({meeG?N}9Pf5z% zv-hftQ1J#Ke-uR9TRC|aXlsf#^XFY**!${!Q+c4j4Y_o(*K;5MwWJ$5D-I&fZ|G0Z zbFd?l2Vs!_!G-a8v)fTfb+`zlil2u9eqI>JPE6<4=uN?@a4o zH#7Y`RM?ASGWQ!fa@^oQnzx`QWt+YZ3vc-_!#-UA8?UFHBegk9RSclQ*r0$aBH%Et zR(8sp-$yLOh_BQj1lBL02!Z{xvh-Zn*nx`I^6~G!FqCqXG3UDj^Rn-sW&J+H8yd?8`|2b3m>ENMXtBEfts3n%K&H&b_+a{1QpRw$w!XcyA9-a4xzCSEF z=HtCNX)fc9q-XG9lGQ$gB#j*5y( z7*MjeZzrt@Z>8Rbf(eGPq{*_hUzAd>TMFbJMmhgc%wF*}3?$ToN^9#! zn72~TBQ@PZhIth2XQhofj-T#r3;bgCMU?O-+IZ=mCg_uKaAl2uIm{V$_pXwzE;B~) zMZjadg0aXMZ$a^1`!|XK1(o({ABulGGkZztWOU7hXXd=1#LrIo z*_4%33bAfx^wlf%wwB|)m89_!9nGAY)4dj;v@>ZYl(+*+gi`?bHC##{YJ7w+B%_%C zyU-w_dH(!47(OF9cEeU`*Xa~yK$PMMthFWm+)CgCz#n$9j7 zgMfW}MkANNX`{6%m$rB>eJhaLLnG)-KWEsL(#BJFu$QS>*GD)66{#}?LIIOUvi9fgB(Vu2Lm%Mxe zpdKQ?U|d$Hh3|d`aKK>{ztmJ!rCvm%b#<%V!5YXaYZne1nr-w=g%sLL)TH8k)QZ`C zwZ?+jEVYMVAB^w_^Gxy#+ZAD9AZIJYfJI;lAb9AUb4f~O9ehpk z9V9tCxM2WN22(cC;P(Id4jP&%`D<@PpLUZF!Yt0iV}6HIv%-T#6HO)VI+K;E=eEMu zaAN9z$;X`P(}n8}YcGwyz@zqA+CV)Y=LO~v-Nila)3%_-jF^rht=?BV4~J5G2CGDd z%&j;7DCeflow@F@D)>TT+8j+;@3Mbaw(l9O#>eijBb)x(vXv_EQ|A<|(Dl<3X5BH5 zX^@?7s#8%mCox-W5UxO?re_y+U#*+2J3loAoa8E}_gi(o%|&DSMyJIa-BnC`_QtAk z3tTn*?%b>YPlH?l_wpP4sLX!4@rnhOo!i%nipp*Io*RDep5erHn=H|B^(LoXK`z(C zn&YEGGGiapupjbbVQP&$pZ-3E4WIX<2#u%^Pt6wcG!_ z=ppS6jwf5Cv#-0!ySvxewrUL)On=4t4%N8< z)*nw_5+A9g=Svp*WvgyI+qaCDLpf+3FO6PP05RMp1bYO<_wv@uWL;&OFNe0CwV_0* z7R?vg4fX++!;G8&I7N!gFmw<1=gzXmqz?RefkuP+K^ndpnIVR4ANQnONGZA%a9(aF zG4MAEcG7N`JoiqO?OL@}S#9kf$Ho=Rz?y2|+l+_)v+4fR=?7+;>VUU*pR?h*wQDI1 z^aUCShqsQAJNNtd?+G`MflIyC)P~b~36I$BDX%}}lisR~9`1np#xG^?JKs*aD$?4~ zCyk;H*1)04-Dvybg1yFdPFPhj7}tKc4HdQdHj9dOXCVs!ikg3Fio{|i2+8NmqX0-; z{N2Xl3jnQ7i*bvQOllUWlFn?I>%sPs3BrOPHK1W2TruR*<0?6L8dMKeGB`lA_0xInhxE1ye-WgDIo zv{|-nTe|t-$BU1LSFNT`nUdCoRtDPCfD?%O%$9Lp>b|kNM*g_X(l+|pu~oCb&+p+k z&iPDL=IHv{_w$pN6{X>k)rck$#=&UwROlT{OWa31$Ut$lt-^4&ES)QlMDdNazeF zHm#T2NW%+Vo&Ecqw<}A^b`Bpkoq!!3)^hY}dTO{4g*cJNj^h3@fo76JGk;Y*?L^$9 zZ^=zF6UJ%y9U@Buoc@K$pC>R80H~%laPAI0U+e0c0M!N};DQ}dg=l{3F2)!{4azWw zf~kfhiJ)ohkkt`Vj=i(kSv-DiyKH4a!APIenDYucLG6X9K)M+4UJ;dkC+SxaknwB6 zjGX1b^^nBI@i`J3Ys`J1?+}xd zRDXuiC%L7Ms-D)Shg20w+?!(*hOYBIr;7q{WnJF^lk;`fe~(v03NbicH!FpMS>?0u zI6v@NUTPY41rdhMgo(|5SGIQrv@a5`+_@0Y^~uPos{ql+=$G9Y5o+q{t4<#+e6aYt zsP&IkpSF|8#pH!^5M0UDxZV?a8mKyYFZ}#XSFyodYQyiKT;HdtGE) zRr`-~6ujw_CK_EZm75W(uN5=xGd7v`IFL)b?_4QD#FFnr2LgopwyA4;w4Pep((p^u zaw6>FO?xdS{*=&2c5<#%4`G&DPpHgyR@3x-;)Ctg1&$v=gs1k7ZkW)puj@Xe9-py> z{2bb=o4&4N?R|Udiq@~-gh$d~U1Q`9J&{*Lp=58p@l*nN3sEX=FxwGWw3 z-*?PrYv_~y{7qDGp-JWf&o477-hahpepyL4@354Rw})>^=^fTu&Xp>q$a)q~KcR18 zc7$2=W0v!d6wlH>1)3Cexs(k>R0<25fA#U!dSqUq-B9+&%|Gha;e!#86(mnRNb_Rm zX_~7K@8P=f7wny~GS_LnxMDu@-8409>F(*{*Ja*Z7x@3GM%LXHS~|e!Z9FU?kbG|?@Q2mEut6&Ig!W#`>z;rVd0_x{a5eHtn$s@vil_x(;RLMK} z6Tu@})d16o2#tdu`TrcLC5u((OBTzn|KJC?Gt)WS%l>Hb=J<=9_}qVTtHj;QB+ZGv z?f%5XbnUfMFA|kM^U2(%EqE0DS@uqsVW~mJ8e7rF#^P)q$ILLIGwYx!D&4TjZ}DwR zi%{&qDl_p6k}N2(#&4pj6J5OEdWB8Qz==s?cJnNaq^)k9D8Nci^;}J8eEM}KTQkhg z>*fec`k^nx(#dxRK!Ksfx85C$7=QhM`Pm^adPMRJVJ+yS=sgi|%&!&P79Kvmee-2y(R9r%W%*g}8`Tq+2qmxT1O_kO>{p93=?mAI zsqpz{Gsj*O!;?0VHY^rW0tFxr(qxSdHtUEJV%Yvt4p#Tv81#oRt+xh z-KLuA&8Rr@fr4l9rkPoys>;Rnr`BI#{D4~Kwrz9=&Ypc{?N1(Z3<)m!>ze+xKC7=4 zzG$DF5-PdPCh_rpglvT2&t&D%vIiFdebNe@nvirhwWmTSC0p*-Hn`7~SNa-xS#@Vv zCU7LZ_fi){Vl6-OMl`Y8D-5=MF&SPuP3EXvJ+rSg=Sw^sb)edX89>_{%nreb!Lbexi|4;|U`yY^iNwkLZfrmQ4_ z4j2H4gb5Pr?VuZ^Tt)M2^Z_|)yf|tT8E$($-S_EcRJD|vbye-V-|p@r8>QyBr0;ik z8=oD$lpRBgLbOV1=HMABBsDc795wGcYR*u#iw7M@Mp-jAx5kpsHJ`ogS%n5z761#v ze7cSNk(AN)p0vJmGt*a!{M3!9I=92|hSXTOrA_EY+C4A%9bR9{^>a%@;ElgqeO=z+ z&*g`2aFJIN77LqPeW*oo_Lr6FQf^AMY;nvU<^?wJ0)+5LWM9u*9k0)ks&MMzj_*P!f0$$M3&Rd+NuU9|VRh06wog(KlrPAc2fx(4i&FnM^ z93S^*44d*QBz{PB^#m|j<5y4H_-H9d*X;byMlzL;=F?|cTbsO_7fhR1ZCt+LG54^vPV!GW8r$<(Uq}7@N~So>P1)roY$VOa3_{q^2*80eTY%YnB; z0d(ip&D|Y?Ik?IXLNm)Zw-t7{#6FE0vFu2>qA|cU?5cqh$Pm3u6ibTi?ITv}GuEIb z>5L?rUAcBoH4#EVR?m+u@--~+cbni?abq9rQFuR8Xml^;N^%o|3jkENX??iMpiLFuxx_Q#|p_<4zZEt@#H#k?iDmJ137B#0I|OG#NX0+v9bLKpH#X zfbHe(f`ZTQqM?l@sus_>Bhqxy8BVhG>nB9xm^(6G+8y@q_@WrAOx*R~21-I-T3KGd z>2PO>)7{LO=(WI%Q8M7Uli%%`e*yyWP!cl8_F&D{30|AqY*2YmsEpEn4IRiFGmvEDZ z%(bc2Uv=HKN1VG6v!#_?f>ZZK8bu;a=!$}@QjG-m*H{u~pmKKcbIy-shG(+h^uzl5 zdj~n9?$}wD#ZZ+59KV1R{tI@a}2Q2^mk*ceOG@3DPPIm{SEA@mhC0t-JwaREM8oWxzbF9 zww-8}^{$@FD#D=7F=Dh9sXBIUaRuq9&lh`?z!1?*T)%iysYbd?_(N^&>S9cm0)W4D zh%hc|E8~)G$Cl|pY%s9;W6Q$9@8dp-MST02boj!$?tQJr4qnqsUk%!x9JF7NUiy}?>x zI$-6_ptdu$dL=9YlT2m>-Or8P8XG5rEzU+A_beFnF(3KwisIRMX>y#yA^~8qky48@ z#liDRNfFq-7BI#1&2mgS>+TvJFD);#Ns$fTC6#bScX;-viE;UKum!4#<&)R9@Bf?~ z#zoW%F?f9h>+XU6>FjE>5mD^0uQ=uhL|*Zp+V>3yn|{%I%Ot$dTUA4an%L`P0m+|3 zmR6@{oXYRTkl^O{`xasYl#-5N3%2Y{KiC#mGg4c$`E8Bo!FwL8K+Dh$Rh7pUnAG(k z+BT8g=ba+(E++PbRp0M|gUCRKDL+{j@0$oD#Zz)5CkgN&P)s_MsAuAY~C zZ@r)|K}}4FdS2pW(#n>uV}&1*;7c!^i>l&y<-;HvT8txFF_pi`>Mh;5gKT1XRv&*0 z1K84<8&^^y?z*k|op!a%Du9Z*3|sRyQ6Z@oiOc-7V)B$~&@N1}l4UyhyYh8K?$fq# zY9o-1It%~e`7HCt_Fk2s1s)F1*mv!ajRoV~)3WlCqQ2(~xL6v+?x1-zI`c*03|V~! zn{9dXw+9jGl?W+M2fXME_~{+ObV}@`uD!MML7Ac2EqwdYd>x*>E!hz~z*3Tvz<}S> z;Ne2H06IoKApc6bToiA3-00~_r#FFUAHj7JF_SN9hFLM)Z?xnj1d~Ebj?x}4`3X9Y^Jv@u1 zWDS*K?Ha-B>vw;s&Bx{m%M*^PCb&Fb6gJEnQiZU}byZknt@%Ou51P;O6#B0^)0AFs zWfgkDLwZp9W=~+cS0b7X<5S&!O-_xG*8neO-FuQjnhhdUAQ7a%m+apbg2z(rL0g5F zckflp4W*i#-}bvMstfn3jE>9AoOcJ2{$IqGs0iS1N-^X>K(($t5!(+uJyt z!z1-^r3WV%x_f6))62yn!>8_-{}>Ul0*plJG%#!T)M8R(e`;)8@Z@4Edp5Ul?yZ2q zMz)$Jv5Yl&hp9~JB8mPC7Saz3dE9`)Sb<7*S-+?!u8H6O-nZOq2i!M(aXR%isgm_YKmv{D;U5GLX7J9 zdjbFAgap8NimtFo>eCEH#aBNM2#EbX^V*<)ZFnZdxxz+f#&mU^&xY_silA(3P!@gk zk=J@UrdVi12m@jJEtDG5Sc7(~yks=5k1)TH-0M-xuh^v1(C|dSU`o3^a=SQ#>oo!{ zSxJ1aoc5N9)Oy;p+I4N$7xnpjao<;Lmr^VeN4<4rPBnXUr?)q){n#QKazRjW&A`vx zrolaOudchSK80i_!e#ZzO8!01XL9`(V@K@Q{@nx4YPjM1duq?T*uG>wGyTQqs@g4y zKiq4I;*~f0?0?ZBuwlr|HuvS$m7rfKDR>i&fZ@0Up-fqYK%=yVUoq!{xoLfpV8)ua zpLhaYY(64Sz(%X79y#|?tE&uPmnR#p+k8m-`vrP*xK-XVU-pr*M&}YM0lN`T%1@V9 zAC$Qyn04fv3nKy~A>?tum+pGxp^_sYKwRK|h7&8Lk15mt{T|+@R~6My-(9CZ;-9GC z?8xb?py9cN|7eeohoKdNE^Z_-Mp+nN;?+W}mD|^a?fcfN$BEQ|E zi||nA##uYCw5`R{)fWKP_h#pp zLMO)S64mjRY`iAvpK+)CDAw`uZC9_7h&5^R7hVd>(DO^Xpn2i(srjP6q4Z1P%pwtM zq;~P@Kp}t#nC`zgXSnKSaeKefcQo+3eLe}Uwl0@yhil!LdzZ<4SI}y!zw=#2Z>=^) zkAC1MQY-jK@54`};-LywbuQ5U>XTm_k!ipWK5IX7uLC9#Ga9yTtQ1bnPMuE`v}9E> z*8IC6Il1X#ciKrVj2JadDCeqd#(-~mE%ih*)1*FXVRh^@wvR3{mR#)!Lt?4i$^378 z&tT(#>m?sa**NoLVt!(TO31i>X1AM6)OO%!L)KsSJ2UZM{%WARZN9&^ec9jt8NnC@$IgzNJEio#0!gj&5I5(%*Zp;hc<}l2(HH{^a{GiRnz9!p z=6Vy7-#7}C?PZy!IaS5IPIMxu*;_pP`HX?Z=0KhSi`z2%QlKH`e@C0?+)uCF~TQ`k2pJqyr4fu)H$J+v5SW%kS{gI@7+5k zkoXpZZMXT2&1OiQX3FSXzAVFJ;6w>}gd|iP4BY!7(5Q>UeFlJvBm;!xP=aJzknmid z5Sbgk6p{RdX;M5?c%bOKft#r&$UN`{dXFD53h9@KyYHxx#-}L!R-JkH;DYUbhyS73 zFY|RffAKYP!^$5S{vH9nc*bzyOFv?)n;Lkg~R2_$Yws$1Lss zJg5Q@9Su+gV6|ds44!zSp9oQm!au8kw_ELiR1S1j*n>b-4lJ$(@*G9C0}*5 zb0!N3HKhwQtqjA~6nX)5uRg7_P|EjhxJSd*6881n}I; z6+9<7F`-O>MY|-mDJ;-5qj?s5-;@w!5xC~9qptqQPERjZ-fIpQZYuD|etL-VzH*zE z!+(w6gH?wjwQ$asf{fm?)=o_|8l$GqywAZP;HoclEzz$ncF3Nce z5uZVqnwjzdn{L0))B$Y;sVgsU9qX!QyLP^{={JI}WD9}}#3w$J@x1k)|9J>rq9NC| zTKnCb;0?BN3xcnNZmgf^j_tbllaax<@^oLq+P?>rYpRaxhc*v4H|6Qv%?E8f26D<} zn53dOPFtZ_2pz#M5Ma@0;QvJZJX0p|6vvs!Gn4M=UF(Eyev4{%SY2#(!f>4p3tid$ z`}ZlJ(iIF*flY7fehs)~z7fxR^7t`Y-Nh9Z6<kd(efHU6_ppd-d*s1mD$$EVjE(Qh^!0Cc#N0AW!P9(_cJy!Ql$~y zS*_~hGj|zJl%b@z#uu#<@C7`*8|jeQq}xvjKJl?3L%9UIN&&Upw{=UTp&a5u)Y$L| z`;mw@I8!ax7tvXk7M~Q2U{`*dj9!J-5c6QbY>^~IbKRNZBmjHDjJN;9% z2>8=W>zB~5ufxk(ntZB|dyfycrE}rOzAd0%fVM8Sr-2uqfm=c%0y@;8z!Ki;SwRKj z3f#r{~yvCfGtz zk7;|tr4Y&g4A65bkbL){6Dx;?uqR~8Ev&6U2{Mxg0Xc4>yg^p?y`6{{=0;aONnLNE zSwKU!@~LwImi*s0wD$ktf6rRLXY5Y6@Ydf!{7e}!OGHKC^3ptjpLZZF1&0$2+ASD(`*%N;Z6m|N6rmA| zMl`I!X<6;YH{vcpjs76ea8O{O?;Hr`cVI8dp;ykw{>n}-_BGUq#q1jRLCA6~J9{69 zCo~fF$DmKEk|+(5`kDL^y_>pElO)^UzPU^IuAPlqXVC|nUPV&#AG{Q z6}<|}e>4vjY}tK?^{A=ogY%%$XZ2;+yxH6U`RoQdt&WUMKE7(1u{VvUvlhsN=xUK678-;N@gg6h!7-@dCMD{z^JGw1TVp0?7#Fq z;`(U^^cn*zEkr3Exs7yag?^O~INDww!WQ!R^YK-}*y`3R*XE8gFoVR+dtgwumJh{7 zKKTW6CGp8BgOV9?Hv7D%UKq20RLUh&4j`hfp@6I$E!nDL&~H^=4##X5!xIPGbU9l&39MucX@83?7S@GQAoY$cm+TK3Q#PU=ZJGjt=GB zh`V>0@k2Bwo)FG)@WQ?j$Nr}auk4*Rjl&P}V!U3uXL)I4IT&D@Dk)G5gwXoW8RWCD zu#gK|3*{SADth ztrqKTSVR?c^QOTtWQz7--er0AaDX`>cv&TE1Zh7^`y@T|^ZN&sTpTJz-iyK5*r(%! zLC0{kJ!D_US&H%^5LxJjG~y4MA)@sw_PuUQou-olbO&_EZ716vVgA4MVa+p%&sD2M4tA0sCR2n9kcLkJQ; z{ZlVxME$Yu^zCcy1b$yKq+#7g|pkt~AvOm6Ur!K#QpW0`1pQ{M!IJGd;9|&g| zWHXZ4U*L@9p}BtL;X|(SU>1LzQK%w!-||`5NqEJU=NgMtj~#PmBWF1er%!_jFV^qQojVk~ z1skifqe3^^&>MOGv7`_zoHc8FN-1GD1TiM;xH~a1Y7Pz#Jyg6WL(!B+7_wVj{I2`N zkAIgZ-))3uyNpaMvFOzjjiUfVP0S6wat0J(6C2w{#L7Qd$*szuj}G|vVO7;NMRWD9 zK=43C_5h25_kI*_O))eL!ov?~Yinm#Nc0vw-b53Uba6=fzrFbl zahH!#LZHFvC;_b#`KHwA*i@I#Q-TQqA}j`h%VQLrPn$V7jCwBNpnS8mvihe+AQ!ri z?)WLyqer=+a*RWmxUk?!c)S{SpTvcwnPpx1AxuN`fUw;|H0A$Xg$NSE#*M|0=sFAT zCc6}D&s12$$f7{Do{p2W{wh$$qyRv)#XIbW$w)^*n^W0(Og>H2DvTssQb-sN^;KLK z&w{lX0|A-{n2#Lt2A=4#BZiBW$#19kt+wg`8l^i@-vJmRKLFLnyMu%-fa&R!fx)lX z6Xf6dlXL?r?O)*Hfd>K8O%IrZ;DL*}9DU~5PCe0h0OmU!uWB8y)bOZ&L$XrD0{oGt=p#KrljWb9Q+RijDb~Y5?(6K7U>W zGR!J3ZxOp_I3Qsw85bkx;To_6`aNo2u!H2RoZNb_7~;<*VSk9~0R*HsL9q<-4el5e zA{b3D9GL;P;)@k4BrC>os_@kl^P0fMGl`0}j}bBWPFTGVeLVcQM=ACp;-x3Y?i~T| z5GcOHgx-!G3=qLC1ev6Tr6mI^>)`Y{%_I?i!@RpN^}pXv9Lt`r<1IyG12)bQ2}xn$ z<2bGEU|*)bkN-qyM+psUs;N;ZCN+h@GA^Tng; z8Cr>U``~$R-`@B-BSYGIK?($7FPsbxb?B3Ga^$J~2((T~d>=7A{ctBZ4|@%qQ6rbr+}5MX=CAKDr27XUXVSSz(#baZup4g$PK)?@BSwq4Dq zPw#|-*A~U8XBP-8CaDo?^LU>u^eG5*(b~e|?@<`Zls)@-J5dZJQQ!)@SW~7Q6YSFD z@Aml=^KW;qs)%CB0n)cpp0mObVNCg41W5}D!TevZ%ju6MsVFNe8=QZ?m27D6%Z&m0 z3y;dn-U>nx$e~f)HEiq~&|}gRrF&HO3h=DrKiBf#2#AYsA@;h{mvR+RpbH2HfD*IT zPa(sb$MY5=@d$&6AXXQnPRu*pndicHT+#zUuei=sfvZr#WF7AI+LFUnz1cSYgY}>JDB!M*) z!H`BG`#p7W1{aNO`J;mb3>4Q-0ZO%bT$Dq)piX49l6Px4x_ z&$U_+zv)Xx=yF?N5l>}-=XI4ZMGvwcx1rWIHsLtkCKTl`uKl*eA#F8*mXLO{F?NKU zT>Bb&dPxd%h{&c+_x4&4hel(PVvzgVwA=A^hU!Z|q*$RACQ5;*wmfIL#m?|nSy@>K z*uO8!D21(ELJ*L*bk-P;RgS4OzD3?;ntEZno((>;{*=rJPsOKD8FQ1VYeVGKI@|QtgnCVp)N`5qG}?vy_}@yNi~*Hkp7tp#d)gRUAd7q}veRYtRLj7B*E z^kAe2EfL#Evd#8u3U!nZYuBy~1{A#QdPs-_#3q9wwP=nYOy%UsJ0yPnwpnw3*POp$ zyAC#BfH@Ptt_hLbLNj(B{va3|j%BFvmE!`_j+6c^yXfc+VUwJ5&2PUnoY@#mA|kAU z%7FvEE72ol?!6;cZr8r@4tKdSlc5E~0ND@)t03tP0qGShs6G|RKVj6=Pcu;r8HvLc zj}M%8DRA9q-(+HdcwYqwlCF^vM{*kkiOM!P6-{?2?#|zuGMXz}<^I-`WI%_f-zO+OSwt7x(mEMYL@pnr>Cis(J0?ZX^nX^K_y{cEh!@t^ucH1bbJJ zy~|ymyq}ZP^tB5q&*mfy%w7M$;+lUC(Z^koon}MvHwY#YuL{x{X2{?c1~h%lfCNvV z&uAvD1ES_ubxCSUhY$Dk^c=zj09b)75o(j5$P@~NfYxbVYcb)t{BsAk5Jm^7temGD zGXNm2gm-vPiXg$k?{H_ZmN+R;$_U5LPxbZo=G-#};+)L5T@47(y3P$baSpl5 z03yFb%1XO_{mVu>CbV|dNJg1dKaXrWTG;fT)%4r8@82I!>WK)Dj~f_hpfZP)6y%N( zhdJD@e*bQQsP~|v;+3am4zDjbI}^rbJR)YY{2*Z)1IA?#Dpr2_v>H2tU`p%w@nS-ohG>5cDM-wIO78~alU zroxzRhOphLVkCu9(B*xx%`T&vVsw1m_HI*61Z+G=CCA3cyD=qzjQlin_?{r%X=_%DDbQv19|N!U%mR!k3IHm)<@v#Ip;C)}LUK<=;e{hJM=zERqMr zhEETtw`M!`x7fYn=Ha0+Ha5<48>vBMQ`5@IDzgIOr$+95Tgz{8Ld4^$-Qqk5!7K5_ zr_4V^5Z*dU9PMPuGZZySik6jr$u~Jr63MK}obM~*(TF}iFu@KCcIWca3>GB{RRfgh z2U@d*d@w4mcH~HM?MLeG_}hJu;)NjNk#K4XRA71l8He<){Nm(S)^O_Y@sy(@6JTJ|2=!xtE#CV%G5Sq6$mV;{MK%Tviy>3s?T zP*NkNC$$2m8v`QYVPPdWGrHLh>;yN=r=8-YZ)cQz|30B>y#YA!r+P*(t)mp8u)H8f ze!*wBwO?PZ1C00sVWm}5qXo7cg}6jC{f)V86H?gzmJRq&8&L1iVMa#I@-s|B#nu^FItIiwKTCjKPi}6-%HIl^dCVJ1& z`gRl`jVy0$j6xhdju$zBj1DY%?^JGJj4@XJ?}4zJ8mvz>3)C@m(JH7VR2*>sCA$+n zOX1l_swyiR*Vbl0RQF)~&p`eRf&baZ6%fv=_5{;0MzJ`&XH9YVe()N&EzF)LB)@_= z-~EofR0QCL63l!r}C zcYulqA@&$y-(zO}z~=b21kAVwPZJN@v|mBg3~0GF%GrY#JzcUK+6S|YiWVCEsMi_1 z--0;rWOfhXt7{a!K@1iwBQZ#|wPeu9z`0tYx3|{`nM|Ym6UAf4LWj!VCdaOWeciEd zACoVk-#ym<>J~ubW8eKZ5cS65T#rL8Dj}@Y#fU9y-R;pc4<>T;FTdeDT2auL#G|Ml zFIS5jUtbIfJz-p}eGZOk;AR)XxmQ!t2nN2(kQz}$XfCQ|>x)sX`^kQK>=M8c!VCOC`6L1~}4 zf!#az?D;?uOelt5zIyiT18hFRIDo~DMfB&uPOdLTF12+0ML+)(c0hjjQFZkf7e9fa z*U%hv&$>@1ba-~`C~+x-Vc#hu(=ydJGBPsAhPk`+OJm)vwKIrOS$wt2><>@Z(v{`b%d|=Lh1uzPoF*8Gd?lF$InN?8_J)ZMGANT>w=nPWSCkl-q=+X zPCq~I$TnoBD|34Pr1LiF!*gRatcv6B9gW{7EDFzFXPl#W6`dMhW-Gf zaO(K}qQzlHO5Job<$DRsu+|j1{E@t`9<>O;2@G=s#7$4nmJNfx{r6y%QQJ@eXv7Q)9|J9j>+ zi|az3l`q$UcG7qL{W_cokx@y0R9A?KNPc1Mr%xB_H#KEg-?ED-h7q~iY%tZo`tkd5 za61mhW*M;|b0u{%;v)O89eZJH2&p_N+ms16<+};ta-4`s_Yb>h8Zj23X@k@%wgIuJSFDgqO;k4 z<*BNvnO5E1yMXFOsw8;6v5QgV^qt`O)a7(LF>%E87rvbGC#qBh39@_kJgX`k7h7F5 zQg?jr#w8nr%o9tehVt;Y>u^~i^6UFsn`IPi2$6a-(n_R{y`i$dowg>XtOJ!kPE{?| z(xx>_01B}$CcnA}81$`ocdu`c*Lm`0&7)VZ93#xDb}J<(4}9v>P_`!yfm+F(C@O6B|3o`#~o*_}O6KsJ(nlDqnv4@3|$6XfOw!=+XJ3 znN~}3gxueGLufKMpW`E6(fC5R`_KltcbIX#LJDb2La7&`g6bMQ&=qJL%xrM^p=Kj` zKTm*verl+Ht}wY>uqFV_Y|K|uOSyf0cH~c@kC(Lv;_Sc5iEs2a?>=_%Xaxe4+98)v z7)QMSe~J+@Rl(Y8U{|)4BYMUO8~qd-)}{q_4BpL9N7^TWGvUu=RpyzEk?fBi_duI&gh3 zSn{UF=Pn}%`nxmuPZ)bSy6b%lUqM~}L+@1jRZv@RXo;m^iZ2x&Cc zcid<4lyl-sWE8hx2q@%Hz*h0-FkTC}a4+Nbi%Z6H$S5IY-~rKde|^V-%$6}^yfok6 z-PoLZnP&a-g^3Xa9;55p{Vis26I5?mf!$?%lhMAn|23 z1skf^iXQA6Tc7#X&NXy&Noi@fW5ld4bicb{TVZWwC5k=->MfK=vRwss&%);N24^?h z@1OhmNGsvhq?H@!gh`2Z%8SLN8{QH`^8XZ*)-S8$!=XF_6KqKd4s&!z;Xf$)L+nw0 zv2#I9)D5nu(Ilfe2g82uzjOO-_gkTqnre29)Vgd_Dvmccw&uGzK790uvfu7i6?!9} z;4cxTBZZ;q-|hNO`=8%Jnac?VuQ*r#uoh}=6zHC|N#8>AD0Ty8d6bg0hU#B1tz!~= zS5a{dR}{N6YO(}QdS}?(MrIswNZS7?TAo)xT=!egkD^AiQeB~hbaKvQ+HR%4w#paT zkqFB1D4=NfJP2%~f?}r4c>}Qv28nCf0!vUm`GZE4Vb9Z_E>p8Tw>Gn}1)*Sw zK=cQSh8OsJqwn3r*7Gbwb%90J)&5@1B32QHpNVhAVxy4b9w``Elvq~6w9a-HjP2f$kB`iwyEbgAUvN3IwRX3ROkUM1kQ67EN)tUCR#Y@nozc!_fy;1E zur8hjVYl|vMmy%fG1w$FUr6vAF6(lFXfeVh(kE+A_zI^72SHjgq&k!Oc>Z^OEz+@z zvWNdHXBRD;QBqN1MJ(;0Fb_MLQ*{YN{E@tlQzuUjF0_DGudgl(B3E2ejkzPO*|!i- z2SZI_6BE;6C>rPH<{~R4DKPShTJ&FZP3IOcqTck{avcGt=wd@cBnYjz^Jol@;w*`h zcj&V)#+%rRIotO^Oc{{e>2U97$3ak4S4R|d9pSDR$g&%EzhD3S-(oo_Qj;PC60z{e z$Ut}i`~?z-XjoKX6!~TD&0ws)3%VK>$k*&!-rPXpAuK*#!^nsOvHiv2mIx%+i3mWy zNXR})y>Q{ehsw(5uZvz=Qj^>woabmac4mK^V__GXETk&T%&dwP#MjdSykc1(ZwbXK z8VurppB?J~j3knFGtX)1EHnyHx*UTmNk`mA61QUs`_)h$z/dev/null; sleep 1; echo done" + ) + run_ssh(target_host, kill_cmd, user, ssh_opts, timeout=15) + run_cmd( + "ps aux | grep '[t]ransfer_engine_bench' | awk '{print $2}' " + "| xargs -r kill 2>/dev/null", + timeout=5, + ) + time.sleep(2) + + +def start_target(target_host, build_dir, buffer_size, user, ssh_opts): + """Start the target process via SSH. Returns target address or None.""" + bench_bin = os.path.join( + build_dir, "mooncake-transfer-engine/example/transfer_engine_bench" + ) + log_file = "/tmp/efa_bench_target.log" + + target_cmd = ( + f"cd {build_dir} && " + f"env MC_METADATA_SERVER=P2PHANDSHAKE " + f"{bench_bin} " + f"--mode=target --protocol=efa --metadata_server=P2PHANDSHAKE " + f"--buffer_size={buffer_size} " + f"> {log_file} 2>&1" + ) + ssh_args = [ + "ssh", "-n", + *ssh_opts.split(), + f"{user}@{target_host}", + target_cmd, + ] + subprocess.Popen( + ssh_args, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + ) + + for _ in range(20): + time.sleep(1) + rc, stdout, _ = run_ssh( + target_host, + f"grep 'listening on' {log_file} 2>/dev/null", + user, ssh_opts, timeout=10, + ) + if rc == 0 and "listening on" in stdout: + match = re.search(r"listening on (\S+:\d+)", stdout) + if match: + return match.group(1) + + _, log_out, _ = run_ssh( + target_host, f"tail -20 {log_file}", user, ssh_opts, timeout=10 + ) + print(f" Target log:\n{log_out}", file=sys.stderr) + return None + + +def run_initiator(initiator_host, build_dir, target_addr, config, + buffer_size, duration, operation, user, ssh_opts): + """Run the initiator benchmark. Returns throughput in GB/s or None.""" + bench_bin = os.path.join( + build_dir, "mooncake-transfer-engine/example/transfer_engine_bench" + ) + flags = config["flags"] + + env_parts = ["MC_METADATA_SERVER=P2PHANDSHAKE"] + for key, val in config.get("env", {}).items(): + env_parts.append(f"{key}={val}") + env_str = " ".join(env_parts) + + bench_cmd = ( + f"cd {build_dir} && " + f"{env_str} {bench_bin} " + f"--mode=initiator --protocol=efa --metadata_server=P2PHANDSHAKE " + f"--segment_id={target_addr} " + f"--operation={operation} " + f"--duration={duration} " + f"--threads={flags['threads']} " + f"--block_size={flags['block_size']} " + f"--batch_size={flags['batch_size']} " + f"--buffer_size={buffer_size} " + f"--report_unit=GB " + f"2>&1" + ) + + timeout = duration + 60 + rc, stdout, stderr = run_ssh( + initiator_host, bench_cmd, user, ssh_opts, timeout=timeout + ) + + combined = stdout + "\n" + stderr + match = re.search(r"throughput\s+([\d.]+)\s+GB/s", combined) + if match: + return float(match.group(1)) + + print(f" WARNING: Could not parse throughput", file=sys.stderr) + lines = combined.strip().split("\n") + for line in lines[-3:]: + print(f" {line}", file=sys.stderr) + return None + + +def plot_results(results, cache_sizes_gb, annotation_gb, output_path): + """Generate and save the latency vs cache size chart.""" + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + fig, ax = plt.subplots(1, 1, figsize=(7, 5)) + + for entry in results: + ax.plot( + cache_sizes_gb, + entry["latencies"], + color=entry["color"], + marker=entry["marker"], + markersize=5, + linewidth=1.5, + label=entry["label"], + ) + + ax.set_xlabel("Cache Size (GB)", fontsize=12) + ax.set_ylabel("Latency (s)", fontsize=12) + ax.set_xlim(0, max(cache_sizes_gb)) + ax.set_ylim(bottom=0) + ax.grid(True, alpha=0.3) + + # Speedup annotation + if len(results) >= 2 and annotation_gb in cache_sizes_gb: + idx = cache_sizes_gb.index(annotation_gb) + tuned_lat = results[0]["latencies"][idx] + default_lat = results[1]["latencies"][idx] + + if tuned_lat > 0: + speedup = default_lat / tuned_lat + ax.axvline(x=annotation_gb, color="gray", linestyle="--", alpha=0.5) + mid_y = (tuned_lat + default_lat) / 2 + ax.annotate( + "", + xy=(annotation_gb + 1, tuned_lat), + xytext=(annotation_gb + 1, default_lat), + arrowprops=dict(arrowstyle="<->", color="black", lw=1.5), + ) + ax.text( + annotation_gb + 3, mid_y, + f"{speedup:.1f}x", + fontsize=11, fontweight="bold", va="center", + ) + + ax.text( + 0.02, 0.98, "8 x 400 Gbps EFA NICs", + transform=ax.transAxes, fontsize=10, + verticalalignment="top", fontweight="bold", + ) + ax.legend(fontsize=10, loc="upper left", bbox_to_anchor=(0.0, 0.90)) + + plt.tight_layout() + plt.savefig(output_path, dpi=150, bbox_inches="tight") + print(f"\nChart saved to: {output_path}") + + +def main(): + args = parse_args() + cache_sizes_gb = [float(x) for x in args.cache_sizes.split(",")] + + print("=" * 60) + print("EFA Latency vs Cache Size Benchmark") + print("=" * 60) + print(f" Target host: {args.target_host}") + print(f" Initiator host: {args.initiator_host}") + print(f" Build dir: {args.build_dir}") + print(f" Duration: {args.duration}s per measurement") + print(f" Operation: {args.operation}") + print(f" Buffer size: {args.buffer_size} bytes") + print(f" Cache sizes: {cache_sizes_gb} GB") + print(f" Output: {args.output}") + print(f" Configs: {len(CONFIGS)}") + n_runs = len(CONFIGS) * len(cache_sizes_gb) + print(f" Total runs: {n_runs} " + f"(each {args.duration}s + overhead)") + print() + + results = [] + + for ci, config in enumerate(CONFIGS): + label = config["label"] + print(f"[{ci+1}/{len(CONFIGS)}] Config: {label}") + print(f" threads={config['flags']['threads']}, " + f"block_size={config['flags']['block_size']}, " + f"batch_size={config['flags']['batch_size']}") + if config.get("env"): + print(f" env: {config['env']}") + print() + + throughputs = [] + latencies = [] + + # Start target once for all cache sizes of this config + kill_bench(args.target_host, args.ssh_user, args.ssh_opts) + print(f" Starting target...", end="", flush=True) + target_addr = start_target( + args.target_host, args.build_dir, args.buffer_size, + args.ssh_user, args.ssh_opts, + ) + if not target_addr: + print(" FAILED") + continue + print(f" ready ({target_addr})") + print() + + for si, cache_gb in enumerate(cache_sizes_gb): + tag = f" [{si+1}/{len(cache_sizes_gb)}] {cache_gb:6.0f} GB" + print(f"{tag} ...", end="", flush=True) + + tp = run_initiator( + args.initiator_host, args.build_dir, + target_addr, config, args.buffer_size, + args.duration, args.operation, + args.ssh_user, args.ssh_opts, + ) + + if tp is None: + print(" FAILED") + throughputs.append(None) + latencies.append(None) + continue + + lat = cache_gb / tp + throughputs.append(tp) + latencies.append(lat) + print(f" {tp:7.2f} GB/s lat={lat:.3f}s") + + # Cleanup + kill_bench(args.target_host, args.ssh_user, args.ssh_opts) + + valid = [t for t in throughputs if t is not None] + if not valid: + print(f" ERROR: No results for {label}\n", file=sys.stderr) + continue + + results.append({ + "label": label, + "throughputs": throughputs, + "latencies": latencies, + "color": config["color"], + "marker": config["marker"], + }) + print() + + if not results: + print("ERROR: No results collected.", file=sys.stderr) + sys.exit(1) + + # Summary table + print("=" * 60) + print("Results Summary") + print("=" * 60) + header = f"{'Cache(GB)':>10}" + for r in results: + header += f" {r['label']:>24}" + print(header) + print("-" * len(header)) + for i, cache_gb in enumerate(cache_sizes_gb): + row = f"{cache_gb:>10.0f}" + for r in results: + tp = r["throughputs"][i] + lat = r["latencies"][i] + if tp is not None: + row += f" {tp:7.2f} GB/s lat={lat:.3f}s" + else: + row += f" {'N/A':>24}" + print(row) + print() + + # Plot only valid points + plot_cache = [] + plot_data = [{**r, "latencies": []} for r in results] + for i, cache_gb in enumerate(cache_sizes_gb): + if all(r["latencies"][i] is not None for r in results): + plot_cache.append(cache_gb) + for j in range(len(results)): + plot_data[j]["latencies"].append(results[j]["latencies"][i]) + + plot_results(plot_data, plot_cache, args.annotation_gb, args.output) + + +if __name__ == "__main__": + main() diff --git a/mooncake-transfer-engine/example/transfer_engine_bench.cpp b/mooncake-transfer-engine/example/transfer_engine_bench.cpp index 6ce1eba6..46f588ff 100644 --- a/mooncake-transfer-engine/example/transfer_engine_bench.cpp +++ b/mooncake-transfer-engine/example/transfer_engine_bench.cpp @@ -88,7 +88,7 @@ DEFINE_string(mode, "initiator", DEFINE_string(operation, "read", "Operation type: read or write"); DEFINE_string(protocol, "rdma", - "Transfer protocol: rdma|barex|tcp|nvlink|nvlink_intra|hip"); + "Transfer protocol: rdma|barex|tcp|efa|nvlink|nvlink_intra|hip"); DEFINE_string(device_name, "mlx5_2", "Device name to use, valid if protocol=rdma"); @@ -458,6 +458,11 @@ static Transport *installTransportFromFlags(TransferEngine *engine) { args.get()[0] = const_cast(nic_priority_matrix.c_str()); args.get()[1] = nullptr; xport = engine->installTransport(FLAGS_protocol.c_str(), args.get()); + } else if (FLAGS_protocol == "efa") { + // EFA needs topology discovery to find devices, but auto_discovery + // would auto-install RDMA transport. Manually discover instead. + engine->getLocalTopology()->discover({}); + xport = engine->installTransport("efa", nullptr); } else if (FLAGS_protocol == "tcp" || FLAGS_protocol == "nvlink" || FLAGS_protocol == "hip" || FLAGS_protocol == "nvlink_intra" || FLAGS_protocol == "ubshmem") { diff --git a/mooncake-transfer-engine/include/transfer_metadata.h b/mooncake-transfer-engine/include/transfer_metadata.h index 3e21c6ff..dae07757 100644 --- a/mooncake-transfer-engine/include/transfer_metadata.h +++ b/mooncake-transfer-engine/include/transfer_metadata.h @@ -117,6 +117,9 @@ class TransferMetadata { #endif std::vector qp_num; std::string reply_msg; // on error +#ifdef USE_EFA + std::string efa_addr; // EFA endpoint address (hex encoded) +#endif }; struct NotifyDesc { diff --git a/mooncake-transfer-engine/include/transport/efa_transport/efa_context.h b/mooncake-transfer-engine/include/transport/efa_transport/efa_context.h new file mode 100644 index 00000000..1a1c056e --- /dev/null +++ b/mooncake-transfer-engine/include/transport/efa_transport/efa_context.h @@ -0,0 +1,177 @@ +// Copyright 2024 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef EFA_CONTEXT_H +#define EFA_CONTEXT_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common.h" +#include "efa_transport.h" +#include "transport/transport.h" + +namespace mooncake { + +class EfaEndPoint; +class EfaTransport; + +struct EfaCq { + EfaCq() : cq(nullptr), outstanding(0) {} + struct fid_cq *cq; + volatile int outstanding; +}; + +struct EfaMemoryRegionMeta { + void *addr; + size_t length; + struct fid_mr *mr; + uint64_t key; +}; + +// Simple endpoint store for EFA +class EfaEndpointStore { + public: + std::shared_ptr get(const std::string &peer_nic_path); + // Atomically get-or-insert: returns existing endpoint or inserts new_ep. + // Prevents duplicate endpoint creation from concurrent callers. + std::shared_ptr getOrInsert( + const std::string &peer_nic_path, std::shared_ptr new_ep); + void add(const std::string &peer_nic_path, + std::shared_ptr endpoint); + void remove(const std::string &peer_nic_path); + int disconnectAll(); + size_t size() const; + + private: + mutable RWSpinlock lock_; + std::unordered_map> endpoints_; +}; + +// EfaContext represents the set of resources controlled by each local EFA +// device, including Memory Region, CQ, EndPoint, etc. using libfabric +class EfaContext { + public: + EfaContext(EfaTransport &engine, const std::string &device_name); + + ~EfaContext(); + + int construct(size_t num_cq_list = 1, size_t num_comp_channels = 1, + uint8_t port = 1, int gid_index = -1, size_t max_cqe = 4096, + int max_endpoints = 256); + + private: + int deconstruct(); + + public: + // Memory Region Management + int registerMemoryRegion(void *addr, size_t length, int access); + int unregisterMemoryRegion(void *addr); + int preTouchMemory(void *addr, size_t length); + uint64_t rkey(void *addr); + uint64_t lkey(void *addr); + void *mrDesc(void *addr); // Get MR descriptor for fi_write local_desc + + private: + int registerMemoryRegionInternal(void *addr, size_t length, int access, + EfaMemoryRegionMeta &mrMeta); + + public: + bool active() const { return active_; } + void set_active(bool flag) { active_ = flag; } + + public: + // EndPoint Management + std::shared_ptr endpoint(const std::string &peer_nic_path); + int deleteEndpoint(const std::string &peer_nic_path); + int disconnectAllEndpoints(); + size_t getTotalQPNumber() const; + + public: + // Access to engine for endpoint handshake + EfaTransport &engine() { return engine_; } + const EfaTransport &engine() const { return engine_; } + + // Submit slices for transfer + int submitPostSend(const std::vector &slice_list); + + // Poll completion queue for completed operations + int pollCq(int max_entries, int cq_index = 0); + + // Get CQ count + size_t cqCount() const { return cq_list_.size(); } + + // Get CQ outstanding count pointer + volatile int *cqOutstandingCount(int cq_index) { + if (cq_index < 0 || (size_t)cq_index >= cq_list_.size()) return nullptr; + return &cq_list_[cq_index]->outstanding; + } + + public: + // Device name, such as `rdmap0s2` + std::string deviceName() const { return device_name_; } + + // NIC Path, such as `192.168.3.76@rdmap0s2` + std::string nicPath() const; + + public: + // Libfabric accessors + struct fid_fabric *fabric() const { return fabric_; } + struct fid_domain *domain() const { return domain_; } + struct fid_av *av() const { return av_; } + struct fi_info *info() const { return fi_info_; } + std::string localAddr() const; + + // Compatibility methods (libfabric doesn't use lid/gid like ibverbs) + uint16_t lid() const { return 0; } + std::string gid() const { return localAddr(); } + + private: + EfaTransport &engine_; + std::string device_name_; + + // Libfabric objects + struct fi_info *fi_info_; + struct fi_info *hints_; + struct fid_fabric *fabric_; + struct fid_domain *domain_; + struct fid_av *av_; // Address vector for peer addressing + + bool active_; + + std::shared_ptr endpoint_store_; + std::vector> cq_list_; + + RWSpinlock mr_lock_; + std::unordered_map mr_map_; +}; + +} // namespace mooncake + +#endif // EFA_CONTEXT_H diff --git a/mooncake-transfer-engine/include/transport/efa_transport/efa_endpoint.h b/mooncake-transfer-engine/include/transport/efa_transport/efa_endpoint.h new file mode 100644 index 00000000..dd810e7a --- /dev/null +++ b/mooncake-transfer-engine/include/transport/efa_transport/efa_endpoint.h @@ -0,0 +1,164 @@ +// Copyright 2024 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef EFA_ENDPOINT_H +#define EFA_ENDPOINT_H + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "common.h" +#include "efa_context.h" +#include "transfer_metadata.h" +#include "transport/transport.h" + +namespace mooncake { + +class EfaContext; + +// Custom context for libfabric operations - stores slice pointer for completion +// handling This struct MUST have fi_context as its first member +struct EfaOpContext { + struct fi_context fi_ctx; // Must be first member + Transport::Slice *slice; // Slice pointer for completion handling + volatile int *wr_depth; // Pointer to endpoint's wr_depth_ for CQ + // completion decrement +}; + +// EfaEndPoint represents a libfabric endpoint for EFA communication. +// Unlike RDMA QPs, EFA uses RDM (Reliable Datagram) endpoints with +// an address vector for peer addressing. +class EfaEndPoint { + public: + using HandShakeDesc = TransferMetadata::HandShakeDesc; + + enum Status { INITIALIZING, UNCONNECTED, CONNECTED }; + + EfaEndPoint(EfaContext &context); + ~EfaEndPoint(); + + // Construct endpoint with specified completion queue + int construct(struct fid_cq *cq, size_t num_qp_list = 1, size_t max_sge = 4, + size_t max_wr = 256, size_t max_inline = 64); + + private: + int deconstruct(); + + public: + void setPeerNicPath(const std::string &peer_nic_path); + + int setupConnectionsByActive(); + + int setupConnectionsByActive(const std::string &peer_nic_path) { + setPeerNicPath(peer_nic_path); + return setupConnectionsByActive(); + } + + int setupConnectionsByPassive(const HandShakeDesc &peer_desc, + HandShakeDesc &local_desc); + + bool hasOutstandingSlice() const; + + bool active() const { return active_; } + + void set_active(bool flag) { + RWSpinlock::WriteGuard guard(lock_); + active_ = flag; + if (!flag) inactive_time_ = getCurrentTimeInNano(); + } + + double inactiveTime() { + if (active_) return 0.0; + return (getCurrentTimeInNano() - inactive_time_) / 1000000000.0; + } + + public: + bool connected() const { + return status_.load(std::memory_order_relaxed) == CONNECTED; + } + + void disconnect(); + int destroyQP(); + + private: + void disconnectUnlocked(); + + public: + const std::string toString() const; + + // Submit RDMA write/read operations via libfabric + int submitPostSend(std::vector &slice_list, + std::vector &failed_slice_list); + + // Get the number of endpoints (always 1 for EFA RDM) + size_t getQPNumber() const { return 1; } + + // Get local endpoint address for handshake + std::string getLocalAddr() const; + + // Get peer's fi_addr + fi_addr_t getPeerFiAddr() const { return peer_fi_addr_; } + + EfaContext &context() { return context_; } + + private: + // Setup connection using peer's address from handshake + int doSetupConnection(const std::string &peer_addr, + std::string *reply_msg = nullptr); + + // Insert peer address into address vector + int insertPeerAddr(const std::string &peer_addr); + + private: + EfaContext &context_; + std::atomic status_; + + RWSpinlock lock_; + std::string peer_nic_path_; + + // Libfabric endpoint + struct fid_ep *ep_; + struct fid_cq *tx_cq_; + struct fid_cq *rx_cq_; + fi_addr_t peer_fi_addr_; // Peer's address in the AV + + // Local endpoint address (for handshake) + std::vector local_addr_; + size_t local_addr_len_; + + volatile int wr_depth_; + int max_wr_depth_; + volatile int *cq_outstanding_; + + // Spinlock to serialize fi_write calls on this endpoint. + // libfabric RDM endpoints are not thread-safe by default. + std::atomic_flag post_lock_ = ATOMIC_FLAG_INIT; + + volatile bool active_; + volatile uint64_t inactive_time_; +}; + +} // namespace mooncake + +#endif // EFA_ENDPOINT_H diff --git a/mooncake-transfer-engine/include/transport/efa_transport/efa_transport.h b/mooncake-transfer-engine/include/transport/efa_transport/efa_transport.h new file mode 100644 index 00000000..50294237 --- /dev/null +++ b/mooncake-transfer-engine/include/transport/efa_transport/efa_transport.h @@ -0,0 +1,149 @@ +// Copyright 2024 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef EFA_TRANSPORT_H_ +#define EFA_TRANSPORT_H_ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "topology.h" +#include "transfer_metadata.h" +#include "transport/transport.h" + +namespace mooncake { + +class EfaContext; +class EfaEndPoint; +class TransferMetadata; + +class EfaTransport : public Transport { + friend class EfaContext; + friend class EfaEndPoint; + + public: + using BufferDesc = TransferMetadata::BufferDesc; + using SegmentDesc = TransferMetadata::SegmentDesc; + using HandShakeDesc = TransferMetadata::HandShakeDesc; + + public: + EfaTransport(); + + ~EfaTransport(); + + int install(std::string &local_server_name, + std::shared_ptr meta, + std::shared_ptr topo) override; + + const char *getName() const override { return "efa"; } + + int registerLocalMemory(void *addr, size_t length, + const std::string &location, bool remote_accessible, + bool update_metadata) override; + + int unregisterLocalMemory(void *addr, bool update_metadata = true) override; + + int registerLocalMemoryBatch(const std::vector &buffer_list, + const std::string &location) override; + + int unregisterLocalMemoryBatch( + const std::vector &addr_list) override; + + private: + // Internal version with force_sequential option to avoid nested parallelism + int registerLocalMemoryInternal(void *addr, size_t length, + const std::string &location, + bool remote_accessible, + bool update_metadata, + bool force_sequential); + + int unregisterLocalMemoryInternal(void *addr, bool update_metadata, + bool force_sequential); + + // TRANSFER + + Status submitTransfer(BatchID batch_id, + const std::vector &entries) override; + + Status submitTransferTask( + const std::vector &task_list) override; + + Status getTransferStatus(BatchID batch_id, + std::vector &status); + + Status getTransferStatus(BatchID batch_id, size_t task_id, + TransferStatus &status) override; + + SegmentID getSegmentID(const std::string &segment_name); + + private: + int allocateLocalSegmentID(); + + int preTouchMemory(void *addr, size_t length); + + public: + int onSetupEfaConnections(const HandShakeDesc &peer_desc, + HandShakeDesc &local_desc); + + int sendHandshake(const std::string &peer_server_name, + const HandShakeDesc &local_desc, + HandShakeDesc &peer_desc) { + return metadata_->sendHandshake(peer_server_name, local_desc, + peer_desc); + } + + const std::string &local_server_name() const { return local_server_name_; } + + std::shared_ptr meta() { return metadata_; } + + private: + int initializeEfaResources(); + + int startHandshakeDaemon(std::string &local_server_name); + + public: + static int selectDevice(SegmentDesc *desc, uint64_t offset, size_t length, + int &buffer_id, int &device_id, int retry_cnt = 0); + static int selectDevice(SegmentDesc *desc, uint64_t offset, size_t length, + std::string_view hint, int &buffer_id, + int &device_id, int retry_cnt = 0); + + private: + // Start/stop CQ polling worker threads + void startWorkerThreads(); + void stopWorkerThreads(); + void workerThreadFunc(int thread_id); + + private: + std::vector> context_list_; + std::shared_ptr local_topology_; + + // CQ polling worker threads + std::atomic worker_running_{false}; + std::vector worker_threads_; +}; + +} // namespace mooncake + +#endif // EFA_TRANSPORT_H_ diff --git a/mooncake-transfer-engine/src/CMakeLists.txt b/mooncake-transfer-engine/src/CMakeLists.txt index 627c9d56..f05401fc 100644 --- a/mooncake-transfer-engine/src/CMakeLists.txt +++ b/mooncake-transfer-engine/src/CMakeLists.txt @@ -2,6 +2,8 @@ file(GLOB ENGINE_SOURCES "*.cpp") add_subdirectory(common) add_subdirectory(transport) +# EFA library path is set globally via common.cmake (LIBFABRIC_LIB_DIR) + set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) if(USE_HIP) @@ -105,3 +107,8 @@ if(USE_INTRA_NVLINK) message(STATUS "Enabled USE_INTRA_NVLINK support") target_compile_definitions(transfer_engine PUBLIC USE_INTRA_NVLINK) endif() + +if(USE_EFA) + message(STATUS "Enabled USE_EFA (AWS Elastic Fabric Adapter) support") + target_link_libraries(transfer_engine PUBLIC fabric efa_transport) +endif() diff --git a/mooncake-transfer-engine/src/multi_transport.cpp b/mooncake-transfer-engine/src/multi_transport.cpp index c9425192..f1445ded 100644 --- a/mooncake-transfer-engine/src/multi_transport.cpp +++ b/mooncake-transfer-engine/src/multi_transport.cpp @@ -52,6 +52,9 @@ #ifdef USE_UBSHMEM #include "transport/ascend_transport/ubshmem_transport/ubshmem_transport.h" #endif +#ifdef USE_EFA +#include "transport/efa_transport/efa_transport.h" +#endif #include @@ -283,6 +286,11 @@ Transport *MultiTransport::installTransport(const std::string &proto, transport = new UBShmemTransport(); } #endif +#ifdef USE_EFA + else if (std::string(proto) == "efa") { + transport = new EfaTransport(); + } +#endif if (!transport) { LOG(ERROR) << "Unsupported transport " << proto diff --git a/mooncake-transfer-engine/src/transfer_metadata.cpp b/mooncake-transfer-engine/src/transfer_metadata.cpp index 47ed804d..c6015006 100644 --- a/mooncake-transfer-engine/src/transfer_metadata.cpp +++ b/mooncake-transfer-engine/src/transfer_metadata.cpp @@ -62,6 +62,9 @@ struct TransferHandshakeUtil { for (const auto &qp : desc.qp_num) qpNums.append(qp); root["qp_num"] = qpNums; root["reply_msg"] = desc.reply_msg; +#ifdef USE_EFA + root["efa_addr"] = desc.efa_addr; // EFA endpoint address +#endif return root; } @@ -74,6 +77,9 @@ struct TransferHandshakeUtil { for (const auto &qp : root["qp_num"]) desc.qp_num.push_back(qp.asUInt()); desc.reply_msg = root["reply_msg"].asString(); +#ifdef USE_EFA + desc.efa_addr = root["efa_addr"].asString(); // EFA endpoint address +#endif return 0; } }; @@ -164,7 +170,8 @@ int TransferMetadata::encodeSegmentDesc(const SegmentDesc &desc, segmentJSON["timestamp"] = getCurrentDateTime(); if (segmentJSON["protocol"] == "rdma" || - segmentJSON["protocol"] == "barex") { + segmentJSON["protocol"] == "barex" || + segmentJSON["protocol"] == "efa") { Json::Value devicesJSON(Json::arrayValue); for (const auto &device : desc.devices) { Json::Value deviceJSON; @@ -325,7 +332,8 @@ TransferMetadata::decodeSegmentDesc(Json::Value &segmentJSON, if (segmentJSON.isMember("timestamp")) desc->timestamp = segmentJSON["timestamp"].asString(); - if (desc->protocol == "rdma" || desc->protocol == "barex") { + if (desc->protocol == "rdma" || desc->protocol == "barex" || + desc->protocol == "efa") { for (const auto &deviceJSON : segmentJSON["devices"]) { DeviceDesc device; device.name = deviceJSON["name"].asString(); diff --git a/mooncake-transfer-engine/src/transport/CMakeLists.txt b/mooncake-transfer-engine/src/transport/CMakeLists.txt index 91e9bb14..1b72a721 100644 --- a/mooncake-transfer-engine/src/transport/CMakeLists.txt +++ b/mooncake-transfer-engine/src/transport/CMakeLists.txt @@ -56,3 +56,9 @@ if (USE_UBSHMEM) add_subdirectory(ascend_transport) target_sources(transport PUBLIC $) endif() + +if (USE_EFA) + add_subdirectory(efa_transport) + target_sources(transport PUBLIC $) + target_link_libraries(transport PRIVATE fabric) +endif() diff --git a/mooncake-transfer-engine/src/transport/efa_transport/CMakeLists.txt b/mooncake-transfer-engine/src/transport/efa_transport/CMakeLists.txt new file mode 100644 index 00000000..ff32f6eb --- /dev/null +++ b/mooncake-transfer-engine/src/transport/efa_transport/CMakeLists.txt @@ -0,0 +1,14 @@ +file(GLOB EFA_SOURCES "*.cpp") +add_library(efa_transport OBJECT ${EFA_SOURCES}) + +# Link against libfabric (fabric) instead of ibverbs for AWS EFA +target_link_libraries(efa_transport PRIVATE fabric glog::glog) + +target_include_directories(efa_transport PRIVATE + ${CMAKE_SOURCE_DIR}/mooncake-transfer-engine/include + ${CMAKE_SOURCE_DIR}/mooncake-transfer-engine/include/transport/efa_transport + ${CMAKE_SOURCE_DIR}/mooncake-common/include + ${LIBFABRIC_INCLUDE_DIR} +) + +# libfabric library path is set globally via common.cmake (LIBFABRIC_LIB_DIR) diff --git a/mooncake-transfer-engine/src/transport/efa_transport/efa_context.cpp b/mooncake-transfer-engine/src/transport/efa_transport/efa_context.cpp new file mode 100644 index 00000000..20dc3f1e --- /dev/null +++ b/mooncake-transfer-engine/src/transport/efa_transport/efa_context.cpp @@ -0,0 +1,562 @@ +// Copyright 2024 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "transport/efa_transport/efa_context.h" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "config.h" +#include "transport/efa_transport/efa_endpoint.h" +#include "transport/efa_transport/efa_transport.h" +#include "transport/transport.h" + +namespace mooncake { + +// EfaEndpointStore implementation +std::shared_ptr EfaEndpointStore::get( + const std::string &peer_nic_path) { + RWSpinlock::ReadGuard guard(lock_); + auto it = endpoints_.find(peer_nic_path); + if (it != endpoints_.end()) { + return it->second; + } + return nullptr; +} + +std::shared_ptr EfaEndpointStore::getOrInsert( + const std::string &peer_nic_path, std::shared_ptr new_ep) { + RWSpinlock::WriteGuard guard(lock_); + auto it = endpoints_.find(peer_nic_path); + if (it != endpoints_.end()) { + return it->second; // Another thread already created it + } + endpoints_[peer_nic_path] = new_ep; + return new_ep; +} + +void EfaEndpointStore::add(const std::string &peer_nic_path, + std::shared_ptr endpoint) { + RWSpinlock::WriteGuard guard(lock_); + endpoints_[peer_nic_path] = endpoint; +} + +void EfaEndpointStore::remove(const std::string &peer_nic_path) { + RWSpinlock::WriteGuard guard(lock_); + endpoints_.erase(peer_nic_path); +} + +int EfaEndpointStore::disconnectAll() { + RWSpinlock::WriteGuard guard(lock_); + for (auto &entry : endpoints_) { + if (entry.second) { + entry.second->disconnect(); + } + } + return 0; +} + +size_t EfaEndpointStore::size() const { + RWSpinlock::ReadGuard guard(lock_); + return endpoints_.size(); +} + +// EfaContext implementation +EfaContext::EfaContext(EfaTransport &engine, const std::string &device_name) + : engine_(engine), + device_name_(device_name), + fi_info_(nullptr), + hints_(nullptr), + fabric_(nullptr), + domain_(nullptr), + av_(nullptr), + active_(true) {} + +EfaContext::~EfaContext() { + if (fabric_) deconstruct(); +} + +int EfaContext::construct(size_t num_cq_list, size_t num_comp_channels, + uint8_t port, int gid_index, size_t max_cqe, + int max_endpoints) { + endpoint_store_ = std::make_shared(); + + // Setup hints for EFA provider + hints_ = fi_allocinfo(); + if (!hints_) { + LOG(ERROR) << "Failed to allocate fi_info hints"; + return ERR_CONTEXT; + } + + hints_->caps = + FI_MSG | FI_RMA | FI_READ | FI_WRITE | FI_REMOTE_READ | FI_REMOTE_WRITE; + hints_->mode = FI_CONTEXT; + hints_->ep_attr->type = FI_EP_RDM; // EFA uses RDM endpoints + hints_->fabric_attr->prov_name = strdup("efa"); + + // Specify the domain (device) name - append "-rdm" for RDM endpoint + std::string domain_name = device_name_ + "-rdm"; + hints_->domain_attr->name = strdup(domain_name.c_str()); + hints_->domain_attr->mr_mode = + FI_MR_LOCAL | FI_MR_VIRT_ADDR | FI_MR_ALLOCATED | FI_MR_PROV_KEY; + hints_->domain_attr->threading = FI_THREAD_SAFE; + + // Get fabric info + int ret = + fi_getinfo(FI_VERSION(1, 14), nullptr, nullptr, 0, hints_, &fi_info_); + if (ret) { + LOG(ERROR) << "fi_getinfo failed for device " << device_name_ << ": " + << fi_strerror(-ret); + fi_freeinfo(hints_); + hints_ = nullptr; + return ERR_CONTEXT; + } + + // Open fabric + ret = fi_fabric(fi_info_->fabric_attr, &fabric_, nullptr); + if (ret) { + LOG(ERROR) << "fi_fabric failed: " << fi_strerror(-ret); + fi_freeinfo(fi_info_); + fi_freeinfo(hints_); + fi_info_ = nullptr; + hints_ = nullptr; + return ERR_CONTEXT; + } + + // Open domain + ret = fi_domain(fabric_, fi_info_, &domain_, nullptr); + if (ret) { + LOG(ERROR) << "fi_domain failed: " << fi_strerror(-ret); + fi_close(&fabric_->fid); + fi_freeinfo(fi_info_); + fi_freeinfo(hints_); + fabric_ = nullptr; + fi_info_ = nullptr; + hints_ = nullptr; + return ERR_CONTEXT; + } + + // Create address vector + struct fi_av_attr av_attr = {}; + av_attr.type = FI_AV_TABLE; + av_attr.count = max_endpoints; + + ret = fi_av_open(domain_, &av_attr, &av_, nullptr); + if (ret) { + LOG(ERROR) << "fi_av_open failed: " << fi_strerror(-ret); + fi_close(&domain_->fid); + fi_close(&fabric_->fid); + fi_freeinfo(fi_info_); + fi_freeinfo(hints_); + domain_ = nullptr; + fabric_ = nullptr; + fi_info_ = nullptr; + hints_ = nullptr; + return ERR_CONTEXT; + } + + // Create completion queues + cq_list_.resize(num_cq_list); + for (size_t i = 0; i < num_cq_list; ++i) { + auto cq = std::make_shared(); + + struct fi_cq_attr cq_attr = {}; + cq_attr.size = max_cqe; + cq_attr.format = FI_CQ_FORMAT_DATA; + cq_attr.wait_obj = FI_WAIT_NONE; + + ret = fi_cq_open(domain_, &cq_attr, &cq->cq, nullptr); + if (ret) { + LOG(ERROR) << "fi_cq_open failed: " << fi_strerror(-ret); + return ERR_CONTEXT; + } + cq_list_[i] = cq; + } + + LOG(INFO) << "EFA device (libfabric): " << device_name_ + << ", domain: " << fi_info_->domain_attr->name + << ", provider: " << fi_info_->fabric_attr->prov_name; + + return 0; +} + +int EfaContext::deconstruct() { + // Destroy all endpoints before closing domain/fabric/AV. + // Endpoints hold fi_ep handles that reference the domain, so they must + // be closed first. + endpoint_store_.reset(); + + { + RWSpinlock::WriteGuard guard(mr_lock_); + for (auto &entry : mr_map_) { + if (entry.second.mr) { + fi_close(&entry.second.mr->fid); + } + } + mr_map_.clear(); + } + + for (auto &cq : cq_list_) { + if (cq && cq->cq) { + fi_close(&cq->cq->fid); + cq->cq = nullptr; + } + } + cq_list_.clear(); + + if (av_) { + fi_close(&av_->fid); + av_ = nullptr; + } + + if (domain_) { + fi_close(&domain_->fid); + domain_ = nullptr; + } + + if (fabric_) { + fi_close(&fabric_->fid); + fabric_ = nullptr; + } + + if (fi_info_) { + fi_freeinfo(fi_info_); + fi_info_ = nullptr; + } + + if (hints_) { + fi_freeinfo(hints_); + hints_ = nullptr; + } + + return 0; +} + +int EfaContext::registerMemoryRegionInternal(void *addr, size_t length, + int access, + EfaMemoryRegionMeta &mrMeta) { + if (length > (size_t)globalConfig().max_mr_size) { + PLOG(WARNING) << "The buffer length exceeds device max_mr_size, " + << "shrink it to " << globalConfig().max_mr_size; + length = (size_t)globalConfig().max_mr_size; + } + + mrMeta.addr = addr; + mrMeta.length = length; + + // Convert access flags to libfabric flags + uint64_t fi_access = 0; + if (access & FI_READ) fi_access |= FI_READ; + if (access & FI_WRITE) fi_access |= FI_WRITE; + if (access & FI_REMOTE_READ) fi_access |= FI_REMOTE_READ; + if (access & FI_REMOTE_WRITE) fi_access |= FI_REMOTE_WRITE; + + // For EFA, we need local read/write and remote read/write + fi_access = FI_READ | FI_WRITE | FI_REMOTE_READ | FI_REMOTE_WRITE; + + int ret = fi_mr_reg(domain_, addr, length, fi_access, 0, 0, 0, &mrMeta.mr, + nullptr); + if (ret) { + LOG(ERROR) << "fi_mr_reg failed for " << addr << ": " + << fi_strerror(-ret); + return ERR_CONTEXT; + } + + mrMeta.key = fi_mr_key(mrMeta.mr); + + return 0; +} + +int EfaContext::registerMemoryRegion(void *addr, size_t length, int access) { + EfaMemoryRegionMeta mrMeta; + int ret = registerMemoryRegionInternal(addr, length, access, mrMeta); + if (ret != 0) { + return ret; + } + RWSpinlock::WriteGuard guard(mr_lock_); + mr_map_[(uint64_t)addr] = mrMeta; + return 0; +} + +int EfaContext::unregisterMemoryRegion(void *addr) { + RWSpinlock::WriteGuard guard(mr_lock_); + auto it = mr_map_.find((uint64_t)addr); + if (it != mr_map_.end()) { + if (it->second.mr) { + int ret = fi_close(&it->second.mr->fid); + if (ret) { + LOG(ERROR) << "Failed to unregister memory " << addr << ": " + << fi_strerror(-ret); + return ERR_CONTEXT; + } + } + mr_map_.erase(it); + } + return 0; +} + +int EfaContext::preTouchMemory(void *addr, size_t length) { + volatile char *ptr = (volatile char *)addr; + for (size_t i = 0; i < length; i += 4096) { + ptr[i] = ptr[i]; + } + return 0; +} + +uint64_t EfaContext::rkey(void *addr) { + RWSpinlock::ReadGuard guard(mr_lock_); + auto it = mr_map_.find((uint64_t)addr); + if (it != mr_map_.end() && it->second.mr) { + return it->second.key; + } + return 0; +} + +uint64_t EfaContext::lkey(void *addr) { + RWSpinlock::ReadGuard guard(mr_lock_); + auto it = mr_map_.find((uint64_t)addr); + if (it != mr_map_.end() && it->second.mr) { + return fi_mr_key(it->second.mr); + } + return 0; +} + +void *EfaContext::mrDesc(void *addr) { + RWSpinlock::ReadGuard guard(mr_lock_); + // Find the MR that contains this address + for (auto &entry : mr_map_) { + if ((uint64_t)addr >= entry.first && + (uint64_t)addr < entry.first + entry.second.length) { + if (entry.second.mr) { + return fi_mr_desc(entry.second.mr); + } + } + } + return nullptr; +} + +std::shared_ptr EfaContext::endpoint( + const std::string &peer_nic_path) { + if (!endpoint_store_) return nullptr; + + // Fast path: endpoint already exists + auto ep = endpoint_store_->get(peer_nic_path); + if (ep) return ep; + + // Slow path: create new endpoint, then atomically insert (or get existing + // if another thread raced us). getOrInsert prevents duplicate endpoints + // and duplicate AV entries for the same peer. + auto new_endpoint = std::make_shared(*this); + if (!cq_list_.empty() && cq_list_[0]) { + int ret = new_endpoint->construct(cq_list_[0]->cq); + if (ret != 0) { + LOG(ERROR) << "Failed to construct EFA endpoint"; + return nullptr; + } + } + new_endpoint->setPeerNicPath(peer_nic_path); + ep = endpoint_store_->getOrInsert(peer_nic_path, new_endpoint); + // If another thread won the race, new_endpoint is discarded (RAII cleanup) + return ep; +} + +int EfaContext::deleteEndpoint(const std::string &peer_nic_path) { + if (endpoint_store_) { + endpoint_store_->remove(peer_nic_path); + } + return 0; +} + +int EfaContext::disconnectAllEndpoints() { + if (endpoint_store_) { + return endpoint_store_->disconnectAll(); + } + return 0; +} + +size_t EfaContext::getTotalQPNumber() const { + return endpoint_store_ ? endpoint_store_->size() : 0; +} + +std::string EfaContext::nicPath() const { + return engine_.local_server_name() + "@" + device_name_; +} + +std::string EfaContext::localAddr() const { + // Return a hex string representation of the local address info + if (!fi_info_ || !fi_info_->src_addr) { + return ""; + } + + std::ostringstream oss; + const uint8_t *addr = static_cast(fi_info_->src_addr); + for (size_t i = 0; i < fi_info_->src_addrlen; ++i) { + oss << std::hex << std::setw(2) << std::setfill('0') << (int)addr[i]; + } + return oss.str(); +} + +int EfaContext::submitPostSend( + const std::vector &slice_list) { + // Route slices to appropriate endpoints for sending + // Group slices by peer NIC path + std::unordered_map> + slices_by_peer; + std::vector failed_slices; + + for (auto *slice : slice_list) { + if (!slice) continue; + + // Get peer segment descriptor to find dest_rkey and peer device info + auto peer_segment_desc = + engine_.meta()->getSegmentDescByID(slice->target_id); + if (!peer_segment_desc) { + LOG(ERROR) << "Cannot get segment descriptor for target " + << slice->target_id; + slice->markFailed(); + continue; + } + + // Find the buffer and device for this destination address + int buffer_id = -1, device_id = -1; + if (EfaTransport::selectDevice(peer_segment_desc.get(), + slice->rdma.dest_addr, slice->length, + buffer_id, device_id)) { + LOG(ERROR) << "Cannot select device for dest_addr " + << (void *)slice->rdma.dest_addr; + slice->markFailed(); + continue; + } + + // Set the remote key from the peer's registered memory region + slice->rdma.dest_rkey = + peer_segment_desc->buffers[buffer_id].rkey[device_id]; + + // Construct peer NIC path: "server_name@device_name" + std::string peer_nic_path = peer_segment_desc->name + "@" + + peer_segment_desc->devices[device_id].name; + slice->peer_nic_path = peer_nic_path; + + slices_by_peer[peer_nic_path].push_back(slice); + } + + // Now send to each peer endpoint + for (auto &entry : slices_by_peer) { + const std::string &peer_nic_path = entry.first; + auto &peer_slices = entry.second; + + // Get or create endpoint for this peer + auto ep = endpoint(peer_nic_path); + if (!ep) { + LOG(ERROR) << "Cannot create endpoint for peer " << peer_nic_path; + for (auto *slice : peer_slices) { + slice->markFailed(); + } + continue; + } + + // Submit to endpoint + std::vector failed_slice_list; + ep->submitPostSend(peer_slices, failed_slice_list); + + // Handle any slices that failed to post + for (auto *slice : failed_slice_list) { + slice->markFailed(); + } + } + + return 0; +} + +int EfaContext::pollCq(int max_entries, int cq_index) { + if (cq_index < 0 || (size_t)cq_index >= cq_list_.size()) { + return 0; + } + + struct fid_cq *cq = cq_list_[cq_index]->cq; + if (!cq) return 0; + + // Use fi_cq_data format for completions + struct fi_cq_data_entry entries[64]; + int to_poll = std::min(max_entries, 64); + + ssize_t ret = fi_cq_read(cq, entries, to_poll); + + if (ret > 0) { + // Process completions outside the lock (markSuccess / delete are safe) + std::unordered_map wr_depth_set; + for (ssize_t i = 0; i < ret; i++) { + EfaOpContext *op_ctx = + reinterpret_cast(entries[i].op_context); + if (op_ctx && op_ctx->slice) { + op_ctx->slice->markSuccess(); + if (op_ctx->wr_depth) { + wr_depth_set[op_ctx->wr_depth]++; + } + delete op_ctx; + } + } + for (auto &entry : wr_depth_set) { + __sync_fetch_and_sub(entry.first, entry.second); + } + __sync_fetch_and_sub(&cq_list_[cq_index]->outstanding, + static_cast(ret)); + return static_cast(ret); + } else if (ret == -FI_EAGAIN) { + return 0; + } else if (ret < 0) { + // CQ error - drain all queued error entries under the domain lock + int err_count = 0; + struct fi_cq_err_entry err_entry; + std::unordered_map wr_depth_set; + + while ((ret = fi_cq_readerr(cq, &err_entry, 0)) > 0) { + EfaOpContext *op_ctx = + reinterpret_cast(err_entry.op_context); + if (op_ctx && op_ctx->slice) { + LOG(ERROR) << "EFA CQ error: " + << fi_cq_strerror(cq, err_entry.prov_errno, + err_entry.err_data, nullptr, 0) + << " for slice at " << op_ctx->slice->source_addr; + op_ctx->slice->markFailed(); + if (op_ctx->wr_depth) { + wr_depth_set[op_ctx->wr_depth]++; + } + delete op_ctx; + } + err_count++; + } + + for (auto &entry : wr_depth_set) { + __sync_fetch_and_sub(entry.first, entry.second); + } + if (err_count > 0) { + __sync_fetch_and_sub(&cq_list_[cq_index]->outstanding, err_count); + } + return err_count; + } + + return 0; +} + +} // namespace mooncake diff --git a/mooncake-transfer-engine/src/transport/efa_transport/efa_endpoint.cpp b/mooncake-transfer-engine/src/transport/efa_transport/efa_endpoint.cpp new file mode 100644 index 00000000..e87fcb58 --- /dev/null +++ b/mooncake-transfer-engine/src/transport/efa_transport/efa_endpoint.cpp @@ -0,0 +1,441 @@ +// Copyright 2024 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "transport/efa_transport/efa_endpoint.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "config.h" + +namespace mooncake { + +EfaEndPoint::EfaEndPoint(EfaContext &context) + : context_(context), + status_(INITIALIZING), + ep_(nullptr), + tx_cq_(nullptr), + rx_cq_(nullptr), + peer_fi_addr_(FI_ADDR_UNSPEC), + local_addr_len_(0), + wr_depth_(0), + max_wr_depth_(0), + cq_outstanding_(nullptr), + active_(true), + inactive_time_(0) {} + +EfaEndPoint::~EfaEndPoint() { + if (ep_) deconstruct(); +} + +int EfaEndPoint::construct(struct fid_cq *cq, size_t num_qp_list, + size_t max_sge, size_t max_wr, size_t max_inline) { + if (status_.load(std::memory_order_relaxed) != INITIALIZING) { + LOG(ERROR) << "EFA Endpoint has already been constructed"; + return ERR_ENDPOINT; + } + + tx_cq_ = cq; + rx_cq_ = cq; // Use same CQ for TX and RX + max_wr_depth_ = max_wr; + cq_outstanding_ = context_.cqOutstandingCount(0); + + // Create endpoint + int ret = fi_endpoint(context_.domain(), context_.info(), &ep_, nullptr); + if (ret) { + LOG(ERROR) << "fi_endpoint failed: " << fi_strerror(-ret); + return ERR_ENDPOINT; + } + + // Bind endpoint to AV + ret = fi_ep_bind(ep_, &context_.av()->fid, 0); + if (ret) { + LOG(ERROR) << "fi_ep_bind (av) failed: " << fi_strerror(-ret); + fi_close(&ep_->fid); + ep_ = nullptr; + return ERR_ENDPOINT; + } + + // Bind endpoint to TX CQ + ret = fi_ep_bind(ep_, &tx_cq_->fid, FI_TRANSMIT); + if (ret) { + LOG(ERROR) << "fi_ep_bind (tx_cq) failed: " << fi_strerror(-ret); + fi_close(&ep_->fid); + ep_ = nullptr; + return ERR_ENDPOINT; + } + + // Bind endpoint to RX CQ + ret = fi_ep_bind(ep_, &rx_cq_->fid, FI_RECV); + if (ret) { + LOG(ERROR) << "fi_ep_bind (rx_cq) failed: " << fi_strerror(-ret); + fi_close(&ep_->fid); + ep_ = nullptr; + return ERR_ENDPOINT; + } + + // Enable endpoint + ret = fi_enable(ep_); + if (ret) { + LOG(ERROR) << "fi_enable failed: " << fi_strerror(-ret); + fi_close(&ep_->fid); + ep_ = nullptr; + return ERR_ENDPOINT; + } + + // Get local endpoint address + local_addr_len_ = 64; // EFA addresses are typically 32 bytes + local_addr_.resize(local_addr_len_); + ret = fi_getname(&ep_->fid, local_addr_.data(), &local_addr_len_); + if (ret) { + LOG(ERROR) << "fi_getname failed: " << fi_strerror(-ret); + fi_close(&ep_->fid); + ep_ = nullptr; + return ERR_ENDPOINT; + } + local_addr_.resize(local_addr_len_); + + status_.store(UNCONNECTED, std::memory_order_relaxed); + return 0; +} + +int EfaEndPoint::deconstruct() { + if (ep_) { + fi_close(&ep_->fid); + ep_ = nullptr; + } + return 0; +} + +int EfaEndPoint::destroyQP() { return deconstruct(); } + +void EfaEndPoint::setPeerNicPath(const std::string &peer_nic_path) { + RWSpinlock::WriteGuard guard(lock_); + if (connected()) { + LOG(WARNING) << "Previous EFA connection will be discarded"; + disconnectUnlocked(); + } + peer_nic_path_ = peer_nic_path; +} + +std::string EfaEndPoint::getLocalAddr() const { + std::ostringstream oss; + for (size_t i = 0; i < local_addr_.size(); ++i) { + oss << std::hex << std::setw(2) << std::setfill('0') + << (int)local_addr_[i]; + } + return oss.str(); +} + +int EfaEndPoint::insertPeerAddr(const std::string &peer_addr) { + // Convert hex string to binary address + std::vector addr_bin; + addr_bin.reserve(peer_addr.size() / 2); + + for (size_t i = 0; i < peer_addr.size(); i += 2) { + std::string byte_str = peer_addr.substr(i, 2); + uint8_t byte = (uint8_t)strtol(byte_str.c_str(), nullptr, 16); + addr_bin.push_back(byte); + } + + // Insert into address vector + int ret = fi_av_insert(context_.av(), addr_bin.data(), 1, &peer_fi_addr_, 0, + nullptr); + if (ret != 1) { + LOG(ERROR) << "fi_av_insert failed: " << fi_strerror(-ret); + return ERR_ENDPOINT; + } + + return 0; +} + +int EfaEndPoint::setupConnectionsByActive() { + RWSpinlock::WriteGuard guard(lock_); + if (connected()) { + LOG(INFO) << "EFA Connection has been established"; + return 0; + } + + // Loopback mode + if (context_.nicPath() == peer_nic_path_) { + // For loopback, insert our own address + int ret = insertPeerAddr(getLocalAddr()); + if (ret != 0) { + return ret; + } + status_.store(CONNECTED, std::memory_order_release); + LOG(INFO) << "EFA loopback connection established: " << toString(); + return 0; + } + + // Exchange addresses via handshake + TransferMetadata::HandShakeDesc local_desc, peer_desc; + local_desc.local_nic_path = context_.nicPath(); + local_desc.peer_nic_path = peer_nic_path_; + // Store our EFA endpoint address in efa_addr field (hex encoded) + local_desc.efa_addr = getLocalAddr(); + + auto peer_server_name = getServerNameFromNicPath(peer_nic_path_); + auto peer_nic_name = getNicNameFromNicPath(peer_nic_path_); + if (peer_server_name.empty() || peer_nic_name.empty()) { + LOG(ERROR) << "Parse peer EFA nic path failed: " << peer_nic_path_; + return ERR_INVALID_ARGUMENT; + } + + int rc = context_.engine().sendHandshake(peer_server_name, local_desc, + peer_desc); + if (rc) return rc; + + if (peer_desc.efa_addr.empty()) { + LOG(ERROR) << "Peer did not provide EFA address in handshake"; + return ERR_REJECT_HANDSHAKE; + } + + // Insert peer's address into our AV + rc = insertPeerAddr(peer_desc.efa_addr); + if (rc != 0) { + return rc; + } + + status_.store(CONNECTED, std::memory_order_release); + VLOG(1) << "EFA connection established: " << toString() + << " peer_fi_addr=" << peer_fi_addr_; + return 0; +} + +int EfaEndPoint::setupConnectionsByPassive(const HandShakeDesc &peer_desc, + HandShakeDesc &local_desc) { + RWSpinlock::WriteGuard guard(lock_); + if (connected()) { + LOG(WARNING) << "Re-establish EFA connection: " << toString(); + disconnectUnlocked(); + } + + if (peer_desc.peer_nic_path != context_.nicPath() || + peer_desc.local_nic_path != peer_nic_path_) { + local_desc.reply_msg = "EFA nic path inconsistency"; + LOG(ERROR) << "Invalid argument: peer EFA nic path inconsistency" + << " peer_nic_path=" << peer_desc.peer_nic_path + << " context_.nicPath()=" << context_.nicPath() + << " local_nic_path=" << peer_desc.local_nic_path + << " peer_nic_path_=" << peer_nic_path_; + return ERR_REJECT_HANDSHAKE; + } + + // Insert peer's address from handshake + if (peer_desc.efa_addr.empty()) { + local_desc.reply_msg = "No EFA address provided"; + LOG(ERROR) << "Peer did not provide EFA address"; + return ERR_REJECT_HANDSHAKE; + } + + int ret = insertPeerAddr(peer_desc.efa_addr); + if (ret != 0) { + local_desc.reply_msg = "Failed to insert peer address"; + return ret; + } + + // Provide our address to peer (using efa_addr field, not reply_msg) + local_desc.local_nic_path = context_.nicPath(); + local_desc.peer_nic_path = peer_nic_path_; + local_desc.efa_addr = getLocalAddr(); + // reply_msg should be empty on success + + status_.store(CONNECTED, std::memory_order_release); + VLOG(1) << "EFA connection established (passive): " << toString(); + return 0; +} + +void EfaEndPoint::disconnect() { + RWSpinlock::WriteGuard guard(lock_); + disconnectUnlocked(); +} + +void EfaEndPoint::disconnectUnlocked() { + // For EFA RDM endpoints, we don't need to reset QP state + // Just remove peer from AV if needed and mark as disconnected + peer_fi_addr_ = FI_ADDR_UNSPEC; + status_.store(UNCONNECTED, std::memory_order_release); +} + +const std::string EfaEndPoint::toString() const { + return "EfaEndPoint[" + context_.nicPath() + " <-> " + peer_nic_path_ + "]"; +} + +bool EfaEndPoint::hasOutstandingSlice() const { return wr_depth_ > 0; } + +int EfaEndPoint::doSetupConnection(const std::string &peer_addr, + std::string *reply_msg) { + int ret = insertPeerAddr(peer_addr); + if (ret != 0) { + if (reply_msg) *reply_msg = "Failed to insert peer address into AV"; + return ret; + } + + status_.store(CONNECTED, std::memory_order_release); + return 0; +} + +int EfaEndPoint::submitPostSend( + std::vector &slice_list, + std::vector &failed_slice_list) { + if (!connected()) { + // Try to establish connection first + int ret = setupConnectionsByActive(); + if (ret != 0) { + // Move all slices to failed list + for (auto *slice : slice_list) { + failed_slice_list.push_back(slice); + } + slice_list.clear(); + return ret; + } + } + + // Process slices - using fi_write for RDMA write operations. + // Use atomic reserve-before-post to prevent CQ overflow when multiple + // threads post to endpoints sharing the same CQ. The CQ has a fixed + // capacity (max_cqe); if more completions arrive than it can hold, the + // provider silently drops them and those slices never complete (hang). + const int kMaxBackoffYields = 100000; + const int cq_limit = static_cast(globalConfig().max_cqe); + + for (auto it = slice_list.begin(); it != slice_list.end();) { + // --- Atomically reserve CQ and WR capacity before posting --- + // This eliminates the TOCTOU race where multiple threads pass the + // capacity check simultaneously and collectively overflow the CQ. + int backoff = 0; + bool reserved = false; + while (!reserved) { + // Try to reserve one WR slot + int cur_wr = wr_depth_; + if (cur_wr >= max_wr_depth_) { + if (++backoff > kMaxBackoffYields) goto timeout; + std::this_thread::yield(); + continue; + } + if (!__sync_bool_compare_and_swap(&wr_depth_, cur_wr, cur_wr + 1)) { + continue; // CAS failed, retry immediately + } + // WR slot reserved. Now try to reserve CQ slot. + if (cq_outstanding_) { + int cur_cq = *cq_outstanding_; + while (cur_cq < cq_limit) { + if (__sync_bool_compare_and_swap(cq_outstanding_, cur_cq, + cur_cq + 1)) { + reserved = true; + break; + } + cur_cq = *cq_outstanding_; + } + if (!reserved) { + // CQ full - release WR reservation and back off + __sync_fetch_and_sub(&wr_depth_, 1); + if (++backoff > kMaxBackoffYields) goto timeout; + std::this_thread::yield(); + continue; + } + } else { + reserved = true; + } + } + + { + Transport::Slice *slice = *it; + + // Get memory region descriptor for the local buffer + void *local_desc = context_.mrDesc(slice->source_addr); + if (!local_desc) { + LOG(ERROR) << "No MR descriptor found for address " + << slice->source_addr; + // Release reservations + __sync_fetch_and_sub(&wr_depth_, 1); + if (cq_outstanding_) __sync_fetch_and_sub(cq_outstanding_, 1); + failed_slice_list.push_back(slice); + it = slice_list.erase(it); + continue; + } + + // Allocate operation context to track the slice for completion + // Note: This memory is freed after CQ completion in pollCq + EfaOpContext *op_ctx = new EfaOpContext(); + memset(op_ctx, 0, sizeof(EfaOpContext)); + op_ctx->slice = slice; + op_ctx->wr_depth = &wr_depth_; + + // Serialize fi_write per-endpoint: concurrent fi_write on the + // same RDM endpoint corrupts provider state. Cross-endpoint + // safety is handled by the FI_THREAD_SAFE hint. + while (post_lock_.test_and_set(std::memory_order_acquire)) { + } + ssize_t ret = fi_write(ep_, + (void *)slice->source_addr, // local buffer + slice->length, local_desc, peer_fi_addr_, + slice->rdma.dest_addr, // remote address + slice->rdma.dest_rkey, // remote key + &op_ctx->fi_ctx); // context for completion + post_lock_.clear(std::memory_order_release); + + if (ret == 0) { + // Successfully posted - do NOT mark success here! + // Success is marked only after CQ completion in pollCq. + // WR and CQ reservations are already accounted for. + slice->status = Transport::Slice::PENDING; + it = slice_list.erase(it); + } else if (ret == -FI_EAGAIN) { + // Provider queue full - release reservations and retry + delete op_ctx; + __sync_fetch_and_sub(&wr_depth_, 1); + if (cq_outstanding_) __sync_fetch_and_sub(cq_outstanding_, 1); + std::this_thread::yield(); + // Don't advance iterator - retry the same slice + } else { + // Hard error - release reservations + LOG(ERROR) << "fi_write failed: " << fi_strerror(-ret) + << " (source=" << slice->source_addr + << ", len=" << slice->length + << ", dest=" << (void *)slice->rdma.dest_addr + << ", rkey=" << slice->rdma.dest_rkey << ")"; + delete op_ctx; + __sync_fetch_and_sub(&wr_depth_, 1); + if (cq_outstanding_) __sync_fetch_and_sub(cq_outstanding_, 1); + failed_slice_list.push_back(slice); + it = slice_list.erase(it); + } + } + continue; + + timeout: + LOG(WARNING) << "EFA submitPostSend: timed out waiting for CQ drain" + << " (wr_depth=" << wr_depth_ << ", max=" << max_wr_depth_ + << ", cq_outstanding=" + << (cq_outstanding_ ? *cq_outstanding_ : -1) + << ", max_cqe=" << cq_limit << ")"; + for (; it != slice_list.end(); ++it) { + failed_slice_list.push_back(*it); + } + slice_list.clear(); + return 0; + } + + return 0; +} + +} // namespace mooncake diff --git a/mooncake-transfer-engine/src/transport/efa_transport/efa_transport.cpp b/mooncake-transfer-engine/src/transport/efa_transport/efa_transport.cpp new file mode 100644 index 00000000..a4a2e3b1 --- /dev/null +++ b/mooncake-transfer-engine/src/transport/efa_transport/efa_transport.cpp @@ -0,0 +1,732 @@ +// Copyright 2024 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "transport/efa_transport/efa_transport.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include + +#include "common.h" +#include "config.h" +#include "memory_location.h" +#include "topology.h" +#include "transport/efa_transport/efa_context.h" +#include "transport/efa_transport/efa_endpoint.h" + +namespace mooncake { + +EfaTransport::EfaTransport() { + LOG(INFO) << "[EFA] AWS Elastic Fabric Adapter transport initialized"; +} + +EfaTransport::~EfaTransport() { + stopWorkerThreads(); + metadata_->removeSegmentDesc(local_server_name_); + batch_desc_set_.clear(); + context_list_.clear(); +} + +void EfaTransport::startWorkerThreads() { + if (worker_running_) return; + + worker_running_ = true; + // One poller thread per context for responsive CQ draining under load + size_t num_threads = context_list_.size(); + for (size_t i = 0; i < num_threads; i++) { + worker_threads_.emplace_back(&EfaTransport::workerThreadFunc, this, i); + } + LOG(INFO) << "EfaTransport: Started " << num_threads + << " CQ polling worker threads"; +} + +void EfaTransport::stopWorkerThreads() { + if (!worker_running_) return; + + worker_running_ = false; + for (auto &thread : worker_threads_) { + if (thread.joinable()) { + thread.join(); + } + } + worker_threads_.clear(); + LOG(INFO) << "EfaTransport: Stopped CQ polling worker threads"; +} + +void EfaTransport::workerThreadFunc(int thread_id) { + const int kPollBatchSize = 64; + + while (worker_running_) { + bool did_work = false; + + // Poll CQs from all contexts + for (size_t ctx_idx = thread_id; ctx_idx < context_list_.size(); + ctx_idx += worker_threads_.size()) { + auto &context = context_list_[ctx_idx]; + if (!context || !context->active()) continue; + + for (size_t cq_idx = 0; cq_idx < context->cqCount(); cq_idx++) { + int completed = context->pollCq(kPollBatchSize, cq_idx); + if (completed > 0) { + did_work = true; + } + } + } + + // If no work was done, yield CPU briefly + if (!did_work) { + std::this_thread::yield(); + } + } +} + +int EfaTransport::install(std::string &local_server_name, + std::shared_ptr meta, + std::shared_ptr topo) { + if (topo == nullptr) { + LOG(ERROR) << "EfaTransport: missing topology"; + return ERR_INVALID_ARGUMENT; + } + + metadata_ = meta; + local_server_name_ = local_server_name; + local_topology_ = topo; + + auto ret = initializeEfaResources(); + if (ret) { + LOG(ERROR) << "EfaTransport: cannot initialize EFA resources"; + return ret; + } + + ret = allocateLocalSegmentID(); + if (ret) { + LOG(ERROR) << "Transfer engine cannot be initialized: cannot " + "allocate local segment"; + return ret; + } + + ret = startHandshakeDaemon(local_server_name); + if (ret) { + LOG(ERROR) << "EfaTransport: cannot start handshake daemon"; + return ret; + } + + ret = metadata_->updateLocalSegmentDesc(); + if (ret) { + LOG(ERROR) << "EfaTransport: cannot publish segments"; + return ret; + } + + // Start CQ polling worker threads + startWorkerThreads(); + + return 0; +} + +int EfaTransport::preTouchMemory(void *addr, size_t length) { + if (context_list_.size() == 0) { + return 0; + } + + auto hwc = std::thread::hardware_concurrency(); + auto num_threads = hwc > 64 ? 16 : std::min(hwc, 8u); + if (length > (size_t)globalConfig().max_mr_size) { + length = (size_t)globalConfig().max_mr_size; + } + size_t block_size = length / num_threads; + if (block_size == 0) { + return 0; + } + + std::vector threads; + threads.reserve(num_threads); + std::vector thread_results(num_threads, 0); + + for (size_t thread_i = 0; thread_i < num_threads; ++thread_i) { + void *block_addr = static_cast(addr) + thread_i * block_size; + threads.emplace_back([this, thread_i, block_addr, block_size, + &thread_results]() { + int ret = context_list_[0]->preTouchMemory(block_addr, block_size); + thread_results[thread_i] = ret; + }); + } + + for (auto &thread : threads) { + thread.join(); + } + + for (size_t i = 0; i < num_threads; ++i) { + if (thread_results[i] != 0) { + return thread_results[i]; + } + } + + return 0; +} + +int EfaTransport::registerLocalMemory(void *addr, size_t length, + const std::string &name, + bool remote_accessible, + bool update_metadata) { + return registerLocalMemoryInternal(addr, length, name, remote_accessible, + update_metadata, false); +} + +int EfaTransport::registerLocalMemoryInternal(void *addr, size_t length, + const std::string &name, + bool remote_accessible, + bool update_metadata, + bool force_sequential) { + (void)remote_accessible; + BufferDesc buffer_desc; + const int kBaseAccessRights = IBV_ACCESS_LOCAL_WRITE | + IBV_ACCESS_REMOTE_WRITE | + IBV_ACCESS_REMOTE_READ; + + int access_rights = kBaseAccessRights; + + bool do_pre_touch = context_list_.size() > 0 && + std::thread::hardware_concurrency() >= 4 && + length >= (size_t)4 * 1024 * 1024 * 1024; + if (do_pre_touch) { + int ret = preTouchMemory(addr, length); + if (ret != 0) { + return ret; + } + } + + int use_parallel_reg = 0; + if (!force_sequential) { + use_parallel_reg = globalConfig().parallel_reg_mr; + if (use_parallel_reg == -1) { + use_parallel_reg = context_list_.size() > 1 && do_pre_touch; + } + } + + auto reg_start = std::chrono::steady_clock::now(); + + if (use_parallel_reg) { + std::vector reg_threads; + reg_threads.reserve(context_list_.size()); + std::vector ret_codes(context_list_.size(), 0); + const int ar = access_rights; + + for (size_t i = 0; i < context_list_.size(); ++i) { + reg_threads.emplace_back([this, &ret_codes, i, addr, length, ar]() { + ret_codes[i] = + context_list_[i]->registerMemoryRegion(addr, length, ar); + }); + } + + for (auto &thread : reg_threads) { + thread.join(); + } + + for (size_t i = 0; i < ret_codes.size(); ++i) { + if (ret_codes[i] != 0) { + LOG(ERROR) + << "Failed to register memory region with EFA context " + << i; + return ret_codes[i]; + } + } + } else { + for (size_t i = 0; i < context_list_.size(); ++i) { + int ret = context_list_[i]->registerMemoryRegion(addr, length, + access_rights); + if (ret) { + LOG(ERROR) + << "Failed to register memory region with EFA context " + << i; + return ret; + } + } + } + + auto reg_end = std::chrono::steady_clock::now(); + auto reg_duration_ms = + std::chrono::duration_cast(reg_end - + reg_start) + .count(); + + if (globalConfig().trace) { + LOG(INFO) << "EFA registerMemoryRegion: addr=" << addr + << ", length=" << length + << ", contexts=" << context_list_.size() + << ", parallel=" << (use_parallel_reg ? "true" : "false") + << ", duration=" << reg_duration_ms << "ms"; + } + + // Collect keys from all contexts + for (auto &context : context_list_) { + buffer_desc.lkey.push_back(context->lkey(addr)); + buffer_desc.rkey.push_back(context->rkey(addr)); + } + + if (name == kWildcardLocation) { + bool only_first_page = true; + const std::vector entries = + getMemoryLocation(addr, length, only_first_page); + if (entries.empty()) return -1; + buffer_desc.name = entries[0].location; + } else { + buffer_desc.name = name; + } + + buffer_desc.addr = (uint64_t)addr; + buffer_desc.length = length; + int rc = metadata_->addLocalMemoryBuffer(buffer_desc, update_metadata); + if (rc) return rc; + return 0; +} + +int EfaTransport::unregisterLocalMemory(void *addr, bool update_metadata) { + return unregisterLocalMemoryInternal(addr, update_metadata, false); +} + +int EfaTransport::unregisterLocalMemoryInternal(void *addr, + bool update_metadata, + bool force_sequential) { + int rc = metadata_->removeLocalMemoryBuffer(addr, update_metadata); + if (rc) return rc; + + int use_parallel_unreg = 0; + if (!force_sequential) { + use_parallel_unreg = globalConfig().parallel_reg_mr; + if (use_parallel_unreg == -1) { + use_parallel_unreg = context_list_.size() > 1; + } + } + + if (use_parallel_unreg) { + std::vector unreg_threads; + unreg_threads.reserve(context_list_.size()); + std::vector ret_codes(context_list_.size(), 0); + + for (size_t i = 0; i < context_list_.size(); ++i) { + unreg_threads.emplace_back([this, &ret_codes, i, addr]() { + ret_codes[i] = context_list_[i]->unregisterMemoryRegion(addr); + }); + } + + for (auto &thread : unreg_threads) { + thread.join(); + } + + for (size_t i = 0; i < ret_codes.size(); ++i) { + if (ret_codes[i] != 0) { + LOG(ERROR) + << "Failed to unregister memory region with EFA context " + << i; + return ret_codes[i]; + } + } + } else { + for (size_t i = 0; i < context_list_.size(); ++i) { + int ret = context_list_[i]->unregisterMemoryRegion(addr); + if (ret) { + LOG(ERROR) + << "Failed to unregister memory region with EFA context " + << i; + return ret; + } + } + } + + return 0; +} + +int EfaTransport::allocateLocalSegmentID() { + auto desc = std::make_shared(); + if (!desc) return ERR_MEMORY; + desc->name = local_server_name_; + desc->protocol = "efa"; + for (auto &entry : context_list_) { + TransferMetadata::DeviceDesc device_desc; + device_desc.name = entry->deviceName(); + device_desc.lid = entry->lid(); + device_desc.gid = entry->gid(); + desc->devices.push_back(device_desc); + } + desc->topology = *(local_topology_.get()); + metadata_->addLocalSegment(LOCAL_SEGMENT_ID, local_server_name_, + std::move(desc)); + return 0; +} + +int EfaTransport::registerLocalMemoryBatch( + const std::vector &buffer_list, + const std::string &location) { + std::vector> results; + for (auto &buffer : buffer_list) { + results.emplace_back( + std::async(std::launch::async, [this, buffer, location]() -> int { + return registerLocalMemoryInternal(buffer.addr, buffer.length, + location, true, false, true); + })); + } + + for (size_t i = 0; i < buffer_list.size(); ++i) { + if (results[i].get()) { + LOG(WARNING) << "EfaTransport: Failed to register memory: addr " + << buffer_list[i].addr << " length " + << buffer_list[i].length; + } + } + + return metadata_->updateLocalSegmentDesc(); +} + +int EfaTransport::unregisterLocalMemoryBatch( + const std::vector &addr_list) { + std::vector> results; + for (auto &addr : addr_list) { + results.emplace_back( + std::async(std::launch::async, [this, addr]() -> int { + return unregisterLocalMemoryInternal(addr, false, true); + })); + } + + for (size_t i = 0; i < addr_list.size(); ++i) { + if (results[i].get()) + LOG(WARNING) << "EfaTransport: Failed to unregister memory: addr " + << addr_list[i]; + } + + return metadata_->updateLocalSegmentDesc(); +} + +Status EfaTransport::submitTransfer( + BatchID batch_id, const std::vector &entries) { + auto &batch_desc = *((BatchDesc *)(batch_id)); + if (batch_desc.task_list.size() + entries.size() > batch_desc.batch_size) { + LOG(ERROR) << "EfaTransport: Exceed the limitation of current batch's " + "capacity"; + return Status::InvalidArgument( + "EfaTransport: Exceed the limitation of capacity, batch id: " + + std::to_string(batch_id)); + } + + size_t task_id = batch_desc.task_list.size(); + batch_desc.task_list.resize(task_id + entries.size()); + std::vector task_list; + for (auto &task : batch_desc.task_list) task_list.push_back(&task); + return submitTransferTask(task_list); +} + +Status EfaTransport::submitTransferTask( + const std::vector &task_list) { + std::unordered_map, std::vector> + slices_to_post; + auto local_segment_desc = metadata_->getSegmentDescByID(LOCAL_SEGMENT_ID); + assert(local_segment_desc.get()); + const size_t kBlockSize = globalConfig().slice_size; + const int kMaxRetryCount = globalConfig().retry_cnt; + const size_t kFragmentSize = globalConfig().fragment_limit; + const size_t kSubmitWatermark = + globalConfig().max_wr * globalConfig().num_qp_per_ep; + uint64_t nr_slices; + for (size_t index = 0; index < task_list.size(); ++index) { + assert(task_list[index]); + auto &task = *task_list[index]; + nr_slices = 0; + assert(task.request); + auto &request = *task.request; + + auto request_buffer_id = -1, request_device_id = -1; + if (selectDevice(local_segment_desc.get(), (uint64_t)request.source, + request.length, request_buffer_id, + request_device_id)) { + request_buffer_id = -1; + request_device_id = -1; + } + + for (uint64_t offset = 0; offset < request.length; + offset += kBlockSize) { + Slice *slice = getSliceCache().allocate(); + assert(slice); + if (!slice->from_cache) { + nr_slices++; + } + + bool merge_final_slice = + request.length - offset <= kBlockSize + kFragmentSize; + + slice->source_addr = (char *)request.source + offset; + slice->length = + merge_final_slice ? request.length - offset : kBlockSize; + slice->opcode = request.opcode; + slice->rdma.dest_addr = request.target_offset + offset; + slice->rdma.retry_cnt = request.advise_retry_cnt; + slice->rdma.max_retry_cnt = kMaxRetryCount; + slice->task = &task; + slice->target_id = request.target_id; + slice->status = Slice::PENDING; + slice->ts = 0; + task.slice_list.push_back(slice); + + int buffer_id = -1, device_id = -1, + retry_cnt = request.advise_retry_cnt; + bool found_device = false; + if (request_buffer_id >= 0 && request_device_id >= 0) { + found_device = true; + buffer_id = request_buffer_id; + device_id = request_device_id; + } + while (retry_cnt < kMaxRetryCount && !found_device) { + if (selectDevice(local_segment_desc.get(), + (uint64_t)slice->source_addr, slice->length, + buffer_id, device_id, retry_cnt++)) + continue; + assert(device_id >= 0 && + static_cast(device_id) < context_list_.size()); + auto &context = context_list_[device_id]; + assert(context.get()); + if (!context->active()) continue; + assert(buffer_id >= 0 && + static_cast(buffer_id) < + local_segment_desc->buffers.size()); + assert(local_segment_desc->buffers[buffer_id].lkey.size() == + context_list_.size()); + found_device = true; + break; + } + if (!found_device) { + auto source_addr = slice->source_addr; + for (auto &entry : slices_to_post) + for (auto s : entry.second) getSliceCache().deallocate(s); + LOG(ERROR) << "Memory region not registered by any active EFA " + "device(s): " + << source_addr; + return Status::AddressNotRegistered( + "Memory region not registered by any active EFA " + "device(s): " + + std::to_string(reinterpret_cast(source_addr))); + } else { + auto &context = context_list_[device_id]; + if (!context->active()) { + LOG(ERROR) + << "EFA Device " << device_id << " is not active"; + return Status::InvalidArgument("EFA Device " + + std::to_string(device_id) + + " is not active"); + } + slice->rdma.source_lkey = + local_segment_desc->buffers[buffer_id].lkey[device_id]; + slices_to_post[context].push_back(slice); + task.total_bytes += slice->length; + __sync_fetch_and_add(&task.slice_count, 1); + } + + if (nr_slices >= kSubmitWatermark) { + for (auto &entry : slices_to_post) + entry.first->submitPostSend(entry.second); + slices_to_post.clear(); + nr_slices = 0; + } + + if (merge_final_slice) { + break; + } + } + } + + for (auto &entry : slices_to_post) + if (!entry.second.empty()) entry.first->submitPostSend(entry.second); + return Status::OK(); +} + +Status EfaTransport::getTransferStatus(BatchID batch_id, + std::vector &status) { + auto &batch_desc = *((BatchDesc *)(batch_id)); + const size_t task_count = batch_desc.task_list.size(); + status.resize(task_count); + for (size_t task_id = 0; task_id < task_count; task_id++) { + auto &task = batch_desc.task_list[task_id]; + status[task_id].transferred_bytes = task.transferred_bytes; + uint64_t success_slice_count = task.success_slice_count; + uint64_t failed_slice_count = task.failed_slice_count; + if (success_slice_count + failed_slice_count == task.slice_count) { + if (failed_slice_count) + status[task_id].s = TransferStatusEnum::FAILED; + else + status[task_id].s = TransferStatusEnum::COMPLETED; + task.is_finished = true; + } else { + status[task_id].s = TransferStatusEnum::WAITING; + } + } + return Status::OK(); +} + +Status EfaTransport::getTransferStatus(BatchID batch_id, size_t task_id, + TransferStatus &status) { + auto &batch_desc = *((BatchDesc *)(batch_id)); + const size_t task_count = batch_desc.task_list.size(); + if (task_id >= task_count) { + return Status::InvalidArgument( + "EfaTransport::getTransportStatus invalid argument, batch id: " + + std::to_string(batch_id)); + } + auto &task = batch_desc.task_list[task_id]; + status.transferred_bytes = task.transferred_bytes; + uint64_t success_slice_count = task.success_slice_count; + uint64_t failed_slice_count = task.failed_slice_count; + if (success_slice_count + failed_slice_count == task.slice_count) { + if (failed_slice_count) + status.s = TransferStatusEnum::FAILED; + else + status.s = TransferStatusEnum::COMPLETED; + task.is_finished = true; + } else { + status.s = TransferStatusEnum::WAITING; + } + return Status::OK(); +} + +EfaTransport::SegmentID EfaTransport::getSegmentID( + const std::string &segment_name) { + return metadata_->getSegmentID(segment_name); +} + +int EfaTransport::onSetupEfaConnections(const HandShakeDesc &peer_desc, + HandShakeDesc &local_desc) { + auto local_nic_name = getNicNameFromNicPath(peer_desc.peer_nic_path); + if (local_nic_name.empty()) return ERR_INVALID_ARGUMENT; + + // Find context by device name instead of using hca_list index, since + // context_list_ only contains EFA devices and may have different + // indexing than the full hca_list. + std::shared_ptr context; + for (auto &entry : context_list_) { + if (entry->deviceName() == local_nic_name) { + context = entry; + break; + } + } + if (!context) return ERR_INVALID_ARGUMENT; + + auto endpoint = context->endpoint(peer_desc.local_nic_path); + if (!endpoint) return ERR_ENDPOINT; + return endpoint->setupConnectionsByPassive(peer_desc, local_desc); +} + +int EfaTransport::initializeEfaResources() { + auto hca_list = local_topology_->getHcaList(); + + // Filter for EFA devices (names typically start with "rdmap" on AWS) + std::vector efa_devices; + std::vector non_efa_devices; + for (auto &device_name : hca_list) { + if (device_name.find("rdmap") != std::string::npos || + device_name.find("efa") != std::string::npos) { + efa_devices.push_back(device_name); + } else { + non_efa_devices.push_back(device_name); + } + } + + if (efa_devices.empty()) { + LOG(WARNING) << "EfaTransport: No EFA devices found, falling back to " + "all devices"; + efa_devices = hca_list; + non_efa_devices.clear(); + } + + // Disable non-EFA devices (e.g. ibp* IB devices) in the topology so that + // topology device indices stay aligned with context_list_ indices. + // Without this, selectDevice() can return an index from the full topology + // (which includes non-EFA devices), causing out-of-bounds access on + // context_list_ which only contains EFA devices. + for (auto &device_name : non_efa_devices) { + local_topology_->disableDevice(device_name); + LOG(INFO) << "EfaTransport: Disabled non-EFA device " << device_name + << " in topology"; + } + + for (auto &device_name : efa_devices) { + auto context = std::make_shared(*this, device_name); + auto &config = globalConfig(); + int ret = context->construct(config.num_cq_per_ctx, + config.num_comp_channels_per_ctx, + config.port, config.gid_index, + config.max_cqe, config.max_ep_per_ctx); + if (ret) { + local_topology_->disableDevice(device_name); + LOG(WARNING) << "EfaTransport: Disable device " << device_name; + } else { + context_list_.push_back(context); + LOG(INFO) << "EfaTransport: Initialized EFA device " << device_name; + } + } + if (context_list_.empty()) { + LOG(ERROR) << "EfaTransport: No available EFA devices"; + return ERR_DEVICE_NOT_FOUND; + } + return 0; +} + +int EfaTransport::startHandshakeDaemon(std::string &local_server_name) { + return metadata_->startHandshakeDaemon( + std::bind(&EfaTransport::onSetupEfaConnections, this, + std::placeholders::_1, std::placeholders::_2), + metadata_->localRpcMeta().rpc_port, metadata_->localRpcMeta().sockfd); +} + +int EfaTransport::selectDevice(SegmentDesc *desc, uint64_t offset, + size_t length, std::string_view hint, + int &buffer_id, int &device_id, + int retry_count) { + if (desc == nullptr) return ERR_ADDRESS_NOT_REGISTERED; + const auto &buffers = desc->buffers; + for (buffer_id = 0; buffer_id < static_cast(buffers.size()); + ++buffer_id) { + const auto &buffer = buffers[buffer_id]; + + if (offset < buffer.addr || length > buffer.length || + offset - buffer.addr > buffer.length - length) { + continue; + } + + device_id = + hint.empty() + ? desc->topology.selectDevice(buffer.name, retry_count) + : desc->topology.selectDevice(buffer.name, hint, retry_count); + if (device_id >= 0) return 0; + device_id = hint.empty() ? desc->topology.selectDevice( + kWildcardLocation, retry_count) + : desc->topology.selectDevice( + kWildcardLocation, hint, retry_count); + if (device_id >= 0) return 0; + } + return ERR_ADDRESS_NOT_REGISTERED; +} + +int EfaTransport::selectDevice(SegmentDesc *desc, uint64_t offset, + size_t length, int &buffer_id, int &device_id, + int retry_count) { + return selectDevice(desc, offset, length, "", buffer_id, device_id, + retry_count); +} + +} // namespace mooncake diff --git a/mooncake-transfer-engine/tests/CMakeLists.txt b/mooncake-transfer-engine/tests/CMakeLists.txt index 12a607b3..ca98ea6c 100644 --- a/mooncake-transfer-engine/tests/CMakeLists.txt +++ b/mooncake-transfer-engine/tests/CMakeLists.txt @@ -54,6 +54,12 @@ if (USE_UBSHMEM) add_test(NAME ubshmem_transport_test COMMAND ubshmem_transport_test) endif() +if (USE_EFA) + add_executable(efa_transport_test ${WORKSPACE}/efa_transport_test.cpp) + target_link_libraries(efa_transport_test PUBLIC transfer_engine gtest gtest_main) + add_test(NAME efa_transport_test COMMAND efa_transport_test) +endif() + add_executable(transfer_metadata_test ${WORKSPACE}/transfer_metadata_test.cpp) target_link_libraries(transfer_metadata_test PUBLIC transfer_engine gtest gtest_main) add_test(NAME transfer_metadata_test COMMAND transfer_metadata_test) diff --git a/mooncake-transfer-engine/tests/efa_transport_test.cpp b/mooncake-transfer-engine/tests/efa_transport_test.cpp new file mode 100644 index 00000000..066421ac --- /dev/null +++ b/mooncake-transfer-engine/tests/efa_transport_test.cpp @@ -0,0 +1,338 @@ +// Copyright 2024 KVCache.AI +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include +#include + +#include "transfer_engine.h" +#include "transport/transport.h" + +using namespace mooncake; + +namespace mooncake { + +static void *allocateMemoryPool(size_t size, int socket_id) { + return numa_alloc_onnode(size, socket_id); +} + +static void freeMemoryPool(void *addr, size_t size) { numa_free(addr, size); } + +// --------------------------------------------------------------------------- +// EFA Transport Test Fixture +// +// This test uses the P2PHANDSHAKE metadata backend and performs loopback +// transfers (local_server_name == segment_id), similar to the TCP transport +// tests. It requires EFA hardware to be present (fi_info -p efa must succeed). +// +// Environment variables: +// MC_METADATA_SERVER - metadata backend (default: P2PHANDSHAKE) +// MC_LOCAL_SERVER_NAME - local server name (default: 127.0.0.1:12345) +// --------------------------------------------------------------------------- +class EFATransportTest : public ::testing::Test { + protected: + void SetUp() override { + google::InitGoogleLogging("EFATransportTest"); + FLAGS_logtostderr = 1; + + const char *env = std::getenv("MC_METADATA_SERVER"); + metadata_server_ = env ? env : "P2PHANDSHAKE"; + LOG(INFO) << "metadata_server: " << metadata_server_; + + env = std::getenv("MC_LOCAL_SERVER_NAME"); + local_server_name_ = env ? env : "127.0.0.1:12345"; + LOG(INFO) << "local_server_name: " << local_server_name_; + } + + void TearDown() override { google::ShutdownGoogleLogging(); } + + // Helper: create engine, install EFA transport, register memory + struct EngineSetup { + std::unique_ptr engine; + Transport *xport; + void *addr; + size_t buffer_size; + SegmentID segment_id; + }; + + EngineSetup createEngine(size_t buffer_size = 1ull << 30) { + EngineSetup s; + s.buffer_size = buffer_size; + + s.engine = std::make_unique(false); + // Manually discover topology to populate EFA device list + // (same pattern as the Python binding in transfer_engine_py.cpp) + s.engine->getLocalTopology()->discover({}); + auto hp = parseHostNameWithPort(local_server_name_); + int rc = s.engine->init(metadata_server_, local_server_name_, + hp.first.c_str(), hp.second); + EXPECT_EQ(rc, 0) << "engine->init failed"; + + s.xport = s.engine->installTransport("efa", nullptr); + EXPECT_NE(s.xport, nullptr) << "installTransport(\"efa\") failed"; + + s.addr = allocateMemoryPool(buffer_size, 0); + EXPECT_NE(s.addr, nullptr) << "allocateMemoryPool failed"; + + rc = s.engine->registerLocalMemory(s.addr, buffer_size, "cpu:0"); + EXPECT_EQ(rc, 0) << "registerLocalMemory failed"; + + // Use actual RPC address (P2PHANDSHAKE picks a random port) + auto actual_addr = s.engine->getLocalIpAndPort(); + s.segment_id = s.engine->openSegment(actual_addr); + return s; + } + + void destroyEngine(EngineSetup &s) { + if (s.engine && s.addr) { + s.engine->unregisterLocalMemory(s.addr); + } + if (s.addr) { + freeMemoryPool(s.addr, s.buffer_size); + s.addr = nullptr; + } + } + + // Helper: submit a single transfer and poll until completion + bool submitAndWait(TransferEngine *engine, SegmentID segment_id, + void *source, uint64_t target_offset, size_t length, + TransferRequest::OpCode opcode) { + auto batch_id = engine->allocateBatchID(1); + + TransferRequest entry; + entry.opcode = opcode; + entry.length = length; + entry.source = (uint8_t *)source; + entry.target_id = segment_id; + entry.target_offset = target_offset; + + Status s = engine->submitTransfer(batch_id, {entry}); + if (!s.ok()) { + LOG(ERROR) << "submitTransfer failed: " << s.ToString(); + engine->freeBatchID(batch_id); + return false; + } + + // Poll for completion with timeout + const int kMaxPollIterations = 1000000; + TransferStatus status; + for (int i = 0; i < kMaxPollIterations; ++i) { + s = engine->getTransferStatus(batch_id, 0, status); + if (!s.ok()) { + LOG(ERROR) << "getTransferStatus failed: " << s.ToString(); + engine->freeBatchID(batch_id); + return false; + } + if (status.s == TransferStatusEnum::COMPLETED) { + engine->freeBatchID(batch_id); + return true; + } + if (status.s == TransferStatusEnum::FAILED) { + LOG(ERROR) << "Transfer FAILED"; + engine->freeBatchID(batch_id); + return false; + } + } + LOG(ERROR) << "Transfer timed out"; + engine->freeBatchID(batch_id); + return false; + } + + std::string metadata_server_; + std::string local_server_name_; +}; + +// Test 1: Verify EFA transport can be installed +TEST_F(EFATransportTest, InstallTransport) { + auto engine = std::make_unique(false); + engine->getLocalTopology()->discover({}); + auto hp = parseHostNameWithPort(local_server_name_); + int rc = engine->init(metadata_server_, local_server_name_, + hp.first.c_str(), hp.second); + ASSERT_EQ(rc, 0); + + Transport *xport = engine->installTransport("efa", nullptr); + ASSERT_NE(xport, nullptr) + << "EFA transport should be installable on EFA hardware"; +} + +// Test 2: Basic loopback write +TEST_F(EFATransportTest, LoopbackWrite) { + auto setup = createEngine(); + + auto segment_desc = + setup.engine->getMetadata()->getSegmentDescByID(setup.segment_id); + ASSERT_NE(segment_desc, nullptr); + uint64_t remote_base = (uint64_t)segment_desc->buffers[0].addr; + + const size_t kDataLength = 4096; + + // Fill source buffer with known data + memset(setup.addr, 0xAB, kDataLength); + + bool ok = submitAndWait(setup.engine.get(), setup.segment_id, setup.addr, + remote_base, kDataLength, TransferRequest::WRITE); + EXPECT_TRUE(ok) << "Loopback write should succeed"; + + destroyEngine(setup); +} + +// Test 3: Write then read, verify data integrity +TEST_F(EFATransportTest, WriteAndRead) { + auto setup = createEngine(); + + auto segment_desc = + setup.engine->getMetadata()->getSegmentDescByID(setup.segment_id); + ASSERT_NE(segment_desc, nullptr); + uint64_t remote_base = (uint64_t)segment_desc->buffers[0].addr; + + const size_t kDataLength = 4096000; + uint8_t *buf = (uint8_t *)setup.addr; + + // Fill first half with random data + for (size_t i = 0; i < kDataLength; ++i) buf[i] = 'a' + lrand48() % 26; + + // Write local -> remote (loopback) + bool ok = submitAndWait(setup.engine.get(), setup.segment_id, buf, + remote_base, kDataLength, TransferRequest::WRITE); + ASSERT_TRUE(ok) << "Write should succeed"; + + // Read remote -> local (into second half of buffer) + ok = submitAndWait(setup.engine.get(), setup.segment_id, buf + kDataLength, + remote_base, kDataLength, TransferRequest::READ); + ASSERT_TRUE(ok) << "Read should succeed"; + + // Verify data integrity + EXPECT_EQ(0, memcmp(buf, buf + kDataLength, kDataLength)) + << "Read-back data should match written data"; + + destroyEngine(setup); +} + +// Test 4: Multiple sequential writes in a batch +TEST_F(EFATransportTest, MultiWrite) { + auto setup = createEngine(); + + auto segment_desc = + setup.engine->getMetadata()->getSegmentDescByID(setup.segment_id); + ASSERT_NE(segment_desc, nullptr); + uint64_t remote_base = (uint64_t)segment_desc->buffers[0].addr; + + const size_t kDataLength = 65536; + const int kBatchSize = 16; + + auto batch_id = setup.engine->allocateBatchID(kBatchSize); + + std::vector requests; + for (int i = 0; i < kBatchSize; ++i) { + TransferRequest entry; + entry.opcode = TransferRequest::WRITE; + entry.length = kDataLength; + entry.source = (uint8_t *)setup.addr + i * kDataLength; + entry.target_id = setup.segment_id; + entry.target_offset = remote_base + i * kDataLength; + requests.push_back(entry); + } + + Status s = setup.engine->submitTransfer(batch_id, requests); + ASSERT_TRUE(s.ok()) << "submitTransfer failed: " << s.ToString(); + + // Poll all tasks until completion + for (int task_id = 0; task_id < kBatchSize; ++task_id) { + TransferStatus status; + const int kMaxPollIterations = 1000000; + for (int i = 0; i < kMaxPollIterations; ++i) { + s = setup.engine->getTransferStatus(batch_id, task_id, status); + ASSERT_TRUE(s.ok()); + if (status.s == TransferStatusEnum::COMPLETED) break; + ASSERT_NE(status.s, TransferStatusEnum::FAILED) + << "Task " << task_id << " failed"; + } + ASSERT_EQ(status.s, TransferStatusEnum::COMPLETED) + << "Task " << task_id << " did not complete"; + } + + s = setup.engine->freeBatchID(batch_id); + ASSERT_TRUE(s.ok()); + + destroyEngine(setup); +} + +// Test 5: Stress test - multiple batches to verify no CQ overflow +TEST_F(EFATransportTest, StressMultipleBatches) { + auto setup = createEngine(); + + auto segment_desc = + setup.engine->getMetadata()->getSegmentDescByID(setup.segment_id); + ASSERT_NE(segment_desc, nullptr); + uint64_t remote_base = (uint64_t)segment_desc->buffers[0].addr; + + const size_t kDataLength = 65536; + const int kBatchSize = 8; + const int kNumBatches = 20; + + for (int batch = 0; batch < kNumBatches; ++batch) { + auto batch_id = setup.engine->allocateBatchID(kBatchSize); + + std::vector requests; + for (int i = 0; i < kBatchSize; ++i) { + TransferRequest entry; + entry.opcode = TransferRequest::WRITE; + entry.length = kDataLength; + entry.source = + (uint8_t *)setup.addr + (i + batch * kBatchSize) * kDataLength; + entry.target_id = setup.segment_id; + entry.target_offset = + remote_base + (i + batch * kBatchSize) * kDataLength; + requests.push_back(entry); + } + + Status s = setup.engine->submitTransfer(batch_id, requests); + ASSERT_TRUE(s.ok()) + << "Batch " << batch << " submitTransfer failed: " << s.ToString(); + + // Wait for all tasks in batch + for (int task_id = 0; task_id < kBatchSize; ++task_id) { + TransferStatus status; + const int kMaxPollIterations = 1000000; + for (int i = 0; i < kMaxPollIterations; ++i) { + s = setup.engine->getTransferStatus(batch_id, task_id, status); + ASSERT_TRUE(s.ok()); + if (status.s == TransferStatusEnum::COMPLETED) break; + ASSERT_NE(status.s, TransferStatusEnum::FAILED) + << "Batch " << batch << " task " << task_id << " failed"; + } + ASSERT_EQ(status.s, TransferStatusEnum::COMPLETED) + << "Batch " << batch << " task " << task_id + << " did not complete"; + } + + s = setup.engine->freeBatchID(batch_id); + ASSERT_TRUE(s.ok()); + } + + destroyEngine(setup); +} + +} // namespace mooncake + +int main(int argc, char **argv) { + gflags::ParseCommandLineFlags(&argc, &argv, false); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}