Te seperated compilation (#1041)

* Add TE Separation Switch

* Fix store compile and link error, add TE installation headers

* Install libmooncake_store.so and libmooncake_common.so when BUILD_SHARED_LIBS=ON

CMakeLists.txt

@@ -12,6 +12,7 @@ if (BUILD_UNIT_TESTS)
   enable_testing()
 endif()
 
+option(WITH_TE "build mooncake transfer engine and sample code" ON)
 option(WITH_STORE "build mooncake store library and sample code" ON)
 option(WITH_P2P_STORE "build p2p store library and sample code" OFF)
 option(WITH_RUST_EXAMPLE "build the Rust interface and sample code for the transfer engine" OFF)
@@ -45,8 +46,10 @@ add_subdirectory(mooncake-common)
 include_directories(mooncake-common/etcd)
 include_directories(mooncake-common/include)
 
-add_subdirectory(mooncake-transfer-engine)
-include_directories(mooncake-transfer-engine/include)
+if (WITH_TE)
+  add_subdirectory(mooncake-transfer-engine)
+  include_directories(mooncake-transfer-engine/include)
+endif()
 
 if (WITH_STORE)
   message(STATUS "Mooncake Store will be built")

mooncake-common/src/CMakeLists.txt

@@ -10,4 +10,8 @@ add_library(mooncake_common
 target_link_libraries(mooncake_common PUBLIC
     yaml-cpp
     jsoncpp
-)
+)
+
+if (BUILD_SHARED_LIBS)
+  install(TARGETS mooncake_common DESTINATION lib)
+endif()

mooncake-integration/CMakeLists.txt

@@ -29,15 +29,17 @@ message("${PYTHON_SYS_PATH}")
 
 set(PYTHON_PACKAGE_NAME "mooncake")
 
-pybind11_add_module(engine ${SOURCES} ${CACHE_ALLOCATOR_SOURCES}
-    transfer_engine/transfer_engine_py.cpp 
-)
+if (WITH_TE)
+    pybind11_add_module(engine ${SOURCES} ${CACHE_ALLOCATOR_SOURCES}
+        transfer_engine/transfer_engine_py.cpp 
+    )
 
-target_link_libraries(engine PUBLIC 
-    transfer_engine 
-    glog::glog
-    gflags::gflags
-)
+    target_link_libraries(engine PUBLIC 
+        transfer_engine 
+        glog::glog
+        gflags::gflags
+    )
+endif()
 
 set(ALLOCATOR_SO_PATH "${CMAKE_BINARY_DIR}/mooncake-transfer-engine/nvlink-allocator/nvlink_allocator.so")
 if(USE_MNNVL)
@@ -113,4 +115,6 @@ if (WITH_STORE)
     install(TARGETS store DESTINATION ${PYTHON_SYS_PATH}/${PYTHON_PACKAGE_NAME})
 endif()
 
-install(TARGETS engine DESTINATION ${PYTHON_SYS_PATH}/${PYTHON_PACKAGE_NAME})
+if (WITH_TE)
+    install(TARGETS engine DESTINATION ${PYTHON_SYS_PATH}/${PYTHON_PACKAGE_NAME})
+endif()

mooncake-store/src/CMakeLists.txt

@@ -48,6 +48,10 @@ if (STORE_USE_ETCD)
     add_dependencies(mooncake_store build_etcd_wrapper)
 endif()
 
+if (BUILD_SHARED_LIBS)
+  install(TARGETS mooncake_store DESTINATION lib)
+endif()
+
 # Master binary
 add_executable(mooncake_master master.cpp)
 
@@ -66,6 +70,7 @@ target_link_libraries(mooncake_master PRIVATE
     mooncake_store
     cachelib_memory_allocator
     pthread
+    ibverbs
     mooncake_common
     ${ETCD_WRAPPER_LIB}
     ${MASTER_EXTRA_LIBS}
@@ -75,4 +80,4 @@ if (STORE_USE_ETCD)
     add_dependencies(mooncake_master build_etcd_wrapper)
 endif()
 
-install(TARGETS mooncake_master DESTINATION bin)
+install(TARGETS mooncake_master DESTINATION bin)

mooncake-store/tests/CMakeLists.txt

@@ -5,6 +5,7 @@ function(add_store_test name)
         cachelib_memory_allocator
         ${ETCD_WRAPPER_LIB}
         glog
+        ibverbs
         gtest
         gtest_main
         pthread

mooncake-transfer-engine/include/CMakeLists.txt

@@ -1 +1,11 @@
 install(FILES transfer_engine_c.h DESTINATION include)
+install(FILES common.h DESTINATION include)
+install(FILES config.h DESTINATION include)
+install(FILES error.h DESTINATION include)
+install(FILES memory_location.h DESTINATION include)
+install(FILES multi_transport.h DESTINATION include)
+install(FILES topology.h DESTINATION include)
+install(FILES transfer_engine.h DESTINATION include)
+install(FILES transfer_metadata.h DESTINATION include)
+install(FILES common/base/status.h DESTINATION include/common/base)
+install(FILES transport/transport.h DESTINATION include/transport)