Make xgboost dep conditional (#466)

Makefile

@@ -75,7 +75,7 @@ ifndef SKIP_BUILDDEPS_CHECK
   endif
 
   # Check if xgboost headers are being built
-  XGBOOST_TARGETS := zli gtests test all
+  XGBOOST_TARGETS := zli gtests test all cli_test
   BUILDING_XGBOOST_TARGETS := $(filter $(XGBOOST_TARGETS),$(MAKECMDGOALS))
   ifeq ($(MAKECMDGOALS),)
     # If no targets are specified, assume we're building everything
@@ -106,10 +106,10 @@ LIBASMSRCS := $(wildcard $(addsuffix /*.S, $(LIBDIRS)))
 LIBOBJS := $(patsubst %.c,%.o,$(LIBCSRCS)) $(patsubst %.S,%.o,$(LIBASMSRCS))
 
 libopenzl.a:
-$(eval $(call static_library,libopenzl.a,$(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A) $(LIBXGBOOST_A) $(LIBDMLC_A)))
+$(eval $(call static_library,libopenzl.a,$(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A)))
 
 libopenzl.so: CFLAGS += -fPIC
-$(eval $(call c_dynamic_library,libopenzl.so,$(LIBOBJS),$(LIBZSTD_SO) $(LIBLZ4_SO) $(LIBXGBOOST_SO)))
+$(eval $(call c_dynamic_library,libopenzl.so,$(LIBOBJS),$(LIBZSTD_SO) $(LIBLZ4_SO)))
 
 .PHONY:lib
 lib: libopenzl.a libopenzl.so
@@ -153,11 +153,15 @@ ML_SELECTOR_CXXOBJS := $(call cxx_objs,$(ML_SELECTOR_DIR))
 # ML selector files depend on xgboost headers
 $(ML_SELECTOR_COBJS) $(ML_SELECTOR_CXXOBJS): | $(XGBOOST_HEADER)
 
+XGBOOST_INCLUDE_PATHS := -Ideps/xgboost/include -Ideps/xgboost/dmlc-core/include # xgboost headers
+
 # Add flags for cross platform compatibility for Windows
 zli: LDFLAGS += $(XGBOOST_LDFLAGS)
+zli: CPPFLAGS += $(XGBOOST_INCLUDE_PATHS)
 zli: LDLIBS += $(XGBOOST_LDLIBS)
 
 gtests: LDFLAGS += $(XGBOOST_LDFLAGS)
+gtests: CPPFLAGS += $(XGBOOST_INCLUDE_PATHS)
 gtests: LDLIBS += $(XGBOOST_LDLIBS)
 
 $(eval $(call cxx_program,zli, \
@@ -218,18 +222,18 @@ zs2_test : examples
 # ********     Tools     ********
 
 UNITBENCH_COBJS := $(foreach DIR,$(UNITBENCH_DIRS),$(call c_objs,$(DIR)))
-$(eval $(call c_program_shared_o,unitBench,tools/time/timefn.o tools/fileio/fileio.o $(UNITBENCH_COBJS) $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A) $(LIBXGBOOST_A) $(LIBDMLC_A)))
+$(eval $(call c_program_shared_o,unitBench,tools/time/timefn.o tools/fileio/fileio.o $(UNITBENCH_COBJS) $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A)))
 
 stream_dump2:
 $(eval $(call c_program_shared_o,stream_dump2, \
-    $(STREAMDUMP_COBJS) tools/fileio/fileio.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A) $(LIBXGBOOST_A) $(LIBDMLC_A)))
+    $(STREAMDUMP_COBJS) tools/fileio/fileio.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A)))
 
 $(eval $(call cxx_program,sddl_compiler, \
 	$(SDDL_COMPILER_DIR)/main.o \
 	$(SDDL_COMPILER_CXXOBJS) \
 	$(ZLCPP_OBJS), \
 	libopenzl.a \
-	$(LIBZSTD_A) $(LIBLZ4_A) $(LIBXGBOOST_A) $(LIBDMLC_A)))
+	$(LIBZSTD_A) $(LIBLZ4_A)))
 
 # Selection of gtest units (by file name convention)
 CXX_FILE_OBJS := $(notdir $(CXX_OBJS))
@@ -301,19 +305,19 @@ $(eval $(call cxx_program,gtests, \
 # ********     Examples     ********
 
 zs2_pipeline:
-$(eval $(call c_program_shared_o,zs2_pipeline,examples/zs2_pipeline.o tools/fileio/fileio.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A) $(LIBXGBOOST_A) $(LIBDMLC_A)))
+$(eval $(call c_program_shared_o,zs2_pipeline,examples/zs2_pipeline.o tools/fileio/fileio.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A)))
 
 zs2_struct:
-$(eval $(call c_program_shared_o,zs2_struct,examples/zs2_struct.o tools/fileio/fileio.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A) $(LIBXGBOOST_A) $(LIBDMLC_A)))
+$(eval $(call c_program_shared_o,zs2_struct,examples/zs2_struct.o tools/fileio/fileio.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A)))
 
 zs2_trygraph:
-$(eval $(call c_program_shared_o,zs2_trygraph,examples/zs2_trygraph.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A) $(LIBXGBOOST_A) $(LIBDMLC_A)))
+$(eval $(call c_program_shared_o,zs2_trygraph,examples/zs2_trygraph.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A)))
 
 zs2_selector:
-$(eval $(call c_program_shared_o,zs2_selector,examples/zs2_selector.o tools/fileio/fileio.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A) $(LIBXGBOOST_A) $(LIBDMLC_A)))
+$(eval $(call c_program_shared_o,zs2_selector,examples/zs2_selector.o tools/fileio/fileio.o $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A)))
 
 zs2_round_trip:
-$(eval $(call cxx_program_shared_o,zs2_round_trip,tests/round_trip.o tools/fileio/fileio.o $(SHARED_COMPONENTS_CXXOBJS) $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A) $(LIBXGBOOST_A) $(LIBDMLC_A)))
+$(eval $(call cxx_program_shared_o,zs2_round_trip,tests/round_trip.o tools/fileio/fileio.o $(SHARED_COMPONENTS_CXXOBJS) $(LIBOBJS),$(LIBZSTD_A) $(LIBLZ4_A)))
 
 # ********     Cleaning     ********
 
@@ -337,7 +341,7 @@ TAR ?= tar
 # Use this target as a work-around if dependencies are not correctly built
 # automatically.
 .PHONY : builddeps
-builddeps : $(LIBGTEST_A) $(LIBZSTD_A) $(LIBZSTD_SO) $(LIBLZ4_A) $(LIBLZ4_SO) $(LIBXGBOOST_A) $(LIBXGBOOST_SO)
+builddeps : $(LIBGTEST_A) $(LIBZSTD_A) $(LIBZSTD_SO) $(LIBLZ4_A) $(LIBLZ4_SO)
 
 .PHONY: cleandeps
 cleandeps:

build-scripts/make/zldefs.make

@@ -36,7 +36,6 @@ LDFLAGS  += $(MOREFLAGS)
 LDLIBS   += -lm # note: to be removed from library once dependency fixed
 CPPFLAGS += -Ideps/zstd/lib/ # "zstd.h"
 CPPFLAGS += -Ideps/lz4/lib/  # "lz4.h"
-CPPFLAGS += -Ideps/xgboost/include -Ideps/xgboost/dmlc-core/include # xgboost headers
 ARFLAGS  += -c # do not print warning message when creating the archive (expected)
 
 # Default build mode

tools/CMakeLists.txt

@@ -8,6 +8,7 @@ set(OPENZL_BUILD_ARG_TOOLS OFF)
 set(OPENZL_BUILD_IO_TOOLS OFF)
 set(OPENZL_BUILD_SDDL_TOOLS OFF)
 set(OPENZL_BUILD_TRAINING_TOOLS OFF)
+set(OPENZL_BUILD_ML_SELECTOR OFF)
 
 if(NOT DEFINED OPENZL_BUILD_LOGGER)
   set(OPENZL_BUILD_LOGGER OFF)
@@ -18,13 +19,15 @@ if(OPENZL_BUILD_TOOLS)
   set(OPENZL_BUILD_IO_TOOLS ON)
   set(OPENZL_BUILD_SDDL_TOOLS ON)
   set(OPENZL_BUILD_TRAINING_TOOLS ON)
+  set(OPENZL_BUILD_ML_SELECTOR ON)
 endif()
 if(OPENZL_BUILD_CLI)
   set(OPENZL_BUILD_ARG_TOOLS ON)
   set(OPENZL_BUILD_IO_TOOLS ON)
   set(OPENZL_BUILD_LOGGER ON)
   set(OPENZL_BUILD_SDDL_TOOLS ON)
   set(OPENZL_BUILD_TRAINING_TOOLS ON)
+  set(OPENZL_BUILD_ML_SELECTOR ON)
 endif()
 if(OPENZL_BUILD_PYTHON_EXT)
   set(OPENZL_BUILD_LOGGER ON)

tools/ml_selector/CMakeLists.txt

@@ -1,119 +1,102 @@
 # Copyright (c) Meta Platforms, Inc. and affiliates.
+if(OPENZL_BUILD_ML_SELECTOR)
+    include(ExternalProject)
+    include(GNUInstallDirs)
+
+    # Build xgboost as an external project
+    set(XGBOOST_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/xgboost-install")
+    set(XGBOOST_LIB_DIR "${XGBOOST_INSTALL_DIR}/lib")
+
+    # Create install directories at configure time
+    file(MAKE_DIRECTORY ${XGBOOST_INSTALL_DIR}/include)
+    file(MAKE_DIRECTORY ${XGBOOST_LIB_DIR})
+
+    # Add XGBoost as external project due to vs build issues
+    # Turn logs on to prevent polluting stdout when building xgboost
+    # Disable DMLC stack trace to avoid dependency on execinfo.h
+    ExternalProject_Add(xgboost_external
+        GIT_REPOSITORY https://github.com/dmlc/xgboost.git
+        GIT_TAG        ccb511768e13d1670c10be07dea89d0edca138f3 # v3.1.0
+        GIT_SUBMODULES "dmlc-core"
+        GIT_SHALLOW    TRUE
+        CMAKE_ARGS
+            -DCMAKE_INSTALL_PREFIX=${XGBOOST_INSTALL_DIR}
+            -DCMAKE_INSTALL_LIBDIR=lib
+            -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+            -DUSE_OPENMP=OFF
+            -DBUILD_TESTING=OFF
+            -DUSE_CUDA=OFF
+            -DUSE_NCCL=OFF
+            -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+            -DBUILD_STATIC_LIB=ON
+            "-DCMAKE_C_FLAGS=-DDMLC_LOG_STACK_TRACE=0"
+            "-DCMAKE_CXX_FLAGS=-DDMLC_LOG_STACK_TRACE=0"
+            ${XGBOOST_EXTRA_CMAKE_ARGS}
+        BUILD_BYPRODUCTS
+            ${XGBOOST_LIB_DIR}/libxgboost${CMAKE_STATIC_LIBRARY_SUFFIX}
+            ${XGBOOST_LIB_DIR}/libdmlc${CMAKE_STATIC_LIBRARY_SUFFIX}
+        LOG_DOWNLOAD ON
+        LOG_CONFIGURE ON
+        LOG_BUILD ON
+        LOG_INSTALL ON
+    )
 
-include(ExternalProject)
-include(GNUInstallDirs)
-
-# Build xgboost as an external project
-set(XGBOOST_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/xgboost-install")
-set(XGBOOST_LIB_DIR "${XGBOOST_INSTALL_DIR}/lib")
-
-# Create install directories at configure time
-file(MAKE_DIRECTORY ${XGBOOST_INSTALL_DIR}/include)
-file(MAKE_DIRECTORY ${XGBOOST_LIB_DIR})
-
-# Add XGBoost as external project due to vs build issues
-# Turn logs on to prevent polluting stdout when building xgboost
-# Disable DMLC stack trace to avoid dependency on execinfo.h
-ExternalProject_Add(xgboost_external
-    GIT_REPOSITORY https://github.com/dmlc/xgboost.git
-    GIT_TAG        ccb511768e13d1670c10be07dea89d0edca138f3 # v3.1.0
-    GIT_SUBMODULES "dmlc-core"
-    GIT_SHALLOW    TRUE
-    CMAKE_ARGS
-        -DCMAKE_INSTALL_PREFIX=${XGBOOST_INSTALL_DIR}
-        -DCMAKE_INSTALL_LIBDIR=lib
-        -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-        -DUSE_OPENMP=OFF
-        -DBUILD_TESTING=OFF
-        -DUSE_CUDA=OFF
-        -DUSE_NCCL=OFF
-        -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-        -DBUILD_STATIC_LIB=ON
-        "-DCMAKE_C_FLAGS=-DDMLC_LOG_STACK_TRACE=0"
-        "-DCMAKE_CXX_FLAGS=-DDMLC_LOG_STACK_TRACE=0"
-        ${XGBOOST_EXTRA_CMAKE_ARGS}
-    BUILD_BYPRODUCTS
-        ${XGBOOST_LIB_DIR}/libxgboost${CMAKE_STATIC_LIBRARY_SUFFIX}
-        ${XGBOOST_LIB_DIR}/libdmlc${CMAKE_STATIC_LIBRARY_SUFFIX}
-    LOG_DOWNLOAD ON
-    LOG_CONFIGURE ON
-    LOG_BUILD ON
-    LOG_INSTALL ON
-)
-
-# Create imported targets
-add_library(xgboost STATIC IMPORTED GLOBAL)
-set_target_properties(xgboost PROPERTIES
-    IMPORTED_LOCATION
-        ${XGBOOST_LIB_DIR}/libxgboost${CMAKE_STATIC_LIBRARY_SUFFIX}
-    INTERFACE_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include
-)
-add_dependencies(xgboost xgboost_external)
-
-if(WIN32 OR MINGW)
+    # Create imported targets
+    add_library(xgboost STATIC IMPORTED GLOBAL)
     set_target_properties(xgboost PROPERTIES
-        INTERFACE_LINK_LIBRARIES "ws2_32")
-endif()
+        IMPORTED_LOCATION
+            ${XGBOOST_LIB_DIR}/libxgboost${CMAKE_STATIC_LIBRARY_SUFFIX}
+        INTERFACE_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include
+    )
+    add_dependencies(xgboost xgboost_external)
 
-add_library(dmlc STATIC IMPORTED GLOBAL)
-set_target_properties(dmlc PROPERTIES
-    IMPORTED_LOCATION ${XGBOOST_LIB_DIR}/libdmlc${CMAKE_STATIC_LIBRARY_SUFFIX}
-    INTERFACE_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include
-)
-add_dependencies(dmlc xgboost_external)
-
-add_library(ml_selector
-    ml_selector_trainer.cpp
-    ml_selector_trainer.h
-    ml_features.cpp
-    ml_features.h
-)
-
-target_include_directories(ml_selector
-    PUBLIC
-        ${PROJECT_SOURCE_DIR}
-        ${XGBOOST_INSTALL_DIR}/include
-)
-
-# Disable DMLC stack trace to avoid dependency on execinfo.h
-target_compile_definitions(ml_selector
-    PUBLIC
-        DMLC_LOG_STACK_TRACE=0
-)
-
-target_link_libraries(ml_selector
-        PUBLIC
-        openzl
-        openzl_cpp
-        xgboost
-        dmlc
+    if(WIN32 OR MINGW)
+        set_target_properties(xgboost PROPERTIES
+            INTERFACE_LINK_LIBRARIES "ws2_32")
+    endif()
+
+    add_library(dmlc STATIC IMPORTED GLOBAL)
+    set_target_properties(dmlc PROPERTIES
+        IMPORTED_LOCATION
+            ${XGBOOST_LIB_DIR}/libdmlc${CMAKE_STATIC_LIBRARY_SUFFIX}
+        INTERFACE_INCLUDE_DIRECTORIES ${XGBOOST_INSTALL_DIR}/include
+    )
+    add_dependencies(dmlc xgboost_external)
+
+    add_library(ml_selector
+        ml_selector_trainer.cpp
+        ml_selector_trainer.h
+        ml_features.cpp
+        ml_features.h
     )
 
-add_dependencies(ml_selector xgboost_external)
+    target_include_directories(ml_selector
+        PUBLIC
+            ${PROJECT_SOURCE_DIR}
+            ${XGBOOST_INSTALL_DIR}/include
+    )
 
-if (OPENZL_BUILD_TESTS)
-    add_executable(test_ml_selector tests/test_mlSelectorGraph.cpp)
-    add_dependencies(test_ml_selector xgboost_external)
+    # Disable DMLC stack trace to avoid dependency on execinfo.h
+    target_compile_definitions(ml_selector
+        PUBLIC
+            DMLC_LOG_STACK_TRACE=0
+    )
 
-    target_link_libraries(test_ml_selector
-        PRIVATE
-            ml_selector
+    target_link_libraries(ml_selector
+            PUBLIC
             openzl
             openzl_cpp
-            openzl_test_support
-            tools_training
-            GTest::gtest_main
-    )
-    apply_openzl_compile_options_to_target(test_ml_selector)
+            xgboost
+            dmlc
+        )
 
-    gtest_discover_tests(test_ml_selector)
+    add_dependencies(ml_selector xgboost_external)
 
-    if (OPENZL_ALLOW_INTROSPECTION)
-        add_executable(test_ml_selector_trainer
-            tests/test_mlSelectorTrainer.cpp)
-        add_dependencies(test_ml_selector_trainer xgboost_external)
+    if (OPENZL_BUILD_TESTS)
+        add_executable(test_ml_selector tests/test_mlSelectorGraph.cpp)
+        add_dependencies(test_ml_selector xgboost_external)
 
-        target_link_libraries(test_ml_selector_trainer
+        target_link_libraries(test_ml_selector
             PRIVATE
                 ml_selector
                 openzl
@@ -122,7 +105,26 @@ if (OPENZL_BUILD_TESTS)
                 tools_training
                 GTest::gtest_main
         )
-        apply_openzl_compile_options_to_target(test_ml_selector_trainer)
-        gtest_discover_tests(test_ml_selector_trainer)
+        apply_openzl_compile_options_to_target(test_ml_selector)
+
+        gtest_discover_tests(test_ml_selector)
+
+        if (OPENZL_ALLOW_INTROSPECTION)
+            add_executable(test_ml_selector_trainer
+                tests/test_mlSelectorTrainer.cpp)
+            add_dependencies(test_ml_selector_trainer xgboost_external)
+
+            target_link_libraries(test_ml_selector_trainer
+                PRIVATE
+                    ml_selector
+                    openzl
+                    openzl_cpp
+                    openzl_test_support
+                    tools_training
+                    GTest::gtest_main
+            )
+            apply_openzl_compile_options_to_target(test_ml_selector_trainer)
+            gtest_discover_tests(test_ml_selector_trainer)
+        endif()
     endif()
 endif()