diff --git a/Makefile b/Makefile index 67e19076c78f..32a5e29b8e1f 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,7 @@ POWERPC_LLVM_CONFIG_LIB=$(if $(WITH_POWERPC), powerpc, ) PTX_CXX_FLAGS=$(if $(WITH_NVPTX), -DWITH_NVPTX, ) PTX_LLVM_CONFIG_LIB=$(if $(WITH_NVPTX), nvptx, ) -PTX_DEVICE_INITIAL_MODULES=$(if $(WITH_NVPTX), libdevice.compute_20.10.bc libdevice.compute_30.10.bc libdevice.compute_35.10.bc, ) +PTX_DEVICE_INITIAL_MODULES=$(if $(WITH_NVPTX), $(BUILD_DIR)/initmod_ptx.libdevice_ll.o, ) OPENCL_CXX_FLAGS=$(if $(WITH_OPENCL), -DWITH_OPENCL, ) OPENCL_LLVM_CONFIG_LIB=$(if $(WITH_OPENCL), , ) @@ -972,7 +972,7 @@ INITIAL_MODULES = $(RUNTIME_CPP_COMPONENTS:%=$(BUILD_DIR)/initmod.%_32.o) \ $(HTML_TEMPLATE_FILES:%=$(BUILD_DIR)/html_template.%.o) \ $(BUILD_DIR)/initmod.inlined_c.o \ $(RUNTIME_LL_COMPONENTS:%=$(BUILD_DIR)/initmod.%_ll.o) \ - $(PTX_DEVICE_INITIAL_MODULES:libdevice.%.bc=$(BUILD_DIR)/initmod_ptx.%_ll.o) + $(PTX_DEVICE_INITIAL_MODULES) TEST_DEPS = $(BIN_DIR)/libHalide.$(SHARED_EXT) $(INCLUDE_DIR)/Halide.h $(RUNTIME_EXPORTED_INCLUDES) ifneq (,$(WITH_EXCEPTIONS)) @@ -1237,8 +1237,8 @@ $(BUILD_DIR)/initmod.%_h.cpp: $(BIN_DIR)/binary2cpp $(SRC_DIR)/runtime/%.h $(BUILD_DIR)/initmod.inlined_c.cpp: $(BIN_DIR)/binary2cpp $(SRC_DIR)/runtime/halide_buffer_t.cpp ./$(BIN_DIR)/binary2cpp halide_internal_initmod_inlined_c < $(SRC_DIR)/runtime/halide_buffer_t.cpp > $@ -$(BUILD_DIR)/initmod_ptx.%_ll.cpp: $(BIN_DIR)/binary2cpp $(SRC_DIR)/runtime/nvidia_libdevice_bitcode/libdevice.%.bc - ./$(BIN_DIR)/binary2cpp halide_internal_initmod_ptx_$(basename $*)_ll < $(SRC_DIR)/runtime/nvidia_libdevice_bitcode/libdevice.$*.bc > $@ +$(BUILD_DIR)/initmod_ptx.libdevice_ll.cpp: $(BIN_DIR)/binary2cpp $(SRC_DIR)/runtime/nvidia_libdevice_bitcode/libdevice.10.bc + ./$(BIN_DIR)/binary2cpp halide_internal_initmod_ptx_libdevice_ll < $(SRC_DIR)/runtime/nvidia_libdevice_bitcode/libdevice.10.bc > $@ $(BUILD_DIR)/c_template.%.cpp: $(BIN_DIR)/binary2cpp $(SRC_DIR)/%.template.cpp ./$(BIN_DIR)/binary2cpp halide_c_template_$* < $(SRC_DIR)/$*.template.cpp > $@ diff --git a/src/LLVM_Runtime_Linker.cpp b/src/LLVM_Runtime_Linker.cpp index 60faee4c47fe..35273e824724 100644 --- a/src/LLVM_Runtime_Linker.cpp +++ b/src/LLVM_Runtime_Linker.cpp @@ -252,9 +252,7 @@ DECLARE_NO_INITMOD(windows_aarch64_cpu_features_arm) #endif // WITH_AARCH64 #ifdef WITH_NVPTX -DECLARE_LL_INITMOD(ptx_compute_20) -DECLARE_LL_INITMOD(ptx_compute_30) -DECLARE_LL_INITMOD(ptx_compute_35) +DECLARE_LL_INITMOD(ptx_libdevice) #endif // WITH_NVPTX #if defined(WITH_D3D12) && defined(WITH_X86) @@ -1412,22 +1410,7 @@ std::unique_ptr get_initial_module_for_ptx_device(Target target, l std::vector> modules; modules.push_back(get_initmod_ptx_dev_ll(c)); - std::unique_ptr module; - - // This table is based on the guidance at: - // http://docs.nvidia.com/cuda/libdevice-users-guide/basic-usage.html#linking-with-libdevice - if (target.has_feature(Target::CUDACapability35)) { - module = get_initmod_ptx_compute_35_ll(c); - } else if (target.features_any_of({Target::CUDACapability32, - Target::CUDACapability50})) { - // For some reason sm_32 and sm_50 use libdevice 20 - module = get_initmod_ptx_compute_20_ll(c); - } else if (target.has_feature(Target::CUDACapability30)) { - module = get_initmod_ptx_compute_30_ll(c); - } else { - module = get_initmod_ptx_compute_20_ll(c); - } - modules.push_back(std::move(module)); + modules.push_back(get_initmod_ptx_libdevice_ll(c)); link_modules(modules, target); diff --git a/src/runtime/CMakeLists.txt b/src/runtime/CMakeLists.txt index 95e78d11e459..f406ff7eeaca 100644 --- a/src/runtime/CMakeLists.txt +++ b/src/runtime/CMakeLists.txt @@ -131,14 +131,6 @@ set(RUNTIME_LL # keep-sorted end ) -set(RUNTIME_BC - # keep-sorted start - compute_20 - compute_30 - compute_35 - # keep-sorted end -) - set(RUNTIME_HEADER_FILES # keep-sorted start HalideBuffer.h @@ -348,19 +340,16 @@ foreach (i IN LISTS RUNTIME_LL) target_sources(Halide_initmod PRIVATE ${INITMOD}) endforeach () -foreach (i IN LISTS RUNTIME_BC) - set(INITMOD "_initmod_ptx_${i}.cpp") - set(RT_BC "${CMAKE_CURRENT_SOURCE_DIR}/nvidia_libdevice_bitcode/libdevice.${i}.10.bc") - - add_custom_command( - OUTPUT "${INITMOD}" - COMMAND - binary2cpp "halide_internal_initmod_ptx_${i}_ll" < "$" > "${INITMOD}" - DEPENDS binary2cpp "${RT_BC}" - VERBATIM - ) - target_sources(Halide_initmod PRIVATE ${INITMOD}) -endforeach () +set(RT_BC "${CMAKE_CURRENT_SOURCE_DIR}/nvidia_libdevice_bitcode/libdevice.10.bc") +add_custom_command( + OUTPUT "_initmod_ptx_libdevice.cpp" + COMMAND + binary2cpp "halide_internal_initmod_ptx_libdevice_ll" < "$" > + "_initmod_ptx_libdevice.cpp" + DEPENDS binary2cpp "${RT_BC}" + VERBATIM +) +target_sources(Halide_initmod PRIVATE "_initmod_ptx_libdevice.cpp") add_custom_command( OUTPUT "_initmod_inlined_c.cpp" diff --git a/src/runtime/nvidia_libdevice_bitcode/README.md b/src/runtime/nvidia_libdevice_bitcode/README.md new file mode 100644 index 000000000000..f25ecf033117 --- /dev/null +++ b/src/runtime/nvidia_libdevice_bitcode/README.md @@ -0,0 +1,26 @@ +# NVIDIA `libdevice` bitcode + +`libdevice.10.bc` is NVIDIA's device-side math library (`sin`, `exp`, `sqrt`, +etc.), distributed as a single, architecture-generic LLVM bitcode module as part +of the CUDA Toolkit. It is vendored here verbatim — Halide does not build it — +and is linked into every GPU kernel module produced by Halide's CUDA/PTX device +codegen. See: + +- `src/CodeGen_PTX_Dev.cpp` — `__nvvm_reflect` / `nvvm-reflect-ftz` handling + that libdevice relies on for its accuracy-vs-speed switches. +- `src/LLVM_Runtime_Linker.cpp` — links this module into the PTX device initial + module. +- `Makefile` / `src/runtime/CMakeLists.txt` — run the file through `binary2cpp` + to embed it as a byte array in the compiler binary. + +## Updating this file + +`libdevice.10.bc` lives at `nvvm/libdevice/libdevice.10.bc` inside any CUDA +Toolkit install (e.g. `$CUDA_HOME/nvvm/libdevice/libdevice.10.bc`). To update +the vendored copy, just copy that file over this one — no other files in this +directory need to change. The current copy was taken from CUDA Toolkit 11.6. + +## License + +This file is distributed under, and its use governed by, NVIDIA's CUDA Toolkit +End User License Agreement: https://docs.nvidia.com/cuda/eula diff --git a/src/runtime/nvidia_libdevice_bitcode/libdevice.10.bc b/src/runtime/nvidia_libdevice_bitcode/libdevice.10.bc new file mode 100644 index 000000000000..ef3ae8d81946 Binary files /dev/null and b/src/runtime/nvidia_libdevice_bitcode/libdevice.10.bc differ diff --git a/src/runtime/nvidia_libdevice_bitcode/libdevice.compute_20.10.bc b/src/runtime/nvidia_libdevice_bitcode/libdevice.compute_20.10.bc deleted file mode 100644 index 455b676b4973..000000000000 Binary files a/src/runtime/nvidia_libdevice_bitcode/libdevice.compute_20.10.bc and /dev/null differ diff --git a/src/runtime/nvidia_libdevice_bitcode/libdevice.compute_30.10.bc b/src/runtime/nvidia_libdevice_bitcode/libdevice.compute_30.10.bc deleted file mode 100644 index 031525e97913..000000000000 Binary files a/src/runtime/nvidia_libdevice_bitcode/libdevice.compute_30.10.bc and /dev/null differ diff --git a/src/runtime/nvidia_libdevice_bitcode/libdevice.compute_35.10.bc b/src/runtime/nvidia_libdevice_bitcode/libdevice.compute_35.10.bc deleted file mode 100644 index 2634126e4b66..000000000000 Binary files a/src/runtime/nvidia_libdevice_bitcode/libdevice.compute_35.10.bc and /dev/null differ