build: add libcurl to contrib build system (#173)

* build: add libcurl to contrib build system

- Add CURL to BUILD_TYPE/VALID_BUILD_TYPES
- Add curl-8.12.1 source archive config (SHA256 validated)
- Add curl.cmake build macro with platform-native TLS backends
  (Schannel on Windows, SecureTransport on macOS, OpenSSL on Linux)
- Add libcurl-dev packages to Dockerfiles
- Disable LDAP and libpsl in curl build

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix: address PR review — fix curl build, integrate curl into Arrow S3

- Fix ZIP_ARGS quoting in curl.cmake (fixes Windows CI extraction failure)
- Use ${BUILD_SHARED_LIBRARIES} instead of hardcoded BUILD_SHARED_LIBS=ON
- Move CURL build before ARROW so Arrow can find contrib-built curl
- Enable ARROW_S3 with AWSSDK_SOURCE=BUNDLED to use contrib curl
- Align BUILD_TYPE help string with VALID_BUILD_TYPES (add EIGEN, HDF5)
- Consolidate Dockerfile yum installs with cache cleanup

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: timosachsenberg <sachsenb@ibminode05.Cs.Uni-Tuebingen.De>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: timosachsenberg <sachsenb@ibminode06.Cs.Uni-Tuebingen.De>

Author: 4 people

CMakeLists.txt

@@ -91,8 +91,8 @@
 ##       - however: exec_process seems to handle our arguments incorrectly (at least on Windows). Check functionality before you commit!
 
 # Determine what to build (all or single library)
-set( BUILD_TYPE LIST CACHE STRING "Can be used to restrict building to a single library: ALL,LIBSVM,XERCESC,BOOST,COINOR,BZIP2,ZLIB,GLPK,KISSFFT,OPENMP,ARROW,LIBZIP")
-set( VALID_BUILD_TYPES "ALL" "LIBSVM" "XERCESC" "BOOST" "COINOR" "BZIP2" "ZLIB" "GLPK" "EIGEN" "KISSFFT" "HDF5" "OPENMP" "ARROW" "LIBZIP")
+set( BUILD_TYPE LIST CACHE STRING "Can be used to restrict building to a single library: ALL,LIBSVM,XERCESC,BOOST,COINOR,BZIP2,ZLIB,GLPK,EIGEN,KISSFFT,HDF5,OPENMP,ARROW,LIBZIP,CURL")
+set( VALID_BUILD_TYPES "ALL" "LIBSVM" "XERCESC" "BOOST" "COINOR" "BZIP2" "ZLIB" "GLPK" "EIGEN" "KISSFFT" "HDF5" "OPENMP" "ARROW" "LIBZIP" "CURL")
 
 # check build type
 if (BUILD_TYPE STREQUAL "HELP" OR BUILD_TYPE STREQUAL "LIST")
@@ -198,6 +198,7 @@ set(HDF5_DIR ${CONTRIB_BIN_SOURCE_DIR}/hdfsrc)
 set(OPENMP_DIR ${CONTRIB_BIN_SOURCE_DIR}/openmp-12.0.1.src)
 set(ARROW_DIR ${CONTRIB_BIN_SOURCE_DIR}/arrow-apache-arrow-23.0.0/cpp)
 set(LIBZIP_DIR ${CONTRIB_BIN_SOURCE_DIR}/libzip-1.11.4)
+set(CURL_DIR ${CONTRIB_BIN_SOURCE_DIR}/curl-8.12.1)
 
 ## source(archive) paths
 ## PLEASE upload all source archives to
@@ -212,7 +213,7 @@ set(LIBZIP_DIR ${CONTRIB_BIN_SOURCE_DIR}/libzip-1.11.4)
 ##  set(ARCHIVE_ZLIB_SHA256 "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
 ## NOTE: for local development you can simply place the *.tar.gz in the binary
 ##       directory to prevent the download and sha256 verification
-## LIBRARIES: BZIP2, ZLIB, BOOST, XERCES, LIBSVM, GLPK, COINOR, EIGEN, KISSFFT, HDF5, OPENMP, ARROW, LIBZIP
+## LIBRARIES: BZIP2, ZLIB, BOOST, XERCES, LIBSVM, GLPK, COINOR, EIGEN, KISSFFT, HDF5, OPENMP, ARROW, LIBZIP, CURL
 
 set(ARCHIVE_BZIP2 bzip2-1.0.5.tar.gz)
 set(ARCHIVE_BZIP2_TAR bzip2-1.0.5.tar)
@@ -266,6 +267,10 @@ set(ARCHIVE_LIBZIP libzip-1.11.4.tar.gz)
 set(ARCHIVE_LIBZIP_TAR libzip-1.11.4.tar)
 set(ARCHIVE_LIBZIP_SHA256 "82e9f2f2421f9d7c2466bbc3173cd09595a88ea37db0d559a9d0a2dc60dc722e")
 
+set(ARCHIVE_CURL curl-8.12.1.tar.gz)
+set(ARCHIVE_CURL_TAR curl-8.12.1.tar)
+set(ARCHIVE_CURL_SHA256 "7b40ea64947e0b440716a4d7f0b7aa56230a5341c8377d7b609649d4aea8dbcf")
+
 ## necessary for clean up .. change if install process of library changes
 set(INCLUDE_DIR_BOOST ${CONTRIB_BIN_INCLUDE_DIR}/boost)
 set(INCLUDE_DIR_XERCES ${CONTRIB_BIN_INCLUDE_DIR}/xercesc)
@@ -275,6 +280,7 @@ set(INCLUDE_DIR_EIGEN ${CONTRIB_BIN_INCLUDE_DIR}/eigen3)
 set(INCLUDE_DIR_KISSFFT ${CONTRIB_BIN_INCLUDE_DIR}/kissfft)
 set(INCLUDE_DIR_OPENMP ${CONTRIB_BIN_INCLUDE_DIR}/openmp.src)
 set(INCLUDE_DIR_ARROW ${CONTRIB_BIN_INCLUDE_DIR}/arrow)
+set(INCLUDE_DIR_CURL ${CONTRIB_BIN_INCLUDE_DIR}/curl)
 
 ## hack for simple libs that do not have include directories
 set(INCLUDE_FILES_LIBSVM ${CONTRIB_BIN_INCLUDE_DIR}/svm.h)
@@ -490,6 +496,7 @@ include ("${CMAKE_SOURCE_DIR}/libraries.cmake/hdf5.cmake")
 include ("${CMAKE_SOURCE_DIR}/libraries.cmake/openmp.cmake")
 include ("${CMAKE_SOURCE_DIR}/libraries.cmake/arrow.cmake")
 include ("${CMAKE_SOURCE_DIR}/libraries.cmake/libzip.cmake")
+include ("${CMAKE_SOURCE_DIR}/libraries.cmake/curl.cmake")
 
 ## build mac os x specific C/C++ flags
 set( OPENMS_CONTRIB_MACOSX_MIXED_MODE 0 CACHE INTERNAL "building multiple architectures on MacOSX" FORCE)
@@ -544,6 +551,15 @@ if (BUILD_TYPE STREQUAL "ALL" OR "BOOST" IN_LIST BUILD_TYPE)
   OPENMS_CONTRIB_BUILD_BOOST()
 endif()
 
+##################################################
+###       CURL                                 ###
+##################################################
+if (BUILD_TYPE STREQUAL "ALL" OR "CURL" IN_LIST BUILD_TYPE)
+  OPENMS_CLEAN_LIB("CURL")
+  OPENMS_CONTRIB_BUILD_CURL()
+  OPENMS_COPY_LIBS("CURL")
+endif()
+
 ##################################################
 ###       ARROW                                ###
 ##################################################

dockerfiles/Dockerfile

@@ -31,6 +31,7 @@ RUN apt-get -y update \
     libqt5opengl5-dev \
     libeigen3-dev \
     coinor-libcoinmp-dev \
+    libcurl4-openssl-dev \
   && rm -rf /var/lib/apt/lists/* \
   && update-ca-certificates
 

dockerfiles/pyopenms/manylinux/ARM64_Dockerfile

@@ -7,9 +7,12 @@ ARG OPENMS_VERSION="latest"
 # Make source files from context available in docker
 COPY . /contrib
 
-RUN yum install -y wget
-RUN yum install -y xz qt6-qtbase-devel qt6-qtsvg-devel
-RUN yum install -y libtool cmake3
+RUN yum install -y \
+    wget \
+    xz qt6-qtbase-devel qt6-qtsvg-devel \
+    libtool cmake3 libcurl-devel \
+  && yum clean all \
+  && rm -rf /var/cache/yum
 
 # Build contrib for the current checked out branch
 # During cleanup: removes archives

dockerfiles/pyopenms/manylinux/Dockerfile

@@ -7,9 +7,12 @@ ARG OPENMS_VERSION="latest"
 # Make source files from context available in docker
 COPY . /contrib
 
-RUN yum install -y wget
-RUN yum install -y xz qt6-qtbase-devel qt6-qtsvg-devel
-RUN yum install -y libtool cmake3
+RUN yum install -y \
+    wget \
+    xz qt6-qtbase-devel qt6-qtsvg-devel \
+    libtool cmake3 libcurl-devel \
+  && yum clean all \
+  && rm -rf /var/cache/yum
 
 # Build contrib for the current checked out branch
 # During cleanup: removes archives

docs/plans/2026-03-06-curl-arrow-integration-design.md

@@ -0,0 +1,44 @@
+# Design: Integrate curl into Arrow's dependency chain
+
+## Problem
+
+jpfeuffer's review on PR #173 asks that the dependency chain
+curl -> aws-sdk -> arrow/parquet uses the same curl built by contrib.
+
+Currently:
+- CURL is built after Arrow (wrong order)
+- Arrow has no S3/curl options enabled
+- Arrow does not bundle curl — it requires it via `find_package(CURL)`
+
+## Approach
+
+Build CURL before ARROW and configure Arrow to use it via the existing
+`CMAKE_PREFIX_PATH` mechanism (same pattern used for boost, zlib, bzip2).
+
+## Changes
+
+### 1. CMakeLists.txt — Reorder build
+
+Move the CURL build block before ARROW. New order:
+... -> BOOST -> CURL -> ARROW -> ...
+
+### 2. libraries.cmake/arrow.cmake — Enable S3 with bundled AWS SDK
+
+Add to both Windows and Linux/macOS cmake invocations:
+- `-DARROW_S3=ON` — enables S3 filesystem, triggers AWS SDK + curl
+- `-DAWSSDK_SOURCE=BUNDLED` — Arrow builds AWS SDK from source, using our curl
+
+Curl is found automatically via `CMAKE_PREFIX_PATH=${PROJECT_BINARY_DIR}`
+which is already set in arrow.cmake. No extra flags needed.
+
+### 3. No other changes needed
+
+- curl.cmake already installs to `${PROJECT_BINARY_DIR}`
+- Dockerfiles already have `libcurl-dev`
+- No new dependency resolution mechanism required
+
+## Key facts
+
+- Arrow 23.0.0 does NOT support `CURL_SOURCE=BUNDLED` — curl must be a system/pre-built dependency
+- Arrow finds curl via standard `find_package(CURL REQUIRED)` in its `find_curl()` macro
+- AWS SDK can be BUNDLED by Arrow, but still requires external curl

libraries.cmake/arrow.cmake

@@ -42,6 +42,8 @@ if (MSVC)
                           -D ARROW_WITH_BZIP2=ON
                           -D ARROW_WITH_ZSTD=ON
                           -D ARROW_WITH_SNAPPY=ON
+                          -D ARROW_S3=ON
+                          -D AWSSDK_SOURCE=BUNDLED
                           -D Snappy_SOURCE=BUNDLED
                           -D zstd_SOURCE=BUNDLED
                           -D Thrift_SOURCE=BUNDLED
@@ -147,6 +149,8 @@ else() ## Linux/MacOS
                           "-DARROW_WITH_BZIP2=ON"
                           "-DARROW_WITH_ZSTD=ON"
                           "-DARROW_WITH_SNAPPY=ON"
+                          "-DARROW_S3=ON"
+                          "-DAWSSDK_SOURCE=BUNDLED"
                           "-DSnappy_SOURCE=BUNDLED"
                           "-Dzstd_SOURCE=BUNDLED"
                           "-DThrift_SOURCE=BUNDLED"

libraries.cmake/curl.cmake

@@ -0,0 +1,92 @@
+##################################################
+###       libcurl                              ###
+##################################################
+
+MACRO( OPENMS_CONTRIB_BUILD_CURL )
+OPENMS_LOGHEADER_LIBRARY("curl")
+
+if(MSVC)
+  set(ZIP_ARGS x -y -osrc)
+else()
+  set(ZIP_ARGS xzf)
+endif()
+OPENMS_SMARTEXTRACT(ZIP_ARGS ARCHIVE_CURL "CURL" "CMakeLists.txt")
+
+# curl doesn't allow insource builds
+set(_CURL_BUILD_DIR "${CURL_DIR}/build")
+file(TO_NATIVE_PATH "${_CURL_BUILD_DIR}" _CURL_NATIVE_BUILD_DIR)
+execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${_CURL_NATIVE_BUILD_DIR})
+
+# Platform-specific TLS backend
+if(MSVC)
+  set(_CURL_TLS_OPTION "-DCURL_USE_SCHANNEL=ON")
+elseif(APPLE)
+  set(_CURL_TLS_OPTION "-DCURL_USE_SECTRANSP=ON")
+else()
+  set(_CURL_TLS_OPTION "-DCURL_USE_OPENSSL=ON")
+endif()
+
+if(APPLE)
+  set(_CURL_EXTRA_ARGS
+      "-DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}"
+      "-DCMAKE_OSX_SYSROOT=${CMAKE_OSX_SYSROOT}"
+      "-DCMAKE_MACOSX_RPATH=TRUE"
+    )
+else()
+  set(_CURL_EXTRA_ARGS "")
+endif()
+
+message(STATUS "Generating curl build system .. ")
+execute_process(COMMAND ${CMAKE_COMMAND}
+                -G "${CMAKE_GENERATOR}"
+                ${ARCHITECTURE_OPTION_CMAKE}
+                -D CMAKE_BUILD_TYPE=Release
+                -D CMAKE_INSTALL_PREFIX=${PROJECT_BINARY_DIR}
+                -D BUILD_SHARED_LIBS=${BUILD_SHARED_LIBRARIES}
+                -D BUILD_CURL_EXE=OFF
+                -D BUILD_TESTING=OFF
+                -D CURL_DISABLE_LDAP=ON
+                -D CURL_USE_LIBPSL=OFF
+                ${_CURL_TLS_OPTION}
+                ${_CURL_EXTRA_ARGS}
+                ${CURL_DIR}
+                WORKING_DIRECTORY ${_CURL_NATIVE_BUILD_DIR}
+                OUTPUT_VARIABLE _CURL_CMAKE_OUT
+                ERROR_VARIABLE _CURL_CMAKE_ERR
+                RESULT_VARIABLE _CURL_CMAKE_SUCCESS)
+
+# output to logfile
+file(APPEND ${LOGFILE} ${_CURL_CMAKE_OUT})
+file(APPEND ${LOGFILE} ${_CURL_CMAKE_ERR})
+
+if (NOT _CURL_CMAKE_SUCCESS EQUAL 0)
+  message(FATAL_ERROR "Generating curl build system .. failed")
+else()
+  message(STATUS "Generating curl build system .. done")
+endif()
+
+# the install target on windows has a different name than on mac/lnx
+if(MSVC)
+    set(_CURL_INSTALL_TARGET "INSTALL")
+else()
+    set(_CURL_INSTALL_TARGET "install")
+endif()
+
+message(STATUS "Building curl (Release) .. ")
+execute_process(COMMAND ${CMAKE_COMMAND} --build ${_CURL_NATIVE_BUILD_DIR} --target ${_CURL_INSTALL_TARGET} --config Release
+                WORKING_DIRECTORY ${_CURL_NATIVE_BUILD_DIR}
+                OUTPUT_VARIABLE _CURL_BUILD_OUT
+                ERROR_VARIABLE _CURL_BUILD_ERR
+                RESULT_VARIABLE _CURL_BUILD_SUCCESS)
+
+# output to logfile
+file(APPEND ${LOGFILE} ${_CURL_BUILD_OUT})
+file(APPEND ${LOGFILE} ${_CURL_BUILD_ERR})
+
+if (NOT _CURL_BUILD_SUCCESS EQUAL 0)
+  message(FATAL_ERROR "Building curl (Release) .. failed")
+else()
+  message(STATUS "Building curl (Release) .. done")
+endif()
+
+ENDMACRO( OPENMS_CONTRIB_BUILD_CURL )