Merge branch 'develop' into feature/isobaricWorkflow

Author: timosachsenberg

.github/workflows/openms_ci_matrix_full.yml

@@ -81,7 +81,6 @@ jobs:
             compiler: g++
             compiler_ver: 11
 
-
           - os: ubuntu-22.04
             compiler: clang++
             compiler_ver: 15
@@ -171,7 +170,7 @@ jobs:
 
     # Create the changelog for the release from our overall changelog
     - id: create_changelog
-      if: matrix.compiler != 'clang++' && startsWith(steps.extract_branch.outputs.RUN_NAME,'release') && startsWith(matrix.os, 'ubuntu')
+      if: matrix.compiler != 'clang++' && startsWith(steps.extract_branch.outputs.RUN_NAME,'release') && startsWith(matrix.os, 'ubuntu') && runner.arch == 'arm64'
       shell: bash
       name: Create changelog for release on Ubuntu + GCC
       run: |
@@ -371,7 +370,7 @@ jobs:
 
    # Upload the changelog to the same artifact that we will add the installers to.
     - name: Upload changelog as artifact
-      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(steps.extract_branch.outputs.RUN_NAME,'release') && startsWith(matrix.os, 'ubuntu')
+      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(steps.extract_branch.outputs.RUN_NAME,'release') && startsWith(matrix.os, 'ubuntu') && runner.arch == 'arm64'
       uses: actions/upload-artifact@v4
       with:
         name: changelog.txt
@@ -448,7 +447,7 @@ jobs:
           BUILD_TYPE: "Release"
           OPENMP: "ON"
           BOOST_USE_STATIC: ${{ steps.set-vars.outputs.static_boost }}
-          COMPILE_PXDS: ${{ startsWith(matrix.os, 'ubuntu') && github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'develop' && 'ON' || 'OFF' }} # test generation of pxds
+          COMPILE_PXDS: ${{ (startsWith(matrix.os, 'ubuntu') && github.event_name == 'pull_request' && github.event.pull_request.base.ref == 'develop' && runner.arch == 'arm64') && 'ON' || 'OFF' }}
           #  BUILD_FLAGS: "-p:CL_MPCount=2" # For VS Generator and MSBuild
           BUILD_FLAGS: "-j${{ steps.cpu-cores.outputs.count }}" # Ninja will otherwise use all cores (doesn't go well in GHA).
           CMAKE_CCACHE_EXE: "ccache"
@@ -543,7 +542,7 @@ jobs:
 
     #Zip the documentation first, since there are :: which make the upload action choke.
     - name: Zip Documentation
-      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu')
+      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu') && runner.arch == 'arm64'
       uses: thedoctor0/zip-release@0.7.6
       with:
         type: 'zip'
@@ -553,7 +552,7 @@ jobs:
 
    # Compress the source
     - name: Create tarball
-      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu')
+      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu') && runner.arch == 'arm64'
       run: |
         tar --exclude='bld/*' \
             --exclude='OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz' \
@@ -565,7 +564,7 @@ jobs:
         
    # Upload the source tar
     - name: Upload source tar as artifact
-      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu')
+      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu') && runner.arch == 'arm64'
       uses: actions/upload-artifact@v4
       with:
         name: OpenMS-${{ steps.create_changelog.outputs.version_number }}.tar.gz
@@ -576,7 +575,7 @@ jobs:
     # Only upload docs when we are building the package, use the ubuntu build simply 'cause its fast
 
     - name: Upload Documentation as artifacts
-      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu')
+      if: steps.set-vars.outputs.pkg_type != 'none' && startsWith(matrix.os, 'ubuntu') && runner.arch == 'arm64'
       uses: actions/upload-artifact@v4
       with:
         name: documentation
@@ -585,7 +584,8 @@ jobs:
 
     - name: Generate KNIME descriptors and payloads
     # We never want to build the KNIME update site on our second Ubuntu + Clang matrix entry even if inputs.knime is true, so we check for that specific os + compiler combo
-      if: steps.set-vars.outputs.pkg_type != 'none' || ( inputs.knime && !(startsWith(matrix.os, 'ubuntu') && startsWith(matrix.compiler, 'clang++') ) )
+    # we also don't want it from both the arm and x86_64 builds
+      if: (steps.set-vars.outputs.pkg_type != 'none' && !(startsWith(matrix.os, 'ubuntu') && runner.arch == 'arm64')) || ( inputs.knime && !(startsWith(matrix.os, 'ubuntu') && (startsWith(matrix.compiler, 'clang++') || runner.arch == 'arm64') ) )
       shell: bash
       run: |
         cd $GITHUB_WORKSPACE/OpenMS/bld/
@@ -595,7 +595,8 @@ jobs:
         
     - name: Upload KNIME payload and descriptors as artifacts
     # We never want to build the KNIME update site on our second Ubuntu + Clang matrix entry even if inputs.knime is true, so we check for that specific os + compiler combo
-      if: steps.set-vars.outputs.pkg_type != 'none' ||  ( inputs.knime && !(startsWith(matrix.os, 'ubuntu') && startsWith(matrix.compiler, 'clang++') ) )
+    # we also don't want it from both the arm and x86_64 builds
+      if: (steps.set-vars.outputs.pkg_type != 'none' && !(startsWith(matrix.os, 'ubuntu') && runner.arch == 'arm64')) ||  ( inputs.knime && !(startsWith(matrix.os, 'ubuntu') && (startsWith(matrix.compiler, 'clang++') || runner.arch == 'arm64') ) )
       uses: actions/upload-artifact@v4
       with:
         name: ${{ format('knime-{0}{1}', steps.set-vars.outputs.tp_folder, runner.arch == 'arm64' && '-arm64' || '') }}

.github/workflows/pyopenms-wheels.yml

@@ -513,7 +513,7 @@ jobs:
 
   build-lnx:
     runs-on: [ubuntu-latest]
-    container: ghcr.io/openms/contrib_manylinux_2_34:latest
+    container: ghcr.io/openms/contrib_manylinux_2_34:latest-x86_64
 
     steps:
     # Cancels older builds if still running
@@ -537,17 +537,139 @@ jobs:
       uses: SimenB/github-actions-cpu-cores@v2
       id: cpu-cores
 
-    - name: Setup cmake
-      uses: jwlawson/actions-setup-cmake@v2
+    - name: Build on manylinux for broad compatibility
+      shell: bash
+      run: |
+        PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
+        yum install -y jq
+        yum install -y ninja-build # Install Ninja for build files
+        yum install -y cmake # Install CMake
+        cmake --version
+
+        mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/
+        mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/before_fix/
+
+        LD_OLD_LIBRARY_PATH=$LD_LIBRARY_PATH
+
+        # compile and configure OpenMS
+        for py in $(echo "${PYTHON_VERSIONS}" | jq -r '.[]'); do
+
+          # Check if there is already and existing build,
+          # if so: clean previous pyopenms libs,
+          # do at the top of the python version loop so that we leave the final libraries for testing
+
+          if [ -d "OpenMS/bld/pyOpenMS/pyopenms" ]; then
+            rm OpenMS/bld/pyOpenMS/pyopenms/_pyopenms*.so
+          fi
+        
+          py=$(echo "$py" | tr -d " \n")
+          pynodot=$(echo "$py" | tr -d ".")
+          PYBIN=$(echo /opt/python/cp*${pynodot})
+          
+          # Create a venv for this python version
+          "$PYBIN/bin/python" -m venv $pynodot
+          source $pynodot/bin/activate
+          
+          "$PYBIN/bin/pip" install --upgrade pip
+          # pip install all the stuff
+          "$PYBIN/bin/pip" install -r $GITHUB_WORKSPACE/OpenMS/src/pyOpenMS/requirements_bld.txt
+
+          # configure (don't copy deps since we use auditwheel)
+
+          export Python_ROOT_DIR=$PYBIN
+          export BUILD_NAME="pyopenms-wheels-${{ steps.extract_branch.outputs.RUN_NAME }}-${{ runner.os }}-${{ runner.arch }}-class-pyopenms-$py-${{ github.run_number }}" #TODO make this more expressive
+
+          ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/cibuild.cmake
+          
+          # ensure auditwheel can find the libraries
+          export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH:`pwd`/OpenMS/bld/lib
+
+          # strip the libraries before repairing
+          strip --strip-all OpenMS/bld/lib/libOpenMS.so
+          strip --strip-all OpenMS/bld/lib/libOpenSwathAlgo.so
+
+          pushd OpenMS/bld/pyOpenMS
+            # Bundle stripped plus external shared libraries into the wheels
+            for whl in dist/pyopenms*.whl; do
+              auditwheel repair "$whl" -w wheelhouse/
+            done
+
+            mv wheelhouse/* $GITHUB_WORKSPACE/data/wheelhouse/
+          popd
+          export LD_LIBRARY_PATH=$LD_OLD_LIBRARY_PATH
+          
+          # deactivate the venv
+          deactivate
+        done
+      env: 
+          SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
+          CMAKE_GENERATOR: "Ninja"
+          OPENMS_CONTRIB_LIBS: "/contrib-build/"
+          BUILD_TYPE: "Release"
+          BUILD_FLAGS: "-j${{ steps.cpu-cores.outputs.count }}"
+          NO_DEPENDENCIES: "ON"
+          Python_FIND_STRATEGY: "LOCATION"
+          PYOPENMS: "ON"
+          PY_NUM_THREADS: ${{ steps.cpu-cores.outputs.count }}
+          CI_PROVIDER: "GitHub-Actions"
+          CMAKE_CXX_COMPILER: "g++"
+          CMAKE_PREFIX_PATH: "/contrib-build/"
+          ENABLE_TUTORIALS: "OFF"
+          ENABLE_DOCS: "OFF"
+          ENABLE_STYLE_TESTING: "OFF"
+          ENABLE_PIPELINE_TESTING: "OFF"
+          ENABLE_TOPP_TESTING: "OFF"
+          ENABLE_CLASS_TESTING: "OFF"
+
+    - name: Test
+      shell: bash
+      run: |
+            ctest --output-on-failure -V -S $GITHUB_WORKSPACE/OpenMS/tools/ci/citest.cmake
+      env:
+          SOURCE_DIRECTORY: "${{ github.workspace }}/OpenMS"
+          CI_PROVIDER: "GitHub-Actions"
+          BUILD_NAME: "${{ steps.extract_branch.outputs.RUN_NAME }}-${{ runner.os }}-${{ runner.arch }}-class-pyopenms-${{ github.run_number }}"
+          # The rest of the vars should be saved in the CMakeCache
+
+    - uses: actions/upload-artifact@v4
+      name: Upload artifacts
       with:
-        cmake-version: '3.25.x'
+        name: ${{ runner.os }}-${{ runner.arch }}-wheels
+        path: data/wheelhouse/
+
+  build-lnx-arm:
+    runs-on: [ubuntu-22.04-arm]
+    container: ghcr.io/openms/contrib_manylinux_2_34:latest-arm64
+
+    steps:
+    # Cancels older builds if still running
+    - uses: rokroskar/workflow-run-cleanup-action@master
+      env:
+        GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+      # Disable warning for this line
+      if: "!startsWith(github.ref, 'refs/tags/') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/develop'" # noqa
+
+    - uses: actions/checkout@v4
+      name: Checkout sources
+      with:
+        path: OpenMS
+
+    - id: extract_branch
+      name: Extract branch/PR infos
+      shell: bash
+      run: echo "RUN_NAME=${{ github.event.pull_request && github.event.pull_request.number || github.ref_name }}" >> $GITHUB_OUTPUT
+
+    - name: Get number of CPU cores
+      uses: SimenB/github-actions-cpu-cores@v2
+      id: cpu-cores
 
     - name: Build on manylinux for broad compatibility
       shell: bash
       run: |
         PYTHON_VERSIONS=$(cat OpenMS/.github/workflows/python_versions.json)
         yum install -y jq
         yum install -y ninja-build # Install Ninja for build files
+        yum install -y cmake # Install CMake
         cmake --version
 
         mkdir -p $GITHUB_WORKSPACE/data/wheelhouse/
@@ -641,15 +763,16 @@ jobs:
         name: ${{ runner.os }}-${{ runner.arch }}-wheels
         path: data/wheelhouse/
 
+
 # Tests the build workflows
   test:
-    needs: [build-win, build-macos, build-lnx, build-macos-arm]
+    needs: [build-win, build-macos, build-lnx, build-macos-arm, build-lnx-arm]
     runs-on: ${{ matrix.os }}
     env:
       SYSTEM_VERSION_COMPAT: 0 #courtesy of https://github.com/actions/setup-python/issues/469 in lieu of an actual solution.
     strategy:
       matrix:
-        os: [ubuntu-latest, macos-15-intel, windows-latest, macos-latest]
+        os: [ubuntu-latest, macos-15-intel, windows-latest, macos-latest, ubuntu-22.04-arm]
     steps:
     - uses: actions/checkout@v4
       name: Checkout sources
@@ -659,7 +782,11 @@ jobs:
     - name: Install Miniconda
       shell: bash
       run: |
-          if [[ "${{ runner.os }}" == "Linux" ]]; then
+          if [[ "${{ runner.os }}" == "Linux" && "${{ runner.arch }}" == "ARM64" ]]; then
+            MINICONDA_FILENAME=Miniconda3-latest-Linux-aarch64.sh
+            curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
+            bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
+          elif [[ "${{ runner.os }}" == "Linux" && "${{ runner.arch }}" == "X64" ]]; then
             MINICONDA_FILENAME=Miniconda3-latest-Linux-x86_64.sh
             curl -o $MINICONDA_FILENAME "https://repo.anaconda.com/miniconda/$MINICONDA_FILENAME"
             bash ${MINICONDA_FILENAME} -b -f -p $HOME/miniconda3
@@ -741,6 +868,11 @@ jobs:
       with:
           name: Linux-X64-wheels
           path: /home/runner/work/OpenMS/OpenMS/Linux-X64-wheels
+    - name: Download artifacts
+      uses: actions/download-artifact@v4
+      with:
+          name: Linux-ARM64-wheels
+          path: /home/runner/work/OpenMS/OpenMS/Linux-ARM64-wheels
     - name: Download artifacts
       uses: actions/download-artifact@v4
       with:

CHANGELOG

@@ -15,17 +15,25 @@ PR - Pull Request (on GitHub), i.e. integration of a new feature or bugfix
 ------------------------------------------------------------------------------------------
 ----                                OpenMS 3.5.0     (under development)              ----
 ------------------------------------------------------------------------------------------
+NOTICE: 3.5.0 is going to be the last official release for MacOS on Intel processors.
+If you need support for OpenMS on that platform moving forward, please contact us.
 
 General:
  - speed improvements:
     - loading .gz files is about 7% faster (#8069)
     - loading of mzML files with more than than m/z+intensity (e.g. ion mobility) is 20-40% faster (#8074)
     - loading of mzML files is 7-25% faster in general (SIMD ASCII conversion) (#8105)
+ - linux arm64 support:
+	- OpenMS releases now include a .deb for arm64 machines
+	- PyOpenMS now has wheels for arm64 linux machines
 
 Dependencies:
     - PyOpenMS now depends on Autowrap 0.23.0
     - PyOpenMS now supports Cython 3.1
 
+PyOpenMS:
+	- PyOpenMS wheels are now available for Python 3.14 on all supported operating systems
+
 TOPP tools:
   Changes:
     FileFilter:
@@ -73,6 +81,7 @@ OpenMS Library:
     - OpenSwath:
       - Added automated iRT calibration using input transition list (#8146)
       - Added automated RT, m/z, and IM extraction window estimation based on iRT calibration (#8188)
+      - Added lowess span grid search params and updated documentation for OpenSwathWorkflow TOPP tool (#8297)
 
 
 

src/openms/source/ANALYSIS/OPENSWATH/MRMTransitionGroupPicker.cpp

@@ -127,6 +127,9 @@ namespace OpenMS
     {
       for (Size k = 0; k < picked_chroms[i].size(); ++k)
       {
+        // Skip peaks already "consumed" by previous iterations
+        if (picked_chroms[i][k].getIntensity() == 0.0) {continue; }
+
         const double left_rt = picked_chroms[i].getFloatDataArrays()[PeakPickerChromatogram::IDX_LEFTBORDER][k];
         const double right_rt = picked_chroms[i].getFloatDataArrays()[PeakPickerChromatogram::IDX_RIGHTBORDER][k];
         const double local_peak_width = right_rt - left_rt;

src/openms/source/ANALYSIS/OPENSWATH/OpenSwathWorkflow.cpp

@@ -272,6 +272,10 @@ namespace OpenMS
     Param model_params;
     model_params.setValue("symmetric_regression", "false");
     model_params.setValue("span", irt_detection_param.getValue("lowess:span"));
+    model_params.setValue("auto_span", irt_detection_param.getValue("lowess:auto_span"));
+    model_params.setValue("auto_span_min", irt_detection_param.getValue("lowess:auto_span_min"));
+    model_params.setValue("auto_span_max", irt_detection_param.getValue("lowess:auto_span_max"));
+    model_params.setValue("auto_span_grid", irt_detection_param.getValue("lowess:auto_span_grid"));
     model_params.setValue("num_nodes", irt_detection_param.getValue("b_spline:num_nodes"));
     String model_type = irt_detection_param.getValue("alignmentMethod").toString();
     trafo_out.fitModel(model_type, model_params);

src/openms/source/APPLICATIONS/OpenSwathBase.cpp

@@ -273,6 +273,10 @@ namespace OpenMS
       Param model_params = getParam_().copy("model:", true);
       model_params.setValue("symmetric_regression", "false");
       model_params.setValue("span", irt_detection_param.getValue("lowess:span"));
+      model_params.setValue("auto_span", irt_detection_param.getValue("lowess:auto_span"));
+      model_params.setValue("auto_span_min", irt_detection_param.getValue("lowess:auto_span_min"));
+      model_params.setValue("auto_span_max", irt_detection_param.getValue("lowess:auto_span_max"));
+      model_params.setValue("auto_span_grid", irt_detection_param.getValue("lowess:auto_span_grid"));
       model_params.setValue("num_nodes", irt_detection_param.getValue("b_spline:num_nodes"));
       String model_type = irt_detection_param.getValue("alignmentMethod").toString();
       trafo_rtnorm.fitModel(model_type, model_params);

src/tests/topp/CMakeLists.txt

@@ -1747,7 +1747,7 @@ ${TOPP_BIN_PATH}/OpenSwathDecoyGenerator -test -in ${DATA_DIR_TOPP}/OpenSwathDec
   set_tests_properties("TOPP_OpenSwathWorkflow_20_out1" PROPERTIES DEPENDS "TOPP_OpenSwathWorkflow_20")
 
   # Test iRT alignment, full workflow
-  add_test("TOPP_OpenSwathWorkflow_21" ${TOPP_BIN_PATH}/OpenSwathWorkflow -in ${DATA_DIR_TOPP}/OpenSwathWorkflow_21_input.mzML -tr ${DATA_DIR_TOPP}/OpenSwathWorkflow_21_input.tsv -Calibration:tr_irt ${DATA_DIR_TOPP}/OpenSwathWorkflow_21_input.irt.TraML -out_features OpenSwathWorkflow_21.tmp.featureXML -Debugging:irt_trafo OpenSwathWorkflow_21.tmp.trafoXML -out_chrom OpenSwathWorkflow_21.tmp.mzML -auto_irt "false" -estimate_extraction_windows none -Calibration:RTNormalization:lowess:span 0.666666666666666666666666666
+  add_test("TOPP_OpenSwathWorkflow_21" ${TOPP_BIN_PATH}/OpenSwathWorkflow -in ${DATA_DIR_TOPP}/OpenSwathWorkflow_21_input.mzML -tr ${DATA_DIR_TOPP}/OpenSwathWorkflow_21_input.tsv -Calibration:tr_irt ${DATA_DIR_TOPP}/OpenSwathWorkflow_21_input.irt.TraML -out_features OpenSwathWorkflow_21.tmp.featureXML -Debugging:irt_trafo OpenSwathWorkflow_21.tmp.trafoXML -out_chrom OpenSwathWorkflow_21.tmp.mzML -auto_irt "false" -estimate_extraction_windows none -Calibration:RTNormalization:lowess:auto_span false -Calibration:RTNormalization:lowess:span 0.666666666666666666666666666
     "-test" "-mz_extraction_window" "0.05" "-mz_extraction_window_unit" "Th" "-ms1_isotopes" "0"
     "-Scoring:Scores:use_total_mi_score"
     "-Scoring:Scores:use_ms1_mi" "false"

src/tests/topp/OpenSwathWorkflow_21_output.trafoXML

@@ -3,20 +3,24 @@
 	<Transformation name="linear">
 		<Param  type="string" name="symmetric_regression" value="false"/>
 		<Param  type="float" name="span" value="0.666666666666667"/>
+		<Param  type="string" name="auto_span" value="false"/>
+		<Param  type="float" name="auto_span_min" value="0.15"/>
+		<Param  type="float" name="auto_span_max" value="0.8"/>
+		<Param  type="string" name="auto_span_grid" value="0.005,0.01,0.05,0.15,0.25,0.30,0.50,0.70,0.90"/>
 		<Param  type="int" name="num_nodes" value="5"/>
 		<Param  type="string" name="x_weight" value="x"/>
 		<Param  type="string" name="y_weight" value="y"/>
-		<Param  type="float" name="x_datum_min" value="1e-15"/>
-		<Param  type="float" name="x_datum_max" value="1e+15"/>
-		<Param  type="float" name="y_datum_min" value="1e-15"/>
-		<Param  type="float" name="y_datum_max" value="1e+15"/>
-		<Param  type="float" name="slope" value="1.00420902693575"/>
-		<Param  type="float" name="intercept" value="-11.4372062523721"/>
+		<Param  type="float" name="x_datum_min" value="1.0e-15"/>
+		<Param  type="float" name="x_datum_max" value="1.0e15"/>
+		<Param  type="float" name="y_datum_min" value="1.0e-15"/>
+		<Param  type="float" name="y_datum_max" value="1.0e15"/>
+		<Param  type="float" name="slope" value="1.004209000652954"/>
+		<Param  type="float" name="intercept" value="-11.437200562330617"/>
 		<Pairs count="4">
 			<Pair from="9.99999952316284" to="0"/>
-			<Pair from="32.404443581899" to="25"/>
-			<Pair from="209" to="200"/>
-			<Pair from="68" to="50"/>
+			<Pair from="32.4044302304585" to="25"/>
+			<Pair from="208.999999523163" to="200"/>
+			<Pair from="67.9999995231628" to="50"/>
 		</Pairs>
 	</Transformation>
 </TrafoXML>