Skip to content

Commit e0c995a

Browse files
edsavageedsavage
committed
[ML] Speed up C++ builds with unity builds, PCH, and Ninja on Windows
Enable CMake unity builds and precompiled headers (PCH) to reduce compilation times. Use Ninja Multi-Config generator on Windows for faster parallel compilation (downloaded on-demand during CI). Unity builds: - Enabled globally via -DCMAKE_UNITY_BUILD=ON (Windows/Linux) - Disabled on macOS (counterproductive on 4-core Orka VMs) - Disabled for MlApi and MlMathsAnalytics (too many symbol conflicts) - Fix anonymous-namespace conflicts in core/ and maths/ sources PCH: - New ML_PCH option adds Boost headers as precompiled header targets - Enabled via -DML_PCH=ON in CI pipeline environments Ninja on Windows: - Set CMAKE_GENERATOR=Ninja Multi-Config in Windows pipeline env - build.ps1 downloads ninja.exe on-demand from GitHub releases when not already on PATH - Remove /Zi in favour of /Z7 (embedded debug info avoids PDB locking issues with Ninja's parallel compilation) Build system: - build.gradle reads CMAKE_FLAGS from environment, allowing CI to pass unity/PCH flags through to cmake configure - Fix ML_DEBUG handling: use env-dict in pipeline generators (not inline PowerShell commands) and restore debug build directory selection in build/test scripts Made-with: Cursor
1 parent 1a6d770 commit e0c995a

23 files changed

Lines changed: 201 additions & 19 deletions

‎.buildkite/pipelines/build_linux.json.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ def main(args):
9393
"key": build_key,
9494
"env": {
9595
**common_env,
96-
"CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake",
96+
"CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
9797
"RUN_TESTS": "false",
9898
},
9999
"notify": [
@@ -118,7 +118,7 @@ def main(args):
118118
"env": {
119119
**common_env,
120120
"BUILD_STEP_KEY": build_key,
121-
"CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake",
121+
"CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
122122
"BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
123123
},
124124
"plugins": {
@@ -151,7 +151,7 @@ def main(args):
151151
"key": build_key,
152152
"env": {
153153
**common_env,
154-
"CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake",
154+
"CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
155155
"RUN_TESTS": "false",
156156
},
157157
"notify": [
@@ -176,7 +176,7 @@ def main(args):
176176
"env": {
177177
**common_env,
178178
"BUILD_STEP_KEY": build_key,
179-
"CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake",
179+
"CMAKE_FLAGS": f"-DCMAKE_TOOLCHAIN_FILE=cmake/linux-{arch}.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
180180
"BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
181181
},
182182
"plugins": {
@@ -212,7 +212,7 @@ def main(args):
212212
"env": {
213213
**common_env,
214214
"ML_DEBUG": "1",
215-
"CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/linux-x86_64.cmake",
215+
"CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/linux-x86_64.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
216216
"RUN_TESTS": "false",
217217
"SKIP_ARTIFACT_UPLOAD": "true",
218218
},
@@ -239,7 +239,7 @@ def main(args):
239239
**common_env,
240240
"BUILD_STEP_KEY": debug_build_key,
241241
"ML_DEBUG": "1",
242-
"CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/linux-x86_64.cmake",
242+
"CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/linux-x86_64.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
243243
"BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
244244
},
245245
"plugins": {

‎.buildkite/pipelines/build_macos.json.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
"PATH": "/opt/homebrew/bin:$PATH",
4444
"ML_DEBUG": "0",
4545
"CPP_CROSS_COMPILE": "",
46-
"CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-aarch64.cmake",
46+
"CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-aarch64.cmake -DML_PCH=ON",
4747
"RUN_TESTS": "true",
4848
"BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
4949
}

‎.buildkite/pipelines/build_windows.json.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
common_env = {
4242
"ML_DEBUG": "0",
4343
"CPP_CROSS_COMPILE": "",
44-
"CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/windows-x86_64.cmake",
44+
"CMAKE_GENERATOR": "Ninja Multi-Config",
45+
"CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/windows-x86_64.cmake -DCMAKE_UNITY_BUILD=ON -DML_PCH=ON",
4546
}
4647

4748
def main(args):

‎.buildkite/scripts/steps/build.ps1‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,23 @@ if ([System.Environment]::OSVersion.Version.Build -lt 20348) {
2020
& "dev-tools\download_windows_deps.ps1"
2121
}
2222

23+
# Install Ninja if CMAKE_GENERATOR requires it and it's not already on PATH
24+
if ((Test-Path Env:CMAKE_GENERATOR) -and $Env:CMAKE_GENERATOR -match "Ninja") {
25+
if (-not (Get-Command ninja -ErrorAction SilentlyContinue)) {
26+
$NinjaVersion = "v1.12.1"
27+
$NinjaUrl = "https://github.com/ninja-build/ninja/releases/download/$NinjaVersion/ninja-win.zip"
28+
$NinjaDir = "$Env:TEMP\ninja"
29+
Write-Output "Installing Ninja $NinjaVersion..."
30+
New-Item -ItemType Directory -Force -Path $NinjaDir | Out-Null
31+
Invoke-WebRequest -Uri $NinjaUrl -OutFile "$NinjaDir\ninja.zip" -UseBasicParsing
32+
Expand-Archive -Path "$NinjaDir\ninja.zip" -DestinationPath $NinjaDir -Force
33+
$Env:PATH = "$NinjaDir;$Env:PATH"
34+
Write-Output "Ninja installed: $(ninja --version)"
35+
} else {
36+
Write-Output "Ninja already available: $(ninja --version)"
37+
}
38+
}
39+
2340
if (!(Test-Path Env:BUILD_SNAPSHOT)) {
2441
$Env:BUILD_SNAPSHOT="true"
2542
}

‎.buildkite/scripts/steps/build_and_test.ps1‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,37 @@ if (Test-Path Env:ML_DEBUG) {
5353
$DebugOption=""
5454
}
5555

56+
# Ensure Ninja is available (required for Ninja Multi-Config generator)
57+
$ninjaCmd = Get-Command ninja -ErrorAction SilentlyContinue
58+
if ($ninjaCmd) {
59+
Write-Output "ninja found: $($ninjaCmd.Source)"
60+
} else {
61+
$ninjaVersion = "1.12.1"
62+
$ninjaDir = "$Env:LOCALAPPDATA\ninja"
63+
$ninjaExe = "$ninjaDir\ninja.exe"
64+
if (Test-Path $ninjaExe) {
65+
Write-Output "ninja already downloaded: $ninjaExe"
66+
} else {
67+
Write-Output "Downloading ninja v${ninjaVersion}..."
68+
$url = "https://github.com/ninja-build/ninja/releases/download/v${ninjaVersion}/ninja-win.zip"
69+
$zipPath = "$Env:TEMP\ninja-win.zip"
70+
if (-not (Test-Path $ninjaDir)) { New-Item -ItemType Directory -Path $ninjaDir | Out-Null }
71+
(New-Object Net.WebClient).DownloadFile($url, $zipPath)
72+
Expand-Archive -Path $zipPath -DestinationPath $ninjaDir -Force
73+
Remove-Item $zipPath -ErrorAction SilentlyContinue
74+
Write-Output "ninja installed: $ninjaExe"
75+
}
76+
if ($Env:PATH -notlike "*$ninjaDir*") {
77+
$Env:PATH = "$ninjaDir;$Env:PATH"
78+
}
79+
}
80+
& ninja --version
81+
82+
# Set up sccache with GCS backend if the bucket env var has been injected
83+
if (Test-Path Env:SCCACHE_GCS_BUCKET) {
84+
. "$PSScriptRoot\..\..\..\dev-tools\setup_sccache.ps1"
85+
}
86+
5687
# The exit code of the gradlew commands is checked explicitly, and their
5788
# stderr is treated as an error by PowerShell without this
5889
$ErrorActionPreference="Continue"
@@ -69,4 +100,10 @@ if ($ExitCode -ne 0) {
69100
Exit $ExitCode
70101
}
71102

103+
# Print sccache stats if it was used
104+
if (Test-Path Env:SCCACHE_PATH) {
105+
& $Env:SCCACHE_PATH --show-stats 2>$null
106+
& $Env:SCCACHE_PATH --stop-server 2>$null
107+
}
108+
72109
buildkite-agent artifact upload "build/distributions/*"

‎CMakeLists.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,20 @@ endif()
6868

6969
project("ML")
7070

71+
if(CMAKE_UNITY_BUILD)
72+
if(NOT DEFINED CMAKE_UNITY_BUILD_BATCH_SIZE)
73+
set(CMAKE_UNITY_BUILD_BATCH_SIZE 16)
74+
endif()
75+
message(STATUS "Unity build enabled (batch size: ${CMAKE_UNITY_BUILD_BATCH_SIZE})")
76+
endif()
77+
78+
option(ML_PCH "Enable precompiled headers for faster compilation" OFF)
79+
if(ML_PCH)
80+
message(STATUS "Precompiled headers enabled")
81+
endif()
82+
83+
include(CTest)
84+
7185
include(CheckPIESupported)
7286
check_pie_supported()
7387

‎build.gradle‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,15 @@ if (isWindows) {
8686
// where the desired build type is specified at build time with the '--config' option. (It's safe to always specify the
8787
// '--config' option as it will simply be ignored by a single-config generated build system)
8888
String cmakeFlags = '--no-warn-unused-cli -D CMAKE_TOOLCHAIN_FILE=cmake/' + artifactClassifier + '.cmake'
89+
// Append any extra CMake flags from the environment (e.g. -DCMAKE_UNITY_BUILD=ON)
90+
String envCmakeFlags = System.env.CMAKE_FLAGS
91+
if (envCmakeFlags != null && !envCmakeFlags.isEmpty()) {
92+
// Strip any toolchain file flag to avoid duplication
93+
envCmakeFlags = envCmakeFlags.replaceAll(/-DCMAKE_TOOLCHAIN_FILE=\\S+/, '').trim()
94+
if (!envCmakeFlags.isEmpty()) {
95+
cmakeFlags += ' ' + envCmakeFlags
96+
}
97+
}
8998
project.ext.cmakeBuildDir = "cmake-build-relwithdebinfo"
9099
project.ext.cmakeBuildType = "RelWithDebInfo"
91100
if (mlDebug.toBoolean()) {

‎cmake/compiler/msvc.cmake‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@ list(APPEND ML_C_FLAGS
3333
"/W4"
3434
"/EHsc"
3535
"/Gw"
36-
"/FS"
3736
"/Zc:inline"
3837
"/diagnostics:caret"
3938
"/utf-8")

‎cmake/functions.cmake‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -190,6 +190,25 @@ function(ml_add_library _target _type)
190190

191191
set_property(TARGET ${_target} PROPERTY POSITION_INDEPENDENT_CODE TRUE)
192192

193+
if(ML_PCH)
194+
target_precompile_headers(${_target} PRIVATE
195+
<string>
196+
<vector>
197+
<memory>
198+
<algorithm>
199+
<cmath>
200+
<cstddef>
201+
<cstdint>
202+
<utility>
203+
<functional>
204+
<optional>
205+
<limits>
206+
<numeric>
207+
<map>
208+
<sstream>
209+
)
210+
endif()
211+
193212
if(ML_LINK_LIBRARIES)
194213
target_link_libraries(${_target} PUBLIC ${ML_LINK_LIBRARIES})
195214
endif()
@@ -363,6 +382,26 @@ function(ml_add_test_executable _target)
363382

364383
set_property(TARGET ml_test_${_target} PROPERTY POSITION_INDEPENDENT_CODE TRUE)
365384

385+
if(ML_PCH)
386+
target_precompile_headers(ml_test_${_target} PRIVATE
387+
<string>
388+
<vector>
389+
<memory>
390+
<algorithm>
391+
<cmath>
392+
<cstddef>
393+
<cstdint>
394+
<utility>
395+
<functional>
396+
<optional>
397+
<limits>
398+
<numeric>
399+
<map>
400+
<sstream>
401+
<boost/test/unit_test.hpp>
402+
)
403+
endif()
404+
366405
target_link_libraries(ml_test_${_target} ${ML_LINK_LIBRARIES})
367406

368407
add_test(ml_test_${_target} ml_test_${_target})

‎cmake/variables.cmake‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -160,9 +160,9 @@ endif()
160160
# Dictate which flags to use for "Release", "RelWithDebinfo", "Debug" and "Sanitizer" builds
161161
if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
162162
set(CMAKE_CXX_FLAGS_RELEASE "/O2 /D NDEBUG /D EXCLUDE_TRACE_LOGGING /Qfast_transcendentals /Qvec-report:1")
163-
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/Zi /O2 /D NDEBUG /D EXCLUDE_TRACE_LOGGING /Qfast_transcendentals /Qvec-report:1")
164-
set(CMAKE_CXX_FLAGS_DEBUG "/Zi /Od /RTC1")
165-
set(CMAKE_CXX_FLAGS_SANITIZER "/fsanitize=address /O2 /Zi" CACHE STRING
163+
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "/Z7 /O2 /D NDEBUG /D EXCLUDE_TRACE_LOGGING /Qfast_transcendentals /Qvec-report:1")
164+
set(CMAKE_CXX_FLAGS_DEBUG "/Z7 /Od /RTC1")
165+
set(CMAKE_CXX_FLAGS_SANITIZER "/fsanitize=address /O2 /Z7" CACHE STRING
166166
"Flags used by the C++ compiler during sanitizer builds."
167167
FORCE)
168168
set(CMAKE_EXE_LINKER_FLAGS_SANITIZER "")

0 commit comments

Comments
 (0)