Skip to content

Commit 35ff41d

Browse files
Andre Ferreiraclaude
andcommitted
fix(cache): enable incremental IR cache for multi-file test suites
Three bugs prevented the incremental IR cache from working with multi-file test suites: 1. IR cache filename collision: get_ir_cache_filename() truncated the 128-bit fingerprint to 64 bits (first 16 hex chars). Since fingerprint_bytes() computes high from CRC32C of the first half of the key data, keys differing only in test_entry_index (last 4 bytes) produced identical filenames, causing files to overwrite each other. Fix: use the full 32-char hex hash. 2. Race condition: parallel suite compilation workers called save_incremental_cache() concurrently, corrupting the shared incr.bin file. Fix: protect with mutex. 3. Cache was disabled: multi-file suites had incremental=false as a workaround for the above bugs. Now enabled since root causes fixed. Result: ~7.5x speedup on cached test runs (92s → 12s for core/fmt). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 6da98ef commit 35ff41d

2 files changed

Lines changed: 12 additions & 7 deletions

File tree

compiler/src/query/query_incr.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -550,7 +550,7 @@ std::string get_ir_cache_filename(const QueryKey& key) {
550550
auto fp = fingerprint_bytes(key_data.data(), key_data.size());
551551
// Include query kind in the name for debugging
552552
auto kind = query_kind(key);
553-
return std::string(query_kind_name(kind)) + "_" + fp.to_hex().substr(0, 16);
553+
return std::string(query_kind_name(kind)) + "_" + fp.to_hex();
554554
}
555555

556556
} // namespace tml::query

compiler/src/testing/testing_compile.cpp

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,7 @@ class ThreadBudget {
9494
static std::string g_clang_path;
9595
static bool g_env_initialized = false;
9696
static std::mutex g_env_mutex;
97+
static std::mutex g_incr_cache_mutex;
9798

9899
/// Pre-built runtime archive path (set by compile_suites_parallel before spawning workers).
99100
/// When non-empty, compile_suite() uses this instead of calling get_runtime_objects() per suite.
@@ -354,10 +355,9 @@ CompileResult compile_suite(const Suite& suite, const CompileConfig& config) {
354355
qopts.verbose = verbose;
355356
qopts.coverage = config.coverage;
356357
qopts.optimization_level = config.optimization_level;
357-
// Incremental IR cache disabled for multi-file suites to prevent
358-
// duplicate symbol errors from stale cache entries.
359-
bool is_multi_file_suite = suite.tests.size() > 1;
360-
qopts.incremental = !is_multi_file_suite;
358+
// Incremental IR cache: test_entry_index in CodegenUnitKey
359+
// differentiates cache entries per file within a suite.
360+
qopts.incremental = true;
361361
qopts.generate_exe_main = false;
362362
qopts.test_entry_index = static_cast<int>(i);
363363

@@ -424,8 +424,10 @@ CompileResult compile_suite(const Suite& suite, const CompileConfig& config) {
424424
continue;
425425
}
426426

427-
// Save incremental cache
427+
// Save incremental cache (mutex protects concurrent suite workers
428+
// from corrupting the shared incr.bin file)
428429
if (qopts.incremental) {
430+
std::lock_guard<std::mutex> lock(g_incr_cache_mutex);
429431
auto build_dir = cli::build::get_build_dir(false);
430432
qctx.save_incremental_cache(build_dir);
431433
}
@@ -450,7 +452,8 @@ CompileResult compile_suite(const Suite& suite, const CompileConfig& config) {
450452
auto cached_obj =
451453
obj_cache_dir / (ir_fp.to_hex().substr(0, 16) + cli::get_object_extension());
452454
bool used_cache = false;
453-
if (fs::exists(cached_obj)) {
455+
bool obj_exists = fs::exists(cached_obj);
456+
if (obj_exists) {
454457
std::error_code ec;
455458
fs::copy_file(cached_obj, obj_path, fs::copy_options::overwrite_existing, ec);
456459
if (!ec) {
@@ -459,6 +462,8 @@ CompileResult compile_suite(const Suite& suite, const CompileConfig& config) {
459462
used_cache = true;
460463
}
461464
}
465+
TML_LOG_DEBUG("test", " [obj-cache] " << suite.tests[i].test_name << " exists="
466+
<< obj_exists << " hit=" << used_cache);
462467
if (!used_cache) {
463468
auto obj_result = cli::compile_ir_string_to_object(
464469
codegen_result.llvm_ir, obj_path, g_clang_path, obj_opts);

0 commit comments

Comments
 (0)