diff --git a/benchmarks/images/mkdd-nopgo-chunk64.png b/benchmarks/images/mkdd-nopgo-chunk64.png new file mode 100644 index 0000000..eae2062 Binary files /dev/null and b/benchmarks/images/mkdd-nopgo-chunk64.png differ diff --git a/benchmarks/images/mkdd-pgo-chunk64.png b/benchmarks/images/mkdd-pgo-chunk64.png new file mode 100644 index 0000000..b6b0234 Binary files /dev/null and b/benchmarks/images/mkdd-pgo-chunk64.png differ diff --git a/src/app/pipeline.c b/src/app/pipeline.c index dfb421b..b8114d8 100644 --- a/src/app/pipeline.c +++ b/src/app/pipeline.c @@ -23,6 +23,7 @@ #include #include #include +#include #ifndef _WIN32 #include #include @@ -51,10 +52,23 @@ static u32 c_chunk_instructions(void) { } #ifdef DOLRECOMP_ENABLE_LLVM -#define DOLLLVM_DEFAULT_CHUNK_INSTRUCTIONS 1024u +// 128, measured. A chunk is one LLVM function, so this is the block count the +// register allocator keeps the whole guest register file live across. Against +// the previous 1024 default this is +57.9% throughput and -66% .text on Mario +// Kart; 64 gains a further 1.4% but its range overlaps 128's, so it is not a +// proven gain. See docs/LLVM-EXPERIMENTS.md E002-E004. +#define DOLLLVM_DEFAULT_CHUNK_INSTRUCTIONS 128u #define DOLLLVM_DEFAULT_WORKER_BATCH 4u // v6 carries the execution budget across generated function calls. -#define DOLLLVM_CACHE_VERSION "dolllvm-v6" +// Any change that alters generated code must bump this, because +// llvm_job_hash() omits the pass pipeline, opt level and LLVM version. +// v7: ps1 preservation fix in dolir_builder (lfd and fmr/fneg/fabs/fnabs/fsel +// no longer splat into the high paired-single slot). Default codegen changed, +// so every cached object from v6 is stale. +#define DOLLLVM_CACHE_VERSION "dolllvm-v7" +// The LLVM optimisation level used for generated objects. Named so it can be +// folded into the cache key; changing it must not reuse cached objects. +#define DOLLLVM_OPT_LEVEL 2 typedef struct { const PPCInst* insts; @@ -70,6 +84,25 @@ typedef struct { char cache_path[1400]; } LLVMChunkJob; +// The floor is 32, not the 128 the C path uses. +// +// A chunk becomes exactly one LLVM function, so this value is the number of +// basic blocks the register allocator has to keep the whole promoted guest +// register file live across -- and that scope is what drives the generated +// code size. Measured on Mario Kart (LLVM-EXPERIMENTS E002/E003), against the +// 1024 default: +// +// 1024 .text 1,012,522,870 speed 0.3288 +// 256 .text 450,227,766 speed 0.4404 +33.9% +// 128 .text 345,215,974 speed 0.5192 +57.9% +// +// monotonic, with disjoint confidence ranges at every step, so 128 was the +// binding constraint rather than the optimum. Smaller chunks do eventually +// cost -- a call that leaves the chunk returns through the dispatcher instead +// of branching -- so this is a curve with a minimum, not a free win. Sweep it +// per title rather than assuming this one's answer. +#define DOLLLVM_MIN_CHUNK_INSTRUCTIONS 32u + static u32 llvm_chunk_instructions(void) { const char* configured = getenv("DOLRECOMP_LLVM_CHUNK_INSTRUCTIONS"); if (!configured || !configured[0]) @@ -77,10 +110,12 @@ static u32 llvm_chunk_instructions(void) { char* end = NULL; errno = 0; unsigned long value = strtoul(configured, &end, 10); - if (errno || !end || *end || value < 128u || value > 4096u) { + if (errno || !end || *end || value < DOLLLVM_MIN_CHUNK_INSTRUCTIONS || + value > 4096u) { fprintf(stderr, - "warning: DOLRECOMP_LLVM_CHUNK_INSTRUCTIONS must be 128..4096; " + "warning: DOLRECOMP_LLVM_CHUNK_INSTRUCTIONS must be %u..4096; " "using %u\n", + DOLLLVM_MIN_CHUNK_INSTRUCTIONS, DOLLLVM_DEFAULT_CHUNK_INSTRUCTIONS); return DOLLLVM_DEFAULT_CHUNK_INSTRUCTIONS; } @@ -207,6 +242,14 @@ static u64 llvm_job_hash(const LLVMChunkJob* job) { if (dolllvm_effective_triple(getenv("DOLRECOMP_LLVM_TARGET"), triple, sizeof(triple))) hash = hash_bytes(hash, triple, strlen(triple)); + // LLVM version, target CPU and features, and the pass pipeline. Without + // these a codegen experiment reuses objects built with the old settings + // and reports them as its result. + char codegen[1024]; + if (dolllvm_codegen_fingerprint(codegen, sizeof(codegen))) + hash = hash_bytes(hash, codegen, strlen(codegen)); + u32 opt_level = (u32)DOLLLVM_OPT_LEVEL; + hash = hash_bytes(hash, &opt_level, sizeof(opt_level)); for (u32 i = 0; i < job->count; i++) { hash = hash_bytes(hash, &job->insts[i].address, sizeof(job->insts[i].address)); @@ -280,8 +323,24 @@ static void cache_llvm_object(const LLVMChunkJob* job) { static int emit_llvm_chunk_job(const void* data, void* user) { const LLVMChunkJob* job = (const LLVMChunkJob*)data; (void)user; - if (reuse_llvm_object(job)) + if (reuse_llvm_object(job)) { +#ifdef _WIN32 + // Say so. A silent reuse is indistinguishable from a regeneration in + // the log, and "the cache was hit" is exactly the thing that must not + // be assumed when checking whether a codegen change was really tested. + printf("[%u/%u] Reusing cached LLVM object %s\n", job->index, + job->total, job->name); + fflush(stdout); +#endif return 1; + } +#ifdef _WIN32 + // See run_llvm_chunk_jobs: on Windows this is the only live progress. + printf("[%u/%u] Emitting LLVM object %s\n", job->index, job->total, + job->name); + fflush(stdout); + time_t started = time(NULL); +#endif char temp_path[1440]; #ifdef _WIN32 int process_id = _getpid(); @@ -303,7 +362,7 @@ static int emit_llvm_chunk_job(const void* data, void* user) { } DolLLVMOptions options = {0}; options.target_triple = getenv("DOLRECOMP_LLVM_TARGET"); - options.optimization_level = 2; + options.optimization_level = DOLLLVM_OPT_LEVEL; options.verify = 1; options.function_ranges = job->ranges; options.function_range_count = job->range_count; @@ -334,6 +393,12 @@ static int emit_llvm_chunk_job(const void* data, void* user) { } if (!ok) remove(temp_path); +#ifdef _WIN32 + printf("[%u/%u] %s LLVM object %s (%llds)\n", job->index, job->total, + ok ? "Finished" : "FAILED", job->name, + (long long)(time(NULL) - started)); + fflush(stdout); +#endif return ok; } @@ -354,11 +419,12 @@ static int run_llvm_chunk_jobs(const LLVMChunkJob* jobs, u32 count, u32 requested_jobs) { u32 workers = effective_chunk_jobs(count, requested_jobs); #ifdef _WIN32 - for (u32 i = 0; i < count; i++) { - printf("[%u/%u] Emitting LLVM object %s\n", - jobs[i].index, jobs[i].total, jobs[i].name); - fflush(stdout); - } + // Progress is reported from inside the job, not dumped up front. The + // Windows path used to print every line before starting any work, so a + // chunk that hung produced a complete-looking log and no indication of + // which chunk was stuck -- one such hang ran 49 minutes with nothing to + // point at. A start line, and a done line carrying elapsed seconds, means + // the stuck chunk is the one with no matching completion. return run_parallel_jobs(jobs, sizeof(*jobs), count, workers, emit_llvm_chunk_job, NULL); #else diff --git a/src/backend/dispatch.c b/src/backend/dispatch.c index 6cce27e..41dca99 100644 --- a/src/backend/dispatch.c +++ b/src/backend/dispatch.c @@ -2,6 +2,55 @@ #include #include +// How dolrecomp_find_original resolves a guest address to a generated chunk. +// +// "linear" is the historical emission and stays the default: a chain of range +// tests, with consecutive equal-stride chunks collapsed into one indexed jump +// table. That is O(1) when every chunk is the same size -- a fixed-128 plan on +// this title emits exactly 2 tests -- and O(chunks) when they are not. The +// control-flow-aligned plans of E008 emitted 2,089 (func/128/512) and 5,224 +// (func/32/256) tests, taken on every dispatch into the module, and those two +// arms rank by chain length exactly as they rank by measured slowness. So no +// throughput number from an irregular-boundary plan is interpretable under +// this lookup. See docs/LLVM-EXPERIMENTS.md W001, "Ops finding". +// +// "indexed" removes that confound: a 4 KiB page index over the covered guest +// range selects a small window of runs, and the window is walked forward. The +// walk is bounded by the number of runs that intersect one page, so the lookup +// is O(1) in the number of chunks regardless of how irregular the plan is. +// +// The default is unchanged because the shipping C module is pinned by hash and +// must stay byte-identical; this selects an experiment, not a new default. +typedef enum { + DISPATCH_LOOKUP_LINEAR = 0, + DISPATCH_LOOKUP_INDEXED = 1 +} DispatchLookupMode; + +// A 4 KiB page is small enough that a page holds only a handful of runs even +// under the most irregular plan measured here (E008a's mean chunk was 87 +// instructions, so ~11 per page), and the whole index is one u32 per page. +#define DISPATCH_PAGE_SHIFT 12u + +// Refuse the page index if the code sections are scattered far enough apart +// that the table would dwarf the thing it indexes. 256 MiB of guest address +// space is 65,536 pages, 256 KiB of table; MEM1 is 24 MiB, so this never fires +// on a GameCube title and exists so a bad input degrades rather than explodes. +#define DISPATCH_MAX_INDEX_PAGES 65536u + +static DispatchLookupMode dispatch_lookup_mode(void) { + const char* configured = getenv("DOLRECOMP_DISPATCH_LOOKUP"); + if (!configured || !configured[0]) + return DISPATCH_LOOKUP_LINEAR; + if (!strcmp(configured, "indexed")) + return DISPATCH_LOOKUP_INDEXED; + if (!strcmp(configured, "linear")) + return DISPATCH_LOOKUP_LINEAR; + fprintf(stderr, + "warning: DOLRECOMP_DISPATCH_LOOKUP must be linear|indexed; using " + "linear\n"); + return DISPATCH_LOOKUP_LINEAR; +} + void emit_chunk_prototype(FILE* out, u32 func_addr) { fprintf(out, "void func_%08X(CPUState* ctx);\n", func_addr); } @@ -97,6 +146,224 @@ static void emit_lookup_run(FILE* out, const FunctionList* funcs, fprintf(out, " }\n"); } +static int range_compare(const void* a, const void* b) { + const FunctionRange* left = (const FunctionRange*)a; + const FunctionRange* right = (const FunctionRange*)b; + if (left->start != right->start) + return left->start < right->start ? -1 : 1; + if (left->end != right->end) + return left->end < right->end ? -1 : 1; + return 0; +} + +// Emit the page-indexed lookup. Returns 0 if this plan cannot be indexed, in +// which case the caller falls back to the linear chain -- correctness never +// depends on which one is emitted. +static int emit_lookup_indexed(FILE* out, const FunctionList* funcs) { + FunctionRange* sorted = NULL; + u32* run_first = NULL; + u32* page_first = NULL; + u32 sorted_count = 0; + u32 run_count = 0; + u32 page_count = 0; + u32 base = 0; + u32 limit = 0; + u32 max_per_page = 0; + FunctionList view = {0}; + + if (funcs->count == 0) + return 0; + + sorted = (FunctionRange*)malloc(funcs->count * sizeof(*sorted)); + if (!sorted) + return 0; + + // An empty range can never match, and it would divide by zero below. + for (u32 i = 0; i < funcs->count; i++) { + if (funcs->ranges[i].start < funcs->ranges[i].end) + sorted[sorted_count++] = funcs->ranges[i]; + } + if (sorted_count == 0) { + free(sorted); + return 0; + } + qsort(sorted, sorted_count, sizeof(*sorted), range_compare); + + // The walk below assumes ranges are disjoint, so that the first run whose + // end passes the address is the only run that can contain it. The linear + // chain instead returns the first range that matches in list order, so on + // an overlapping plan the two would disagree. Refuse rather than differ. + for (u32 i = 1; i < sorted_count; i++) { + if (sorted[i].start < sorted[i - 1].end) { + fprintf(stderr, + "warning: DOLRECOMP_DISPATCH_LOOKUP=indexed needs disjoint " + "chunk ranges (0x%08X overlaps 0x%08X); using linear\n", + sorted[i].start, sorted[i - 1].start); + free(sorted); + return 0; + } + } + + base = sorted[0].start & ~((1u << DISPATCH_PAGE_SHIFT) - 1u); + limit = sorted[sorted_count - 1].end; + page_count = ((limit - base) >> DISPATCH_PAGE_SHIFT) + 1u; + if (page_count > DISPATCH_MAX_INDEX_PAGES) { + fprintf(stderr, + "warning: DOLRECOMP_DISPATCH_LOOKUP=indexed needs the code " + "sections within %u pages (got %u); using linear\n", + DISPATCH_MAX_INDEX_PAGES, page_count); + free(sorted); + return 0; + } + + // Reuse uniform_run_end so the runs -- and therefore the emitted function + // table -- are exactly the ones the linear chain would have produced. + view.ranges = sorted; + view.count = sorted_count; + view.capacity = sorted_count; + + run_first = (u32*)malloc((sorted_count + 1u) * sizeof(*run_first)); + page_first = (u32*)malloc(page_count * sizeof(*page_first)); + if (!run_first || !page_first) { + free(run_first); + free(page_first); + free(sorted); + return 0; + } + + for (u32 first = 0; first < sorted_count;) { + run_first[run_count++] = first; + first = uniform_run_end(&view, first); + } + run_first[run_count] = sorted_count; + + // page_first[p] is the first run whose end passes the start of page p, so + // the forward walk from it can only skip runs that also intersect the page. + { + u32 run = 0; + for (u32 page = 0; page < page_count; page++) { + u32 page_start = base + (page << DISPATCH_PAGE_SHIFT); + while (run < run_count && + sorted[run_first[run + 1u] - 1u].end <= page_start) + run++; + page_first[page] = run; + } + } + + // The walk length is bounded by the runs intersecting one page. Report it, + // because it is the number that makes the lookup O(1) rather than O(runs). + { + u32 run = 0; + for (u32 page = 0; page < page_count; page++) { + u32 page_end = base + ((page + 1u) << DISPATCH_PAGE_SHIFT); + u32 here = 0; + for (run = page_first[page]; + run < run_count && sorted[run_first[run]].start < page_end; + run++) + here++; + if (here > max_per_page) + max_per_page = here; + } + } + + fprintf(out, "\n#define DOLRECOMP_LOOKUP_RUNS %uu\n", run_count); + fprintf(out, "#define DOLRECOMP_LOOKUP_BASE 0x%08Xu\n", base); + fprintf(out, "#define DOLRECOMP_LOOKUP_PAGES %uu\n", page_count); + fprintf(out, "#define DOLRECOMP_LOOKUP_PAGE_SHIFT %uu\n", + DISPATCH_PAGE_SHIFT); + + fprintf(out, + "\nstatic const u32 dolrecomp_run_start[DOLRECOMP_LOOKUP_RUNS] " + "DOLRECOMP_UNUSED = {\n"); + for (u32 i = 0; i < run_count; i++) + fprintf(out, " 0x%08Xu,\n", sorted[run_first[i]].start); + fprintf(out, "};\n"); + + fprintf(out, + "\nstatic const u32 dolrecomp_run_end[DOLRECOMP_LOOKUP_RUNS] " + "DOLRECOMP_UNUSED = {\n"); + for (u32 i = 0; i < run_count; i++) + fprintf(out, " 0x%08Xu,\n", sorted[run_first[i + 1u] - 1u].end); + fprintf(out, "};\n"); + + // A run of one chunk gets its own width as the stride, so the division + // always yields index 0 and no branch is needed to special-case it. + fprintf(out, + "\nstatic const u32 dolrecomp_run_stride[DOLRECOMP_LOOKUP_RUNS] " + "DOLRECOMP_UNUSED = {\n"); + for (u32 i = 0; i < run_count; i++) { + const FunctionRange* head = &sorted[run_first[i]]; + fprintf(out, " 0x%08Xu,\n", head->end - head->start); + } + fprintf(out, "};\n"); + + fprintf(out, + "\nstatic const u32 dolrecomp_run_base[DOLRECOMP_LOOKUP_RUNS] " + "DOLRECOMP_UNUSED = {\n"); + for (u32 i = 0; i < run_count; i++) + fprintf(out, " %uu,\n", run_first[i]); + fprintf(out, "};\n"); + + fprintf(out, + "\nstatic const DolRecompFunction " + "dolrecomp_run_chunks[%uu] DOLRECOMP_UNUSED = {\n", + sorted_count); + for (u32 i = 0; i < sorted_count; i++) + fprintf(out, " func_%08X,\n", sorted[i].start); + fprintf(out, "};\n"); + + fprintf(out, + "\nstatic const u32 dolrecomp_page_first[DOLRECOMP_LOOKUP_PAGES] " + "DOLRECOMP_UNUSED = {\n"); + for (u32 i = 0; i < page_count; i++) + fprintf(out, " %uu,\n", page_first[i]); + fprintf(out, "};\n"); + + fprintf(out, "\nstatic inline DolRecompFunction dolrecomp_find_original(u32 address) {\n"); + fprintf(out, " u32 page;\n"); + fprintf(out, " u32 run;\n"); + fprintf(out, " u32 offset;\n"); + fprintf(out, " if (address < DOLRECOMP_LOOKUP_BASE) return NULL;\n"); + fprintf(out, + " page = (address - DOLRECOMP_LOOKUP_BASE) >> " + "DOLRECOMP_LOOKUP_PAGE_SHIFT;\n"); + fprintf(out, " if (page >= DOLRECOMP_LOOKUP_PAGES) return NULL;\n"); + fprintf(out, " run = dolrecomp_page_first[page];\n"); + fprintf(out, + " while (run < DOLRECOMP_LOOKUP_RUNS && " + "dolrecomp_run_end[run] <= address) run++;\n"); + fprintf(out, + " if (run >= DOLRECOMP_LOOKUP_RUNS || " + "address < dolrecomp_run_start[run]) return NULL;\n"); + fprintf(out, " offset = address - dolrecomp_run_start[run];\n"); + fprintf(out, " if ((offset & 3u) != 0u) return NULL;\n"); + fprintf(out, + " return dolrecomp_run_chunks[dolrecomp_run_base[run] + " + "offset / dolrecomp_run_stride[run]];\n"); + fprintf(out, "}\n"); + + printf(" dispatch lookup: indexed, %u chunks in %u runs, %u pages, " + "at most %u run%s walked per lookup\n", + sorted_count, run_count, page_count, max_per_page, + max_per_page == 1u ? "" : "s"); + + free(run_first); + free(page_first); + free(sorted); + return 1; +} + +static void emit_lookup_linear(FILE* out, const FunctionList* funcs) { + fprintf(out, "\nstatic inline DolRecompFunction dolrecomp_find_original(u32 address) {\n"); + for (u32 first = 0; first < funcs->count;) { + u32 end = uniform_run_end(funcs, first); + emit_lookup_run(out, funcs, first, end); + first = end; + } + fprintf(out, " return NULL;\n"); + fprintf(out, "}\n"); +} + void emit_dispatch_helpers(FILE* out, const FunctionList* funcs, u32 entry_point) { fprintf(out, "\n#define DOLRECOMP_ENTRY_POINT 0x%08Xu\n", entry_point); fprintf(out, "\ntypedef void (*DolRecompFunction)(CPUState* ctx);\n"); @@ -114,14 +381,9 @@ void emit_dispatch_helpers(FILE* out, const FunctionList* funcs, u32 entry_point fprintf(out, " return 0;\n"); fprintf(out, "}\n"); fprintf(out, "#endif\n"); - fprintf(out, "\nstatic inline DolRecompFunction dolrecomp_find_original(u32 address) {\n"); - for (u32 first = 0; first < funcs->count;) { - u32 end = uniform_run_end(funcs, first); - emit_lookup_run(out, funcs, first, end); - first = end; - } - fprintf(out, " return NULL;\n"); - fprintf(out, "}\n"); + if (dispatch_lookup_mode() != DISPATCH_LOOKUP_INDEXED || + !emit_lookup_indexed(out, funcs)) + emit_lookup_linear(out, funcs); fprintf(out, "\nstatic inline int dolrecomp_call_original(CPUState* ctx, u32 address) {\n"); fprintf(out, " DolRecompFunction fn = dolrecomp_find_original(address);\n"); fprintf(out, " if (!fn) return 0;\n"); diff --git a/src/backend/llvm/llvm_backend.cpp b/src/backend/llvm/llvm_backend.cpp index 05ecf2f..93e48ed 100644 --- a/src/backend/llvm/llvm_backend.cpp +++ b/src/backend/llvm/llvm_backend.cpp @@ -14,15 +14,24 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include +#include +#include #include #include +#include + +#include +#include +#include namespace { @@ -34,6 +43,164 @@ static std::string resolveTriple(const char *requested) { : llvm::sys::getDefaultTargetTriple(); } +// Everything below feeds both codegen and the object cache key. Changing any of +// it must invalidate cached objects, so keep it reachable from one place: the +// fingerprint is built from these same accessors, and an edit that misses the +// key produces a build that silently reuses objects from the old settings and +// reports them as a result. +// +// DOLRECOMP_LLVM_CPU and DOLRECOMP_LLVM_FEATURES override the target machine's +// CPU and feature string; "native" resolves to the host. The default stays +// "generic" with no features, which is the portable x86-64 baseline -- a +// released module must run on any x86-64 host, so raising the target has to +// stay opt-in. +// +// That baseline already includes SSE2, which is not incidental here: E001 +// measured the vectorizers as worth 1.37x precisely because Gekko +// paired-singles are 2-wide f32 pairs that map onto SSE2. So a wider target is +// a live hypothesis rather than a long shot. +static constexpr const char *kDefaultTargetCPU = "generic"; +static constexpr const char *kDefaultTargetFeatures = ""; + +static std::string targetCPU() { + const char *cpu = getenv("DOLRECOMP_LLVM_CPU"); + if (!cpu || !cpu[0]) + return kDefaultTargetCPU; + // createTargetMachine does not expand "native" itself. + if (!strcmp(cpu, "native")) + return llvm::sys::getHostCPUName().str(); + return cpu; +} + +static std::string targetFeatures() { + const char *features = getenv("DOLRECOMP_LLVM_FEATURES"); + return features ? features : kDefaultTargetFeatures; +} + +// instcombine's fixpoint check is a self-diagnostic for the pass, not a +// correctness property of the IR. Recompiled Gekko functions contain long +// straight-line integer and condition-flag sequences that can still be changing +// after one iteration, which makes the pass call report_fatal_error and take the +// whole recompilation down. Suppressing the check leaves the optimization +// itself intact. +// +// Do not remove the vectorizers. Measured (LLVM-EXPERIMENTS E001): dropping +// loop-vectorize, slp-vectorizer and vector-combine costs **-27%** throughput +// and makes the module 4.6% *larger*. The intuition that they cannot pay off +// against CPU "generic" with an empty feature string is wrong -- SSE2 is part +// of the x86-64 baseline, and Gekko paired-singles are inherently 2-wide f32 +// pairs, so SLP has real work to do at that baseline. +static constexpr const char *kPassPipeline = + "function(mem2reg,early-cse,instcombine," + "simplifycfg,sccp," + "correlated-propagation,jump-threading,gvn,dse,adce,loop-simplify," + "loop-rotate,loop-mssa(licm),loop-vectorize,slp-vectorizer,vector-" + "combine," + "tailcallelim),cgscc(inline),ipsccp,globaldce"; + +// The profile path, and a content hash of it, resolved once. The hash is what +// goes in the cache key: two different profiles written to the same path must +// not share objects, and a profile regenerated in place by a collection script +// makes the path alone a non-identity -- exactly the silent stale-reuse the +// fingerprint exists to prevent. +static const std::string &pgoProfilePath() { + static const std::string path = [] { + const char *file = getenv("DOLRECOMP_LLVM_PROFILE"); + return std::string(file ? file : ""); + }(); + return path; +} + +static const std::string &pgoProfileFingerprint() { + static const std::string fingerprint = [] { + if (pgoProfilePath().empty()) + return std::string("none"); + FILE *file = fopen(pgoProfilePath().c_str(), "rb"); + if (!file) + return std::string("missing"); + unsigned long long hash = 1469598103934665603ull; + unsigned long long size = 0; + unsigned char buffer[65536]; + size_t read; + while ((read = fread(buffer, 1, sizeof(buffer), file)) > 0) { + size += read; + for (size_t i = 0; i < read; i++) { + hash ^= buffer[i]; + hash *= 1099511628211ull; + } + } + fclose(file); + char text[64]; + snprintf(text, sizeof(text), "%016llx/%llu", hash, size); + return std::string(text); + }(); + return fingerprint; +} + +static void reportPgoStaleSummary(); + +// Off by default, so the default module stays byte-identical to an unprofiled +// build and the existing object cache keeps its meaning. "use" without a +// readable DOLRECOMP_LLVM_PROFILE is refused rather than silently degraded to +// an unprofiled build -- an untrained PGO build that looks trained is the one +// failure mode that corrupts a measurement instead of stopping it. +extern "C" int dolllvm_pgo_mode(void) { + static const int mode = [] { + const char *requested = getenv("DOLRECOMP_LLVM_PGO"); + if (!requested || !requested[0] || !strcmp(requested, "0") || + !strcmp(requested, "off")) + return (int)DOLLLVM_PGO_OFF; + if (!strcmp(requested, "gen")) + return (int)DOLLLVM_PGO_GEN; + if (strcmp(requested, "use")) { + fprintf(stderr, "dolllvm: DOLRECOMP_LLVM_PGO must be gen, use or off\n"); + abort(); + } + if (pgoProfilePath().empty() || pgoProfileFingerprint() == "missing") { + fprintf(stderr, + "dolllvm: DOLRECOMP_LLVM_PGO=use needs a readable " + "DOLRECOMP_LLVM_PROFILE (.profdata)\n"); + abort(); + } + fprintf(stderr, "dolllvm: PGO use, profile %s (%s)\n", + pgoProfilePath().c_str(), pgoProfileFingerprint().c_str()); + fflush(stderr); + // One summary per process, at exit, so a `warn`-policy build ends with a + // total rather than with thousands of individually ignorable lines. + atexit(reportPgoStaleSummary); + return (int)DOLLLVM_PGO_USE; + }(); + return mode; +} + +// Process-wide tallies for the staleness gate. The job runner is threads in one +// process (run_parallel_jobs), so these are atomic and the summary is printed +// once, from an atexit hook registered when use mode is first resolved. Without +// the summary a stale profile under the `warn` policy is thousands of +// individually ignorable lines and no total. +static std::atomic pgoMatchedFunctions{0}; +static std::atomic pgoUnmatchedFunctions{0}; +static std::atomic pgoStaleChunks{0}; + +static void reportPgoStaleSummary() { + const unsigned long long matched = pgoMatchedFunctions.load(); + const unsigned long long unmatched = pgoUnmatchedFunctions.load(); + const unsigned long long total = matched + unmatched; + if (total == 0) + return; + fprintf(stderr, + "dolllvm: PGO profile match: %llu/%llu functions matched, %llu " + "unmatched across %llu stale chunks\n", + matched, total, unmatched, pgoStaleChunks.load()); + if (unmatched != 0) + fprintf(stderr, + "dolllvm: PROFILE IS STALE against this DOL -- %.4f%% of emitted " + "functions carry no profile record. Re-collect the profile against " + "this binary.\n", + 100.0 * (double)unmatched / (double)total); + fflush(stderr); +} + static CodeGenOptLevel codegenLevel(int level) { if (level <= 0) return CodeGenOptLevel::None; @@ -52,8 +219,8 @@ static TargetMachine *targetMachine(const Target *target, if (!cachedMachine || cachedTriple != tripleName || cachedOpt != opt) { TargetOptions options; cachedMachine.reset(target->createTargetMachine( - tripleName, "generic", "", options, Reloc::PIC_, std::nullopt, - codegenLevel(opt))); + tripleName, targetCPU(), targetFeatures(), options, Reloc::PIC_, + std::nullopt, codegenLevel(opt))); cachedTriple = tripleName; cachedOpt = opt; } @@ -129,33 +296,130 @@ extern "C" bool dolllvm_emit_object(const DolIRModule *source, llvm::FunctionAnalysisManager fam; llvm::CGSCCAnalysisManager cgam; llvm::ModuleAnalysisManager mam; - llvm::PassBuilder passBuilder(machine); + + // DOLRECOMP_LLVM_TRACE_PASSES names the last pass and IR unit to start. + // Without it an optimizer that fails to converge is indistinguishable from + // one that is merely slow: the historical instcombine hang on this title + // spun for 49 minutes at 1.00 core with nothing identifying the function. + // Each line is flushed, so the last line printed is where it stopped. + llvm::PassInstrumentationCallbacks callbacks; + const int pgo = dolllvm_pgo_mode(); + const bool tracePasses = getenv("DOLRECOMP_LLVM_TRACE_PASSES") != nullptr; + // P002. Counted here, acted on after the pipeline has run. The callback is + // observational -- pass instrumentation cannot change what the passes do -- + // so the gate costs the emitted objects nothing and the fingerprint does + // not move. + unsigned long long matchedHere = 0; + unsigned long long unmatchedHere = 0; + const bool gatePgo = + pgo == DOLLLVM_PGO_USE && + dolllvm_pgo_stale_policy() != DOLLLVM_PGO_STALE_OFF; + if (gatePgo) { + callbacks.registerAfterPassCallback( + [&matchedHere, &unmatchedHere](llvm::StringRef pass, llvm::Any ir, + const llvm::PreservedAnalyses &) { + if (pass != "PGOInstrumentationUse") + return; + const llvm::Module *const *mod = + llvm::any_cast(&ir); + if (!mod || !*mod) + return; + // Immediately after the Use pass, so the count reflects what the + // profile matched and not what later passes went on to create. + for (const llvm::Function &function : **mod) { + if (function.isDeclaration()) + continue; + if (function.getEntryCount()) + matchedHere++; + else + unmatchedHere++; + } + }); + } + if (tracePasses) { + callbacks.registerBeforeNonSkippedPassCallback( + [](llvm::StringRef pass, llvm::Any ir) { + // Any holds a pointer to the IR unit, so any_cast on + // the Any* yields const T *const *. + std::string unit = ""; + const llvm::Function *const *fn = + llvm::any_cast(&ir); + const llvm::Module *const *mod = + llvm::any_cast(&ir); + if (fn && *fn) + unit = (*fn)->getName().str(); + else if (mod && *mod) + unit = (*mod)->getName().str(); + fprintf(stderr, "dolllvm: pass %s on %s\n", pass.str().c_str(), + unit.c_str()); + fflush(stderr); + }); + } + llvm::PassBuilder passBuilder(machine, llvm::PipelineTuningOptions(), + std::nullopt, + (tracePasses || gatePgo) ? &callbacks + : nullptr); passBuilder.registerModuleAnalyses(mam); passBuilder.registerCGSCCAnalyses(cgam); passBuilder.registerFunctionAnalyses(fam); passBuilder.registerLoopAnalyses(lam); passBuilder.crossRegisterProxies(lam, fam, cgam, mam); llvm::ModulePassManager passes; - std::string pipeline = - // instcombine's fixpoint check is a self-diagnostic for the pass, not a - // correctness property of the IR. Recompiled Gekko functions contain - // long straight-line integer and condition-flag sequences that can still - // be changing after one iteration, which makes the pass call - // report_fatal_error and take the whole recompilation down. Suppressing - // the check leaves the optimization itself intact. - "function(mem2reg,early-cse,instcombine," - "simplifycfg,sccp," - "correlated-propagation,jump-threading,gvn,dse,adce,loop-simplify," - "loop-rotate,loop-mssa(licm),loop-vectorize,slp-vectorizer,vector-" - "combine," - "tailcallelim),cgscc(inline),ipsccp,globaldce"; - if (llvm::Error error = passBuilder.parsePassPipeline(passes, pipeline)) { + // P001. Front of the pipeline, before anything has touched the emitter's + // output. Gen and Use must observe byte-identical IR or the CFG hashes they + // key on disagree and every function silently goes unprofiled; running both + // here makes that identity structural rather than a property of the passes + // in between, which are free to change without invalidating a profile. + if (pgo == DOLLLVM_PGO_GEN) { +#if LLVM_VERSION_MAJOR >= 20 + passes.addPass(llvm::PGOInstrumentationGen( + llvm::PGOInstrumentationType::FDO)); +#else + passes.addPass(llvm::PGOInstrumentationGen(/*IsCS=*/false)); +#endif + } else if (pgo == DOLLLVM_PGO_USE) { + passes.addPass(llvm::PGOInstrumentationUse(pgoProfilePath())); + } + if (llvm::Error error = + passBuilder.parsePassPipeline(passes, kPassPipeline)) { fprintf(diagnostics, "dolllvm: cannot construct optimization pipeline: %s\n", llvm::toString(std::move(error)).c_str()); return false; } + // The counter intrinsics have to survive the optimizer as intrinsics: once + // lowered they are a load, an add and a store on a global, and GVN or DSE + // will happily fold two iterations of a loop into one increment. Lower them + // last, which is where clang lowers them and for the same reason. + if (pgo == DOLLLVM_PGO_GEN) + passes.addPass(llvm::InstrProfilingLoweringPass(llvm::InstrProfOptions(), + /*IsCS=*/false)); passes.run(module, mam); + + // P002. The verdict. Under `error` a stale profile stops the build here, + // which is the point: the failure this gate exists for is a build that + // SUCCEEDS while training on records that no longer describe it, and every + // downstream number then belongs to a module nobody meant to measure. + if (gatePgo) { + pgoMatchedFunctions += matchedHere; + pgoUnmatchedFunctions += unmatchedHere; + if (unmatchedHere != 0) { + pgoStaleChunks++; + fprintf(diagnostics, + "dolllvm: PGO profile stale for %s: %llu of %llu functions " + "have no profile record\n", + object_path ? object_path : "", unmatchedHere, + matchedHere + unmatchedHere); + fflush(diagnostics); + if (dolllvm_pgo_stale_policy() == DOLLLVM_PGO_STALE_ERROR) { + fprintf(diagnostics, + "dolllvm: refusing to emit against a stale profile. Re-collect " + "it, or set DOLRECOMP_LLVM_PGO_STALE=warn to build anyway.\n"); + fflush(diagnostics); + return false; + } + } + } } if (llvm::verifyModule(module, &diagnosticStream)) { diagnosticStream.flush(); @@ -226,3 +490,44 @@ extern "C" bool dolllvm_object_matches_triple(const char *path, return magic[0] == 0x7F && magic[1] == 'E' && magic[2] == 'L' && magic[3] == 'F'; } + +// Default `error`: a stale profile is a wrong measurement, not a slow one, and +// a profile that looks applied and is not costs a whole build cycle. +extern "C" int dolllvm_pgo_stale_policy(void) { + static const int policy = [] { + const char *requested = getenv("DOLRECOMP_LLVM_PGO_STALE"); + if (!requested || !requested[0] || !strcmp(requested, "error")) + return (int)DOLLLVM_PGO_STALE_ERROR; + if (!strcmp(requested, "warn")) + return (int)DOLLLVM_PGO_STALE_WARN; + if (!strcmp(requested, "off") || !strcmp(requested, "0")) + return (int)DOLLLVM_PGO_STALE_OFF; + fprintf(stderr, + "dolllvm: DOLRECOMP_LLVM_PGO_STALE must be error, warn or off\n"); + abort(); + return (int)DOLLLVM_PGO_STALE_ERROR; + }(); + return policy; +} + +extern "C" bool dolllvm_codegen_fingerprint(char *out, size_t size) { + if (!out || size == 0) + return false; + // Every codegen-affecting input the object cache key would otherwise miss. + // The triple is hashed separately by the caller, which already had it. + // Keyed to the PGO mode, and in use mode to the profile's CONTENT -- a + // profile rewritten in place by a collection script makes its path a + // non-identity. Absent entirely when PGO is off, which is what keeps the + // default objects byte-identical and the existing cache valid. + const std::string fingerprint = + std::string(LLVM_VERSION_STRING) + "|" + targetCPU() + "|" + + targetFeatures() + "|" + "pic|small|" + kPassPipeline + + (dolllvm_pgo_mode() == DOLLLVM_PGO_GEN ? "|pgo=gen" : "") + + (dolllvm_pgo_mode() == DOLLLVM_PGO_USE + ? "|pgo=use:" + pgoProfileFingerprint() + : ""); + if (fingerprint.size() + 1 > size) + return false; + memcpy(out, fingerprint.c_str(), fingerprint.size() + 1); + return true; +} diff --git a/src/backend/llvm/llvm_backend.h b/src/backend/llvm/llvm_backend.h index 51743e2..07fa1cc 100644 --- a/src/backend/llvm/llvm_backend.h +++ b/src/backend/llvm/llvm_backend.h @@ -31,6 +31,74 @@ bool dolllvm_effective_triple(const char* requested, char* out, size_t size); // Validate an object's magic against the effective target triple. bool dolllvm_object_matches_triple(const char* path, const char* requested); +// Profile-guided optimization for the LLVM backend. +// +// The C backend gets PGO for free: its chunks are C source, so clang's own +// -fprofile-generate / -fprofile-use reach them through CFLAGS. Nothing reaches +// these objects the same way, because this backend emits IR and codegens it +// in-process -- clang never sees a translation unit. Sample/AutoFDO is +// genuinely unavailable (the emitter attaches no DILocation, so a sample +// profile has nothing to bind to), but IR-level *instrumentation* PGO needs no +// debug info: it is two LLVM passes run over the module the backend builds. +// +// DOLRECOMP_LLVM_PGO=gen instrument +// DOLRECOMP_LLVM_PGO=use apply DOLRECOMP_LLVM_PROFILE +// DOLRECOMP_LLVM_PROFILE= merged .profdata (use mode only) +// +// Both are placed at the very front of the pipeline, on the raw emitter output, +// so the CFG hashes PGOInstrumentationUse matches against are computed on +// exactly the IR PGOInstrumentationGen saw. The lowering pass runs last, as +// clang does it, so the counter intrinsics stay opaque to the optimizer and +// cannot be sunk, merged or eliminated into wrong counts. +// +// -fprofile-generate must still reach the module link line, which is where the +// profiling runtime comes from. A C half of the same module instrumented by +// clang through CFLAGS writes IR-level counters into the same profile, so the +// two merge without special handling. +typedef enum { + DOLLLVM_PGO_OFF = 0, + DOLLLVM_PGO_GEN = 1, + DOLLLVM_PGO_USE = 2 +} DolLLVMPGOMode; + +int dolllvm_pgo_mode(void); + +// The profile-vs-DOL staleness gate. +// +// A profile that no longer describes the emitted CFG does not fail. It +// degrades: PGOInstrumentationUse rejects the mismatched records one function +// at a time and leaves those functions unprofiled, so the build succeeds, the +// module looks trained, and the measurement is quietly against something +// between a profiled and an unprofiled arm. Clang's own warning for the C half +// (-Wprofile-instr-out-of-date) was suppressed on this project, and the LLVM +// half never had one at all. +// +// The gate is a POSITIVE check rather than a warning scrape: after +// PGOInstrumentationUse has run -- observed through a pass-instrumentation +// callback, so nothing about the pipeline moves -- every defined function that +// matched its profile record carries entry-count metadata, and every function +// that did not carries none. On a profile collected from this same module that +// second set is empty, because IR instrumentation records EVERY function at +// gen time, whether or not the scene ever executed it. So a non-zero count +// means the profile and the DOL have diverged, not that the scene was narrow. +// +// DOLRECOMP_LLVM_PGO_STALE=error fail the emit (the default) +// DOLRECOMP_LLVM_PGO_STALE=warn report and keep building +// DOLRECOMP_LLVM_PGO_STALE=off no check +typedef enum { + DOLLLVM_PGO_STALE_OFF = 0, + DOLLLVM_PGO_STALE_WARN = 1, + DOLLLVM_PGO_STALE_ERROR = 2 +} DolLLVMPGOStalePolicy; + +int dolllvm_pgo_stale_policy(void); + +// Every codegen-affecting input that is not already in the object cache key: +// LLVM version, target CPU and feature string, relocation and code model, and +// the pass pipeline. Hash this alongside the instruction words, or a codegen +// change silently reuses objects built with the old settings. +bool dolllvm_codegen_fingerprint(char* out, size_t size); + #ifdef __cplusplus } #endif diff --git a/src/ir/dolir_builder.c b/src/ir/dolir_builder.c index e463bc7..93cdc6a 100644 --- a/src/ir/dolir_builder.c +++ b/src/ir/dolir_builder.c @@ -798,7 +798,15 @@ static bool lower_float_memory(Builder* b) { if (single) value = unary(b, DOLIR_OP_FPEXT, DOLIR_TYPE_F64, value); set_fpr(b, i->rD, value); - set_ps1(b, i->rD, value); + // Gekko splits the float loads: lfs fills BOTH slots of the pair + // (Interpreter::lfs -> Fill), while lfd writes ps0 only and leaves + // ps1 architecturally intact (Interpreter::lfd -> SetPS0) for a + // later ps_ op or psq_st to read. Splatting unconditionally left + // the int-to-double bias (0x4330000000000000) from the guest's + // stw/stw/lfd conversion idiom sitting in ps1. The C emitter has + // always had this guard -- see emit_fload/emit_floadx. + if (single) + set_ps1(b, i->rD, value); } else { DolIRValue value = fpr(b, i->rS); if (single) @@ -974,8 +982,10 @@ static bool lower_float(Builder* b) { } default: return false; } + // fmr, fneg, fabs, fnabs and fsel are all ps0-only on Gekko -- + // Interpreter_FloatMisc uses SetPS0 for every one of them -- so ps1 is + // preserved across them. The C emitter never wrote ps1 here either. set_fpr(b, i->rD, value); - set_ps1(b, i->rD, value); if (i->rc) set_cr1_from_fpscr(b); return true; diff --git a/tests/test_dispatch.c b/tests/test_dispatch.c index 84eba52..5b4f843 100644 --- a/tests/test_dispatch.c +++ b/tests/test_dispatch.c @@ -17,6 +17,23 @@ static void check(int condition, const char* name) { fail_count++; } +// MSVC has no setenv, and _putenv_s is not portable back the other way. +static int set_lookup_mode(const char* value) { +#if defined(_WIN32) + char buffer[64]; + snprintf(buffer, sizeof(buffer), "DOLRECOMP_DISPATCH_LOOKUP=%s", + value ? value : ""); + return _putenv(buffer) == 0; +#else + if (!value) + return unsetenv("DOLRECOMP_DISPATCH_LOOKUP") == 0; + return setenv("DOLRECOMP_DISPATCH_LOOKUP", value, 1) == 0; +#endif +} + +// The four ranges below cover the three shapes the lookup has to get right: +// a contiguous equal-stride run (0x3000..0x3080), a short chunk closing that +// run (0x3080..0x30A0), and an isolated chunk a page away (0x4000..0x4020). static char* emit_dispatch_to_string(void) { FunctionList funcs = {0}; FILE* f = NULL; @@ -101,6 +118,43 @@ int main(void) { "public dispatcher can fall back to original code"); free(code); + + // DOLRECOMP_DISPATCH_LOOKUP=indexed. The linear chain is O(chunks) on an + // irregular plan and that confounded E008; the indexed form must replace + // it without changing which chunk an address resolves to. + if (set_lookup_mode("indexed")) { + char* indexed = emit_dispatch_to_string(); + if (!indexed) { + check(0, "indexed: emit dispatch helpers"); + } else { + check(strstr(indexed, "dolrecomp_page_first[DOLRECOMP_LOOKUP_PAGES]") != NULL && + strstr(indexed, "run = dolrecomp_page_first[page];") != NULL, + "indexed: page index selects the run window"); + check(strstr(indexed, "if (address >= 0x80003000u && address < 0x80003040u") == NULL, + "indexed: no linear range-test chain remains"); + check(strstr(indexed, "#define DOLRECOMP_LOOKUP_RUNS 2u") != NULL, + "indexed: collapses the contiguous chunks into one run"); + // 0x80003000..0x800040a0 spans two 4 KiB pages plus the boundary page. + check(strstr(indexed, "#define DOLRECOMP_LOOKUP_BASE 0x80003000u") != NULL && + strstr(indexed, "#define DOLRECOMP_LOOKUP_PAGES 2u") != NULL, + "indexed: page table covers exactly the emitted code"); + check(strstr(indexed, "func_80003000,") != NULL && + strstr(indexed, "func_80003040,") != NULL && + strstr(indexed, "func_80003080,") != NULL && + strstr(indexed, "func_80004000,") != NULL, + "indexed: chunk table covers generated chunks"); + check(strstr(indexed, "if ((offset & 3u) != 0u) return NULL;") != NULL, + "indexed: keeps the instruction-alignment check"); + check(strstr(indexed, "ctx->pc = address;") != NULL && + strstr(indexed, "dolrecomp_physical_pc_alias") != NULL, + "indexed: leaves the rest of the dispatcher alone"); + free(indexed); + } + set_lookup_mode(NULL); + } else { + check(0, "indexed: set DOLRECOMP_DISPATCH_LOOKUP"); + } + printf("DISPATCH,total,%d passed %d failed\n", pass_count, fail_count); return fail_count == 0 ? 0 : 1; }