chore(bench): add javac compilation step to Java fixture for javacg-static comparison

docs/benchmarks/RESOLUTION-COMPARISON.md

@@ -177,10 +177,35 @@ Established Java static call graph tools all require compiled bytecode:
 | [Soot](https://github.com/soot-oss/soot) | CHA / RTA / VTA / Spark | Needs compiled `.class` files |
 | [javacg-static](https://github.com/gousiosg/java-callgraph) | CHA | Lightweight, reads JARs |
 
-The fixture contains raw `.java` source with no build system. Running these
-tools requires a `javac` compilation step (tracked in #1307).
+The fixture now includes a `Makefile` that compiles all `.java` sources into
+`fixture.jar`:
 
-**Current codegraph Java metrics** (`scripts/resolution-benchmark.ts`):
+```bash
+cd tests/benchmarks/resolution/fixtures/java && make
+```
+
+`scripts/compare-javacg.mjs` then runs
+[javacg-static](https://github.com/gousiosg/java-callgraph) on the compiled JAR.
+javacg-static uses CHA to enumerate all possible call targets at virtual,
+interface, and static call sites.
+
+```bash
+node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar
+# or: JAVACG_JAR=... node scripts/compare-javacg.mjs
+```
+
+Download javacg-static from
+[github.com/gousiosg/java-callgraph/releases](https://github.com/gousiosg/java-callgraph/releases)
+or build with `mvn package -DskipTests`.
+
+**Name mapping:** javacg-static uses `pkg.ClassName:method(JVM-descriptors)` form.
+`compare-javacg.mjs` maps this to `ClassName.method` and matches
+`ClassName.ClassName` (source constructor) / `ClassName` (target constructor)
+against the expected-edges.json convention. Only edges where both class names
+appear in the fixture source files are counted; JDK calls (`String`, `HashMap`,
+`System.out`, …) are filtered out.
+
+**Codegraph Java metrics** (`scripts/resolution-benchmark.ts`):
 
 | Mode | Codegraph |
 |------|:---------:|
@@ -192,6 +217,19 @@ tools requires a `javac` compilation step (tracked in #1307).
 | `class-inheritance` (3 edges) | 0/3 (0%) |
 | **Total** | **9/17 (53%)** · precision=100% |
 
+**javacg-static comparison** — run `node scripts/compare-javacg.mjs` to populate:
+
+| Tool | Precision | Recall | TP | FP | FN |
+|------|:---------:|:------:|---:|---:|---:|
+| Codegraph | 100% | 53% | 9 | 0 | 8 |
+| javacg-static (CHA) | — | — | — | — | — |
+
+javacg-static uses CHA and reads compiled bytecode, so it should resolve
+`class-inheritance` (inherited `log()` calls) and `interface-dispatched`
+(virtual dispatch via `UserRepository` interface) edges that codegraph
+currently misses at the source-level. `static` and `same-file` calls
+(`invokestatic` in bytecode) should also be fully captured.
+
 ---
 
 ## Conclusions
@@ -231,8 +269,9 @@ tools requires a `javac` compilation step (tracked in #1307).
    2 `class-inheritance` edges (+7 recall on TS fixture).
 2. **Property-assignment type tracking** (#1306) — Track `this.prop = new Foo()`
    writes. Recovers 3 JS `receiver-typed` FN.
-3. **Java comparison with javacg-static** (#1307) — Add `javac` compilation to
-   the Java fixture so a bytecode-level tool can validate Java recall claims.
+3. **Java recall gaps** — `same-file` (0/2) and `static` (0/2) are `invokestatic`
+   patterns that javacg-static will expose; `class-inheritance` (0/3) requires
+   tracking inherited method calls from superclass to subclass invocation site.
 
 ---
 
@@ -300,6 +339,10 @@ npx tsx scripts/resolution-benchmark.ts | jq '{javascript, typescript, java}'
 npm install @cs-au-dk/jelly @persper/js-callgraph
 node scripts/compare-tools.mjs --all
 
+# javacg-static comparison (Java)
+cd tests/benchmarks/resolution/fixtures/java && make && cd -
+node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar
+
 # Full resolution test suite
 npx vitest run tests/benchmarks/resolution/resolution-benchmark.test.ts
 ```

scripts/compare-javacg.mjs

@@ -0,0 +1,297 @@
+#!/usr/bin/env node
+/**
+ * javacg-static vs Codegraph: Java fixture call graph comparison
+ *
+ * Runs javacg-static (gousiosg/java-callgraph) on the compiled fixture JAR,
+ * parses its output, maps class:method names to ClassName.method form, and
+ * computes precision/recall against expected-edges.json.
+ *
+ * javacg-static output format:
+ *   M:pkg.ClassName:method(argDescriptors)  (T)pkg.ClassName:method(argDescriptors)
+ * where T is: C=virtual, S=static, O=special (constructors, super), I=interface, D=dynamic
+ *
+ * Name mapping to expected-edges.json convention:
+ *   source <init>  → ClassName.ClassName  (constructor-as-method)
+ *   target <init>  → ClassName            (constructor target = class name only)
+ *   other method   → ClassName.method
+ *
+ * Prerequisites:
+ *   1. Java runtime (java -jar must work)
+ *   2. javacg-static JAR — download from:
+ *        https://github.com/gousiosg/java-callgraph/releases
+ *      or build: `cd java-callgraph && mvn package -DskipTests`
+ *      Pass via --jar or set JAVACG_JAR, or place at scripts/lib/javacg-static.jar
+ *   3. Compiled fixture JAR:
+ *        cd tests/benchmarks/resolution/fixtures/java && make
+ *
+ * Usage:
+ *   node scripts/compare-javacg.mjs
+ *   node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar
+ *   node scripts/compare-javacg.mjs --json
+ */
+
+import { execFileSync } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(__dirname, '..');
+const FIXTURE_DIR = path.join(ROOT, 'tests/benchmarks/resolution/fixtures/java');
+
+// ── CLI ──────────────────────────────────────────────────────────────────────
+
+const args = process.argv.slice(2);
+const jsonFlag = args.includes('--json');
+const jarArgIdx = args.indexOf('--jar');
+const jarArgNext = jarArgIdx !== -1 ? (args[jarArgIdx + 1] ?? null) : null;
+const jarArgPath = jarArgNext && !jarArgNext.startsWith('--') ? jarArgNext : null;
+if (jarArgIdx !== -1 && !jarArgPath) {
+  console.error('Error: --jar requires a path argument');
+  process.exit(1);
+}
+
+// ── Tool discovery ───────────────────────────────────────────────────────────
+
+function findJavacgJar() {
+  if (jarArgPath) return jarArgPath;
+  if (process.env.JAVACG_JAR) return process.env.JAVACG_JAR;
+  // Glob for any jar with "javacg" in the name under scripts/lib/
+  const libDir = path.join(__dirname, 'lib');
+  if (fs.existsSync(libDir)) {
+    const jar = fs.readdirSync(libDir).find((f) => f.includes('javacg') && f.endsWith('.jar'));
+    if (jar) return path.join(libDir, jar);
+  }
+  return null;
+}
+
+// ── Name mapping ─────────────────────────────────────────────────────────────
+
+/**
+ * Scan .java source files to build SimpleClassName → filename.java map.
+ * Used to resolve file fields in the edge key format "name@file".
+ *
+ * Maps the first top-level type per file — inner classes are not indexed.
+ * Handles common modifiers (public, abstract, final, sealed, non-sealed, strictfp)
+ * and type keywords (class, interface, enum, record).
+ */
+function buildClassFileMap(fixtureDir) {
+  const map = new Map();
+  const javaFiles = fs.readdirSync(fixtureDir).filter((f) => f.endsWith('.java'));
+  for (const filename of javaFiles) {
+    const src = fs.readFileSync(path.join(fixtureDir, filename), 'utf8');
+    // Match any combination of access/modifier keywords before the type keyword.
+    // Anchored to line start (^…/m) so Javadoc comments containing the word
+    // "class" before the actual declaration don't produce a false match.
+    const m = src.match(
+      /^(?:(?:public|protected|private|abstract|final|sealed|non-sealed|strictfp)\s+)*(?:class|interface|enum|record)\s+(\w+)/m,
+    );
+    if (m) {
+      map.set(m[1], filename);
+    } else {
+      console.warn(`[warn] buildClassFileMap: no type declaration found in ${filename} — edges involving this file will be filtered out`);
+    }
+  }
+  // Validate: every .java file should map to exactly one class name
+  if (map.size !== javaFiles.length) {
+    console.warn(
+      `[warn] buildClassFileMap: ${javaFiles.length} .java files but only ${map.size} class names resolved — precision/recall may be skewed`,
+    );
+  }
+  return map;
+}
+
+/**
+ * Parse "pkg.ClassName:methodName(descriptors)" into { className, methodName }.
+ * Works with both "." and "/" as package separators (javacg uses ".").
+ */
+function parseMethodSpec(spec) {
+  // Strip argument descriptor — everything from "(" onwards
+  const parenIdx = spec.indexOf('(');
+  const withoutArgs = parenIdx !== -1 ? spec.slice(0, parenIdx) : spec;
+  const colonIdx = withoutArgs.indexOf(':');
+  if (colonIdx === -1) return null;
+  const classPart = withoutArgs.slice(0, colonIdx);
+  const methodName = withoutArgs.slice(colonIdx + 1);
+  // Simple class name: last segment after "." or "/"
+  const className = classPart.split(/[./]/).at(-1);
+  if (!className) return null;
+  return { className, methodName };
+}
+
+/** Source side: "<init>" method maps to ClassName.ClassName. */
+function toSourceName({ className, methodName }) {
+  return methodName === '<init>' ? `${className}.${className}` : `${className}.${methodName}`;
+}
+
+/** Target side: "<init>" method maps to just ClassName (constructor target). */
+function toTargetName({ className, methodName }) {
+  return methodName === '<init>' ? className : `${className}.${methodName}`;
+}
+
+// ── Ground truth ─────────────────────────────────────────────────────────────
+
+function loadGroundTruth(fixtureDir) {
+  const manifest = JSON.parse(
+    fs.readFileSync(path.join(fixtureDir, 'expected-edges.json'), 'utf8'),
+  );
+  const set = new Set(
+    manifest.edges.map(
+      (e) =>
+        `${e.source.name}@${path.basename(e.source.file)}→${e.target.name}@${path.basename(e.target.file)}`,
+    ),
+  );
+  return set;
+}
+
+// ── Run javacg-static ────────────────────────────────────────────────────────
+
+function runJavacg(javacgJar, fixtureDir) {
+  const fixtureJar = path.join(fixtureDir, 'fixture.jar');
+  if (!fs.existsSync(fixtureJar)) {
+    console.error(`fixture.jar not found at ${fixtureJar}`);
+    console.error(`Build it with: cd ${fixtureDir} && make`);
+    process.exit(1);
+  }
+  try {
+    return execFileSync('java', ['-jar', javacgJar, fixtureJar], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+  } catch (err) {
+    // javacg-static may exit non-zero but still produce useful stdout
+    if (err.stdout?.trim()) return err.stdout;
+    console.error(`javacg-static failed: ${err.message}`);
+    process.exit(1);
+  }
+}
+
+/**
+ * Parse javacg-static text output into a Set of edge keys.
+ *
+ * Line format:
+ *   M:pkg.Class:method(args)  (T)pkg.Class:method(args)
+ *
+ * Only edges where both class names appear in classFileMap are included —
+ * this filters out JDK / stdlib calls (HashMap, String, System.out, etc.).
+ */
+function parseJavacgOutput(output, classFileMap) {
+  // M: caller (T) callee — the space between caller and (T) may vary
+  // T values: C=virtual, S=static, O=special (constructors/super), I=interface, D=dynamic (invokedynamic)
+  const lineRe = /^M:(\S+)\s+\(([CSOID])\)(\S+)$/;
+  const edges = new Set();
+
+  for (const rawLine of output.split('\n')) {
+    const line = rawLine.trim();
+    if (!line.startsWith('M:')) continue;
+
+    const m = line.match(lineRe);
+    if (!m) continue;
+
+    const [, sourceSpec, , targetSpec] = m;
+
+    const sourceParsed = parseMethodSpec(sourceSpec);
+    const targetParsed = parseMethodSpec(targetSpec);
+    if (!sourceParsed || !targetParsed) continue;
+
+    const sourceFile = classFileMap.get(sourceParsed.className);
+    const targetFile = classFileMap.get(targetParsed.className);
+    // Skip edges to/from classes outside the fixture (JDK, etc.)
+    if (!sourceFile || !targetFile) continue;
+
+    const sourceName = toSourceName(sourceParsed);
+    const targetName = toTargetName(targetParsed);
+
+    const key = `${sourceName}@${sourceFile}→${targetName}@${targetFile}`;
+    // Skip self-edges (e.g. recursive calls not in expected-edges)
+    if (sourceName === targetName && sourceFile === targetFile) continue;
+    edges.add(key);
+  }
+  return edges;
+}
+
+// ── Metrics ──────────────────────────────────────────────────────────────────
+
+function computeMetrics(predicted, groundTruth) {
+  let tp = 0;
+  const fp = [];
+  const fn = [];
+  for (const edge of predicted) (groundTruth.has(edge) ? tp++ : fp.push(edge));
+  for (const edge of groundTruth) if (!predicted.has(edge)) fn.push(edge);
+  return {
+    precision: predicted.size === 0 ? 0 : tp / predicted.size,
+    recall: groundTruth.size === 0 ? 0 : tp / groundTruth.size,
+    tp,
+    fp: fp.length,
+    fn: fn.length,
+    totalPredicted: predicted.size,
+    totalExpected: groundTruth.size,
+    fpEdges: fp,
+    fnEdges: fn,
+  };
+}
+
+// ── Main ─────────────────────────────────────────────────────────────────────
+
+const javacgJar = findJavacgJar();
+if (!javacgJar) {
+  console.error('javacg-static JAR not found.');
+  console.error('Download from: https://github.com/gousiosg/java-callgraph/releases');
+  console.error('Then use one of:');
+  console.error('  node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar');
+  console.error('  JAVACG_JAR=/path/to/javacg-0.1-SNAPSHOT.jar node scripts/compare-javacg.mjs');
+  console.error('  cp /path/to/javacg-0.1-SNAPSHOT.jar scripts/lib/javacg-static.jar');
+  process.exit(1);
+}
+
+const classFileMap = buildClassFileMap(FIXTURE_DIR);
+const groundTruth = loadGroundTruth(FIXTURE_DIR);
+
+console.error(`\n── JAVA ──────────────────────────────────────────────────`);
+console.error(`  Ground truth: ${groundTruth.size} edges`);
+console.error(`  Running javacg-static on fixture.jar...`);
+
+const rawOutput = runJavacg(javacgJar, FIXTURE_DIR);
+const predictedEdges = parseJavacgOutput(rawOutput, classFileMap);
+
+console.error(`  javacg-static: ${predictedEdges.size} named benchmark edges`);
+
+const metrics = computeMetrics(predictedEdges, groundTruth);
+
+console.error(
+  `  precision=${metrics.precision.toFixed(2)} recall=${metrics.recall.toFixed(2)} ` +
+    `TP=${metrics.tp} FP=${metrics.fp} FN=${metrics.fn}`,
+);
+
+if (metrics.fpEdges.length) {
+  console.error(`  FP (edges not in expected-edges.json):`);
+  for (const e of metrics.fpEdges) console.error(`    - ${e}`);
+}
+if (metrics.fnEdges.length) {
+  console.error(`  FN (expected edges missed):`);
+  for (const e of metrics.fnEdges) console.error(`    - ${e}`);
+}
+
+if (jsonFlag) {
+  console.log(
+    JSON.stringify(
+      {
+        java: {
+          groundTruth: groundTruth.size,
+          javacgEdges: predictedEdges.size,
+          metrics,
+        },
+      },
+      null,
+      2,
+    ),
+  );
+} else {
+  console.log('\n## javacg-static vs expected-edges.json Ground Truth\n');
+  console.log('| Language | Tool | Precision | Recall | TP | FP | FN |');
+  console.log('|----------|------|:---------:|:------:|---:|---:|---:|');
+  console.log(
+    `| Java | javacg-static (CHA) | ${(metrics.precision * 100).toFixed(0)}% | ` +
+      `${(metrics.recall * 100).toFixed(0)}% | ${metrics.tp} | ${metrics.fp} | ${metrics.fn} |`,
+  );
+}

tests/benchmarks/resolution/fixtures/java/.gitignore

@@ -0,0 +1,2 @@
+fixture.jar
+benchmark/

tests/benchmarks/resolution/fixtures/java/Makefile

@@ -0,0 +1,15 @@
+SRCS = $(wildcard *.java)
+JAR  = fixture.jar
+
+.PHONY: all clean
+
+all: $(JAR)
+
+$(JAR): $(SRCS)
+	rm -rf benchmark/
+	javac -d . $(SRCS)
+	jar cf $(JAR) benchmark/
+
+clean:
+	rm -f $(JAR)
+	rm -rf benchmark/