diff --git a/docs/benchmarks/RESOLUTION-COMPARISON.md b/docs/benchmarks/RESOLUTION-COMPARISON.md index 79789bbd0..6b2801d86 100644 --- a/docs/benchmarks/RESOLUTION-COMPARISON.md +++ b/docs/benchmarks/RESOLUTION-COMPARISON.md @@ -177,10 +177,35 @@ Established Java static call graph tools all require compiled bytecode: | [Soot](https://github.com/soot-oss/soot) | CHA / RTA / VTA / Spark | Needs compiled `.class` files | | [javacg-static](https://github.com/gousiosg/java-callgraph) | CHA | Lightweight, reads JARs | -The fixture contains raw `.java` source with no build system. Running these -tools requires a `javac` compilation step (tracked in #1307). +The fixture now includes a `Makefile` that compiles all `.java` sources into +`fixture.jar`: -**Current codegraph Java metrics** (`scripts/resolution-benchmark.ts`): +```bash +cd tests/benchmarks/resolution/fixtures/java && make +``` + +`scripts/compare-javacg.mjs` then runs +[javacg-static](https://github.com/gousiosg/java-callgraph) on the compiled JAR. +javacg-static uses CHA to enumerate all possible call targets at virtual, +interface, and static call sites. + +```bash +node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar +# or: JAVACG_JAR=... node scripts/compare-javacg.mjs +``` + +Download javacg-static from +[github.com/gousiosg/java-callgraph/releases](https://github.com/gousiosg/java-callgraph/releases) +or build with `mvn package -DskipTests`. + +**Name mapping:** javacg-static uses `pkg.ClassName:method(JVM-descriptors)` form. +`compare-javacg.mjs` maps this to `ClassName.method` and matches +`ClassName.ClassName` (source constructor) / `ClassName` (target constructor) +against the expected-edges.json convention. Only edges where both class names +appear in the fixture source files are counted; JDK calls (`String`, `HashMap`, +`System.out`, …) are filtered out. + +**Codegraph Java metrics** (`scripts/resolution-benchmark.ts`): | Mode | Codegraph | |------|:---------:| @@ -192,6 +217,19 @@ tools requires a `javac` compilation step (tracked in #1307). | `class-inheritance` (3 edges) | 0/3 (0%) | | **Total** | **9/17 (53%)** · precision=100% | +**javacg-static comparison** — run `node scripts/compare-javacg.mjs` to populate: + +| Tool | Precision | Recall | TP | FP | FN | +|------|:---------:|:------:|---:|---:|---:| +| Codegraph | 100% | 53% | 9 | 0 | 8 | +| javacg-static (CHA) | — | — | — | — | — | + +javacg-static uses CHA and reads compiled bytecode, so it should resolve +`class-inheritance` (inherited `log()` calls) and `interface-dispatched` +(virtual dispatch via `UserRepository` interface) edges that codegraph +currently misses at the source-level. `static` and `same-file` calls +(`invokestatic` in bytecode) should also be fully captured. + --- ## Conclusions @@ -231,8 +269,9 @@ tools requires a `javac` compilation step (tracked in #1307). 2 `class-inheritance` edges (+7 recall on TS fixture). 2. **Property-assignment type tracking** (#1306) — Track `this.prop = new Foo()` writes. Recovers 3 JS `receiver-typed` FN. -3. **Java comparison with javacg-static** (#1307) — Add `javac` compilation to - the Java fixture so a bytecode-level tool can validate Java recall claims. +3. **Java recall gaps** — `same-file` (0/2) and `static` (0/2) are `invokestatic` + patterns that javacg-static will expose; `class-inheritance` (0/3) requires + tracking inherited method calls from superclass to subclass invocation site. --- @@ -300,6 +339,10 @@ npx tsx scripts/resolution-benchmark.ts | jq '{javascript, typescript, java}' npm install @cs-au-dk/jelly @persper/js-callgraph node scripts/compare-tools.mjs --all +# javacg-static comparison (Java) +cd tests/benchmarks/resolution/fixtures/java && make && cd - +node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar + # Full resolution test suite npx vitest run tests/benchmarks/resolution/resolution-benchmark.test.ts ``` diff --git a/scripts/compare-javacg.mjs b/scripts/compare-javacg.mjs new file mode 100644 index 000000000..bd9941ecd --- /dev/null +++ b/scripts/compare-javacg.mjs @@ -0,0 +1,297 @@ +#!/usr/bin/env node +/** + * javacg-static vs Codegraph: Java fixture call graph comparison + * + * Runs javacg-static (gousiosg/java-callgraph) on the compiled fixture JAR, + * parses its output, maps class:method names to ClassName.method form, and + * computes precision/recall against expected-edges.json. + * + * javacg-static output format: + * M:pkg.ClassName:method(argDescriptors) (T)pkg.ClassName:method(argDescriptors) + * where T is: C=virtual, S=static, O=special (constructors, super), I=interface, D=dynamic + * + * Name mapping to expected-edges.json convention: + * source → ClassName.ClassName (constructor-as-method) + * target → ClassName (constructor target = class name only) + * other method → ClassName.method + * + * Prerequisites: + * 1. Java runtime (java -jar must work) + * 2. javacg-static JAR — download from: + * https://github.com/gousiosg/java-callgraph/releases + * or build: `cd java-callgraph && mvn package -DskipTests` + * Pass via --jar or set JAVACG_JAR, or place at scripts/lib/javacg-static.jar + * 3. Compiled fixture JAR: + * cd tests/benchmarks/resolution/fixtures/java && make + * + * Usage: + * node scripts/compare-javacg.mjs + * node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar + * node scripts/compare-javacg.mjs --json + */ + +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const ROOT = path.resolve(__dirname, '..'); +const FIXTURE_DIR = path.join(ROOT, 'tests/benchmarks/resolution/fixtures/java'); + +// ── CLI ────────────────────────────────────────────────────────────────────── + +const args = process.argv.slice(2); +const jsonFlag = args.includes('--json'); +const jarArgIdx = args.indexOf('--jar'); +const jarArgNext = jarArgIdx !== -1 ? (args[jarArgIdx + 1] ?? null) : null; +const jarArgPath = jarArgNext && !jarArgNext.startsWith('--') ? jarArgNext : null; +if (jarArgIdx !== -1 && !jarArgPath) { + console.error('Error: --jar requires a path argument'); + process.exit(1); +} + +// ── Tool discovery ─────────────────────────────────────────────────────────── + +function findJavacgJar() { + if (jarArgPath) return jarArgPath; + if (process.env.JAVACG_JAR) return process.env.JAVACG_JAR; + // Glob for any jar with "javacg" in the name under scripts/lib/ + const libDir = path.join(__dirname, 'lib'); + if (fs.existsSync(libDir)) { + const jar = fs.readdirSync(libDir).find((f) => f.includes('javacg') && f.endsWith('.jar')); + if (jar) return path.join(libDir, jar); + } + return null; +} + +// ── Name mapping ───────────────────────────────────────────────────────────── + +/** + * Scan .java source files to build SimpleClassName → filename.java map. + * Used to resolve file fields in the edge key format "name@file". + * + * Maps the first top-level type per file — inner classes are not indexed. + * Handles common modifiers (public, abstract, final, sealed, non-sealed, strictfp) + * and type keywords (class, interface, enum, record). + */ +function buildClassFileMap(fixtureDir) { + const map = new Map(); + const javaFiles = fs.readdirSync(fixtureDir).filter((f) => f.endsWith('.java')); + for (const filename of javaFiles) { + const src = fs.readFileSync(path.join(fixtureDir, filename), 'utf8'); + // Match any combination of access/modifier keywords before the type keyword. + // Anchored to line start (^…/m) so Javadoc comments containing the word + // "class" before the actual declaration don't produce a false match. + const m = src.match( + /^(?:(?:public|protected|private|abstract|final|sealed|non-sealed|strictfp)\s+)*(?:class|interface|enum|record)\s+(\w+)/m, + ); + if (m) { + map.set(m[1], filename); + } else { + console.warn(`[warn] buildClassFileMap: no type declaration found in ${filename} — edges involving this file will be filtered out`); + } + } + // Validate: every .java file should map to exactly one class name + if (map.size !== javaFiles.length) { + console.warn( + `[warn] buildClassFileMap: ${javaFiles.length} .java files but only ${map.size} class names resolved — precision/recall may be skewed`, + ); + } + return map; +} + +/** + * Parse "pkg.ClassName:methodName(descriptors)" into { className, methodName }. + * Works with both "." and "/" as package separators (javacg uses "."). + */ +function parseMethodSpec(spec) { + // Strip argument descriptor — everything from "(" onwards + const parenIdx = spec.indexOf('('); + const withoutArgs = parenIdx !== -1 ? spec.slice(0, parenIdx) : spec; + const colonIdx = withoutArgs.indexOf(':'); + if (colonIdx === -1) return null; + const classPart = withoutArgs.slice(0, colonIdx); + const methodName = withoutArgs.slice(colonIdx + 1); + // Simple class name: last segment after "." or "/" + const className = classPart.split(/[./]/).at(-1); + if (!className) return null; + return { className, methodName }; +} + +/** Source side: "" method maps to ClassName.ClassName. */ +function toSourceName({ className, methodName }) { + return methodName === '' ? `${className}.${className}` : `${className}.${methodName}`; +} + +/** Target side: "" method maps to just ClassName (constructor target). */ +function toTargetName({ className, methodName }) { + return methodName === '' ? className : `${className}.${methodName}`; +} + +// ── Ground truth ───────────────────────────────────────────────────────────── + +function loadGroundTruth(fixtureDir) { + const manifest = JSON.parse( + fs.readFileSync(path.join(fixtureDir, 'expected-edges.json'), 'utf8'), + ); + const set = new Set( + manifest.edges.map( + (e) => + `${e.source.name}@${path.basename(e.source.file)}→${e.target.name}@${path.basename(e.target.file)}`, + ), + ); + return set; +} + +// ── Run javacg-static ──────────────────────────────────────────────────────── + +function runJavacg(javacgJar, fixtureDir) { + const fixtureJar = path.join(fixtureDir, 'fixture.jar'); + if (!fs.existsSync(fixtureJar)) { + console.error(`fixture.jar not found at ${fixtureJar}`); + console.error(`Build it with: cd ${fixtureDir} && make`); + process.exit(1); + } + try { + return execFileSync('java', ['-jar', javacgJar, fixtureJar], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + } catch (err) { + // javacg-static may exit non-zero but still produce useful stdout + if (err.stdout?.trim()) return err.stdout; + console.error(`javacg-static failed: ${err.message}`); + process.exit(1); + } +} + +/** + * Parse javacg-static text output into a Set of edge keys. + * + * Line format: + * M:pkg.Class:method(args) (T)pkg.Class:method(args) + * + * Only edges where both class names appear in classFileMap are included — + * this filters out JDK / stdlib calls (HashMap, String, System.out, etc.). + */ +function parseJavacgOutput(output, classFileMap) { + // M: caller (T) callee — the space between caller and (T) may vary + // T values: C=virtual, S=static, O=special (constructors/super), I=interface, D=dynamic (invokedynamic) + const lineRe = /^M:(\S+)\s+\(([CSOID])\)(\S+)$/; + const edges = new Set(); + + for (const rawLine of output.split('\n')) { + const line = rawLine.trim(); + if (!line.startsWith('M:')) continue; + + const m = line.match(lineRe); + if (!m) continue; + + const [, sourceSpec, , targetSpec] = m; + + const sourceParsed = parseMethodSpec(sourceSpec); + const targetParsed = parseMethodSpec(targetSpec); + if (!sourceParsed || !targetParsed) continue; + + const sourceFile = classFileMap.get(sourceParsed.className); + const targetFile = classFileMap.get(targetParsed.className); + // Skip edges to/from classes outside the fixture (JDK, etc.) + if (!sourceFile || !targetFile) continue; + + const sourceName = toSourceName(sourceParsed); + const targetName = toTargetName(targetParsed); + + const key = `${sourceName}@${sourceFile}→${targetName}@${targetFile}`; + // Skip self-edges (e.g. recursive calls not in expected-edges) + if (sourceName === targetName && sourceFile === targetFile) continue; + edges.add(key); + } + return edges; +} + +// ── Metrics ────────────────────────────────────────────────────────────────── + +function computeMetrics(predicted, groundTruth) { + let tp = 0; + const fp = []; + const fn = []; + for (const edge of predicted) (groundTruth.has(edge) ? tp++ : fp.push(edge)); + for (const edge of groundTruth) if (!predicted.has(edge)) fn.push(edge); + return { + precision: predicted.size === 0 ? 0 : tp / predicted.size, + recall: groundTruth.size === 0 ? 0 : tp / groundTruth.size, + tp, + fp: fp.length, + fn: fn.length, + totalPredicted: predicted.size, + totalExpected: groundTruth.size, + fpEdges: fp, + fnEdges: fn, + }; +} + +// ── Main ───────────────────────────────────────────────────────────────────── + +const javacgJar = findJavacgJar(); +if (!javacgJar) { + console.error('javacg-static JAR not found.'); + console.error('Download from: https://github.com/gousiosg/java-callgraph/releases'); + console.error('Then use one of:'); + console.error(' node scripts/compare-javacg.mjs --jar /path/to/javacg-0.1-SNAPSHOT.jar'); + console.error(' JAVACG_JAR=/path/to/javacg-0.1-SNAPSHOT.jar node scripts/compare-javacg.mjs'); + console.error(' cp /path/to/javacg-0.1-SNAPSHOT.jar scripts/lib/javacg-static.jar'); + process.exit(1); +} + +const classFileMap = buildClassFileMap(FIXTURE_DIR); +const groundTruth = loadGroundTruth(FIXTURE_DIR); + +console.error(`\n── JAVA ──────────────────────────────────────────────────`); +console.error(` Ground truth: ${groundTruth.size} edges`); +console.error(` Running javacg-static on fixture.jar...`); + +const rawOutput = runJavacg(javacgJar, FIXTURE_DIR); +const predictedEdges = parseJavacgOutput(rawOutput, classFileMap); + +console.error(` javacg-static: ${predictedEdges.size} named benchmark edges`); + +const metrics = computeMetrics(predictedEdges, groundTruth); + +console.error( + ` precision=${metrics.precision.toFixed(2)} recall=${metrics.recall.toFixed(2)} ` + + `TP=${metrics.tp} FP=${metrics.fp} FN=${metrics.fn}`, +); + +if (metrics.fpEdges.length) { + console.error(` FP (edges not in expected-edges.json):`); + for (const e of metrics.fpEdges) console.error(` - ${e}`); +} +if (metrics.fnEdges.length) { + console.error(` FN (expected edges missed):`); + for (const e of metrics.fnEdges) console.error(` - ${e}`); +} + +if (jsonFlag) { + console.log( + JSON.stringify( + { + java: { + groundTruth: groundTruth.size, + javacgEdges: predictedEdges.size, + metrics, + }, + }, + null, + 2, + ), + ); +} else { + console.log('\n## javacg-static vs expected-edges.json Ground Truth\n'); + console.log('| Language | Tool | Precision | Recall | TP | FP | FN |'); + console.log('|----------|------|:---------:|:------:|---:|---:|---:|'); + console.log( + `| Java | javacg-static (CHA) | ${(metrics.precision * 100).toFixed(0)}% | ` + + `${(metrics.recall * 100).toFixed(0)}% | ${metrics.tp} | ${metrics.fp} | ${metrics.fn} |`, + ); +} diff --git a/tests/benchmarks/resolution/fixtures/java/.gitignore b/tests/benchmarks/resolution/fixtures/java/.gitignore new file mode 100644 index 000000000..af4220f73 --- /dev/null +++ b/tests/benchmarks/resolution/fixtures/java/.gitignore @@ -0,0 +1,2 @@ +fixture.jar +benchmark/ diff --git a/tests/benchmarks/resolution/fixtures/java/Makefile b/tests/benchmarks/resolution/fixtures/java/Makefile new file mode 100644 index 000000000..9f9a51448 --- /dev/null +++ b/tests/benchmarks/resolution/fixtures/java/Makefile @@ -0,0 +1,15 @@ +SRCS = $(wildcard *.java) +JAR = fixture.jar + +.PHONY: all clean + +all: $(JAR) + +$(JAR): $(SRCS) + rm -rf benchmark/ + javac -d . $(SRCS) + jar cf $(JAR) benchmark/ + +clean: + rm -f $(JAR) + rm -rf benchmark/