Skip to content

feat(crashtracking): include JVM distribution details in crash report #11715

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gyuheon0h wants to merge 1 commit into
base: master
Choose a base branch
from
Open

feat(crashtracking): include JVM distribution details in crash report #11715

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -577,6 +577,21 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool
"os.version")); // this has been restructured under OsInfo so taking raw here
writer.endObject();
}
// runtime info (JDK vendor and build details from the crash log)
if (payload.runtimeInfo != null) {
writer.name("runtime_info");
writer.beginObject();
if (payload.runtimeInfo.jreVersion != null) {
writer.name("jre_version").value(payload.runtimeInfo.jreVersion);
}
if (payload.runtimeInfo.javaVm != null) {
writer.name("java_vm").value(payload.runtimeInfo.javaVm);
}
if (payload.runtimeInfo.vmInfo != null) {
writer.name("vm_info").value(payload.runtimeInfo.vmInfo);
}
writer.endObject();
}
// experimental
if (payload.experimental != null
&& (payload.experimental.ucontext != null
Expand Down
36 changes: 36 additions & 0 deletions dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ public final class CrashLog {

public final Experimental experimental;

@Json(name = "runtime_info")
public final RuntimeInfo runtimeInfo;

/**
* Useful files for triage and debugging (e.g. {@code /proc/self/maps}, {@code
* dynamic_libraries}).
Expand Down Expand Up @@ -68,6 +71,7 @@ public CrashLog(
sigInfo,
dataSchemaVersion,
null,
null,
null);
}

Expand All @@ -83,6 +87,34 @@ public CrashLog(
String dataSchemaVersion,
Experimental experimental,
DynamicLibs files) {
this(
uuid,
incomplete,
timestamp,
error,
metadata,
osInfo,
procInfo,
sigInfo,
dataSchemaVersion,
experimental,
null,
files);
}

public CrashLog(
String uuid,
boolean incomplete,
String timestamp,
ErrorData error,
Metadata metadata,
OSInfo osInfo,
ProcInfo procInfo,
SigInfo sigInfo,
String dataSchemaVersion,
Experimental experimental,
RuntimeInfo runtimeInfo,
DynamicLibs files) {
this.uuid = uuid != null ? uuid : RandomUtils.randomUUID().toString();
this.incomplete = incomplete;
this.timestamp = timestamp;
Expand All @@ -93,6 +125,7 @@ public CrashLog(
this.sigInfo = sigInfo;
this.dataSchemaVersion = dataSchemaVersion;
this.experimental = experimental;
this.runtimeInfo = runtimeInfo;
this.files = files;
}

Expand Down Expand Up @@ -123,6 +156,7 @@ public boolean equals(Object o) {
&& Objects.equals(sigInfo, crashLog.sigInfo)
&& Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion)
&& Objects.equals(experimental, crashLog.experimental)
&& Objects.equals(runtimeInfo, crashLog.runtimeInfo)
&& Objects.equals(files, crashLog.files);
}

Expand All @@ -140,6 +174,7 @@ public int hashCode() {
version,
dataSchemaVersion,
experimental,
runtimeInfo,
files);
}

Expand All @@ -161,6 +196,7 @@ public boolean equalsForTest(Object o) {
&& Objects.equals(sigInfo, crashLog.sigInfo)
&& Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion)
&& Objects.equals(experimental, crashLog.experimental)
&& Objects.equals(runtimeInfo, crashLog.runtimeInfo)
&& Objects.equals(files, crashLog.files);
}
}
45 changes: 45 additions & 0 deletions dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/RuntimeInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package datadog.crashtracking.dto;

import com.squareup.moshi.Json;
import java.util.Objects;

/**
* JDK runtime information extracted from the hs_err crash log header and vm_info line. This
* captures the exact JDK vendor and build so crash reports can be correlated with the specific
* binaries in use.
*/
public final class RuntimeInfo {
@Json(name = "jre_version")
public final String jreVersion;

@Json(name = "java_vm")
public final String javaVm;

@Json(name = "vm_info")
public final String vmInfo;

public RuntimeInfo(String jreVersion, String javaVm, String vmInfo) {
this.jreVersion = jreVersion;
this.javaVm = javaVm;
this.vmInfo = vmInfo;
}

@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
RuntimeInfo that = (RuntimeInfo) o;
return Objects.equals(jreVersion, that.jreVersion)
&& Objects.equals(javaVm, that.javaVm)
&& Objects.equals(vmInfo, that.vmInfo);
}

@Override
public int hashCode() {
return Objects.hash(jreVersion, javaVm, vmInfo);
}
}
21 changes: 21 additions & 0 deletions ...gent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import datadog.crashtracking.dto.Metadata;
import datadog.crashtracking.dto.OSInfo;
import datadog.crashtracking.dto.ProcInfo;
import datadog.crashtracking.dto.RuntimeInfo;
import datadog.crashtracking.dto.SigInfo;
import datadog.crashtracking.dto.StackFrame;
import datadog.crashtracking.dto.StackTrace;
Expand Down Expand Up @@ -44,6 +45,9 @@
*/
public final class HotspotCrashLogParser {
private static final String HOTSPOT_JVM_ARGS_PREFIX = "jvm_args:";
private static final String JRE_VERSION_PREFIX = "# JRE version: ";
private static final String JAVA_VM_PREFIX = "# Java VM: ";
private static final String VM_INFO_PREFIX = "vm_info: ";
private static final DateTimeFormatter ZONED_DATE_TIME_FORMATTER =
DateTimeFormatter.ofPattern("EEE MMM ppd HH:mm:ss yyyy zzz", Locale.getDefault());
private static final DateTimeFormatter OFFSET_DATE_TIME_FORMATTER =
Expand Down Expand Up @@ -362,6 +366,9 @@ public CrashLog parse(String uuid, String crashLog) {
String dynamicLibraryKey = null;
boolean previousLineBlank = false;
State nextThreadSectionState = null;
String jreVersion = null;
String javaVm = null;
String vmInfo = null;

String[] lines = NEWLINE_SPLITTER.split(crashLog);
outer:
Expand Down Expand Up @@ -392,6 +399,11 @@ public CrashLog parse(String uuid, String crashLog) {
}
}
}
if (jreVersion == null && line.startsWith(JRE_VERSION_PREFIX)) {
jreVersion = line.substring(JRE_VERSION_PREFIX.length()).trim();
} else if (javaVm == null && line.startsWith(JAVA_VM_PREFIX)) {
javaVm = line.substring(JAVA_VM_PREFIX.length()).trim();
}
break;
case HEADER:
if (line.contains("S U M M A R Y")) {
Expand Down Expand Up @@ -486,6 +498,8 @@ public CrashLog parse(String uuid, String crashLog) {
state = State.DYNAMIC_LIBRARIES;
} else if (line.contains("S Y S T E M")) {
state = State.SYSTEM;
} else if (vmInfo == null && line.startsWith(VM_INFO_PREFIX)) {
vmInfo = line.substring(VM_INFO_PREFIX.length()).trim();
} else if (line.equals("END.")) {
state = State.DONE;
}
Expand Down Expand Up @@ -527,6 +541,8 @@ public CrashLog parse(String uuid, String crashLog) {
datetimeRaw = line.substring(6).trim();
} else if (datetime == null && datetimeRaw != null && line.startsWith("timezone: ")) {
datetime = dateTimeToISO(datetimeRaw + " " + line.substring(10).trim());
} else if (vmInfo == null && line.startsWith(VM_INFO_PREFIX)) {
vmInfo = line.substring(VM_INFO_PREFIX.length()).trim();
}
break;
case DONE:
Expand Down Expand Up @@ -615,6 +631,10 @@ public CrashLog parse(String uuid, String crashLog) {
|| (runtimeArgs != null && !runtimeArgs.isEmpty())
? new Experimental(registers, resolvedMapping, runtimeArgs)
: null;
RuntimeInfo runtimeInfo =
(jreVersion != null || javaVm != null || vmInfo != null)
? new RuntimeInfo(jreVersion, javaVm, vmInfo)
: null;
DynamicLibs files =
(dynamicLibraryLines != null && !dynamicLibraryLines.isEmpty())
? new DynamicLibs(dynamicLibraryKey, dynamicLibraryLines)
Expand All @@ -630,6 +650,7 @@ public CrashLog parse(String uuid, String crashLog) {
sigInfo,
"1.0",
experimental,
runtimeInfo,
files);
}

Expand Down
15 changes: 15 additions & 0 deletions ...ent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/J9JavacoreParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import datadog.crashtracking.dto.Metadata;
import datadog.crashtracking.dto.OSInfo;
import datadog.crashtracking.dto.ProcInfo;
import datadog.crashtracking.dto.RuntimeInfo;
import datadog.crashtracking.dto.SigInfo;
import datadog.crashtracking.dto.StackFrame;
import datadog.crashtracking.dto.StackTrace;
Expand Down Expand Up @@ -46,6 +47,8 @@
*/
public final class J9JavacoreParser {
private static final String J9_USER_ARG_PREFIX = "2CIUSERARG";
private static final String J9_JAVA_VERSION_PREFIX = "1CIJAVAVERSION ";
private static final String J9_VM_VERSION_PREFIX = "1CIVMVERSION";

private final BuildIdCollector buildIdCollector;

Expand Down Expand Up @@ -119,6 +122,8 @@ public CrashLog parse(String uuid, String javacoreContent) {

Map<String, String> registers = null;
RuntimeArgs j9UserArgs = new RuntimeArgs();
String j9JavaVersion = null;
String j9VmVersion = null;

String[] lines = NEWLINE_SPLITTER.split(javacoreContent);

Expand Down Expand Up @@ -179,6 +184,11 @@ public CrashLog parse(String uuid, String javacoreContent) {
if (pidMatcher.matches()) {
pid = pidMatcher.group(1);
}
if (j9JavaVersion == null && line.startsWith(J9_JAVA_VERSION_PREFIX)) {
j9JavaVersion = line.substring(J9_JAVA_VERSION_PREFIX.length()).trim();
} else if (j9VmVersion == null && line.startsWith(J9_VM_VERSION_PREFIX)) {
j9VmVersion = line.substring(J9_VM_VERSION_PREFIX.length()).trim();
}
break;

case THREADS:
Expand Down Expand Up @@ -304,6 +314,10 @@ public CrashLog parse(String uuid, String javacoreContent) {
|| (runtimeArgs != null && !runtimeArgs.isEmpty())
? new Experimental(registers, runtimeArgs)
: null;
RuntimeInfo runtimeInfo =
(j9JavaVersion != null || j9VmVersion != null)
? new RuntimeInfo(j9JavaVersion, null, j9VmVersion)
: null;

return new CrashLog(
uuid,
Expand All @@ -316,6 +330,7 @@ public CrashLog parse(String uuid, String javacoreContent) {
sigInfo,
"1.0",
experimental,
runtimeInfo,
null);
}

Expand Down
21 changes: 21 additions & 0 deletions ...-crashtracking/src/test/java/datadog/crashtracking/parsers/HotspotCrashLogParserTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,27 @@ public void testParseCurrentThreadName(String line, String expected) {
HotspotCrashLogParser.parseCurrentThreadName(line));
}

@TableTest({
"scenario | filename | expectedJreVersion | expectedVmInfo ",
"Zulu 17 | sample-crash-for-telemetry.txt | OpenJDK Runtime Environment Zulu17.42+20-SA (17.0.7+7) (build 17.0.7+7-LTS) | OpenJDK 64-Bit Server VM (17.0.7+7-LTS) for linux-amd64 JRE (17.0.7+7-LTS) (Zulu17.42+20-SA), built on Apr 11 2023 11:39:51 by \"zulu_re\" with gcc 8.3.0 ",
"Temurin 22 | sample-crash-for-telemetry-2.txt | OpenJDK Runtime Environment Temurin-22.0.1+8 (22.0.1+8) (build 22.0.1+8) | OpenJDK 64-Bit Server VM (22.0.1+8) for linux-amd64 JRE (22.0.1+8), built on 2024-04-16T00:00:00Z by \"admin\" with gcc 11.3.0 ",
"Zulu 8 | sample-crash-for-telemetry-3.txt | OpenJDK Runtime Environment (Zulu 8.70.0.23-CA-macos-aarch64) (8.0_372-b07) (build 1.8.0_372-b07) | OpenJDK 64-Bit Server VM (25.372-b07) for bsd-aarch64 JRE (Zulu 8.70.0.23-CA-macos-aarch64) (1.8.0_372-b07), built on Apr 18 2023 01:36:20 by \"zulu_re\" with gcc Apple LLVM 12.0.0 (clang-1200.0.32.28)",
"Corretto 21 | sample-crash-linux-aarch64.txt | OpenJDK Runtime Environment Corretto-21.0.7.6.1 (21.0.7+6) (build 21.0.7+6-LTS) | OpenJDK 64-Bit Server VM (21.0.7+6-LTS) for linux-aarch64-musl JRE (21.0.7+6-LTS), built on 2025-04-09T23:34:45Z by \"jenkins\" with gcc 12.2.1 20220924 ",
"OpenJDK 25 | sample-crash-macos-aarch64.txt | OpenJDK Runtime Environment (25.0.2+10) (build 25.0.2+10-69) | OpenJDK 64-Bit Server VM (25.0.2+10-69) for bsd-aarch64 JRE (25.0.2+10-69), built on 2025-12-18T11:36:35Z with clang Apple LLVM 15.0.0 (clang-1500.3.9.4) "
})
public void testRuntimeInfoParsing(
String filename, String expectedJreVersion, String expectedVmInfo) throws Exception {
CrashLog crashLog =
new HotspotCrashLogParser().parse(UUID.randomUUID().toString(), readFileAsString(filename));

assertNotNull(crashLog.runtimeInfo, "runtimeInfo should be populated");
assertNotNull(crashLog.runtimeInfo.jreVersion, "jreVersion should be populated");
assertNotNull(crashLog.runtimeInfo.javaVm, "javaVm should be populated");
assertNotNull(crashLog.runtimeInfo.vmInfo, "vmInfo should be populated");
assertEquals(expectedJreVersion, crashLog.runtimeInfo.jreVersion);
assertEquals(expectedVmInfo, crashLog.runtimeInfo.vmInfo);
}

@Test
public void testNoSignalProducesInternalError() throws Exception {
// A crash log that reaches the PROCESS section but has no siginfo line
Expand Down
16 changes: 16 additions & 0 deletions ...agent-crashtracking/src/test/java/datadog/crashtracking/parsers/J9JavacoreParserTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,6 +212,22 @@ public void testDateTimeParsing() throws Exception {
"Expected ISO-8601 format, got: " + crashLog.timestamp);
}

@TableTest({
"scenario | filename | expectedJreVersion ",
"OpenJ9 11 GPF | sample-j9-javacore-gpf.txt | JRE 11.0.12 Linux amd64-64 ",
"OpenJ9 17 OOM | sample-j9-javacore-oom.txt | JRE 17.0.6 Linux amd64-64 ",
"OpenJ9 11 aarch | sample-openj9-11-javacore-gpf.txt | JRE 11 Linux aarch64-64 (build 11.0.28+6) ",
"IBM J9 8 | sample-ibmj9-8-javacore-gpf.txt | JRE 1.8.0 Linux amd64-64 (build 8.0.8.51 - pxa6480sr8fp51-20250819_01(SR8 FP51))"
})
public void testRuntimeInfoParsing(String filename, String expectedJreVersion) throws Exception {
CrashLog crashLog =
new J9JavacoreParser().parse(UUID.randomUUID().toString(), readFileAsString(filename));

assertNotNull(crashLog.runtimeInfo, "runtimeInfo should be populated");
assertNotNull(crashLog.runtimeInfo.jreVersion, "jreVersion should be populated");
assertEquals(expectedJreVersion, crashLog.runtimeInfo.jreVersion);
}

@Test
public void testNoSignalProducesInternalError() throws Exception {
// A javacore with a THREADS section but no 1TISIGINFO line
Expand Down
5 changes: 5 additions & 0 deletions ...t-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,11 @@
"si_pid": 554848,
"si_uid": 1000
},
"runtime_info": {
"java_vm": "OpenJDK 64-Bit Server VM Temurin-22.0.1+8 (22.0.1+8, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)",
"jre_version": "OpenJDK Runtime Environment Temurin-22.0.1+8 (22.0.1+8) (build 22.0.1+8)",
"vm_info": "OpenJDK 64-Bit Server VM (22.0.1+8) for linux-amd64 JRE (22.0.1+8), built on 2024-04-16T00:00:00Z by \"admin\" with gcc 11.3.0"
},
"experimental": {
"ucontext": {
"RAX": "0x00000000000000ca",
Expand Down
5 changes: 5 additions & 0 deletions ...t-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-3.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,11 @@
]
}
},
"runtime_info": {
"java_vm": "OpenJDK 64-Bit Server VM (25.372-b07 mixed mode bsd-aarch64 compressed oops)",
"jre_version": "OpenJDK Runtime Environment (Zulu 8.70.0.23-CA-macos-aarch64) (8.0_372-b07) (build 1.8.0_372-b07)",
"vm_info": "OpenJDK 64-Bit Server VM (25.372-b07) for bsd-aarch64 JRE (Zulu 8.70.0.23-CA-macos-aarch64) (1.8.0_372-b07), built on Apr 18 2023 01:36:20 by \"zulu_re\" with gcc Apple LLVM 12.0.0 (clang-1200.0.32.28)"
},
"files": {
"dynamic_libraries": [
"0x00000001ae54a000 \t/System/Library/Frameworks/Cocoa.framework/Versions/A/Cocoa",
Expand Down
5 changes: 5 additions & 0 deletions ...ent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,11 @@
"si_code": 1,
"si_code_human_readable": "SEGV_MAPERR"
},
"runtime_info": {
"java_vm": "OpenJDK 64-Bit Server VM Zulu17.42+20-SA (17.0.7+7-LTS, mixed mode, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)",
"jre_version": "OpenJDK Runtime Environment Zulu17.42+20-SA (17.0.7+7) (build 17.0.7+7-LTS)",
"vm_info": "OpenJDK 64-Bit Server VM (17.0.7+7-LTS) for linux-amd64 JRE (17.0.7+7-LTS) (Zulu17.42+20-SA), built on Apr 11 2023 11:39:51 by \"zulu_re\" with gcc 8.3.0"
},
"experimental": {
"ucontext": {
"RAX": "0x00007f36ccfbf170",
Expand Down
5 changes: 5 additions & 0 deletions dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample_oom.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,5 +114,10 @@
}
]
}
},
"runtime_info": {
"java_vm": "OpenJDK 64-Bit Server VM Zulu21.30+15-CA (21.0.1+12-LTS, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, g1 gc, bsd-aarch64)",
"jre_version": "OpenJDK Runtime Environment Zulu21.30+15-CA (21.0.1+12) (build 21.0.1+12-LTS)",
"vm_info": "OpenJDK 64-Bit Server VM (21.0.1+12-LTS) for bsd-aarch64 JRE (21.0.1+12-LTS) (Zulu21.30+15-CA), built on 2023-10-11T21:40:40Z by \"zulu_re\" with clang Apple LLVM 12.0.0 (clang-1200.0.32.28)"
}
}
5 changes: 5 additions & 0 deletions ...agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-2.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@
"proc_info": {
"pid": 576034
},
"runtime_info": {
"java_vm": "OpenJDK 64-Bit Server VM Temurin-22.0.1+8 (22.0.1+8, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, g1 gc, linux-amd64)",
"jre_version": "OpenJDK Runtime Environment Temurin-22.0.1+8 (22.0.1+8) (build 22.0.1+8)",
"vm_info": "OpenJDK 64-Bit Server VM (22.0.1+8) for linux-amd64 JRE (22.0.1+8), built on 2024-04-16T00:00:00Z by \"admin\" with gcc 11.3.0"
},
"sig_info": {
"si_code": 0,
"si_code_human_readable": "SI_USER",
Expand Down
Loading
Loading