From 672d7047648d312fd75122517ee2b9bccfd0183f Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 24 Apr 2026 09:38:06 +0200 Subject: [PATCH 01/10] Make event module a file --- crates/symbolicli/src/event.rs | 331 ++++++++++++++++++++++++++++++++ crates/symbolicli/src/main.rs | 335 +-------------------------------- 2 files changed, 332 insertions(+), 334 deletions(-) create mode 100644 crates/symbolicli/src/event.rs diff --git a/crates/symbolicli/src/event.rs b/crates/symbolicli/src/event.rs new file mode 100644 index 000000000..ed4ce88c7 --- /dev/null +++ b/crates/symbolicli/src/event.rs @@ -0,0 +1,331 @@ +use std::sync::Arc; + +use anyhow::bail; +use serde::Deserialize; +use symbolic::common::Language; +use symbolicator_js::interface::{ + JsFrame, JsFrameData, JsModule, JsStacktrace, SymbolicateJsStacktraces, +}; +use symbolicator_native::interface::{ + AddrMode, CompleteObjectInfo, FrameTrust, RawFrame, RawStacktrace, Signal, StacktraceOrigin, + SymbolicateStacktraces, +}; +use symbolicator_service::types::{FrameOrder, Platform, RawObjectInfo, Scope, ScrapingConfig}; +use symbolicator_service::utils::hex::HexValue; +use symbolicator_sources::{SentrySourceConfig, SourceConfig}; + +pub fn create_js_symbolication_request( + scope: Scope, + source: Arc, + event: Event, + scraping_enabled: bool, +) -> anyhow::Result { + let Event { + platform, + debug_meta, + exception, + threads, + release, + dist, + .. + } = event; + + let mut stacktraces = vec![]; + if let Some(mut excs) = exception.map(|excs| excs.values) { + stacktraces.extend( + excs.iter_mut() + .filter_map(|exc| exc.raw_stacktrace.take().or_else(|| exc.stacktrace.take())), + ); + } + if let Some(mut threads) = threads.map(|threads| threads.values) { + stacktraces.extend(threads.iter_mut().filter_map(|thread| { + thread + .raw_stacktrace + .take() + .or_else(|| thread.stacktrace.take()) + })); + } + + let stacktraces: Vec<_> = stacktraces + .into_iter() + .map(JsStacktrace::from) + .filter(|stacktrace| !stacktrace.frames.is_empty()) + .collect(); + + let modules: Vec<_> = debug_meta + .images + .into_iter() + .filter_map(|module| match module { + Module::Sourcemap(m) => Some(m), + _ => None, + }) + .collect(); + + if stacktraces.is_empty() { + bail!("Event has no usable frames"); + }; + + Ok(SymbolicateJsStacktraces { + platform: Some(platform), + scope, + source, + release, + dist, + scraping: ScrapingConfig { + enabled: scraping_enabled, + ..Default::default() + }, + apply_source_context: true, + // we manually reversed the frames when we created the stacktraces, so this is + // "callee first" + frame_order: FrameOrder::CalleeFirst, + + stacktraces, + modules, + }) +} + +pub fn create_native_symbolication_request( + scope: Scope, + sources: Arc<[SourceConfig]>, + event: Event, +) -> anyhow::Result { + let Event { + debug_meta, + exception, + threads, + signal, + .. + } = event; + + let mut stacktraces = vec![]; + if let Some(mut excs) = exception.map(|excs| excs.values) { + stacktraces.extend( + excs.iter_mut() + .filter_map(|exc| exc.raw_stacktrace.take().or_else(|| exc.stacktrace.take())), + ); + } + if let Some(mut threads) = threads.map(|threads| threads.values) { + stacktraces.extend(threads.iter_mut().filter_map(|thread| { + thread + .raw_stacktrace + .take() + .or_else(|| thread.stacktrace.take()) + })); + } + + let stacktraces: Vec<_> = stacktraces + .into_iter() + .map(RawStacktrace::from) + .filter(|stacktrace| !stacktrace.frames.is_empty()) + .collect(); + + let modules: Vec<_> = debug_meta + .images + .into_iter() + .filter_map(|module| match module { + Module::Object(m) => Some(CompleteObjectInfo::from(m)), + _ => None, + }) + .collect(); + + if modules.is_empty() { + bail!("Event has no debug images"); + }; + + if stacktraces.is_empty() { + bail!("Event has no usable frames"); + }; + + Ok(SymbolicateStacktraces { + platform: Some(event.platform), + scope, + signal, + sources, + origin: StacktraceOrigin::Symbolicate, + stacktraces, + modules, + apply_source_context: true, + scraping: Default::default(), + rewrite_first_module: Default::default(), + // we manually reversed the frames when we created the stacktraces, so this is + // "callee first" + frame_order: FrameOrder::CalleeFirst, + }) +} + +#[derive(Debug, Deserialize)] +pub struct Event { + #[serde(default)] + pub platform: Platform, + #[serde(default)] + debug_meta: DebugMeta, + exception: Option, + threads: Option, + signal: Option, + release: Option, + dist: Option, +} + +#[derive(Debug, Deserialize, Default)] +struct DebugMeta { + images: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +pub enum Module { + Object(RawObjectInfo), + Sourcemap(JsModule), +} + +#[derive(Debug, Deserialize)] +struct Exceptions { + values: Vec, +} + +#[derive(Debug, Deserialize)] +struct Exception { + raw_stacktrace: Option, + stacktrace: Option, +} + +#[derive(Debug, Deserialize)] +struct Threads { + values: Vec, +} + +#[derive(Debug, Deserialize)] +struct Thread { + raw_stacktrace: Option, + stacktrace: Option, +} + +#[derive(Debug, Deserialize)] +struct Stacktrace { + frames: Vec, + #[serde(default)] + is_requesting: bool, +} + +impl From for RawStacktrace { + fn from(stacktrace: Stacktrace) -> Self { + let frames = stacktrace + .frames + .into_iter() + .filter_map(to_raw_frame) + .rev() + .collect(); + + Self { + is_requesting: Some(stacktrace.is_requesting), + frames, + ..Default::default() + } + } +} + +impl From for JsStacktrace { + fn from(stacktrace: Stacktrace) -> Self { + let frames = stacktrace + .frames + .into_iter() + .filter_map(to_js_frame) + .rev() + .collect(); + + Self { frames } + } +} + +#[derive(Debug, Deserialize)] +struct Frame { + platform: Option, + #[serde(default)] + addr_mode: AddrMode, + + instruction_addr: Option, + + #[serde(default)] + function_id: Option, + + #[serde(default)] + package: Option, + + lang: Option, + + symbol: Option, + + sym_addr: Option, + + function: Option, + + filename: Option, + + abs_path: Option, + + lineno: Option, + + colno: Option, + + #[serde(default)] + pre_context: Vec, + + context_line: Option, + + #[serde(default)] + post_context: Vec, + + module: Option, + + source_link: Option, + + in_app: Option, + + #[serde(default)] + trust: FrameTrust, + + #[serde(default)] + data: JsFrameData, +} + +fn to_raw_frame(value: Frame) -> Option { + Some(RawFrame { + platform: value.platform, + addr_mode: value.addr_mode, + instruction_addr: value.instruction_addr?, + adjust_instruction_addr: None, + function_id: value.function_id, + package: value.package, + lang: value.lang, + symbol: value.symbol, + sym_addr: value.sym_addr, + function: value.function, + filename: value.filename, + abs_path: value.abs_path, + lineno: value.lineno, + pre_context: value.pre_context, + context_line: value.context_line, + post_context: value.post_context, + source_link: value.source_link, + in_app: value.in_app, + trust: value.trust, + }) +} + +fn to_js_frame(value: Frame) -> Option { + Some(JsFrame { + platform: value.platform, + function: value.function, + filename: value.filename, + module: value.module, + abs_path: value.abs_path?, + lineno: value.lineno?, + colno: value.colno, + pre_context: value.pre_context, + context_line: value.context_line, + post_context: value.post_context, + token_name: None, + data: value.data, + }) +} diff --git a/crates/symbolicli/src/main.rs b/crates/symbolicli/src/main.rs index 201e2f5df..dd4cfb9b8 100644 --- a/crates/symbolicli/src/main.rs +++ b/crates/symbolicli/src/main.rs @@ -28,6 +28,7 @@ use tracing_subscriber::prelude::*; use crate::output::CompletedResponse; +mod event; mod output; mod settings; @@ -449,337 +450,3 @@ mod remote { } } } - -mod event { - use std::sync::Arc; - - use anyhow::bail; - use serde::Deserialize; - use symbolic::common::Language; - use symbolicator_js::interface::{ - JsFrame, JsFrameData, JsModule, JsStacktrace, SymbolicateJsStacktraces, - }; - use symbolicator_native::interface::{ - AddrMode, CompleteObjectInfo, FrameTrust, RawFrame, RawStacktrace, Signal, - StacktraceOrigin, SymbolicateStacktraces, - }; - use symbolicator_service::types::{FrameOrder, Platform, RawObjectInfo, Scope, ScrapingConfig}; - use symbolicator_service::utils::hex::HexValue; - use symbolicator_sources::{SentrySourceConfig, SourceConfig}; - - pub fn create_js_symbolication_request( - scope: Scope, - source: Arc, - event: Event, - scraping_enabled: bool, - ) -> anyhow::Result { - let Event { - platform, - debug_meta, - exception, - threads, - release, - dist, - .. - } = event; - - let mut stacktraces = vec![]; - if let Some(mut excs) = exception.map(|excs| excs.values) { - stacktraces.extend( - excs.iter_mut() - .filter_map(|exc| exc.raw_stacktrace.take().or_else(|| exc.stacktrace.take())), - ); - } - if let Some(mut threads) = threads.map(|threads| threads.values) { - stacktraces.extend(threads.iter_mut().filter_map(|thread| { - thread - .raw_stacktrace - .take() - .or_else(|| thread.stacktrace.take()) - })); - } - - let stacktraces: Vec<_> = stacktraces - .into_iter() - .map(JsStacktrace::from) - .filter(|stacktrace| !stacktrace.frames.is_empty()) - .collect(); - - let modules: Vec<_> = debug_meta - .images - .into_iter() - .filter_map(|module| match module { - Module::Sourcemap(m) => Some(m), - _ => None, - }) - .collect(); - - if stacktraces.is_empty() { - bail!("Event has no usable frames"); - }; - - Ok(SymbolicateJsStacktraces { - platform: Some(platform), - scope, - source, - release, - dist, - scraping: ScrapingConfig { - enabled: scraping_enabled, - ..Default::default() - }, - apply_source_context: true, - // we manually reversed the frames when we created the stacktraces, so this is - // "callee first" - frame_order: FrameOrder::CalleeFirst, - - stacktraces, - modules, - }) - } - - pub fn create_native_symbolication_request( - scope: Scope, - sources: Arc<[SourceConfig]>, - event: Event, - ) -> anyhow::Result { - let Event { - debug_meta, - exception, - threads, - signal, - .. - } = event; - - let mut stacktraces = vec![]; - if let Some(mut excs) = exception.map(|excs| excs.values) { - stacktraces.extend( - excs.iter_mut() - .filter_map(|exc| exc.raw_stacktrace.take().or_else(|| exc.stacktrace.take())), - ); - } - if let Some(mut threads) = threads.map(|threads| threads.values) { - stacktraces.extend(threads.iter_mut().filter_map(|thread| { - thread - .raw_stacktrace - .take() - .or_else(|| thread.stacktrace.take()) - })); - } - - let stacktraces: Vec<_> = stacktraces - .into_iter() - .map(RawStacktrace::from) - .filter(|stacktrace| !stacktrace.frames.is_empty()) - .collect(); - - let modules: Vec<_> = debug_meta - .images - .into_iter() - .filter_map(|module| match module { - Module::Object(m) => Some(CompleteObjectInfo::from(m)), - _ => None, - }) - .collect(); - - if modules.is_empty() { - bail!("Event has no debug images"); - }; - - if stacktraces.is_empty() { - bail!("Event has no usable frames"); - }; - - Ok(SymbolicateStacktraces { - platform: Some(event.platform), - scope, - signal, - sources, - origin: StacktraceOrigin::Symbolicate, - stacktraces, - modules, - apply_source_context: true, - scraping: Default::default(), - rewrite_first_module: Default::default(), - // we manually reversed the frames when we created the stacktraces, so this is - // "callee first" - frame_order: FrameOrder::CalleeFirst, - }) - } - - #[derive(Debug, Deserialize)] - pub struct Event { - #[serde(default)] - pub platform: Platform, - #[serde(default)] - debug_meta: DebugMeta, - exception: Option, - threads: Option, - signal: Option, - release: Option, - dist: Option, - } - - #[derive(Debug, Deserialize, Default)] - struct DebugMeta { - images: Vec, - } - - #[derive(Debug, Clone, Deserialize)] - #[serde(untagged)] - pub enum Module { - Object(RawObjectInfo), - Sourcemap(JsModule), - } - - #[derive(Debug, Deserialize)] - struct Exceptions { - values: Vec, - } - - #[derive(Debug, Deserialize)] - struct Exception { - raw_stacktrace: Option, - stacktrace: Option, - } - - #[derive(Debug, Deserialize)] - struct Threads { - values: Vec, - } - - #[derive(Debug, Deserialize)] - struct Thread { - raw_stacktrace: Option, - stacktrace: Option, - } - - #[derive(Debug, Deserialize)] - struct Stacktrace { - frames: Vec, - #[serde(default)] - is_requesting: bool, - } - - impl From for RawStacktrace { - fn from(stacktrace: Stacktrace) -> Self { - let frames = stacktrace - .frames - .into_iter() - .filter_map(to_raw_frame) - .rev() - .collect(); - - Self { - is_requesting: Some(stacktrace.is_requesting), - frames, - ..Default::default() - } - } - } - - impl From for JsStacktrace { - fn from(stacktrace: Stacktrace) -> Self { - let frames = stacktrace - .frames - .into_iter() - .filter_map(to_js_frame) - .rev() - .collect(); - - Self { frames } - } - } - - #[derive(Debug, Deserialize)] - struct Frame { - platform: Option, - #[serde(default)] - addr_mode: AddrMode, - - instruction_addr: Option, - - #[serde(default)] - function_id: Option, - - #[serde(default)] - package: Option, - - lang: Option, - - symbol: Option, - - sym_addr: Option, - - function: Option, - - filename: Option, - - abs_path: Option, - - lineno: Option, - - colno: Option, - - #[serde(default)] - pre_context: Vec, - - context_line: Option, - - #[serde(default)] - post_context: Vec, - - module: Option, - - source_link: Option, - - in_app: Option, - - #[serde(default)] - trust: FrameTrust, - - #[serde(default)] - data: JsFrameData, - } - - fn to_raw_frame(value: Frame) -> Option { - Some(RawFrame { - platform: value.platform, - addr_mode: value.addr_mode, - instruction_addr: value.instruction_addr?, - adjust_instruction_addr: None, - function_id: value.function_id, - package: value.package, - lang: value.lang, - symbol: value.symbol, - sym_addr: value.sym_addr, - function: value.function, - filename: value.filename, - abs_path: value.abs_path, - lineno: value.lineno, - pre_context: value.pre_context, - context_line: value.context_line, - post_context: value.post_context, - source_link: value.source_link, - in_app: value.in_app, - trust: value.trust, - }) - } - - fn to_js_frame(value: Frame) -> Option { - Some(JsFrame { - platform: value.platform, - function: value.function, - filename: value.filename, - module: value.module, - abs_path: value.abs_path?, - lineno: value.lineno?, - colno: value.colno, - pre_context: value.pre_context, - context_line: value.context_line, - post_context: value.post_context, - token_name: None, - data: value.data, - }) - } -} From 31e72ecdf4447f8cafee8f2f1d202feb1e25a3f5 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Mon, 27 Apr 2026 17:17:34 +0200 Subject: [PATCH 02/10] WIP --- Cargo.lock | 2 + crates/symbolicli/Cargo.toml | 2 + crates/symbolicli/src/js_local_source.rs | 161 +++++++++++++++++++++++ crates/symbolicli/src/main.rs | 1 + 4 files changed, 166 insertions(+) create mode 100644 crates/symbolicli/src/js_local_source.rs diff --git a/Cargo.lock b/Cargo.lock index 1a4c7dfe8..2e3671877 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5038,6 +5038,8 @@ dependencies = [ "toml", "tracing", "tracing-subscriber", + "walkdir", + "zip 5.1.0", ] [[package]] diff --git a/crates/symbolicli/Cargo.toml b/crates/symbolicli/Cargo.toml index 3c02e9cd3..3f8c88dbb 100644 --- a/crates/symbolicli/Cargo.toml +++ b/crates/symbolicli/Cargo.toml @@ -29,3 +29,5 @@ tokio = { workspace = true, features = [ toml = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +walkdir = { workspace = true } +zip = { workspace = true } diff --git a/crates/symbolicli/src/js_local_source.rs b/crates/symbolicli/src/js_local_source.rs new file mode 100644 index 000000000..fd06b1f19 --- /dev/null +++ b/crates/symbolicli/src/js_local_source.rs @@ -0,0 +1,161 @@ +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use symbolic::common::DebugId; +use symbolic::debuginfo::sourcebundle::{SourceBundle, SourceFileInfo, SourceFileType}; +use symbolicator_js::interface::ResolvedWith; +use walkdir::WalkDir; +use zip::ZipArchive; + +#[derive(Debug, Clone)] +struct LookupKey { + release: Option>, + dist: Option>, + url: Option>, + debug_id: Option, +} + +#[derive(Debug, Clone)] +struct FoundBundle { + resolved_with: ResolvedWith, + id: usize, + path: PathBuf, +} + +#[derive(Debug, Clone)] +struct LookupResponse(Box<[FoundBundle]>); + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct ReleaseDistUrl { + release: Option>, + dist: Option>, + url: Arc, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct SourceBundleManifest { + #[serde(default)] + pub files: HashMap, + #[serde(default)] + pub attributes: HashMap>, +} + +#[derive(Debug, Clone, Default)] +struct Index { + by_debug_id: HashMap>, + by_url: HashMap>, + bundles: Vec, +} + +impl Index { + fn build(path: impl AsRef) -> Result { + let mut out = Self::default(); + for entry in WalkDir::new(path) { + let entry = entry.context("Accessing files")?; + + if !entry.file_type().is_file() + || entry.path().extension().is_none_or(|ext| ext != "zip") + { + continue; + } + + let archive = std::fs::File::open(entry.path())?; + let mut archive = ZipArchive::new(archive)?; + + let manifest = archive.by_name("manifest.json")?; + let manifest: SourceBundleManifest = serde_json::from_reader(manifest)?; + + dbg!(&manifest); + let release = manifest.attributes.get("release").cloned(); + let dist = manifest.attributes.get("dist").cloned(); + + for file in manifest.files.values() { + if file.ty() != Some(SourceFileType::MinifiedSource) { + continue; + } + + let Some(url) = file.url() else { + continue; + }; + + out.by_url + .entry(ReleaseDistUrl { + release: release.clone(), + dist: dist.clone(), + url: url.into(), + }) + .or_default() + .insert(out.bundles.len()); + + if let Some(debug_id) = file.debug_id() { + out.by_debug_id + .entry(debug_id.to_owned()) + .or_default() + .insert(out.bundles.len()); + } + } + + out.bundles.push(entry.path().to_owned()); + } + + Ok(out) + } + + fn lookup(&self, key: LookupKey) -> LookupResponse { + let mut out = Vec::new(); + + if let Some(debug_id) = key.debug_id { + for id in self + .by_debug_id + .get(&debug_id) + .into_iter() + .flat_map(|s| s.iter().copied()) + { + out.push(FoundBundle { + resolved_with: ResolvedWith::DebugId, + id, + path: self.bundles[id].clone(), + }); + } + } + + if let Some(url) = key.url { + for id in self + .by_url + .get(&ReleaseDistUrl { + release: key.release.clone(), + dist: key.dist.clone(), + url: url.clone(), + }) + .into_iter() + .flat_map(|s| s.iter().copied()) + { + out.push(FoundBundle { + resolved_with: ResolvedWith::Release, + id, + path: self.bundles[id].clone(), + }); + } + } + + LookupResponse(out.into()) + } +} + +#[test] +fn test_existing_bundle() { + let index = Index::build( + "/Users/sebastian/code/symbolicator/tests/fixtures/sourcemaps/e2e_node_debugid", + ) + .unwrap(); + + dbg!(index.lookup(LookupKey { + release: None, + dist: None, + url: None, + debug_id: Some("2f259f80-58b7-44cb-d7cd-de1505e7e718".parse().unwrap()), + })); +} diff --git a/crates/symbolicli/src/main.rs b/crates/symbolicli/src/main.rs index dd4cfb9b8..e38dd6fe5 100644 --- a/crates/symbolicli/src/main.rs +++ b/crates/symbolicli/src/main.rs @@ -29,6 +29,7 @@ use tracing_subscriber::prelude::*; use crate::output::CompletedResponse; mod event; +mod js_local_source; mod output; mod settings; From 9f5329a7410f96d01f564744d6c3d85bf6bd45ce Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 22 May 2026 17:07:50 +0200 Subject: [PATCH 03/10] WIP --- Cargo.lock | 4 + Cargo.toml | 1 + crates/symbolicli/Cargo.toml | 4 + crates/symbolicli/src/event.rs | 2 +- crates/symbolicli/src/js_local_source.rs | 246 ++++++++++++++++------- crates/symbolicli/src/main.rs | 63 ++++-- crates/symbolicli/src/settings.rs | 4 +- 7 files changed, 231 insertions(+), 93 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2e3671877..21cc1b6cb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5021,6 +5021,7 @@ name = "symbolicli" version = "26.5.0" dependencies = [ "anyhow", + "axum", "clap", "dirs", "prettytable-rs", @@ -5036,8 +5037,11 @@ dependencies = [ "tempfile", "tokio", "toml", + "tower-http", "tracing", "tracing-subscriber", + "url", + "urlencoding", "walkdir", "zip 5.1.0", ] diff --git a/Cargo.toml b/Cargo.toml index 9b8d8cd65..c74736d20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,6 +147,7 @@ tower-service = "0.3" tracing = "0.1.34" tracing-subscriber = { version = "0.3.17", features = ["env-filter", "time"] } url = { version = "2.2.0", features = ["serde"] } +urlencoding = "2.1.3" uuid = { version = "1.0.0", features = ["v4", "serde"] } walkdir = "2.3.1" wasmbin = { version = "0.8.1", features = ["exception-handling"] } diff --git a/crates/symbolicli/Cargo.toml b/crates/symbolicli/Cargo.toml index 3f8c88dbb..8a0aa4d85 100644 --- a/crates/symbolicli/Cargo.toml +++ b/crates/symbolicli/Cargo.toml @@ -7,6 +7,7 @@ license-file = "../../LICENSE.md" [dependencies] anyhow = { workspace = true } +axum = { workspace = true } clap = { workspace = true } dirs = { workspace = true } prettytable-rs = { workspace = true } @@ -27,7 +28,10 @@ tokio = { workspace = true, features = [ "sync", ] } toml = { workspace = true } +tower-http = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } +url = { workspace = true } +urlencoding = { workspace = true } walkdir = { workspace = true } zip = { workspace = true } diff --git a/crates/symbolicli/src/event.rs b/crates/symbolicli/src/event.rs index ed4ce88c7..79fe7a17f 100644 --- a/crates/symbolicli/src/event.rs +++ b/crates/symbolicli/src/event.rs @@ -175,8 +175,8 @@ struct DebugMeta { #[derive(Debug, Clone, Deserialize)] #[serde(untagged)] pub enum Module { - Object(RawObjectInfo), Sourcemap(JsModule), + Object(RawObjectInfo), } #[derive(Debug, Deserialize)] diff --git a/crates/symbolicli/src/js_local_source.rs b/crates/symbolicli/src/js_local_source.rs index fd06b1f19..1521d5006 100644 --- a/crates/symbolicli/src/js_local_source.rs +++ b/crates/symbolicli/src/js_local_source.rs @@ -1,33 +1,74 @@ use std::collections::{HashMap, HashSet}; -use std::path::{Path, PathBuf}; +use std::net::{SocketAddr, TcpListener}; +use std::path::Path; use std::sync::Arc; use anyhow::{Context, Result}; +use axum::routing::{get, get_service}; +use axum::{Json, Router, extract}; use serde::{Deserialize, Serialize}; use symbolic::common::DebugId; -use symbolic::debuginfo::sourcebundle::{SourceBundle, SourceFileInfo, SourceFileType}; +use symbolic::debuginfo::sourcebundle::{SourceFileInfo, SourceFileType}; use symbolicator_js::interface::ResolvedWith; +use symbolicator_sources::{SentrySourceConfig, SentryToken, SourceId}; +use tower_http::services::ServeDir; +use tower_http::trace::TraceLayer; +use url::Url; use walkdir::WalkDir; use zip::ZipArchive; -#[derive(Debug, Clone)] +/// Start a server that mimicks the Sentry artifact lookup endpoint, serving artifact +/// bundles from the given directory. +pub fn start_server(path: impl AsRef + Clone) -> anyhow::Result { + let addr = SocketAddr::from(([127, 0, 0, 1], 0)); + let listener = TcpListener::bind(addr).unwrap(); + listener.set_nonblocking(true).unwrap(); + let socket = listener.local_addr().unwrap(); + let index = Index::new(path.clone(), socket)?; + let index = Arc::new(index); + let source_url = index.url("lookup"); + + let router = Router::new() + .route("/lookup", get(lookup)) + .nest_service("/bundles", get_service(ServeDir::new(path))) + .layer(TraceLayer::new_for_http()) + .with_state(index); + + let handle = tokio::spawn(async move { + let listener = tokio::net::TcpListener::from_std(listener).unwrap(); + axum::serve(listener, router).await.unwrap(); + }); + + std::mem::forget(handle); + + Ok(SentrySourceConfig { + id: SourceId::new("local"), + url: source_url, + token: SentryToken(String::new()), + }) +} + +/// A key with which to look up an artifact bundle. +/// +/// This is deserialized from a query in [`lookup`]. +#[derive(Debug, Clone, Deserialize)] struct LookupKey { + /// The release name. + /// + /// This is only relevant in conjunction with `url`. release: Option>, + /// The dist name. + /// + /// This is only relevant in conjunction with `url`. dist: Option>, + /// The URL/abs_path. url: Option>, + /// The debug ID. debug_id: Option, } -#[derive(Debug, Clone)] -struct FoundBundle { - resolved_with: ResolvedWith, - id: usize, - path: PathBuf, -} - -#[derive(Debug, Clone)] -struct LookupResponse(Box<[FoundBundle]>); - +/// A combination of release + dist + URL for +/// path-based lookups. #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct ReleaseDistUrl { release: Option>, @@ -35,6 +76,7 @@ struct ReleaseDistUrl { url: Arc, } +/// Simple representation of the manifest of a sourcebundle (including artifact bundles.). #[derive(Clone, Debug, Default, Serialize, Deserialize)] struct SourceBundleManifest { #[serde(default)] @@ -43,17 +85,31 @@ struct SourceBundleManifest { pub attributes: HashMap>, } -#[derive(Debug, Clone, Default)] +/// Application state for the pretend Sentry source. +/// +/// This is immutable after initial construction. +#[derive(Debug, Clone)] struct Index { - by_debug_id: HashMap>, - by_url: HashMap>, - bundles: Vec, + /// The socket on which the server listens. + socket: SocketAddr, + /// A map from debug IDs to bundles containing them. + by_debug_id: HashMap>>, + /// A map from release/dist/URL combinations to bundles containing them. + by_url: HashMap>>, } impl Index { - fn build(path: impl AsRef) -> Result { - let mut out = Self::default(); - for entry in WalkDir::new(path) { + /// Build a new index containing bundles within the given `base_path` + /// and serving downloads on the given `socket`. + fn new(base_path: impl AsRef, socket: SocketAddr) -> Result { + let base_path = base_path.as_ref(); + let mut out = Self { + socket, + by_debug_id: Default::default(), + by_url: Default::default(), + }; + + for entry in WalkDir::new(base_path) { let entry = entry.context("Accessing files")?; if !entry.file_type().is_file() @@ -62,13 +118,15 @@ impl Index { continue; } - let archive = std::fs::File::open(entry.path())?; + let bundle_path: Arc = entry.path().into(); + let relative_path: Arc = bundle_path.strip_prefix(base_path).unwrap().into(); + + let archive = std::fs::File::open(&bundle_path)?; let mut archive = ZipArchive::new(archive)?; let manifest = archive.by_name("manifest.json")?; let manifest: SourceBundleManifest = serde_json::from_reader(manifest)?; - dbg!(&manifest); let release = manifest.attributes.get("release").cloned(); let dist = manifest.attributes.get("dist").cloned(); @@ -88,74 +146,120 @@ impl Index { url: url.into(), }) .or_default() - .insert(out.bundles.len()); + .insert(Arc::clone(&relative_path)); if let Some(debug_id) = file.debug_id() { out.by_debug_id .entry(debug_id.to_owned()) .or_default() - .insert(out.bundles.len()); + .insert(Arc::clone(&relative_path)); } } - - out.bundles.push(entry.path().to_owned()); } Ok(out) } - fn lookup(&self, key: LookupKey) -> LookupResponse { - let mut out = Vec::new(); + /// Returns a full URL pointing to the given path. + fn url(&self, path: &str) -> Url { + let path = path.trim_start_matches('/'); + format!("http://{}/{}", self.socket, path).parse().unwrap() + } +} - if let Some(debug_id) = key.debug_id { - for id in self - .by_debug_id - .get(&debug_id) - .into_iter() - .flat_map(|s| s.iter().copied()) - { - out.push(FoundBundle { - resolved_with: ResolvedWith::DebugId, - id, - path: self.bundles[id].clone(), - }); +/// A value returned by artifact lookup. +/// +/// Currently this only supports bundles. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum LookupResult { + Bundle { + /// The bundle's ID. + /// + /// We use the lossy version of the file's path for this. + id: String, + /// The URL where the bundle can be downloaded. + url: Url, + /// How the bundle was resolved. + resolved_with: ResolvedWith, + }, +} + +async fn lookup( + extract::State(index): extract::State>, + extract::Query(key): extract::Query, +) -> Json> { + let mut out = Vec::new(); + let mut found_bundles = HashSet::new(); + + if let Some(debug_id) = key.debug_id { + for path in index + .by_debug_id + .get(&debug_id) + .into_iter() + .flat_map(|s| s.iter().cloned()) + { + if found_bundles.contains(&path) { + continue; } + let path_encoded = urlencoding::encode_binary(path.as_os_str().as_encoded_bytes()); + out.push(LookupResult::Bundle { + id: path.to_string_lossy().to_string(), + url: index.url(&format!("bundles/{path_encoded}")), + resolved_with: ResolvedWith::DebugId, + }); + found_bundles.insert(path); } + } - if let Some(url) = key.url { - for id in self - .by_url - .get(&ReleaseDistUrl { - release: key.release.clone(), - dist: key.dist.clone(), - url: url.clone(), - }) - .into_iter() - .flat_map(|s| s.iter().copied()) - { - out.push(FoundBundle { - resolved_with: ResolvedWith::Release, - id, - path: self.bundles[id].clone(), - }); + if let Some(url) = key.url { + for path in index + .by_url + .get(&ReleaseDistUrl { + release: key.release.clone(), + dist: key.dist.clone(), + url: url.clone(), + }) + .into_iter() + .flat_map(|s| s.iter().cloned()) + { + if found_bundles.contains(&path) { + continue; } + let path_encoded = urlencoding::encode_binary(path.as_os_str().as_encoded_bytes()); + out.push(LookupResult::Bundle { + id: path.to_string_lossy().to_string(), + url: index.url(&format!("bundles/{path_encoded}")), + resolved_with: ResolvedWith::Release, + }); + found_bundles.insert(path); } - - LookupResponse(out.into()) } + + Json(out.into()) } -#[test] -fn test_existing_bundle() { - let index = Index::build( - "/Users/sebastian/code/symbolicator/tests/fixtures/sourcemaps/e2e_node_debugid", - ) - .unwrap(); - - dbg!(index.lookup(LookupKey { - release: None, - dist: None, - url: None, - debug_id: Some("2f259f80-58b7-44cb-d7cd-de1505e7e718".parse().unwrap()), - })); +#[tokio::test] +async fn test_existing_bundle() { + let source = start_server("/Users/sebastian/code/symbolicator/tests/fixtures/").unwrap(); + + let mut lookup_url = source.url.clone(); + lookup_url.set_query(Some("debug_id=2f259f80-58b7-44cb-d7cd-de1505e7e718")); + + let results: Box<[LookupResult]> = reqwest::get(lookup_url) + .await + .unwrap() + .json() + .await + .unwrap(); + + let LookupResult::Bundle { url, .. } = &results[0]; + + assert!( + reqwest::get(url.clone()) + .await + .unwrap() + .status() + .is_success() + ); } diff --git a/crates/symbolicli/src/main.rs b/crates/symbolicli/src/main.rs index e38dd6fe5..c1d4edefe 100644 --- a/crates/symbolicli/src/main.rs +++ b/crates/symbolicli/src/main.rs @@ -42,6 +42,7 @@ async fn main() -> Result<()> { log_level, mode, symbols, + scraping_enabled, } = settings::Settings::get()?; // We depend on `rustls` with both the `aws-lc-rs` and @@ -118,25 +119,7 @@ async fn main() -> Result<()> { let res = match payload { Payload::Event(event) if event.platform.is_js() => { - let Mode::Online { - ref org, - ref project, - ref base_url, - ref auth_token, - scraping_enabled, - } = mode - else { - anyhow::bail!("JavaScript symbolication is not supported in offline mode."); - }; - - let source = Arc::new(SentrySourceConfig { - id: SourceId::new("sentry:project"), - token: SentryToken(auth_token.clone()), - url: base_url - .join(&format!("projects/{org}/{project}/artifact-lookup/")) - .unwrap(), - }); - + let source = prepare_sourcemap_source(mode, symbols)?; let request = create_js_symbolication_request(scope, source, event, scraping_enabled) .context("Event cannot be symbolicated")?; @@ -155,6 +138,7 @@ async fn main() -> Result<()> { let res = native.symbolicate(request).await?; CompletedResponse::NativeSymbolication(res) } + Payload::Minidump(minidump_file) => { let dsym_sources = prepare_dsym_sources(mode, &symbolicator_config, symbols); tracing::info!("symbolicating minidump"); @@ -170,6 +154,7 @@ async fn main() -> Result<()> { .await?; CompletedResponse::NativeSymbolication(res) } + Payload::AppleCrashReport(file) => { let dsym_sources = prepare_dsym_sources(mode, &symbolicator_config, symbols); tracing::info!("symbolicating apple crash report"); @@ -184,6 +169,7 @@ async fn main() -> Result<()> { .await?; CompletedResponse::NativeSymbolication(res) } + Payload::Event(event) => anyhow::bail!( "Cannot symbolicate event: invalid platform {}", event.platform @@ -251,6 +237,45 @@ fn prepare_dsym_sources( Arc::from(dsym_sources.into_boxed_slice()) } +fn prepare_sourcemap_source( + mode: Mode, + local_symbols: Option, +) -> Result> { + match mode { + Mode::Online { + ref org, + ref project, + ref base_url, + ref auth_token, + } => { + if local_symbols.is_some() { + tracing::warn!("Local symbol source will not be used in online mode"); + } + + Ok(Arc::new(SentrySourceConfig { + id: SourceId::new("sentry:project"), + token: SentryToken(auth_token.clone()), + url: base_url + .join(&format!("projects/{org}/{project}/artifact-lookup/")) + .unwrap(), + })) + } + + Mode::Offline => { + let Some(SymbolsPath { path, .. }) = local_symbols else { + anyhow::bail!( + "In JS offline mode, you must provide a local symbol directory with --symbols" + ); + }; + + let source = js_local_source::start_server(&path) + .context("Failed to start local symbol server")?; + + Ok(Arc::new(source)) + } + } +} + #[derive(Debug)] enum Payload { Event(event::Event), diff --git a/crates/symbolicli/src/settings.rs b/crates/symbolicli/src/settings.rs index 9f8b4ead0..54c1976fe 100644 --- a/crates/symbolicli/src/settings.rs +++ b/crates/symbolicli/src/settings.rs @@ -40,7 +40,6 @@ pub enum Mode { project: String, auth_token: String, base_url: reqwest::Url, - scraping_enabled: bool, }, } @@ -190,6 +189,7 @@ pub struct Settings { pub log_level: LevelFilter, pub mode: Mode, pub symbols: Option, + pub scraping_enabled: bool, } impl Settings { @@ -252,7 +252,6 @@ impl Settings { org, project, auth_token, - scraping_enabled: !cli.no_scrape, } }; @@ -288,6 +287,7 @@ impl Settings { log_level: cli.log_level, mode, symbols: cli.symbols, + scraping_enabled: !cli.no_scrape, }; Ok(args) From 1122691cb7f2fff3e9ae09803524a0e8c99bcae0 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 22 May 2026 18:24:07 +0200 Subject: [PATCH 04/10] Simplify handle --- crates/symbolicli/src/js_local_source.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/symbolicli/src/js_local_source.rs b/crates/symbolicli/src/js_local_source.rs index 1521d5006..073a3a87d 100644 --- a/crates/symbolicli/src/js_local_source.rs +++ b/crates/symbolicli/src/js_local_source.rs @@ -34,13 +34,11 @@ pub fn start_server(path: impl AsRef + Clone) -> anyhow::Result Date: Fri, 22 May 2026 18:24:13 +0200 Subject: [PATCH 05/10] Fix test --- crates/symbolicli/src/js_local_source.rs | 42 ++++++++++++++---------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/crates/symbolicli/src/js_local_source.rs b/crates/symbolicli/src/js_local_source.rs index 073a3a87d..e4de7a0e5 100644 --- a/crates/symbolicli/src/js_local_source.rs +++ b/crates/symbolicli/src/js_local_source.rs @@ -237,27 +237,33 @@ async fn lookup( Json(out.into()) } -#[tokio::test] -async fn test_existing_bundle() { - let source = start_server("/Users/sebastian/code/symbolicator/tests/fixtures/").unwrap(); +#[cfg(test)] +mod tests { + use super::*; - let mut lookup_url = source.url.clone(); - lookup_url.set_query(Some("debug_id=2f259f80-58b7-44cb-d7cd-de1505e7e718")); + #[tokio::test] + async fn test_existing_bundle() { + let source = + start_server(concat!(env!("CARGO_MANIFEST_DIR"), "/../../tests/fixtures")).unwrap(); - let results: Box<[LookupResult]> = reqwest::get(lookup_url) - .await - .unwrap() - .json() - .await - .unwrap(); + let mut lookup_url = source.url.clone(); + lookup_url.set_query(Some("debug_id=2f259f80-58b7-44cb-d7cd-de1505e7e718")); - let LookupResult::Bundle { url, .. } = &results[0]; - - assert!( - reqwest::get(url.clone()) + let results: Box<[LookupResult]> = reqwest::get(lookup_url) .await .unwrap() - .status() - .is_success() - ); + .json() + .await + .unwrap(); + + let LookupResult::Bundle { url, .. } = &results[0]; + + assert!( + reqwest::get(url.clone()) + .await + .unwrap() + .status() + .is_success() + ); + } } From 0b8c6970a64251b2634eb09dd7a25d8cdd38ef7d Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 22 May 2026 18:31:06 +0200 Subject: [PATCH 06/10] readme --- crates/symbolicli/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/symbolicli/README.md b/crates/symbolicli/README.md index f9c1abfec..7f80924ae 100644 --- a/crates/symbolicli/README.md +++ b/crates/symbolicli/README.md @@ -20,7 +20,7 @@ symbolicli --offline In offline mode `symbolicli` will not attempt to access a Sentry server, which means you can only process local events. -*NB*: JavaScript symbolication is not supported in offline mode. +*NB*: JavaScript symbolication is supported in offline mode, but you have to pass a directory containing artifact bundles with `--symbols`. # Configuration @@ -48,6 +48,6 @@ You can control the level of logging output by passing the desired log level to Available levels are `off`, `error`, `warn`, `info`, `debug`, `trace`. The default is `info`. # Local Symbols -The `--symbols` option allows you to supply a local directory containing debug files to use -in addition to the configured sources. The directory must be sorted according to the -`unified` layout. The easiest way to accomplish that is using `symsorter`. +The `--symbols` option allows you to supply local debug information. +- For native events, pass a local directory containing debug files to use in addition to the configured sources. The directory must be sorted according to the `unified` layout. The easiest way to accomplish that is using `symsorter`. +- For JS events, pass a directory containing artifact bundles (individual files are not supported right now). Note that this only works in offline mode; in online mode, only files from Sentry will be used. From 3083cb918f3310404343ca2788a7ad61fb2d4365 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 22 May 2026 19:05:44 +0200 Subject: [PATCH 07/10] Fix path-based lookups --- crates/symbolicator-js/src/lib.rs | 1 + crates/symbolicli/src/js_local_source.rs | 55 ++++++++++++++++-------- 2 files changed, 37 insertions(+), 19 deletions(-) diff --git a/crates/symbolicator-js/src/lib.rs b/crates/symbolicator-js/src/lib.rs index 3ccd87a4b..5ff638aa7 100644 --- a/crates/symbolicator-js/src/lib.rs +++ b/crates/symbolicator-js/src/lib.rs @@ -9,3 +9,4 @@ mod symbolication; mod utils; pub use service::SourceMapService; +pub use utils::extract_file_stem; diff --git a/crates/symbolicli/src/js_local_source.rs b/crates/symbolicli/src/js_local_source.rs index e4de7a0e5..3f0162729 100644 --- a/crates/symbolicli/src/js_local_source.rs +++ b/crates/symbolicli/src/js_local_source.rs @@ -69,9 +69,12 @@ struct LookupKey { /// path-based lookups. #[derive(Debug, Clone, PartialEq, Eq, Hash)] struct ReleaseDistUrl { - release: Option>, - dist: Option>, + /// A URL is mandatory for path-based lookups. url: Arc, + /// A release is mandatory for path-based lookups. + release: Arc, + /// A dist is optional for path-based lookups. + dist: Option>, } /// Simple representation of the manifest of a sourcebundle (including artifact bundles.). @@ -129,22 +132,29 @@ impl Index { let dist = manifest.attributes.get("dist").cloned(); for file in manifest.files.values() { - if file.ty() != Some(SourceFileType::MinifiedSource) { + if !matches!( + file.ty(), + Some(SourceFileType::MinifiedSource | SourceFileType::Source), + ) { continue; } - let Some(url) = file.url() else { - continue; - }; + if let Some(release) = release.as_ref() { + let Some(url) = file.url() else { + continue; + }; - out.by_url - .entry(ReleaseDistUrl { - release: release.clone(), - dist: dist.clone(), - url: url.into(), - }) - .or_default() - .insert(Arc::clone(&relative_path)); + let url = symbolicator_js::extract_file_stem(url); + + out.by_url + .entry(ReleaseDistUrl { + release: release.clone(), + dist: dist.clone(), + url: url.into(), + }) + .or_default() + .insert(Arc::clone(&relative_path)); + } if let Some(debug_id) = file.debug_id() { out.by_debug_id @@ -185,12 +195,17 @@ enum LookupResult { async fn lookup( extract::State(index): extract::State>, - extract::Query(key): extract::Query, + extract::Query(LookupKey { + release, + dist, + url, + debug_id, + }): extract::Query, ) -> Json> { let mut out = Vec::new(); let mut found_bundles = HashSet::new(); - if let Some(debug_id) = key.debug_id { + if let Some(debug_id) = debug_id { for path in index .by_debug_id .get(&debug_id) @@ -210,12 +225,14 @@ async fn lookup( } } - if let Some(url) = key.url { + if let Some(url) = url + && let Some(release) = release + { for path in index .by_url .get(&ReleaseDistUrl { - release: key.release.clone(), - dist: key.dist.clone(), + release: release.clone(), + dist: dist.clone(), url: url.clone(), }) .into_iter() From d7fb5404ce9ec29c5856d6694a842848289c50a8 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Fri, 22 May 2026 19:07:06 +0200 Subject: [PATCH 08/10] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d42e50708..dbb74834f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## Unreleased + +### Features + +- (symbolicli) Support local JavaScript symbolication by @loewenheim in [#1956](https://github.com/getsentry/symbolicator/pull/1956) + ## 26.5.0 ### Bug Fixes 🐛 From d5cc6f902b4b7e867e5db199d81ed61bb9818519 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 26 May 2026 10:29:59 +0200 Subject: [PATCH 09/10] Update crates/symbolicli/README.md Co-authored-by: David Herberth --- crates/symbolicli/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/symbolicli/README.md b/crates/symbolicli/README.md index 7f80924ae..d62a34852 100644 --- a/crates/symbolicli/README.md +++ b/crates/symbolicli/README.md @@ -50,4 +50,4 @@ Available levels are `off`, `error`, `warn`, `info`, `debug`, `trace`. The defau # Local Symbols The `--symbols` option allows you to supply local debug information. - For native events, pass a local directory containing debug files to use in addition to the configured sources. The directory must be sorted according to the `unified` layout. The easiest way to accomplish that is using `symsorter`. -- For JS events, pass a directory containing artifact bundles (individual files are not supported right now). Note that this only works in offline mode; in online mode, only files from Sentry will be used. +- For JS events, pass a directory containing artifact bundles (individual files are not supported right now). Note that this only works in offline mode; in online mode, only files from Sentry are used. From 017f0061e29493acb03e8b1bf6b2ab51910491f5 Mon Sep 17 00:00:00 2001 From: Sebastian Zivota Date: Tue, 26 May 2026 10:32:40 +0200 Subject: [PATCH 10/10] Remove some unwraps --- crates/symbolicli/src/js_local_source.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/symbolicli/src/js_local_source.rs b/crates/symbolicli/src/js_local_source.rs index 3f0162729..d90da9bcc 100644 --- a/crates/symbolicli/src/js_local_source.rs +++ b/crates/symbolicli/src/js_local_source.rs @@ -21,9 +21,9 @@ use zip::ZipArchive; /// bundles from the given directory. pub fn start_server(path: impl AsRef + Clone) -> anyhow::Result { let addr = SocketAddr::from(([127, 0, 0, 1], 0)); - let listener = TcpListener::bind(addr).unwrap(); - listener.set_nonblocking(true).unwrap(); - let socket = listener.local_addr().unwrap(); + let listener = TcpListener::bind(addr).context("Failed to bind listener")?; + listener.set_nonblocking(true)?; + let socket = listener.local_addr()?; let index = Index::new(path.clone(), socket)?; let index = Arc::new(index); let source_url = index.url("lookup");