-
-
Notifications
You must be signed in to change notification settings - Fork 239
feat(preprod): Add snapshots subcommand #3110
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from all commits
95876c4
e45f588
f3cfadb
4794046
c6e67e9
1cd38b7
15a89af
1bc440d
3417b53
3f206dd
e902bfc
e5bd2d7
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -962,6 +962,13 @@ impl AuthenticatedApi<'_> { | |
| } | ||
| Ok(rv) | ||
| } | ||
|
|
||
| /// Fetch organization details | ||
| pub fn fetch_organization_details(&self, org: &str) -> ApiResult<OrganizationDetails> { | ||
| let path = format!("/api/0/organizations/{}/", PathArg(org)); | ||
| self.get(&path)? | ||
| .convert_rnf(ApiErrorKind::OrganizationNotFound) | ||
| } | ||
| } | ||
|
|
||
| /// Available datasets for fetching organization events | ||
|
|
@@ -1761,6 +1768,18 @@ pub struct Organization { | |
| pub features: Vec<String>, | ||
| } | ||
|
|
||
| #[derive(Deserialize, Debug)] | ||
| pub struct OrganizationLinks { | ||
| #[serde(rename = "regionUrl")] | ||
| pub region_url: String, | ||
| } | ||
|
|
||
| #[derive(Deserialize, Debug)] | ||
| pub struct OrganizationDetails { | ||
| pub id: String, | ||
| pub links: OrganizationLinks, | ||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This endpoint has a bunch more information but what we really care about is the If I understand correctly I think this information might already be somehow embedded in org tokens, but not in personal tokens. Or maybe I'm confusing this with something else. |
||
| } | ||
|
|
||
| #[derive(Deserialize, Debug)] | ||
| pub struct Team { | ||
| #[expect(dead_code)] | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,210 @@ | ||
| use std::fs; | ||
| use std::path::Path; | ||
|
|
||
| use anyhow::{Context as _, Result}; | ||
| use clap::{Arg, ArgMatches, Command}; | ||
| use console::style; | ||
| use log::{debug, info}; | ||
| use objectstore_client::{Client, Usecase}; | ||
| use sha2::{Digest as _, Sha256}; | ||
| use walkdir::WalkDir; | ||
|
|
||
| use crate::api::Api; | ||
| use crate::config::Config; | ||
| use crate::utils::api::get_org_project_id; | ||
| use crate::utils::args::ArgExt as _; | ||
| use crate::utils::objectstore::get_objectstore_url; | ||
|
|
||
| const EXPERIMENTAL_WARNING: &str = | ||
| "[EXPERIMENTAL] The \"build snapshots\" command is experimental. \ | ||
| The command is subject to breaking changes, including removal, in any Sentry CLI release."; | ||
|
|
||
| pub fn make_command(command: Command) -> Command { | ||
| command | ||
| .about("[EXPERIMENTAL] Upload build snapshots to a project.") | ||
| .long_about(format!( | ||
| "Upload build snapshots to a project.\n\n{EXPERIMENTAL_WARNING}" | ||
| )) | ||
| .org_arg() | ||
| .project_arg(false) | ||
| .arg( | ||
| Arg::new("path") | ||
| .value_name("PATH") | ||
| .help("The path to the folder containing build snapshots.") | ||
| .required(true), | ||
| ) | ||
| .arg( | ||
| Arg::new("snapshot_id") | ||
| .long("snapshot-id") | ||
| .value_name("ID") | ||
| .help("The snapshot identifier to associate with the upload.") | ||
| .required(true), | ||
| ) | ||
| } | ||
|
|
||
| pub fn execute(matches: &ArgMatches) -> Result<()> { | ||
| eprintln!("{EXPERIMENTAL_WARNING}"); | ||
|
|
||
| let config = Config::current(); | ||
| let org = config.get_org(matches)?; | ||
| let project = config.get_project(matches)?; | ||
|
|
||
| let path = matches | ||
| .get_one::<String>("path") | ||
| .expect("path argument is required"); | ||
| let snapshot_id = matches | ||
| .get_one::<String>("snapshot_id") | ||
| .expect("snapshot_id argument is required"); | ||
|
|
||
| info!("Processing build snapshots from: {path}"); | ||
| info!("Using snapshot ID: {snapshot_id}"); | ||
| info!("Organization: {org}"); | ||
| info!("Project: {project}"); | ||
|
|
||
| // Collect files to upload | ||
| let files = collect_files(Path::new(path))?; | ||
|
|
||
| if files.is_empty() { | ||
| println!("{} No files found to upload", style("!").yellow()); | ||
| return Ok(()); | ||
| } | ||
|
|
||
| println!( | ||
| "{} Found {} {} to upload", | ||
| style(">").dim(), | ||
| style(files.len()).yellow(), | ||
| if files.len() == 1 { "file" } else { "files" } | ||
| ); | ||
|
|
||
| // Upload files using objectstore client | ||
| upload_files(&files, &org, &project, snapshot_id)?; | ||
|
|
||
| Ok(()) | ||
| } | ||
|
|
||
| fn collect_files(path: &Path) -> Result<Vec<std::path::PathBuf>> { | ||
| if !path.exists() { | ||
| anyhow::bail!("Path does not exist: {}", path.display()); | ||
| } | ||
|
|
||
| let mut files = Vec::new(); | ||
|
|
||
| if path.is_file() { | ||
| // Only add if not hidden | ||
| if !is_hidden_file(path) { | ||
| files.push(path.to_path_buf()); | ||
| } | ||
| } else if path.is_dir() { | ||
| for entry in WalkDir::new(path) | ||
| .follow_links(true) | ||
| .into_iter() | ||
| .filter_map(Result::ok) | ||
| { | ||
| if entry.metadata()?.is_file() { | ||
| let entry_path = entry.path(); | ||
| // Skip hidden files | ||
| if !is_hidden_file(entry_path) { | ||
| files.push(entry_path.to_path_buf()); | ||
| } | ||
| } | ||
| } | ||
| } else { | ||
| anyhow::bail!("Path is neither a file nor directory: {}", path.display()); | ||
| } | ||
|
|
||
| Ok(files) | ||
| } | ||
|
|
||
| fn is_hidden_file(path: &Path) -> bool { | ||
| path.file_name() | ||
| .and_then(|name| name.to_str()) | ||
| .map(|name| name.starts_with('.')) | ||
| .unwrap_or(false) | ||
| } | ||
|
|
||
| fn is_json_file(path: &Path) -> bool { | ||
| path.extension() | ||
| .and_then(|ext| ext.to_str()) | ||
| .map(|ext| ext.eq_ignore_ascii_case("json")) | ||
| .unwrap_or(false) | ||
| } | ||
|
|
||
| fn upload_files( | ||
| files: &[std::path::PathBuf], | ||
| org: &str, | ||
| project: &str, | ||
| snapshot_id: &str, | ||
| ) -> Result<()> { | ||
| let url = get_objectstore_url(Api::current(), org)?; | ||
| let client = Client::new(url)?; | ||
|
|
||
| let (org, project) = get_org_project_id(Api::current(), org, project)?; | ||
| let session = Usecase::new("preprod") | ||
| .for_project(org, project) | ||
| .session(&client)?; | ||
|
|
||
| let runtime = tokio::runtime::Builder::new_current_thread() | ||
| .enable_all() | ||
| .build() | ||
| .context("Failed to create tokio runtime")?; | ||
|
|
||
| let mut many_builder = session.many(); | ||
|
|
||
| for file_path in files { | ||
| debug!("Processing file: {}", file_path.display()); | ||
|
|
||
| let contents = fs::read(file_path) | ||
| .with_context(|| format!("Failed to read file: {}", file_path.display()))?; | ||
|
|
||
| let key = if is_json_file(file_path) { | ||
| // For JSON files, use {org}/{snapshotId}/{filename} | ||
| let filename = file_path | ||
| .file_name() | ||
| .and_then(|name| name.to_str()) | ||
| .unwrap_or("unknown.json"); | ||
| format!("{org}/{snapshot_id}/{filename}") | ||
| } else { | ||
| // For other files, use {org}/{project}/{hash} | ||
| let hash = compute_sha256_hash(&contents); | ||
| format!("{org}/{project}/{hash}") | ||
| }; | ||
|
|
||
| info!("Queueing {} as {key}", file_path.display()); | ||
|
|
||
| many_builder = many_builder.push(session.put(contents).key(&key)); | ||
| } | ||
|
|
||
| let upload = runtime | ||
| .block_on(async { many_builder.send().await }) | ||
| .context("Failed to upload files")?; | ||
|
|
||
| match upload.error_for_failures() { | ||
| Ok(()) => { | ||
| println!( | ||
| "{} Uploaded {} {}", | ||
| style(">").dim(), | ||
| style(files.len()).yellow(), | ||
| if files.len() == 1 { "file" } else { "files" } | ||
| ); | ||
| Ok(()) | ||
| } | ||
| Err(errors) => { | ||
| eprintln!("There were errors uploading files:"); | ||
| for error in &errors { | ||
| eprintln!(" {}", style(error).red()); | ||
| } | ||
| anyhow::bail!( | ||
| "Failed to upload {} out of {} files", | ||
| errors.len(), | ||
| files.len() | ||
| ) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| fn compute_sha256_hash(data: &[u8]) -> String { | ||
| let mut hasher = Sha256::new(); | ||
| hasher.update(data); | ||
| let result = hasher.finalize(); | ||
| format!("{result:x}") | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| //! Utilities that rely on the Sentry API. | ||
|
|
||
| use crate::api::{Api, AuthenticatedApi}; | ||
| use anyhow::{Context as _, Result}; | ||
|
|
||
| /// Given an org and project slugs or IDs, returns the IDs of both. | ||
| pub fn get_org_project_id(api: impl AsRef<Api>, org: &str, project: &str) -> Result<(u64, u64)> { | ||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As far as I know, for all commands a user can provide
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It might be weird to take in |
||
| let authenticated_api = api.as_ref().authenticated()?; | ||
| let org_id = get_org_id(authenticated_api, org)?; | ||
| let authenticated_api = api.as_ref().authenticated()?; | ||
| let project_id = get_project_id(authenticated_api, org, project)?; | ||
| Ok((org_id, project_id)) | ||
| } | ||
|
|
||
| /// Given an org slug or ID, returns its ID as a number. | ||
| fn get_org_id(api: AuthenticatedApi<'_>, org: &str) -> Result<u64> { | ||
| if let Ok(id) = org.parse::<u64>() { | ||
| return Ok(id); | ||
| } | ||
| let details = api.fetch_organization_details(org)?; | ||
| details.id.parse::<u64>().context("Unable to parse org id") | ||
| } | ||
|
|
||
| /// Given an org and project slugs or IDs, returns the project ID. | ||
| fn get_project_id(api: AuthenticatedApi<'_>, org: &str, project: &str) -> Result<u64> { | ||
| if let Ok(id) = project.parse::<u64>() { | ||
| return Ok(id); | ||
| } | ||
|
|
||
| let projects = api.list_organization_projects(org)?; | ||
| for p in projects { | ||
| if p.slug == project { | ||
| return p.id.parse::<u64>().context("Unable to parse project id"); | ||
| } | ||
| } | ||
|
|
||
| anyhow::bail!("Project not found") | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,11 @@ | ||
| //! Utilities to work with the Objectstore service. | ||
|
|
||
| use anyhow::Result; | ||
|
|
||
| use crate::api::Api; | ||
|
|
||
| pub fn get_objectstore_url(api: impl AsRef<Api>, org: &str) -> Result<String> { | ||
| let api = api.as_ref().authenticated()?; | ||
| let base = api.fetch_organization_details(org)?.links.region_url; | ||
| Ok(format!("{base}/api/0/objectstore")) | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This indirectly adds a dependency to
reqwestand we need to double check what the implications of that are, especially in regard torustlsvsnative-tlswhich could be problematic.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It also adds a bunch of deps but I think most of them are inevitable.