From 17b6437011b52de918aae5f7707c606dbf56385c Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Thu, 10 Jul 2025 13:03:11 +0800 Subject: [PATCH 01/11] refactor: providers trait & network --- Cargo.lock | 12 + Cargo.toml | 1 + src/cli/mod.rs | 11 +- src/cli/progress.rs | 44 + src/core/mod.rs | 2 - src/core/net.rs | 1272 ----------------------------- src/lib.rs | 2 + src/net/mod.rs | 520 ++++++++++++ src/net/providers/archive.rs | 227 +++++ src/net/providers/azure_devops.rs | 197 +++++ src/net/providers/bitbucket.rs | 214 +++++ src/net/providers/codeberg.rs | 214 +++++ src/net/providers/gitea.rs | 192 +++++ src/net/providers/github.rs | 303 +++++++ src/net/providers/gitlab.rs | 242 ++++++ src/net/providers/mod.rs | 17 + src/net/providers/sourceforge.rs | 191 +++++ src/net/stream.rs | 379 +++++++++ src/net/traits.rs | 410 ++++++++++ src/server/mod.rs | 2 +- src/worker.rs | 4 +- tests/cli_tests.rs | 88 +- tests/integration_tests.rs | 5 +- 23 files changed, 3214 insertions(+), 1335 deletions(-) delete mode 100644 src/core/net.rs create mode 100644 src/net/mod.rs create mode 100644 src/net/providers/archive.rs create mode 100644 src/net/providers/azure_devops.rs create mode 100644 src/net/providers/bitbucket.rs create mode 100644 src/net/providers/codeberg.rs create mode 100644 src/net/providers/gitea.rs create mode 100644 src/net/providers/github.rs create mode 100644 src/net/providers/gitlab.rs create mode 100644 src/net/providers/mod.rs create mode 100644 src/net/providers/sourceforge.rs create mode 100644 src/net/stream.rs create mode 100644 src/net/traits.rs diff --git a/Cargo.lock b/Cargo.lock index 73fe0e8..2838bcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -82,6 +82,17 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "backtrace" version = "0.3.75" @@ -126,6 +137,7 @@ name = "bytes-radar" version = "0.6.0" dependencies = [ "anyhow", + "async-trait", "bytes", "clap", "colored", diff --git a/Cargo.toml b/Cargo.toml index a450091..89cbb3e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ tar = "0.4" flate2 = "1.1" futures-util = "0.3" bytes = "1.5" +async-trait = "0.1" # CLI dependencies colored = { version = "2.2", optional = true } diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 56dda52..92f1892 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -10,6 +10,8 @@ mod url_parser; #[cfg(feature = "cli")] use crate::core::*; #[cfg(feature = "cli")] +use crate::net::RemoteAnalyzer; +#[cfg(feature = "cli")] use clap::Parser; #[cfg(feature = "cli")] use std::time::Instant; @@ -54,12 +56,17 @@ async fn analyze_remote_archive(url: &str, cli: &Cli) -> Result<()> { let mut analyzer = RemoteAnalyzer::new(); if let Some(token) = &cli.token { - analyzer.set_github_token(token); + let mut credentials = std::collections::HashMap::new(); + credentials.insert("token".to_string(), token.clone()); + analyzer.set_provider_credentials("github", credentials); } analyzer.set_timeout(cli.timeout); analyzer.set_allow_insecure(cli.allow_insecure); - analyzer.set_progress_bar(progress_bar.clone()); + + if let Some(pb) = progress_bar.clone() { + analyzer.set_progress_hook(progress::ProgressBarHook::new(pb)); + } if cli.aggressive_filter { analyzer.set_aggressive_filtering(true); diff --git a/src/cli/progress.rs b/src/cli/progress.rs index 6a074b3..af2eb4c 100644 --- a/src/cli/progress.rs +++ b/src/cli/progress.rs @@ -1,3 +1,4 @@ +use crate::net::traits::ProgressHook; use indicatif::{ProgressBar, ProgressStyle}; use std::time::Duration; @@ -60,3 +61,46 @@ pub fn format_bytes(bytes: u64) -> String { format!("{:.1} {}", size, UNITS[unit_index]) } } + +pub struct ProgressBarHook { + progress_bar: ProgressBar, +} + +impl ProgressBarHook { + pub fn new(progress_bar: ProgressBar) -> Self { + Self { progress_bar } + } +} + +impl ProgressHook for ProgressBarHook { + fn on_download_progress(&self, downloaded: u64, total: Option) { + if let Some(total_size) = total { + self.progress_bar.set_style( + ProgressStyle::default_bar() + .template("[{elapsed_precise}] [{wide_bar:.cyan/blue}] {decimal_bytes_per_sec} {binary_bytes}/{binary_total_bytes} ({eta}) {msg}") + .unwrap_or_else(|_| ProgressStyle::default_bar()) + .progress_chars("#>-"), + ); + self.progress_bar.set_length(total_size); + self.progress_bar.set_position(downloaded); + } else { + let formatted = format_bytes(downloaded); + self.progress_bar + .set_message(format!("Downloaded {}...", formatted)); + } + } + + fn on_processing_start(&self, message: &str) { + self.progress_bar.set_message(message.to_string()); + } + + fn on_processing_progress(&self, current: usize, total: usize) { + if total > 0 { + let percentage = (current * 100) / total; + self.progress_bar.set_message(format!( + "Processing files: {}% ({}/{})", + percentage, current, total + )); + } + } +} diff --git a/src/core/mod.rs b/src/core/mod.rs index 1dabc79..2525dc1 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -1,10 +1,8 @@ pub mod analysis; pub mod error; pub mod filter; -pub mod net; pub mod registry; pub use analysis::*; pub use error::*; -pub use net::*; pub use registry::*; diff --git a/src/core/net.rs b/src/core/net.rs deleted file mode 100644 index e3e83a7..0000000 --- a/src/core/net.rs +++ /dev/null @@ -1,1272 +0,0 @@ -use crate::core::{ - analysis::{FileMetrics, ProjectAnalysis}, - error::{AnalysisError, Result}, - filter::{FilterStats, IntelligentFilter}, - registry::LanguageRegistry, -}; -use flate2::read::GzDecoder; -use futures_util::StreamExt; -use reqwest::Client; -use serde::Deserialize; -use std::io::{Cursor, Read}; -use tar::Archive; - -#[cfg(not(target_arch = "wasm32"))] -use tokio::task; - -static USER_AGENT: &str = "bytes-radar/1.0.0"; - -#[derive(Deserialize)] -struct GitHubRepoInfo { - default_branch: String, -} - -#[cfg(feature = "cli")] -use indicatif::ProgressBar; - -pub struct RemoteAnalyzer { - client: Client, - github_token: Option, - timeout: u64, - allow_insecure: bool, - filter: IntelligentFilter, - #[cfg(feature = "cli")] - progress_bar: Option, -} - -impl RemoteAnalyzer { - pub fn new() -> Self { - let mut builder = Client::builder().user_agent(USER_AGENT); - - #[cfg(not(target_arch = "wasm32"))] - { - builder = builder.timeout(std::time::Duration::from_secs(300)); - } - - let client = builder.build().expect("Failed to create HTTP client"); - - Self { - client, - github_token: None, - timeout: 300, - allow_insecure: false, - filter: IntelligentFilter::default(), - #[cfg(feature = "cli")] - progress_bar: None, - } - } - - #[cfg(feature = "cli")] - pub fn set_progress_bar(&mut self, progress_bar: Option) { - self.progress_bar = progress_bar; - } - - pub fn set_github_token(&mut self, token: &str) { - self.github_token = Some(token.to_string()); - self.rebuild_client(); - } - - pub fn set_timeout(&mut self, timeout: u64) { - self.timeout = timeout; - self.rebuild_client(); - } - - pub fn set_allow_insecure(&mut self, allow_insecure: bool) { - self.allow_insecure = allow_insecure; - self.rebuild_client(); - } - - pub fn set_filter(&mut self, filter: IntelligentFilter) { - self.filter = filter; - } - - pub fn set_aggressive_filtering(&mut self, enabled: bool) { - if enabled { - self.filter = IntelligentFilter::aggressive(); - } else { - self.filter = IntelligentFilter::default(); - } - } - - fn rebuild_client(&mut self) { - let mut builder = Client::builder().user_agent(USER_AGENT); - - #[cfg(not(target_arch = "wasm32"))] - { - builder = builder.timeout(std::time::Duration::from_secs(self.timeout)); - } - - #[cfg(not(target_arch = "wasm32"))] - if self.allow_insecure { - builder = builder.danger_accept_invalid_certs(true); - } - - if let Some(token) = &self.github_token { - let mut headers = reqwest::header::HeaderMap::new(); - let auth_value = format!("token {}", token); - headers.insert( - reqwest::header::AUTHORIZATION, - auth_value.parse().expect("Invalid token format"), - ); - builder = builder.default_headers(headers); - } - - self.client = builder.build().expect("Failed to create HTTP client"); - } - - pub async fn analyze_url(&self, url: &str) -> Result { - let download_urls = self.resolve_git_url(url).await?; - - let mut url_errors: Vec = Vec::new(); - let mut total_attempts = 0u32; - for download_url in download_urls { - total_attempts += 1; - match self.analyze_tarball_with_name(&download_url, url).await { - Ok(analysis) => return Ok(analysis), - Err(e) => { - #[cfg(feature = "cli")] - log::debug!("Failed to download from {}: {}", download_url, e); - - let error_info = crate::core::error::DownloadUrlError { - url: download_url.clone(), - error_message: format!("{}", e), - error_type: match e { - AnalysisError::NetworkError { .. } => "NetworkError".to_string(), - AnalysisError::ArchiveError { .. } => "ArchiveError".to_string(), - _ => "UnknownError".to_string(), - }, - http_status_code: self.extract_http_status_code(&e), - retry_count: 1, - }; - - url_errors.push(error_info); - continue; - } - } - } - - Err(AnalysisError::network( - "All download URLs failed".to_string(), - )) - } - - async fn resolve_git_url(&self, url: &str) -> Result> { - if url.ends_with(".tar.gz") || url.ends_with(".tgz") { - return Ok(vec![url.to_string()]); - } - - let expanded_url = self.expand_url(url); - - if expanded_url.starts_with("http://") || expanded_url.starts_with("https://") { - if !expanded_url.contains("github.com") - && !expanded_url.contains("gitlab.com") - && !expanded_url.contains("gitlab.") - && !expanded_url.contains("bitbucket.org") - && !expanded_url.contains("codeberg.org") - { - if expanded_url.ends_with(".tar.gz") || expanded_url.ends_with(".tgz") { - return Ok(vec![expanded_url.to_string()]); - } else { - return Ok(vec![expanded_url.to_string()]); - } - } - } - - let mut download_urls = Vec::new(); - - if let Some(github_url) = self.parse_github_url_with_branch(&expanded_url) { - download_urls.push(github_url); - } - - if let Some(gitlab_url) = self.parse_gitlab_url_with_branch(&expanded_url) { - download_urls.push(gitlab_url); - } - - if let Some(bitbucket_url) = self.parse_bitbucket_url_with_branch(&expanded_url) { - download_urls.push(bitbucket_url); - } - - if let Some(codeberg_url) = self.parse_codeberg_url_with_branch(&expanded_url) { - download_urls.push(codeberg_url); - } - - if download_urls.is_empty() { - let mut branches = vec![ - "main".to_string(), - "master".to_string(), - "develop".to_string(), - "dev".to_string(), - ]; - - #[cfg(not(target_arch = "wasm32"))] - if expanded_url.contains("github.com") { - if let Some(default_branch) = self.get_github_default_branch(&expanded_url).await { - branches.insert(0, default_branch); - branches.dedup(); - } - } - - #[cfg(target_arch = "wasm32")] - if expanded_url.contains("github.com") { - branches = vec![ - "main".to_string(), - "master".to_string(), - "develop".to_string(), - "dev".to_string(), - ]; - } - - for branch in &branches { - if let Some(github_url) = self.parse_github_url(&expanded_url, branch) { - download_urls.push(github_url); - } - - if let Some(gitlab_url) = self.parse_gitlab_url(&expanded_url, branch) { - download_urls.push(gitlab_url); - } - - if let Some(bitbucket_url) = self.parse_bitbucket_url(&expanded_url, branch) { - download_urls.push(bitbucket_url); - } - - if let Some(codeberg_url) = self.parse_codeberg_url(&expanded_url, branch) { - download_urls.push(codeberg_url); - } - } - } - - if download_urls.is_empty() { - return Err(AnalysisError::url_parsing(format!( - "Unsupported URL format or no accessible branch found: {}. Please provide a direct tar.gz URL or a supported repository URL.", - expanded_url - ))); - } - - Ok(download_urls) - } - - fn parse_github_url_with_branch(&self, url: &str) -> Option { - if url.contains("github.com") { - if url.contains("/tree/") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(tree_pos) = parts.iter().position(|&x| x == "tree") { - if tree_pos + 1 < parts.len() && tree_pos >= 2 { - let owner = parts[tree_pos - 2]; - let repo = parts[tree_pos - 1]; - let branch = parts[tree_pos + 1]; - return Some(format!( - "https://github.com/{}/{}/archive/refs/heads/{}.tar.gz", - owner, repo, branch - )); - } - } - } - - if url.contains("/commit/") { - return self.extract_github_commit_url(url); - } - } - None - } - - fn parse_gitlab_url_with_branch(&self, url: &str) -> Option { - if url.contains("gitlab.com") || url.contains("gitlab.") { - if url.contains("/-/tree/") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(tree_pos) = parts.iter().position(|&x| x == "tree") { - if tree_pos + 1 < parts.len() && tree_pos >= 3 { - let gitlab_pos = parts.iter().position(|&x| x.contains("gitlab")).unwrap(); - let host = parts[gitlab_pos]; - let owner = parts[gitlab_pos + 1]; - let repo = parts[gitlab_pos + 2]; - let branch = parts[tree_pos + 1]; - return Some(format!( - "https://{}/{}{}/-/archive/{}/{}-{}.tar.gz", - host, - owner, - if parts.len() > gitlab_pos + 3 && parts[gitlab_pos + 3] != "-" { - format!("/{}", parts[gitlab_pos + 3..tree_pos - 1].join("/")) - } else { - String::new() - }, - branch, - repo, - branch - )); - } - } - } - } - None - } - - fn parse_bitbucket_url_with_branch(&self, url: &str) -> Option { - if url.contains("bitbucket.org") { - if url.contains("/commits/") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(commits_pos) = parts.iter().position(|&x| x == "commits") { - if commits_pos + 1 < parts.len() && commits_pos >= 2 { - let owner = parts[commits_pos - 2]; - let repo = parts[commits_pos - 1]; - let commit = parts[commits_pos + 1]; - return Some(format!( - "https://bitbucket.org/{}/{}/get/{}.tar.gz", - owner, repo, commit - )); - } - } - } - - if url.contains("/branch/") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(branch_pos) = parts.iter().position(|&x| x == "branch") { - if branch_pos + 1 < parts.len() && branch_pos >= 2 { - let owner = parts[branch_pos - 2]; - let repo = parts[branch_pos - 1]; - let branch = parts[branch_pos + 1]; - return Some(format!( - "https://bitbucket.org/{}/{}/get/{}.tar.gz", - owner, repo, branch - )); - } - } - } - } - None - } - - fn parse_codeberg_url_with_branch(&self, url: &str) -> Option { - if url.contains("codeberg.org") { - if url.contains("/commit/") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(commit_pos) = parts.iter().position(|&x| x == "commit") { - if commit_pos + 1 < parts.len() && commit_pos >= 2 { - let owner = parts[commit_pos - 2]; - let repo = parts[commit_pos - 1]; - let commit = parts[commit_pos + 1]; - return Some(format!( - "https://codeberg.org/{}/{}/archive/{}.tar.gz", - owner, repo, commit - )); - } - } - } - - if url.contains("/src/branch/") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(branch_pos) = parts.iter().position(|&x| x == "branch") { - if branch_pos + 1 < parts.len() && branch_pos >= 3 { - let owner = parts[branch_pos - 3]; - let repo = parts[branch_pos - 2]; - let branch = parts[branch_pos + 1]; - return Some(format!( - "https://codeberg.org/{}/{}/archive/{}.tar.gz", - owner, repo, branch - )); - } - } - } - } - None - } - - fn parse_bitbucket_url(&self, url: &str, branch: &str) -> Option { - if url.contains("bitbucket.org") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(bitbucket_pos) = parts.iter().position(|&x| x == "bitbucket.org") { - if bitbucket_pos + 2 < parts.len() { - let owner = parts[bitbucket_pos + 1]; - let repo = parts[bitbucket_pos + 2]; - return Some(format!( - "https://bitbucket.org/{}/{}/get/{}.tar.gz", - owner, repo, branch - )); - } - } - } - None - } - - fn parse_codeberg_url(&self, url: &str, branch: &str) -> Option { - if url.contains("codeberg.org") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(codeberg_pos) = parts.iter().position(|&x| x == "codeberg.org") { - if codeberg_pos + 2 < parts.len() { - let owner = parts[codeberg_pos + 1]; - let repo = parts[codeberg_pos + 2]; - return Some(format!( - "https://codeberg.org/{}/{}/archive/{}.tar.gz", - owner, repo, branch - )); - } - } - } - None - } - - async fn check_url_exists(&self, url: &str) -> bool { - if let Ok(response) = self.client.head(url).send().await { - response.status().is_success() - } else { - false - } - } - - fn extract_http_status_code(&self, error: &AnalysisError) -> Option { - match error { - AnalysisError::NetworkError { message } => { - if message.contains("HTTP request failed with status: ") { - if let Some(start) = message.find("HTTP request failed with status: ") { - let status_start = start + "HTTP request failed with status: ".len(); - let status_str = &message[status_start..]; - if let Some(end) = status_str.find(' ') { - status_str[..end].parse().ok() - } else { - status_str.parse().ok() - } - } else { - None - } - } else { - None - } - } - _ => None, - } - } - - fn expand_url(&self, url: &str) -> String { - if url.starts_with("http://") || url.starts_with("https://") { - return url.to_string(); - } - - if url.contains('/') && !url.starts_with("http://") && !url.starts_with("https://") { - let parts: Vec<&str> = url.split('@').collect(); - let repo_part = parts[0]; - let branch_or_commit = parts.get(1); - - let path_parts: Vec<&str> = repo_part.split('/').collect(); - if path_parts.len() == 2 { - if let Some(branch) = branch_or_commit { - if branch.len() >= 7 && branch.chars().all(|c| c.is_ascii_hexdigit()) { - return format!("https://github.com/{}/commit/{}", repo_part, branch); - } else { - return format!("https://github.com/{}/tree/{}", repo_part, branch); - } - } else { - return format!("https://github.com/{}", repo_part); - } - } - } - - url.to_string() - } - - #[cfg(not(target_arch = "wasm32"))] - async fn get_github_default_branch(&self, url: &str) -> Option { - let (owner, repo) = self.extract_github_owner_repo(url)?; - - let api_url = format!("https://api.github.com/repos/{}/{}", owner, repo); - - match self.client.get(&api_url).send().await { - Ok(response) => { - if response.status().is_success() { - match response.json::().await { - Ok(repo_info) => { - #[cfg(feature = "cli")] - log::debug!( - "GitHub API: Found default branch '{}' for {}/{}", - repo_info.default_branch, - owner, - repo - ); - Some(repo_info.default_branch) - } - Err(_) => { - #[cfg(feature = "cli")] - log::debug!( - "GitHub API: Failed to parse response for {}/{}", - owner, - repo - ); - None - } - } - } else { - #[cfg(feature = "cli")] - log::debug!( - "GitHub API: Request failed with status {} for {}/{}", - response.status(), - owner, - repo - ); - None - } - } - Err(_) => { - #[cfg(feature = "cli")] - log::debug!("GitHub API: Network error for {}/{}", owner, repo); - None - } - } - } - - fn extract_github_owner_repo(&self, url: &str) -> Option<(String, String)> { - let url = url.trim_end_matches('/'); - - if let Some(github_url) = url.strip_prefix("https://github.com/") { - let parts: Vec<&str> = github_url.split('/').collect(); - if parts.len() >= 2 { - return Some((parts[0].to_string(), parts[1].to_string())); - } - } - - if url.contains("github.com") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(github_pos) = parts.iter().position(|&x| x == "github.com") { - if github_pos + 2 < parts.len() { - return Some(( - parts[github_pos + 1].to_string(), - parts[github_pos + 2].to_string(), - )); - } - } - } - - let parts: Vec<&str> = url.split('@').collect(); - let repo_part = parts[0]; - let path_parts: Vec<&str> = repo_part.split('/').collect(); - if path_parts.len() == 2 { - return Some((path_parts[0].to_string(), path_parts[1].to_string())); - } - - None - } - - fn parse_github_url(&self, url: &str, branch: &str) -> Option { - let url = url.trim_end_matches('/'); - - if url.contains("github.com") { - if let Some(commit_url) = self.extract_github_commit_url(url) { - return Some(commit_url); - } - - if let Some(repo_url) = self.extract_github_repo_url(url, branch) { - return Some(repo_url); - } - } - - None - } - - fn extract_github_commit_url(&self, url: &str) -> Option { - if url.contains("/commit/") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(commit_pos) = parts.iter().position(|&x| x == "commit") { - if commit_pos + 1 < parts.len() { - let owner = parts.get(parts.len() - 4)?; - let repo = parts.get(parts.len() - 3)?; - let commit = parts.get(commit_pos + 1)?; - return Some(format!( - "https://github.com/{}/{}/archive/{}.tar.gz", - owner, repo, commit - )); - } - } - } - None - } - - fn extract_github_repo_url(&self, url: &str, branch: &str) -> Option { - let parts: Vec<&str> = url.split('/').collect(); - if parts.len() >= 2 && parts.contains(&"github.com") { - if let Some(github_pos) = parts.iter().position(|&x| x == "github.com") { - if github_pos + 2 < parts.len() { - let owner = parts[github_pos + 1]; - let repo = parts[github_pos + 2]; - return Some(format!( - "https://github.com/{}/{}/archive/refs/heads/{}.tar.gz", - owner, repo, branch - )); - } - } - } - None - } - - fn parse_gitlab_url(&self, url: &str, branch: &str) -> Option { - let url = url.trim_end_matches('/'); - - if url.contains("gitlab.com") || url.contains("gitlab.") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(gitlab_pos) = parts.iter().position(|&x| x.contains("gitlab")) { - if gitlab_pos + 2 < parts.len() { - let host = parts[gitlab_pos]; - let owner = parts[gitlab_pos + 1]; - let repo = parts[gitlab_pos + 2]; - return Some(format!( - "https://{}/{}{}/-/archive/{}/{}-{}.tar.gz", - host, - owner, - if parts.len() > gitlab_pos + 3 { - format!("/{}", parts[gitlab_pos + 3..].join("/")) - } else { - String::new() - }, - branch, - repo, - branch - )); - } - } - } - - None - } - - async fn analyze_tarball_with_name( - &self, - download_url: &str, - original_url: &str, - ) -> Result { - let project_name = self.extract_project_name_from_original(original_url); - let mut project_analysis = ProjectAnalysis::new(project_name); - - let response = self - .client - .get(download_url) - .send() - .await - .map_err(|e| AnalysisError::network(format!("Failed to fetch URL: {}", e)))?; - - if !response.status().is_success() { - return Err(AnalysisError::network(format!( - "HTTP request failed with status: {}", - response.status() - ))); - } - - let total_size = response.content_length(); - - #[cfg(feature = "cli")] - if let Some(pb) = &self.progress_bar { - if let Some(size) = total_size { - use indicatif::ProgressStyle; - pb.set_style( - ProgressStyle::default_bar() - .template("[{elapsed_precise}] [{wide_bar:.cyan/blue}] {decimal_bytes_per_sec} {binary_bytes}/{binary_total_bytes} ({eta}) {msg}") - .unwrap_or_else(|_| ProgressStyle::default_bar()) - .progress_chars("#>-"), - ); - pb.set_length(size); - pb.set_message("Downloading and processing..."); - } else { - pb.set_message("Downloading and processing..."); - pb.enable_steady_tick(std::time::Duration::from_millis(120)); - } - } - - #[cfg(target_arch = "wasm32")] - { - let bytes = response - .bytes() - .await - .map_err(|e| AnalysisError::network(format!("Failed to read response: {}", e)))?; - - #[cfg(feature = "cli")] - if let Some(pb) = &self.progress_bar { - pb.set_message("Processing archive..."); - } - - self.process_tarball_bytes(&bytes, &mut project_analysis) - .await?; - } - - #[cfg(not(target_arch = "wasm32"))] - { - let stream = response.bytes_stream(); - let stream_reader = StreamReader::new( - stream, - #[cfg(feature = "cli")] - self.progress_bar.clone(), - total_size, - ); - - #[cfg(feature = "cli")] - if let Some(pb) = &self.progress_bar { - pb.set_message("Processing archive..."); - } - - self.process_tarball_stream(stream_reader, &mut project_analysis) - .await?; - } - - Ok(project_analysis) - } - - async fn process_tarball_stream( - &self, - stream_reader: StreamReader, - project_analysis: &mut ProjectAnalysis, - ) -> Result<()> { - #[cfg(not(target_arch = "wasm32"))] - { - let filter = self.filter.clone(); - let metrics_result = task::spawn_blocking(move || { - let decoder = GzDecoder::new(stream_reader); - let mut archive = Archive::new(decoder); - - let entries = archive.entries().map_err(|e| { - AnalysisError::archive(format!("Failed to read tar entries: {}", e)) - })?; - - let mut collected_metrics = Vec::new(); - let mut stats = FilterStats::new(); - - for entry in entries { - let entry = entry.map_err(|e| { - AnalysisError::archive(format!("Failed to read tar entry: {}", e)) - })?; - - if let Ok(metrics) = Self::process_tar_entry_sync(entry, &filter, &mut stats) { - collected_metrics.push(metrics); - } - } - - #[cfg(feature = "cli")] - log::info!( - "Filter stats: processed {}/{} files ({:.1}% filtered), saved {}", - stats.processed, - stats.total_entries, - stats.filter_ratio() * 100.0, - stats.format_bytes_saved() - ); - - Ok::, AnalysisError>(collected_metrics) - }) - .await - .map_err(|e| AnalysisError::archive(format!("Task join error: {}", e)))??; - - for metrics in metrics_result { - project_analysis.add_file_metrics(metrics)?; - } - } - - #[cfg(target_arch = "wasm32")] - { - let decoder = GzDecoder::new(stream_reader); - let mut archive = Archive::new(decoder); - - let entries = archive.entries().map_err(|e| { - AnalysisError::archive(format!("Failed to read tar entries: {}", e)) - })?; - - let mut stats = FilterStats::new(); - - for entry in entries { - let entry = entry.map_err(|e| { - AnalysisError::archive(format!("Failed to read tar entry: {}", e)) - })?; - - if let Ok(metrics) = Self::process_tar_entry_sync(entry, &self.filter, &mut stats) { - project_analysis.add_file_metrics(metrics)?; - } - } - - web_sys::console::log_1( - &format!( - "Filter stats: processed {}/{} files ({:.1}% filtered), saved {}", - stats.processed, - stats.total_entries, - stats.filter_ratio() * 100.0, - stats.format_bytes_saved() - ) - .into(), - ); - } - - Ok(()) - } - - #[cfg(target_arch = "wasm32")] - async fn process_tarball_bytes( - &self, - bytes: &bytes::Bytes, - project_analysis: &mut ProjectAnalysis, - ) -> Result<()> { - let cursor = Cursor::new(bytes.as_ref()); - let decoder = GzDecoder::new(cursor); - let mut archive = Archive::new(decoder); - - let entries = archive - .entries() - .map_err(|e| AnalysisError::archive(format!("Failed to read tar entries: {}", e)))?; - - let mut stats = FilterStats::new(); - - for entry in entries { - let entry = entry - .map_err(|e| AnalysisError::archive(format!("Failed to read tar entry: {}", e)))?; - - if let Ok(metrics) = Self::process_tar_entry_sync(entry, &self.filter, &mut stats) { - project_analysis.add_file_metrics(metrics)?; - } - } - - web_sys::console::log_1( - &format!( - "Filter stats: processed {}/{} files ({:.1}% filtered), saved {}", - stats.processed, - stats.total_entries, - stats.filter_ratio() * 100.0, - stats.format_bytes_saved() - ) - .into(), - ); - - Ok(()) - } - - fn process_tar_entry_sync( - mut entry: tar::Entry<'_, R>, - filter: &IntelligentFilter, - stats: &mut FilterStats, - ) -> Result { - let header = entry.header(); - let path = header - .path() - .map_err(|e| AnalysisError::archive(format!("Invalid path in tar entry: {}", e)))?; - - let file_path = path.to_string_lossy().to_string(); - - if !header.entry_type().is_file() || header.size().unwrap_or(0) == 0 { - return Err(AnalysisError::archive("Not a file or empty".to_string())); - } - - let file_size = header.size().unwrap_or(0); - - let should_process = filter.should_process_file(&file_path, file_size); - stats.record_entry(file_size, !should_process); - - if !should_process { - return Err(AnalysisError::archive("File filtered out".to_string())); - } - - let language = LanguageRegistry::detect_by_path(&file_path) - .map(|l| l.name.clone()) - .unwrap_or_else(|| "Text".to_string()); - - let mut content = String::new(); - if entry.read_to_string(&mut content).is_err() { - return Err(AnalysisError::archive( - "Failed to read file content".to_string(), - )); - } - - analyze_file_content(&file_path, &content, &language, file_size) - } - - fn extract_project_name_from_original(&self, url: &str) -> String { - if url.starts_with("http://") || url.starts_with("https://") { - let url = url.trim_end_matches('/'); - - if url.contains("/tree/") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(tree_pos) = parts.iter().position(|&x| x == "tree") { - if tree_pos > 1 { - let repo = parts[tree_pos - 1]; - let branch = parts.get(tree_pos + 1).unwrap_or(&"unknown"); - return format!("{}@{}", repo, branch); - } - } - } - - if url.contains("/commit/") { - let parts: Vec<&str> = url.split('/').collect(); - if let Some(commit_pos) = parts.iter().position(|&x| x == "commit") { - if commit_pos > 1 { - let repo = parts[commit_pos - 1]; - let commit = parts.get(commit_pos + 1).unwrap_or(&"unknown"); - return format!("{}@{}", repo, &commit[..7.min(commit.len())]); - } - } - } - - let parts: Vec<&str> = url.split('/').collect(); - if parts.len() >= 2 { - let repo = parts[parts.len() - 1]; - return format!("{}@main", repo); - } - } else if url.contains('/') && !url.contains('.') { - let parts: Vec<&str> = url.split('@').collect(); - let repo_part = parts[0]; - let branch = parts.get(1).unwrap_or(&"main"); - - if let Some(repo_name) = repo_part.split('/').last() { - return format!("{}@{}", repo_name, branch); - } - } - - "remote-project".to_string() - } - - #[allow(dead_code)] - fn extract_project_name(&self, url: &str) -> String { - let url_path = url.trim_end_matches('/'); - - if let Some(filename) = url_path.split('/').last() { - if filename.ends_with(".tar.gz") { - return filename.trim_end_matches(".tar.gz").to_string(); - } - if filename.ends_with(".tgz") { - return filename.trim_end_matches(".tgz").to_string(); - } - return filename.to_string(); - } - - "remote-project".to_string() - } - - fn format_bytes_simple(bytes: u64) -> String { - const UNITS: &[&str] = &["B", "KiB", "MiB", "GiB", "TiB"]; - const THRESHOLD: f64 = 1024.0; - - if bytes == 0 { - return "0 B".to_string(); - } - - let mut size = bytes as f64; - let mut unit_index = 0; - - while size >= THRESHOLD && unit_index < UNITS.len() - 1 { - size /= THRESHOLD; - unit_index += 1; - } - - if unit_index == 0 { - format!("{} {}", bytes, UNITS[unit_index]) - } else { - format!("{:.1} {}", size, UNITS[unit_index]) - } - } -} - -impl Default for RemoteAnalyzer { - fn default() -> Self { - Self::new() - } -} - -fn analyze_file_content( - file_path: &str, - content: &str, - language: &str, - file_size: u64, -) -> Result { - let lines: Vec<&str> = content.lines().collect(); - let total_lines = lines.len(); - - let mut code_lines = 0; - let mut comment_lines = 0; - let mut blank_lines = 0; - - let lang_def = LanguageRegistry::get_language(language); - let empty_line_comments = vec![]; - let empty_multi_line_comments = vec![]; - let line_comments = lang_def - .map(|l| &l.line_comments) - .unwrap_or(&empty_line_comments); - let multi_line_comments = lang_def - .map(|l| &l.multi_line_comments) - .unwrap_or(&empty_multi_line_comments); - - let mut in_multi_line_comment = false; - - for line in lines { - let trimmed = line.trim(); - - if trimmed.is_empty() { - blank_lines += 1; - continue; - } - - let mut is_comment = false; - - if !in_multi_line_comment { - for comment_start in line_comments { - if trimmed.starts_with(comment_start) { - is_comment = true; - break; - } - } - - for (start, end) in multi_line_comments { - if trimmed.starts_with(start) { - is_comment = true; - if !trimmed.ends_with(end) { - in_multi_line_comment = true; - } - break; - } - } - } else { - is_comment = true; - for (_, end) in multi_line_comments { - if trimmed.ends_with(end) { - in_multi_line_comment = false; - break; - } - } - } - - if is_comment { - comment_lines += 1; - } else { - code_lines += 1; - } - } - - let metrics = FileMetrics::new( - file_path, - language.to_string(), - total_lines, - code_lines, - comment_lines, - blank_lines, - )? - .with_size_bytes(file_size); - - Ok(metrics) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_github_url_parsing() { - let analyzer = RemoteAnalyzer::new(); - - assert_eq!( - analyzer.parse_github_url("https://github.com/user/repo", "main"), - Some("https://github.com/user/repo/archive/refs/heads/main.tar.gz".to_string()) - ); - - assert_eq!( - analyzer.parse_github_url("https://github.com/user/repo/commit/abc123", "main"), - Some("https://github.com/user/repo/archive/abc123.tar.gz".to_string()) - ); - } - - #[test] - fn test_bitbucket_url_parsing() { - let analyzer = RemoteAnalyzer::new(); - - assert_eq!( - analyzer.parse_bitbucket_url("https://bitbucket.org/user/repo", "main"), - Some("https://bitbucket.org/user/repo/get/main.tar.gz".to_string()) - ); - } - - #[test] - fn test_codeberg_url_parsing() { - let analyzer = RemoteAnalyzer::new(); - - assert_eq!( - analyzer.parse_codeberg_url("https://codeberg.org/user/repo", "main"), - Some("https://codeberg.org/user/repo/archive/main.tar.gz".to_string()) - ); - } - - #[test] - fn test_extract_project_name() { - let analyzer = RemoteAnalyzer::new(); - - assert_eq!( - analyzer.extract_project_name("https://example.com/project.tar.gz"), - "project" - ); - - assert_eq!( - analyzer.extract_project_name("https://github.com/user/repo/archive/main.tar.gz"), - "main" - ); - } -} - -use tokio::sync::mpsc; - -struct StreamReader { - receiver: mpsc::Receiver>, - current_chunk: Option>, - finished: bool, -} - -impl StreamReader { - #[cfg(not(target_arch = "wasm32"))] - fn new( - stream: impl futures_util::Stream> + Send + 'static, - #[cfg(feature = "cli")] progress_bar: Option, - total_size: Option, - ) -> Self { - let (tx, rx) = mpsc::channel(32); - - tokio::spawn(async move { - let mut downloaded = 0u64; - let mut stream = Box::pin(stream); - - while let Some(chunk_result) = stream.next().await { - match chunk_result { - Ok(chunk) => { - downloaded += chunk.len() as u64; - - #[cfg(feature = "cli")] - if let Some(pb) = &progress_bar { - if let Some(_total) = total_size { - pb.set_position(downloaded); - } else { - let formatted = RemoteAnalyzer::format_bytes_simple(downloaded); - pb.set_message(format!("Downloaded {}...", formatted)); - } - } - - if tx.send(Ok(chunk)).await.is_err() { - break; - } - } - Err(e) => { - let _ = tx - .send(Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("Stream error: {}", e), - ))) - .await; - break; - } - } - } - }); - - Self { - receiver: rx, - current_chunk: None, - finished: false, - } - } - - #[cfg(target_arch = "wasm32")] - fn new( - stream: impl futures_util::Stream> + 'static, - #[cfg(feature = "cli")] _progress_bar: Option, - _total_size: Option, - ) -> Self { - let (tx, rx) = mpsc::channel(32); - - wasm_bindgen_futures::spawn_local(async move { - let mut downloaded = 0u64; - let mut stream = Box::pin(stream); - - while let Some(chunk_result) = stream.next().await { - match chunk_result { - Ok(chunk) => { - downloaded += chunk.len() as u64; - - #[cfg(feature = "cli")] - if let Some(pb) = &_progress_bar { - if let Some(_total) = _total_size { - pb.set_position(downloaded); - } else { - let formatted = RemoteAnalyzer::format_bytes_simple(downloaded); - pb.set_message(format!("Downloaded {}...", formatted)); - } - } - - if tx.send(Ok(chunk)).await.is_err() { - break; - } - } - Err(e) => { - let _ = tx - .send(Err(std::io::Error::new( - std::io::ErrorKind::Other, - format!("Stream error: {}", e), - ))) - .await; - break; - } - } - } - }); - - Self { - receiver: rx, - current_chunk: None, - finished: false, - } - } -} - -impl Read for StreamReader { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - if let Some(ref mut cursor) = self.current_chunk { - let read = cursor.read(buf)?; - if read > 0 { - return Ok(read); - } - self.current_chunk = None; - } - - if self.finished { - return Ok(0); - } - - match self.receiver.try_recv() { - Ok(Ok(chunk)) => { - self.current_chunk = Some(Cursor::new(chunk)); - if let Some(ref mut cursor) = self.current_chunk { - cursor.read(buf) - } else { - Ok(0) - } - } - Ok(Err(e)) => { - self.finished = true; - Err(e) - } - Err(mpsc::error::TryRecvError::Empty) => { - #[cfg(not(target_arch = "wasm32"))] - { - match self.receiver.blocking_recv() { - Some(Ok(chunk)) => { - self.current_chunk = Some(Cursor::new(chunk)); - if let Some(ref mut cursor) = self.current_chunk { - cursor.read(buf) - } else { - Ok(0) - } - } - Some(Err(e)) => { - self.finished = true; - Err(e) - } - None => { - self.finished = true; - Ok(0) - } - } - } - #[cfg(target_arch = "wasm32")] - { - Err(std::io::Error::new( - std::io::ErrorKind::WouldBlock, - "Would block in WASM", - )) - } - } - Err(mpsc::error::TryRecvError::Disconnected) => { - self.finished = true; - Ok(0) - } - } - } -} diff --git a/src/lib.rs b/src/lib.rs index d2a64c6..5c1c0f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,7 +2,9 @@ pub mod worker; pub mod core; +pub mod net; pub use core::*; +pub use net::RemoteAnalyzer; #[cfg(feature = "cli")] pub mod cli; diff --git a/src/net/mod.rs b/src/net/mod.rs new file mode 100644 index 0000000..27ec794 --- /dev/null +++ b/src/net/mod.rs @@ -0,0 +1,520 @@ +pub mod providers; +pub mod stream; +pub mod traits; + +use crate::core::{analysis::ProjectAnalysis, error::Result, filter::IntelligentFilter}; +use providers::*; +use reqwest::Client; +use std::collections::HashMap; +use traits::{GitProvider, NoOpProgressHook}; + +pub use traits::{ParsedRepository, ProgressHook, ProviderConfig}; + +/// Remote repository analyzer with comprehensive configuration support +/// +/// The RemoteAnalyzer supports multiple Git hosting providers and allows +/// extensive customization of HTTP requests, authentication, and processing behavior. +/// +/// # Examples +/// +/// ```rust +/// use bytes_radar::net::{RemoteAnalyzer, ProviderConfig}; +/// +/// // Basic usage +/// let mut analyzer = RemoteAnalyzer::new(); +/// +/// // With custom configuration +/// let config = ProviderConfig::new() +/// .with_timeout(120) +/// .with_header("X-Custom-Header", "value") +/// .with_credential("token", "your-token"); +/// +/// analyzer.set_global_config(config); +/// ``` +pub struct RemoteAnalyzer { + providers: Vec>, + global_config: ProviderConfig, + filter: IntelligentFilter, + progress_hook: Box, + provider_configs: HashMap, +} + +impl RemoteAnalyzer { + /// Create a new analyzer with default configuration + pub fn new() -> Self { + let mut analyzer = Self { + providers: Vec::new(), + global_config: ProviderConfig::default(), + filter: IntelligentFilter::default(), + progress_hook: Box::new(NoOpProgressHook), + provider_configs: HashMap::new(), + }; + + analyzer.register_default_providers(); + analyzer + } + + /// Register all default Git providers + fn register_default_providers(&mut self) { + self.providers.push(Box::new(GitHubProvider::new())); + self.providers.push(Box::new(GitLabProvider::new())); + self.providers.push(Box::new(BitbucketProvider::new())); + self.providers.push(Box::new(CodebergProvider::new())); + self.providers.push(Box::new(GiteaProvider::new())); + self.providers.push(Box::new(SourceForgeProvider::new())); + self.providers.push(Box::new(AzureDevOpsProvider::new())); + self.providers.push(Box::new(ArchiveProvider::new())); + } + + /// Set a progress hook for monitoring operations + /// + /// # Arguments + /// * `hook` - Progress hook implementation + /// + /// # Examples + /// ```rust + /// use bytes_radar::net::{RemoteAnalyzer, ProgressHook}; + /// + /// struct MyHook; + /// impl ProgressHook for MyHook { + /// fn on_download_progress(&self, downloaded: u64, total: Option) { + /// println!("Downloaded: {} bytes", downloaded); + /// } + /// fn on_processing_start(&self, message: &str) { + /// println!("Processing: {}", message); + /// } + /// fn on_processing_progress(&self, current: usize, total: usize) { + /// println!("Progress: {}/{}", current, total); + /// } + /// } + /// + /// let mut analyzer = RemoteAnalyzer::new(); + /// analyzer.set_progress_hook(MyHook); + /// ``` + pub fn set_progress_hook(&mut self, hook: H) { + self.progress_hook = Box::new(hook); + } + + /// Set global configuration that applies to all providers + /// + /// # Arguments + /// * `config` - Global configuration + /// + /// # Examples + /// ```rust + /// use bytes_radar::net::{RemoteAnalyzer, ProviderConfig}; + /// + /// let config = ProviderConfig::new() + /// .with_timeout(300) + /// .with_user_agent("my-app/1.0.0") + /// .with_header("X-API-Key", "secret"); + /// + /// let mut analyzer = RemoteAnalyzer::new(); + /// analyzer.set_global_config(config); + /// ``` + pub fn set_global_config(&mut self, config: ProviderConfig) { + self.global_config = config; + self.apply_config_to_providers(); + } + + /// Set configuration for a specific provider + /// + /// # Arguments + /// * `provider_name` - Name of the provider (e.g., "github", "gitlab") + /// * `config` - Provider-specific configuration + /// + /// # Examples + /// ```rust + /// use bytes_radar::net::{RemoteAnalyzer, ProviderConfig}; + /// + /// let github_config = ProviderConfig::new() + /// .with_credential("token", "github-token") + /// .with_header("Accept", "application/vnd.github.v3+json"); + /// + /// let mut analyzer = RemoteAnalyzer::new(); + /// analyzer.set_provider_config("github", github_config); + /// ``` + pub fn set_provider_config(&mut self, provider_name: &str, config: ProviderConfig) { + self.provider_configs + .insert(provider_name.to_string(), config); + self.apply_config_to_providers(); + } + + /// Apply configurations to all providers + fn apply_config_to_providers(&mut self) { + for provider in &mut self.providers { + let provider_name = provider.name(); + + // Start with global config + let mut config = self.global_config.clone(); + + // Override with provider-specific config if exists + if let Some(provider_config) = self.provider_configs.get(provider_name) { + // Merge configurations (provider-specific takes precedence) + config.headers.extend(provider_config.headers.clone()); + config + .credentials + .extend(provider_config.credentials.clone()); + config + .provider_settings + .extend(provider_config.provider_settings.clone()); + + if provider_config.timeout.is_some() { + config.timeout = provider_config.timeout; + } + if provider_config.max_redirects.is_some() { + config.max_redirects = provider_config.max_redirects; + } + if provider_config.user_agent.is_some() { + config.user_agent = provider_config.user_agent.clone(); + } + if provider_config.max_file_size.is_some() { + config.max_file_size = provider_config.max_file_size; + } + if provider_config.proxy.is_some() { + config.proxy = provider_config.proxy.clone(); + } + + config.accept_invalid_certs = provider_config.accept_invalid_certs; + config.use_compression = provider_config.use_compression; + } + + provider.apply_config(&config); + } + } + + /// Set file filtering configuration + /// + /// # Arguments + /// * `filter` - File filter configuration + pub fn set_filter(&mut self, filter: IntelligentFilter) { + self.filter = filter; + } + + /// Enable or disable aggressive file filtering + /// + /// # Arguments + /// * `enabled` - Whether to enable aggressive filtering + pub fn set_aggressive_filtering(&mut self, enabled: bool) { + if enabled { + self.filter = IntelligentFilter::aggressive(); + } else { + self.filter = IntelligentFilter::default(); + } + } + + // Legacy methods for backward compatibility + + /// Set timeout for all providers (legacy method) + /// + /// # Arguments + /// * `timeout` - Timeout in seconds + pub fn set_timeout(&mut self, timeout: u64) { + self.global_config.timeout = Some(timeout); + self.apply_config_to_providers(); + } + + /// Set whether to accept invalid SSL certificates (legacy method) + /// + /// # Arguments + /// * `allow_insecure` - Whether to accept invalid certificates + pub fn set_allow_insecure(&mut self, allow_insecure: bool) { + self.global_config.accept_invalid_certs = allow_insecure; + self.apply_config_to_providers(); + } + + /// Set credentials for a specific provider (legacy method) + /// + /// # Arguments + /// * `provider_name` - Name of the provider + /// * `credentials` - Credentials map + pub fn set_provider_credentials( + &mut self, + provider_name: &str, + credentials: HashMap, + ) { + let config = self + .provider_configs + .entry(provider_name.to_string()) + .or_insert_with(ProviderConfig::default); + + config.credentials.extend(credentials); + self.apply_config_to_providers(); + } + + /// Analyze a repository from its URL + /// + /// # Arguments + /// * `url` - Repository URL or shorthand notation + /// + /// # Examples + /// ```rust,no_run + /// use bytes_radar::net::RemoteAnalyzer; + /// + /// #[tokio::main] + /// async fn main() -> Result<(), Box> { + /// let analyzer = RemoteAnalyzer::new(); + /// + /// // Full URLs + /// let analysis = analyzer.analyze_url("https://github.com/user/repo").await?; + /// + /// // Shorthand notation + /// let analysis = analyzer.analyze_url("user/repo@main").await?; + /// + /// // Direct archive + /// let analysis = analyzer.analyze_url("https://example.com/project.tar.gz").await?; + /// + /// Ok(()) + /// } + /// ``` + pub async fn analyze_url(&self, url: &str) -> Result { + let expanded_url = self.expand_url(url); + + // Try direct archive first for better performance + if expanded_url.ends_with(".tar.gz") || expanded_url.ends_with(".tgz") { + return self.analyze_direct_tarball(&expanded_url).await; + } + + // Try each provider + for provider in &self.providers { + if provider.can_handle(&expanded_url) { + if let Some(parsed) = provider.parse_url(&expanded_url) { + match self.analyze_with_provider(provider.as_ref(), &parsed).await { + Ok(analysis) => return Ok(analysis), + Err(e) => { + #[cfg(feature = "cli")] + log::debug!( + "Provider {} failed for {}: {}", + provider.name(), + expanded_url, + e + ); + continue; + } + } + } + } + } + + Err(crate::core::error::AnalysisError::url_parsing(format!( + "Unsupported URL format: {}. Supported formats include GitHub, GitLab, Bitbucket, Codeberg, Gitea, SourceForge, Azure DevOps, and direct archive URLs.", + expanded_url + ))) + } + + /// Analyze using a specific provider + async fn analyze_with_provider( + &self, + provider: &dyn GitProvider, + parsed: &ParsedRepository, + ) -> Result { + let mut download_urls = provider.build_download_urls(parsed); + + // If no URLs and no specific branch/commit, try common branches + if download_urls.is_empty() && parsed.branch_or_commit.is_none() { + let mut branches = vec![ + "main".to_string(), + "master".to_string(), + "develop".to_string(), + "dev".to_string(), + ]; + + // Try to get default branch from API + #[cfg(not(target_arch = "wasm32"))] + { + let config = self.get_effective_config(provider.name()); + if let Ok(client) = provider.build_client(&config) { + if let Some(default_branch) = provider.get_default_branch(&client, parsed).await + { + branches.insert(0, default_branch); + branches.dedup(); + } + } + } + + // Generate URLs for each branch + for branch in branches { + let mut branch_parsed = parsed.clone(); + branch_parsed.branch_or_commit = Some(branch); + download_urls.extend(provider.build_download_urls(&branch_parsed)); + } + } + + // Try each download URL + for download_url in download_urls { + match self + .analyze_direct_tarball_with_name(&download_url, &parsed.project_name) + .await + { + Ok(analysis) => return Ok(analysis), + Err(e) => { + #[cfg(feature = "cli")] + log::debug!("Failed to download from {}: {}", download_url, e); + continue; + } + } + } + + Err(crate::core::error::AnalysisError::network( + "All download URLs failed".to_string(), + )) + } + + /// Get effective configuration for a provider + fn get_effective_config(&self, provider_name: &str) -> ProviderConfig { + let mut config = self.global_config.clone(); + + if let Some(provider_config) = self.provider_configs.get(provider_name) { + // Merge configurations + config.headers.extend(provider_config.headers.clone()); + config + .credentials + .extend(provider_config.credentials.clone()); + config + .provider_settings + .extend(provider_config.provider_settings.clone()); + + if provider_config.timeout.is_some() { + config.timeout = provider_config.timeout; + } + if provider_config.max_redirects.is_some() { + config.max_redirects = provider_config.max_redirects; + } + if provider_config.user_agent.is_some() { + config.user_agent = provider_config.user_agent.clone(); + } + if provider_config.max_file_size.is_some() { + config.max_file_size = provider_config.max_file_size; + } + if provider_config.proxy.is_some() { + config.proxy = provider_config.proxy.clone(); + } + + config.accept_invalid_certs = provider_config.accept_invalid_certs; + config.use_compression = provider_config.use_compression; + } + + config + } + + /// Analyze a direct archive URL + async fn analyze_direct_tarball(&self, url: &str) -> Result { + let project_name = self.extract_project_name_from_url(url); + self.analyze_direct_tarball_with_name(url, &project_name) + .await + } + + /// Analyze a direct archive URL with custom project name + async fn analyze_direct_tarball_with_name( + &self, + url: &str, + project_name: &str, + ) -> Result { + let mut project_analysis = ProjectAnalysis::new(project_name); + + // Use global config to build client for direct downloads + let client = self.build_global_client()?; + + let response = client.get(url).send().await.map_err(|e| { + crate::core::error::AnalysisError::network(format!("Failed to fetch URL: {}", e)) + })?; + + if !response.status().is_success() { + return Err(crate::core::error::AnalysisError::network(format!( + "HTTP request failed with status: {}", + response.status() + ))); + } + + let total_size = response.content_length(); + self.progress_hook.on_download_progress(0, total_size); + + let stream = response.bytes_stream(); + let stream_reader = stream::StreamReader::new( + stream, + Box::new(|_downloaded, _total| { + // Progress handled by progress hook in stream processing + }), + total_size, + ); + + self.progress_hook + .on_processing_start("Processing archive..."); + stream::process_tarball_stream( + stream_reader, + &mut project_analysis, + &self.filter, + &*self.progress_hook, + ) + .await?; + + Ok(project_analysis) + } + + /// Build HTTP client using global configuration + fn build_global_client(&self) -> Result { + // Use archive provider to build client (it has good defaults) + let archive_provider = ArchiveProvider::new(); + archive_provider + .build_client(&self.global_config) + .map_err(|e| { + crate::core::error::AnalysisError::network(format!( + "Failed to build HTTP client: {}", + e + )) + }) + } + + /// Expand shorthand URLs to full URLs + fn expand_url(&self, url: &str) -> String { + if url.starts_with("http://") || url.starts_with("https://") { + return url.to_string(); + } + + // Handle shorthand notation like "user/repo@branch" + if url.contains('/') && !url.starts_with("http://") && !url.starts_with("https://") { + let parts: Vec<&str> = url.split('@').collect(); + let repo_part = parts[0]; + let branch_or_commit = parts.get(1); + + let path_parts: Vec<&str> = repo_part.split('/').collect(); + if path_parts.len() == 2 { + if let Some(branch) = branch_or_commit { + // Check if it looks like a commit hash + if branch.len() >= 7 && branch.chars().all(|c| c.is_ascii_hexdigit()) { + return format!("https://github.com/{}/commit/{}", repo_part, branch); + } else { + return format!("https://github.com/{}/tree/{}", repo_part, branch); + } + } else { + return format!("https://github.com/{}", repo_part); + } + } + } + + url.to_string() + } + + /// Extract project name from a direct URL + fn extract_project_name_from_url(&self, url: &str) -> String { + let url_path = url.trim_end_matches('/'); + + if let Some(filename) = url_path.split('/').last() { + if filename.ends_with(".tar.gz") { + return filename.trim_end_matches(".tar.gz").to_string(); + } + if filename.ends_with(".tgz") { + return filename.trim_end_matches(".tgz").to_string(); + } + return filename.to_string(); + } + + "remote-project".to_string() + } +} + +impl Default for RemoteAnalyzer { + fn default() -> Self { + Self::new() + } +} diff --git a/src/net/providers/archive.rs b/src/net/providers/archive.rs new file mode 100644 index 0000000..c7ed80e --- /dev/null +++ b/src/net/providers/archive.rs @@ -0,0 +1,227 @@ +use crate::net::traits::{GitProvider, ParsedRepository, ProviderConfig}; +use async_trait::async_trait; +use reqwest::Client; +use std::collections::HashMap; + +pub struct ArchiveProvider { + credentials: HashMap, +} + +impl ArchiveProvider { + pub fn new() -> Self { + Self { + credentials: HashMap::new(), + } + } +} + +#[async_trait] +impl GitProvider for ArchiveProvider { + fn name(&self) -> &'static str { + "archive" + } + + fn can_handle(&self, url: &str) -> bool { + url.ends_with(".tar.gz") + || url.ends_with(".tgz") + || url.ends_with(".tar.bz2") + || url.ends_with(".tar.xz") + || url.ends_with(".zip") + || url.contains("/archive/") + || url.contains("/tarball/") + || url.contains("/zipball/") + } + + fn parse_url(&self, url: &str) -> Option { + if !self.can_handle(url) { + return None; + } + + let filename = url.split('/').last()?; + let name = self.extract_name_from_filename(filename); + + Some( + ParsedRepository::new("archive".to_string(), name.clone()) + .with_host(self.extract_host_from_url(url)), + ) + } + + fn build_download_urls(&self, _parsed: &ParsedRepository) -> Vec { + vec![] + } + + async fn get_default_branch( + &self, + _client: &Client, + _parsed: &ParsedRepository, + ) -> Option { + None + } + + fn apply_config(&mut self, config: &ProviderConfig) { + self.credentials = config.credentials.clone(); + } + + fn get_project_name(&self, url: &str) -> String { + if let Some(filename) = url.split('/').last() { + return self.extract_name_from_filename(filename); + } + + "archive-project".to_string() + } +} + +impl ArchiveProvider { + fn extract_name_from_filename(&self, filename: &str) -> String { + let name = if filename.ends_with(".tar.gz") { + filename.trim_end_matches(".tar.gz") + } else if filename.ends_with(".tgz") { + filename.trim_end_matches(".tgz") + } else if filename.ends_with(".tar.bz2") { + filename.trim_end_matches(".tar.bz2") + } else if filename.ends_with(".tar.xz") { + filename.trim_end_matches(".tar.xz") + } else if filename.ends_with(".zip") { + filename.trim_end_matches(".zip") + } else { + filename + }; + + name.to_string() + } + + fn extract_host_from_url(&self, url: &str) -> String { + if let Some(start) = url.find("://") { + let after_protocol = &url[start + 3..]; + if let Some(end) = after_protocol.find('/') { + return after_protocol[..end].to_string(); + } + return after_protocol.to_string(); + } + "unknown".to_string() + } +} + +impl Default for ArchiveProvider { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_can_handle() { + let provider = ArchiveProvider::new(); + assert!(provider.can_handle("https://example.com/project.tar.gz")); + assert!(provider.can_handle("https://example.com/project.tgz")); + assert!(provider.can_handle("https://example.com/project.tar.bz2")); + assert!(provider.can_handle("https://example.com/project.tar.xz")); + assert!(provider.can_handle("https://example.com/project.zip")); + assert!(provider.can_handle("https://example.com/archive/main.tar.gz")); + assert!(provider.can_handle("https://example.com/tarball/main")); + assert!(provider.can_handle("https://example.com/zipball/main")); + assert!(!provider.can_handle("https://github.com/user/repo")); + } + + #[test] + fn test_parse_tar_gz_url() { + let provider = ArchiveProvider::new(); + + let parsed = provider + .parse_url("https://example.com/myproject.tar.gz") + .unwrap(); + assert_eq!(parsed.owner, "archive"); + assert_eq!(parsed.repo, "myproject"); + assert_eq!(parsed.project_name, "myproject@main"); + assert_eq!(parsed.branch_or_commit, None); + assert!(!parsed.is_commit); + assert_eq!(parsed.host.as_ref().unwrap(), "example.com"); + } + + #[test] + fn test_parse_tgz_url() { + let provider = ArchiveProvider::new(); + + let parsed = provider + .parse_url("https://cdn.example.com/releases/v1.0.0.tgz") + .unwrap(); + assert_eq!(parsed.owner, "archive"); + assert_eq!(parsed.repo, "v1.0.0"); + assert_eq!(parsed.project_name, "v1.0.0@main"); + assert_eq!(parsed.host.as_ref().unwrap(), "cdn.example.com"); + } + + #[test] + fn test_parse_zip_url() { + let provider = ArchiveProvider::new(); + + let parsed = provider + .parse_url("https://releases.example.com/project-v2.0.zip") + .unwrap(); + assert_eq!(parsed.owner, "archive"); + assert_eq!(parsed.repo, "project-v2.0"); + assert_eq!(parsed.project_name, "project-v2.0@main"); + assert_eq!(parsed.host.as_ref().unwrap(), "releases.example.com"); + } + + #[test] + fn test_extract_name_from_filename() { + let provider = ArchiveProvider::new(); + + assert_eq!( + provider.extract_name_from_filename("project.tar.gz"), + "project" + ); + assert_eq!( + provider.extract_name_from_filename("mylib-v1.0.0.tgz"), + "mylib-v1.0.0" + ); + assert_eq!( + provider.extract_name_from_filename("source.tar.bz2"), + "source" + ); + assert_eq!( + provider.extract_name_from_filename("archive.tar.xz"), + "archive" + ); + assert_eq!( + provider.extract_name_from_filename("release.zip"), + "release" + ); + } + + #[test] + fn test_extract_host_from_url() { + let provider = ArchiveProvider::new(); + + assert_eq!( + provider.extract_host_from_url("https://example.com/file.tar.gz"), + "example.com" + ); + assert_eq!( + provider.extract_host_from_url("http://cdn.example.org/releases/v1.0.tgz"), + "cdn.example.org" + ); + assert_eq!( + provider.extract_host_from_url("https://api.github.com/repos/user/repo/tarball/main"), + "api.github.com" + ); + } + + #[test] + fn test_get_project_name() { + let provider = ArchiveProvider::new(); + + assert_eq!( + provider.get_project_name("https://example.com/myproject.tar.gz"), + "myproject" + ); + assert_eq!( + provider.get_project_name("https://releases.example.com/v1.2.3.tgz"), + "v1.2.3" + ); + } +} diff --git a/src/net/providers/azure_devops.rs b/src/net/providers/azure_devops.rs new file mode 100644 index 0000000..784cbee --- /dev/null +++ b/src/net/providers/azure_devops.rs @@ -0,0 +1,197 @@ +use crate::net::traits::{GitProvider, ParsedRepository, ProviderConfig}; +use async_trait::async_trait; +use reqwest::Client; +use std::collections::HashMap; + +pub struct AzureDevOpsProvider { + credentials: HashMap, +} + +impl AzureDevOpsProvider { + pub fn new() -> Self { + Self { + credentials: HashMap::new(), + } + } +} + +#[async_trait] +impl GitProvider for AzureDevOpsProvider { + fn name(&self) -> &'static str { + "azure_devops" + } + + fn can_handle(&self, url: &str) -> bool { + url.contains("dev.azure.com") || url.contains("visualstudio.com") || url.contains("_git/") + } + + fn parse_url(&self, url: &str) -> Option { + if !self.can_handle(url) { + return None; + } + + let url = url.trim_end_matches('/'); + + if url.contains("?version=GB") { + return self.parse_branch_url(url); + } + + if url.contains("?version=GC") { + return self.parse_commit_url(url); + } + + self.parse_basic_url(url) + } + + fn build_download_urls(&self, parsed: &ParsedRepository) -> Vec { + let mut urls = Vec::new(); + + if let Some(ref branch_or_commit) = parsed.branch_or_commit { + let host = parsed.host.as_deref().unwrap_or("dev.azure.com"); + + if parsed.is_commit { + urls.push(format!( + "https://{}/{}/{}/_apis/git/repositories/{}/items?path=/&versionDescriptor.version={}&$format=zip", + host, parsed.owner, parsed.repo.split('/').next().unwrap_or(&parsed.repo), parsed.repo, branch_or_commit + )); + } else { + urls.push(format!( + "https://{}/{}/{}/_apis/git/repositories/{}/items?path=/&versionDescriptor.versionType=branch&versionDescriptor.version={}&$format=zip", + host, parsed.owner, parsed.repo.split('/').next().unwrap_or(&parsed.repo), parsed.repo, branch_or_commit + )); + } + } + + urls + } + + async fn get_default_branch( + &self, + _client: &Client, + _parsed: &ParsedRepository, + ) -> Option { + None + } + + fn apply_config(&mut self, config: &ProviderConfig) { + self.credentials = config.credentials.clone(); + } + + fn get_project_name(&self, url: &str) -> String { + if let Some(parsed) = self.parse_url(url) { + return parsed.project_name; + } + + if let Some(filename) = url.split('/').last() { + if filename.ends_with(".zip") { + return filename.trim_end_matches(".zip").to_string(); + } + return filename.to_string(); + } + + "azure-devops-project".to_string() + } +} + +impl AzureDevOpsProvider { + fn parse_commit_url(&self, url: &str) -> Option { + if let Some(commit_start) = url.find("?version=GC") { + let base_url = &url[..commit_start]; + let commit = &url[commit_start + "?version=GC".len()..]; + + if let Some(parsed_base) = self.parse_basic_url(base_url) { + return Some(parsed_base.with_commit(commit.to_string())); + } + } + None + } + + fn parse_branch_url(&self, url: &str) -> Option { + if let Some(branch_start) = url.find("?version=GB") { + let base_url = &url[..branch_start]; + let branch = &url[branch_start + "?version=GB".len()..]; + + if let Some(parsed_base) = self.parse_basic_url(base_url) { + return Some(parsed_base.with_branch(branch.to_string())); + } + } + None + } + + fn parse_basic_url(&self, url: &str) -> Option { + if url.contains("dev.azure.com") { + let parts: Vec<&str> = url.split('/').collect(); + if parts.len() >= 7 && parts.contains(&"_git") { + let host = parts[2].to_string(); + let org = parts[3].to_string(); + let project = parts[4].to_string(); + let repo = parts[6].to_string(); + + return Some( + ParsedRepository::new(format!("{}/{}", org, project), repo).with_host(host), + ); + } + } else if url.contains("visualstudio.com") { + let parts: Vec<&str> = url.split('/').collect(); + if parts.len() >= 6 && parts.contains(&"_git") { + let host = parts[2].to_string(); + let org = parts[2].split('.').next().unwrap_or("").to_string(); + let project = parts[4].to_string(); + let repo = parts[6].to_string(); + + return Some( + ParsedRepository::new(format!("{}/{}", org, project), repo).with_host(host), + ); + } + } + None + } +} + +impl Default for AzureDevOpsProvider { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_can_handle() { + let provider = AzureDevOpsProvider::new(); + assert!(provider.can_handle("https://dev.azure.com/org/project/_git/repo")); + assert!(provider.can_handle("https://org.visualstudio.com/project/_git/repo")); + assert!(!provider.can_handle("https://github.com/user/repo")); + } + + #[test] + fn test_parse_basic_url() { + let provider = AzureDevOpsProvider::new(); + + let parsed = provider + .parse_url("https://dev.azure.com/myorg/myproject/_git/myrepo") + .unwrap(); + assert_eq!(parsed.owner, "myorg/myproject"); + assert_eq!(parsed.repo, "myrepo"); + assert_eq!(parsed.project_name, "myrepo@main"); + assert_eq!(parsed.branch_or_commit, None); + assert!(!parsed.is_commit); + assert_eq!(parsed.host.as_ref().unwrap(), "dev.azure.com"); + } + + #[test] + fn test_parse_branch_url() { + let provider = AzureDevOpsProvider::new(); + + let parsed = provider + .parse_url("https://dev.azure.com/myorg/myproject/_git/myrepo?version=GBdevelop") + .unwrap(); + assert_eq!(parsed.owner, "myorg/myproject"); + assert_eq!(parsed.repo, "myrepo"); + assert_eq!(parsed.project_name, "myrepo@develop"); + assert_eq!(parsed.branch_or_commit, Some("develop".to_string())); + assert!(!parsed.is_commit); + } +} diff --git a/src/net/providers/bitbucket.rs b/src/net/providers/bitbucket.rs new file mode 100644 index 0000000..aa62c25 --- /dev/null +++ b/src/net/providers/bitbucket.rs @@ -0,0 +1,214 @@ +use crate::net::traits::{GitProvider, ParsedRepository, ProviderConfig}; +use async_trait::async_trait; +use reqwest::Client; +use std::collections::HashMap; + +pub struct BitbucketProvider { + credentials: HashMap, +} + +impl BitbucketProvider { + pub fn new() -> Self { + Self { + credentials: HashMap::new(), + } + } +} + +#[async_trait] +impl GitProvider for BitbucketProvider { + fn name(&self) -> &'static str { + "bitbucket" + } + + fn can_handle(&self, url: &str) -> bool { + url.contains("bitbucket.org") + } + + fn parse_url(&self, url: &str) -> Option { + if !self.can_handle(url) { + return None; + } + + let url = url.trim_end_matches('/'); + + if url.contains("/commits/") { + return self.parse_commit_url(url); + } + + if url.contains("/branch/") { + return self.parse_branch_url(url); + } + + self.parse_basic_url(url) + } + + fn build_download_urls(&self, parsed: &ParsedRepository) -> Vec { + let mut urls = Vec::new(); + + if let Some(ref branch_or_commit) = parsed.branch_or_commit { + urls.push(format!( + "https://bitbucket.org/{}/{}/get/{}.tar.gz", + parsed.owner, parsed.repo, branch_or_commit + )); + } + + urls + } + + async fn get_default_branch( + &self, + _client: &Client, + _parsed: &ParsedRepository, + ) -> Option { + None + } + + fn apply_config(&mut self, config: &ProviderConfig) { + self.credentials = config.credentials.clone(); + } + + fn get_project_name(&self, url: &str) -> String { + if let Some(parsed) = self.parse_url(url) { + return parsed.project_name; + } + + if let Some(filename) = url.split('/').last() { + if filename.ends_with(".tar.gz") { + return filename.trim_end_matches(".tar.gz").to_string(); + } + if filename.ends_with(".tgz") { + return filename.trim_end_matches(".tgz").to_string(); + } + return filename.to_string(); + } + + "bitbucket-project".to_string() + } +} + +impl BitbucketProvider { + fn parse_commit_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(commits_pos) = parts.iter().position(|&x| x == "commits") { + if commits_pos + 1 < parts.len() && commits_pos >= 2 { + let owner = parts[commits_pos - 2].to_string(); + let repo = parts[commits_pos - 1].to_string(); + let commit = parts[commits_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_commit(commit) + .with_host("bitbucket.org".to_string()), + ); + } + } + None + } + + fn parse_branch_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(branch_pos) = parts.iter().position(|&x| x == "branch") { + if branch_pos + 1 < parts.len() && branch_pos >= 2 { + let owner = parts[branch_pos - 2].to_string(); + let repo = parts[branch_pos - 1].to_string(); + let branch = parts[branch_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_branch(branch) + .with_host("bitbucket.org".to_string()), + ); + } + } + None + } + + fn parse_basic_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(bitbucket_pos) = parts.iter().position(|&x| x == "bitbucket.org") { + if bitbucket_pos + 2 < parts.len() { + let owner = parts[bitbucket_pos + 1].to_string(); + let repo = parts[bitbucket_pos + 2].to_string(); + + return Some( + ParsedRepository::new(owner, repo).with_host("bitbucket.org".to_string()), + ); + } + } + None + } +} + +impl Default for BitbucketProvider { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_can_handle() { + let provider = BitbucketProvider::new(); + assert!(provider.can_handle("https://bitbucket.org/user/repo")); + assert!(provider.can_handle("https://bitbucket.org/user/repo/commits/abc123")); + assert!(!provider.can_handle("https://github.com/user/repo")); + } + + #[test] + fn test_parse_basic_url() { + let provider = BitbucketProvider::new(); + + let parsed = provider + .parse_url("https://bitbucket.org/user/repo") + .unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@main"); + assert_eq!(parsed.branch_or_commit, None); + assert!(!parsed.is_commit); + assert_eq!(parsed.host.as_ref().unwrap(), "bitbucket.org"); + } + + #[test] + fn test_parse_branch_url() { + let provider = BitbucketProvider::new(); + + let parsed = provider + .parse_url("https://bitbucket.org/user/repo/branch/develop") + .unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@develop"); + assert_eq!(parsed.branch_or_commit, Some("develop".to_string())); + assert!(!parsed.is_commit); + } + + #[test] + fn test_parse_commit_url() { + let provider = BitbucketProvider::new(); + + let parsed = provider + .parse_url("https://bitbucket.org/user/repo/commits/abc1234567890") + .unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@abc1234"); + assert_eq!(parsed.branch_or_commit, Some("abc1234567890".to_string())); + assert!(parsed.is_commit); + } + + #[test] + fn test_build_download_urls() { + let provider = BitbucketProvider::new(); + + let parsed = ParsedRepository::new("user".to_string(), "repo".to_string()) + .with_branch("main".to_string()); + + let urls = provider.build_download_urls(&parsed); + assert!(urls.contains(&"https://bitbucket.org/user/repo/get/main.tar.gz".to_string())); + } +} diff --git a/src/net/providers/codeberg.rs b/src/net/providers/codeberg.rs new file mode 100644 index 0000000..586ecb5 --- /dev/null +++ b/src/net/providers/codeberg.rs @@ -0,0 +1,214 @@ +use crate::net::traits::{GitProvider, ParsedRepository, ProviderConfig}; +use async_trait::async_trait; +use reqwest::Client; +use std::collections::HashMap; + +pub struct CodebergProvider { + credentials: HashMap, +} + +impl CodebergProvider { + pub fn new() -> Self { + Self { + credentials: HashMap::new(), + } + } +} + +#[async_trait] +impl GitProvider for CodebergProvider { + fn name(&self) -> &'static str { + "codeberg" + } + + fn can_handle(&self, url: &str) -> bool { + url.contains("codeberg.org") + } + + fn parse_url(&self, url: &str) -> Option { + if !self.can_handle(url) { + return None; + } + + let url = url.trim_end_matches('/'); + + if url.contains("/commit/") { + return self.parse_commit_url(url); + } + + if url.contains("/src/branch/") { + return self.parse_branch_url(url); + } + + self.parse_basic_url(url) + } + + fn build_download_urls(&self, parsed: &ParsedRepository) -> Vec { + let mut urls = Vec::new(); + + if let Some(ref branch_or_commit) = parsed.branch_or_commit { + urls.push(format!( + "https://codeberg.org/{}/{}/archive/{}.tar.gz", + parsed.owner, parsed.repo, branch_or_commit + )); + } + + urls + } + + async fn get_default_branch( + &self, + _client: &Client, + _parsed: &ParsedRepository, + ) -> Option { + None + } + + fn apply_config(&mut self, config: &ProviderConfig) { + self.credentials = config.credentials.clone(); + } + + fn get_project_name(&self, url: &str) -> String { + if let Some(parsed) = self.parse_url(url) { + return parsed.project_name; + } + + if let Some(filename) = url.split('/').last() { + if filename.ends_with(".tar.gz") { + return filename.trim_end_matches(".tar.gz").to_string(); + } + if filename.ends_with(".tgz") { + return filename.trim_end_matches(".tgz").to_string(); + } + return filename.to_string(); + } + + "codeberg-project".to_string() + } +} + +impl CodebergProvider { + fn parse_commit_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(commit_pos) = parts.iter().position(|&x| x == "commit") { + if commit_pos + 1 < parts.len() && commit_pos >= 2 { + let owner = parts[commit_pos - 2].to_string(); + let repo = parts[commit_pos - 1].to_string(); + let commit = parts[commit_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_commit(commit) + .with_host("codeberg.org".to_string()), + ); + } + } + None + } + + fn parse_branch_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(branch_pos) = parts.iter().position(|&x| x == "branch") { + if branch_pos + 1 < parts.len() && branch_pos >= 3 { + let owner = parts[branch_pos - 3].to_string(); + let repo = parts[branch_pos - 2].to_string(); + let branch = parts[branch_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_branch(branch) + .with_host("codeberg.org".to_string()), + ); + } + } + None + } + + fn parse_basic_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(codeberg_pos) = parts.iter().position(|&x| x == "codeberg.org") { + if codeberg_pos + 2 < parts.len() { + let owner = parts[codeberg_pos + 1].to_string(); + let repo = parts[codeberg_pos + 2].to_string(); + + return Some( + ParsedRepository::new(owner, repo).with_host("codeberg.org".to_string()), + ); + } + } + None + } +} + +impl Default for CodebergProvider { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_can_handle() { + let provider = CodebergProvider::new(); + assert!(provider.can_handle("https://codeberg.org/user/repo")); + assert!(provider.can_handle("https://codeberg.org/user/repo/commit/abc123")); + assert!(!provider.can_handle("https://github.com/user/repo")); + } + + #[test] + fn test_parse_basic_url() { + let provider = CodebergProvider::new(); + + let parsed = provider + .parse_url("https://codeberg.org/user/repo") + .unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@main"); + assert_eq!(parsed.branch_or_commit, None); + assert!(!parsed.is_commit); + assert_eq!(parsed.host.as_ref().unwrap(), "codeberg.org"); + } + + #[test] + fn test_parse_branch_url() { + let provider = CodebergProvider::new(); + + let parsed = provider + .parse_url("https://codeberg.org/user/repo/src/branch/develop") + .unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@develop"); + assert_eq!(parsed.branch_or_commit, Some("develop".to_string())); + assert!(!parsed.is_commit); + } + + #[test] + fn test_parse_commit_url() { + let provider = CodebergProvider::new(); + + let parsed = provider + .parse_url("https://codeberg.org/user/repo/commit/abc1234567890") + .unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@abc1234"); + assert_eq!(parsed.branch_or_commit, Some("abc1234567890".to_string())); + assert!(parsed.is_commit); + } + + #[test] + fn test_build_download_urls() { + let provider = CodebergProvider::new(); + + let parsed = ParsedRepository::new("user".to_string(), "repo".to_string()) + .with_branch("main".to_string()); + + let urls = provider.build_download_urls(&parsed); + assert!(urls.contains(&"https://codeberg.org/user/repo/archive/main.tar.gz".to_string())); + } +} diff --git a/src/net/providers/gitea.rs b/src/net/providers/gitea.rs new file mode 100644 index 0000000..b0fe9dc --- /dev/null +++ b/src/net/providers/gitea.rs @@ -0,0 +1,192 @@ +use crate::net::traits::{GitProvider, ParsedRepository, ProviderConfig}; +use async_trait::async_trait; +use reqwest::Client; +use std::collections::HashMap; + +pub struct GiteaProvider { + credentials: HashMap, +} + +impl GiteaProvider { + pub fn new() -> Self { + Self { + credentials: HashMap::new(), + } + } +} + +#[async_trait] +impl GitProvider for GiteaProvider { + fn name(&self) -> &'static str { + "gitea" + } + + fn can_handle(&self, url: &str) -> bool { + url.contains("gitea.") + || url.contains("/gitea") + || url.contains("git.") + || self.is_likely_gitea(url) + } + + fn parse_url(&self, url: &str) -> Option { + if !self.can_handle(url) { + return None; + } + + let url = url.trim_end_matches('/'); + + if url.contains("/commit/") { + return self.parse_commit_url(url); + } + + if url.contains("/src/branch/") { + return self.parse_branch_url(url); + } + + self.parse_basic_url(url) + } + + fn build_download_urls(&self, parsed: &ParsedRepository) -> Vec { + let mut urls = Vec::new(); + + if let Some(ref branch_or_commit) = parsed.branch_or_commit { + let host = parsed.host.as_deref().unwrap_or("gitea.com"); + + urls.push(format!( + "https://{}/{}/{}/archive/{}.tar.gz", + host, parsed.owner, parsed.repo, branch_or_commit + )); + } + + urls + } + + async fn get_default_branch( + &self, + _client: &Client, + _parsed: &ParsedRepository, + ) -> Option { + None + } + + fn apply_config(&mut self, config: &ProviderConfig) { + self.credentials = config.credentials.clone(); + } + + fn get_project_name(&self, url: &str) -> String { + if let Some(parsed) = self.parse_url(url) { + return parsed.project_name; + } + + if let Some(filename) = url.split('/').last() { + if filename.ends_with(".tar.gz") { + return filename.trim_end_matches(".tar.gz").to_string(); + } + if filename.ends_with(".tgz") { + return filename.trim_end_matches(".tgz").to_string(); + } + return filename.to_string(); + } + + "gitea-project".to_string() + } +} + +impl GiteaProvider { + fn is_likely_gitea(&self, _url: &str) -> bool { + false + } + + fn parse_commit_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(commit_pos) = parts.iter().position(|&x| x == "commit") { + if commit_pos + 1 < parts.len() && commit_pos >= 4 { + let host = parts[2].to_string(); + let owner = parts[commit_pos - 2].to_string(); + let repo = parts[commit_pos - 1].to_string(); + let commit = parts[commit_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_commit(commit) + .with_host(host), + ); + } + } + None + } + + fn parse_branch_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(branch_pos) = parts.iter().position(|&x| x == "branch") { + if branch_pos + 1 < parts.len() && branch_pos >= 5 { + let host = parts[2].to_string(); + let owner = parts[branch_pos - 3].to_string(); + let repo = parts[branch_pos - 2].to_string(); + let branch = parts[branch_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_branch(branch) + .with_host(host), + ); + } + } + None + } + + fn parse_basic_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if parts.len() >= 5 && parts[0] == "https:" { + let host = parts[2].to_string(); + let owner = parts[3].to_string(); + let repo = parts[4].to_string(); + + return Some(ParsedRepository::new(owner, repo).with_host(host)); + } + None + } +} + +impl Default for GiteaProvider { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_can_handle() { + let provider = GiteaProvider::new(); + assert!(provider.can_handle("https://gitea.com/user/repo")); + assert!(provider.can_handle("https://git.company.com/user/repo")); + } + + #[test] + fn test_parse_basic_url() { + let provider = GiteaProvider::new(); + + let parsed = provider.parse_url("https://gitea.com/user/repo").unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@main"); + assert_eq!(parsed.branch_or_commit, None); + assert!(!parsed.is_commit); + assert_eq!(parsed.host.as_ref().unwrap(), "gitea.com"); + } + + #[test] + fn test_build_download_urls() { + let provider = GiteaProvider::new(); + + let parsed = ParsedRepository::new("user".to_string(), "repo".to_string()) + .with_branch("main".to_string()) + .with_host("gitea.com".to_string()); + + let urls = provider.build_download_urls(&parsed); + assert!(urls.contains(&"https://gitea.com/user/repo/archive/main.tar.gz".to_string())); + } +} diff --git a/src/net/providers/github.rs b/src/net/providers/github.rs new file mode 100644 index 0000000..fa6a90f --- /dev/null +++ b/src/net/providers/github.rs @@ -0,0 +1,303 @@ +use crate::net::traits::{GitProvider, ParsedRepository, ProviderConfig}; +use async_trait::async_trait; +use reqwest::Client; +use serde::Deserialize; + +#[derive(Deserialize)] +struct GitHubRepoInfo { + default_branch: String, +} + +pub struct GitHubProvider { + token: Option, +} + +impl GitHubProvider { + pub fn new() -> Self { + Self { token: None } + } +} + +#[async_trait] +impl GitProvider for GitHubProvider { + fn name(&self) -> &'static str { + "github" + } + + fn can_handle(&self, url: &str) -> bool { + url.contains("github.com") + } + + fn parse_url(&self, url: &str) -> Option { + if !self.can_handle(url) { + return None; + } + + let url = url.trim_end_matches('/'); + + if url.contains("/tree/") { + return self.parse_tree_url(url); + } + + if url.contains("/commit/") { + return self.parse_commit_url(url); + } + + self.parse_basic_url(url) + } + + fn build_download_urls(&self, parsed: &ParsedRepository) -> Vec { + let mut urls = Vec::new(); + + if let Some(ref branch_or_commit) = parsed.branch_or_commit { + if parsed.is_commit { + urls.push(format!( + "https://github.com/{}/{}/archive/{}.tar.gz", + parsed.owner, parsed.repo, branch_or_commit + )); + } else { + urls.push(format!( + "https://github.com/{}/{}/archive/refs/heads/{}.tar.gz", + parsed.owner, parsed.repo, branch_or_commit + )); + urls.push(format!( + "https://github.com/{}/{}/archive/refs/tags/{}.tar.gz", + parsed.owner, parsed.repo, branch_or_commit + )); + } + } + + urls + } + + async fn get_default_branch( + &self, + client: &Client, + parsed: &ParsedRepository, + ) -> Option { + #[cfg(not(target_arch = "wasm32"))] + { + let api_url = format!( + "https://api.github.com/repos/{}/{}", + parsed.owner, parsed.repo + ); + + let mut request = client.get(&api_url); + + if let Some(ref token) = self.token { + request = request.header("Authorization", format!("token {}", token)); + } + + match request.send().await { + Ok(response) => { + if response.status().is_success() { + match response.json::().await { + Ok(repo_info) => { + #[cfg(feature = "cli")] + log::debug!( + "GitHub API: Found default branch '{}' for {}/{}", + repo_info.default_branch, + parsed.owner, + parsed.repo + ); + Some(repo_info.default_branch) + } + Err(_) => { + #[cfg(feature = "cli")] + log::debug!( + "GitHub API: Failed to parse response for {}/{}", + parsed.owner, + parsed.repo + ); + None + } + } + } else { + #[cfg(feature = "cli")] + log::debug!( + "GitHub API: Request failed with status {} for {}/{}", + response.status(), + parsed.owner, + parsed.repo + ); + None + } + } + Err(_) => { + #[cfg(feature = "cli")] + log::debug!( + "GitHub API: Network error for {}/{}", + parsed.owner, + parsed.repo + ); + None + } + } + } + + #[cfg(target_arch = "wasm32")] + None + } + + fn apply_config(&mut self, config: &ProviderConfig) { + self.token = config.credentials.get("token").cloned(); + } + + fn get_project_name(&self, url: &str) -> String { + if let Some(parsed) = self.parse_url(url) { + return parsed.project_name; + } + + if let Some(filename) = url.split('/').last() { + if filename.ends_with(".tar.gz") { + return filename.trim_end_matches(".tar.gz").to_string(); + } + if filename.ends_with(".tgz") { + return filename.trim_end_matches(".tgz").to_string(); + } + return filename.to_string(); + } + + "github-project".to_string() + } +} + +impl GitHubProvider { + fn parse_tree_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(tree_pos) = parts.iter().position(|&x| x == "tree") { + if tree_pos + 1 < parts.len() && tree_pos >= 2 { + let owner = parts[tree_pos - 2].to_string(); + let repo = parts[tree_pos - 1].to_string(); + let branch = parts[tree_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_branch(branch) + .with_host("github.com".to_string()), + ); + } + } + None + } + + fn parse_commit_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(commit_pos) = parts.iter().position(|&x| x == "commit") { + if commit_pos + 1 < parts.len() && commit_pos >= 2 { + let owner = parts[commit_pos - 2].to_string(); + let repo = parts[commit_pos - 1].to_string(); + let commit = parts[commit_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_commit(commit) + .with_host("github.com".to_string()), + ); + } + } + None + } + + fn parse_basic_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(github_pos) = parts.iter().position(|&x| x == "github.com") { + if github_pos + 2 < parts.len() { + let owner = parts[github_pos + 1].to_string(); + let repo = parts[github_pos + 2].to_string(); + + return Some( + ParsedRepository::new(owner, repo).with_host("github.com".to_string()), + ); + } + } + + if let Some(stripped) = url.strip_prefix("https://github.com/") { + let parts: Vec<&str> = stripped.split('/').collect(); + if parts.len() >= 2 { + let owner = parts[0].to_string(); + let repo = parts[1].to_string(); + + return Some( + ParsedRepository::new(owner, repo).with_host("github.com".to_string()), + ); + } + } + + None + } +} + +impl Default for GitHubProvider { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_can_handle() { + let provider = GitHubProvider::new(); + assert!(provider.can_handle("https://github.com/user/repo")); + assert!(provider.can_handle("https://github.com/user/repo/tree/main")); + assert!(!provider.can_handle("https://gitlab.com/user/repo")); + } + + #[test] + fn test_parse_basic_url() { + let provider = GitHubProvider::new(); + + let parsed = provider.parse_url("https://github.com/user/repo").unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@main"); + assert_eq!(parsed.branch_or_commit, None); + assert!(!parsed.is_commit); + assert_eq!(parsed.host.as_ref().unwrap(), "github.com"); + } + + #[test] + fn test_parse_tree_url() { + let provider = GitHubProvider::new(); + + let parsed = provider + .parse_url("https://github.com/user/repo/tree/develop") + .unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@develop"); + assert_eq!(parsed.branch_or_commit, Some("develop".to_string())); + assert!(!parsed.is_commit); + } + + #[test] + fn test_parse_commit_url() { + let provider = GitHubProvider::new(); + + let parsed = provider + .parse_url("https://github.com/user/repo/commit/abc1234567890") + .unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@abc1234"); + assert_eq!(parsed.branch_or_commit, Some("abc1234567890".to_string())); + assert!(parsed.is_commit); + } + + #[test] + fn test_build_download_urls() { + let provider = GitHubProvider::new(); + + let parsed = ParsedRepository::new("user".to_string(), "repo".to_string()) + .with_branch("main".to_string()); + + let urls = provider.build_download_urls(&parsed); + assert!(urls + .contains(&"https://github.com/user/repo/archive/refs/heads/main.tar.gz".to_string())); + assert!(urls + .contains(&"https://github.com/user/repo/archive/refs/tags/main.tar.gz".to_string())); + } +} diff --git a/src/net/providers/gitlab.rs b/src/net/providers/gitlab.rs new file mode 100644 index 0000000..72e3d73 --- /dev/null +++ b/src/net/providers/gitlab.rs @@ -0,0 +1,242 @@ +use crate::net::traits::{GitProvider, ParsedRepository, ProviderConfig}; +use async_trait::async_trait; +use reqwest::Client; + +pub struct GitLabProvider { + token: Option, +} + +impl GitLabProvider { + pub fn new() -> Self { + Self { token: None } + } +} + +#[async_trait] +impl GitProvider for GitLabProvider { + fn name(&self) -> &'static str { + "gitlab" + } + + fn can_handle(&self, url: &str) -> bool { + url.contains("gitlab.com") || url.contains("gitlab.") + } + + fn parse_url(&self, url: &str) -> Option { + if !self.can_handle(url) { + return None; + } + + let url = url.trim_end_matches('/'); + + if url.contains("/-/tree/") { + return self.parse_tree_url(url); + } + + if url.contains("/-/commit/") { + return self.parse_commit_url(url); + } + + self.parse_basic_url(url) + } + + fn build_download_urls(&self, parsed: &ParsedRepository) -> Vec { + let mut urls = Vec::new(); + + if let Some(ref branch_or_commit) = parsed.branch_or_commit { + let host = parsed.host.as_deref().unwrap_or("gitlab.com"); + + if parsed.is_commit { + urls.push(format!( + "https://{}/{}/-/archive/{}/{}-{}.tar.gz", + host, + self.build_project_path(&parsed.owner, &parsed.repo), + branch_or_commit, + parsed.repo, + branch_or_commit + )); + } else { + urls.push(format!( + "https://{}/{}/-/archive/{}/{}-{}.tar.gz", + host, + self.build_project_path(&parsed.owner, &parsed.repo), + branch_or_commit, + parsed.repo, + branch_or_commit + )); + } + } + + urls + } + + async fn get_default_branch( + &self, + _client: &Client, + _parsed: &ParsedRepository, + ) -> Option { + None + } + + fn apply_config(&mut self, config: &ProviderConfig) { + self.token = config + .credentials + .get("token") + .cloned() + .or_else(|| config.credentials.get("private_token").cloned()); + } + + fn get_project_name(&self, url: &str) -> String { + if let Some(parsed) = self.parse_url(url) { + return parsed.project_name; + } + + if let Some(filename) = url.split('/').last() { + if filename.ends_with(".tar.gz") { + return filename.trim_end_matches(".tar.gz").to_string(); + } + if filename.ends_with(".tgz") { + return filename.trim_end_matches(".tgz").to_string(); + } + return filename.to_string(); + } + + "gitlab-project".to_string() + } +} + +impl GitLabProvider { + fn parse_tree_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(tree_pos) = parts.iter().position(|&x| x == "tree") { + if tree_pos + 1 < parts.len() && tree_pos >= 3 { + let gitlab_pos = parts.iter().position(|&x| x.contains("gitlab"))?; + let host = parts[gitlab_pos].to_string(); + let owner = parts[gitlab_pos + 1].to_string(); + let repo = parts[gitlab_pos + 2].to_string(); + let branch = parts[tree_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_branch(branch) + .with_host(host), + ); + } + } + None + } + + fn parse_commit_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(commit_pos) = parts.iter().position(|&x| x == "commit") { + if commit_pos + 1 < parts.len() && commit_pos >= 3 { + let gitlab_pos = parts.iter().position(|&x| x.contains("gitlab"))?; + let host = parts[gitlab_pos].to_string(); + let owner = parts[gitlab_pos + 1].to_string(); + let repo = parts[gitlab_pos + 2].to_string(); + let commit = parts[commit_pos + 1].to_string(); + + return Some( + ParsedRepository::new(owner, repo) + .with_commit(commit) + .with_host(host), + ); + } + } + None + } + + fn parse_basic_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(gitlab_pos) = parts.iter().position(|&x| x.contains("gitlab")) { + if gitlab_pos + 2 < parts.len() { + let host = parts[gitlab_pos].to_string(); + let owner = parts[gitlab_pos + 1].to_string(); + let repo = parts[gitlab_pos + 2].to_string(); + + return Some(ParsedRepository::new(owner, repo).with_host(host)); + } + } + + None + } + + fn build_project_path(&self, owner: &str, repo: &str) -> String { + if owner.contains("/") { + format!("{}/{}", owner, repo) + } else { + format!("{}/{}", owner, repo) + } + } +} + +impl Default for GitLabProvider { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_can_handle() { + let provider = GitLabProvider::new(); + assert!(provider.can_handle("https://gitlab.com/user/repo")); + assert!(provider.can_handle("https://gitlab.example.com/user/repo")); + assert!(!provider.can_handle("https://github.com/user/repo")); + } + + #[test] + fn test_parse_basic_url() { + let provider = GitLabProvider::new(); + + let parsed = provider.parse_url("https://gitlab.com/user/repo").unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@main"); + assert_eq!(parsed.branch_or_commit, None); + assert!(!parsed.is_commit); + assert_eq!(parsed.host.as_ref().unwrap(), "gitlab.com"); + } + + #[test] + fn test_parse_tree_url() { + let provider = GitLabProvider::new(); + + let parsed = provider + .parse_url("https://gitlab.com/user/repo/-/tree/develop") + .unwrap(); + assert_eq!(parsed.owner, "user"); + assert_eq!(parsed.repo, "repo"); + assert_eq!(parsed.project_name, "repo@develop"); + assert_eq!(parsed.branch_or_commit, Some("develop".to_string())); + assert!(!parsed.is_commit); + } + + #[test] + fn test_build_download_urls() { + let provider = GitLabProvider::new(); + + let parsed = ParsedRepository::new("user".to_string(), "repo".to_string()) + .with_branch("main".to_string()) + .with_host("gitlab.com".to_string()); + + let urls = provider.build_download_urls(&parsed); + assert!(urls + .contains(&"https://gitlab.com/user/repo/-/archive/main/repo-main.tar.gz".to_string())); + } + + #[test] + fn test_self_hosted_gitlab() { + let provider = GitLabProvider::new(); + + let parsed = provider + .parse_url("https://gitlab.company.com/team/project") + .unwrap(); + assert_eq!(parsed.owner, "team"); + assert_eq!(parsed.repo, "project"); + assert_eq!(parsed.host.as_ref().unwrap(), "gitlab.company.com"); + } +} diff --git a/src/net/providers/mod.rs b/src/net/providers/mod.rs new file mode 100644 index 0000000..287c08b --- /dev/null +++ b/src/net/providers/mod.rs @@ -0,0 +1,17 @@ +pub mod archive; +pub mod azure_devops; +pub mod bitbucket; +pub mod codeberg; +pub mod gitea; +pub mod github; +pub mod gitlab; +pub mod sourceforge; + +pub use archive::ArchiveProvider; +pub use azure_devops::AzureDevOpsProvider; +pub use bitbucket::BitbucketProvider; +pub use codeberg::CodebergProvider; +pub use gitea::GiteaProvider; +pub use github::GitHubProvider; +pub use gitlab::GitLabProvider; +pub use sourceforge::SourceForgeProvider; diff --git a/src/net/providers/sourceforge.rs b/src/net/providers/sourceforge.rs new file mode 100644 index 0000000..be7cb2c --- /dev/null +++ b/src/net/providers/sourceforge.rs @@ -0,0 +1,191 @@ +use crate::net::traits::{GitProvider, ParsedRepository, ProviderConfig}; +use async_trait::async_trait; +use reqwest::Client; +use std::collections::HashMap; + +pub struct SourceForgeProvider { + credentials: HashMap, +} + +impl SourceForgeProvider { + pub fn new() -> Self { + Self { + credentials: HashMap::new(), + } + } +} + +#[async_trait] +impl GitProvider for SourceForgeProvider { + fn name(&self) -> &'static str { + "sourceforge" + } + + fn can_handle(&self, url: &str) -> bool { + url.contains("sourceforge.net") || url.contains("sf.net") + } + + fn parse_url(&self, url: &str) -> Option { + if !self.can_handle(url) { + return None; + } + + let url = url.trim_end_matches('/'); + + if url.contains("/ci/") && url.contains("/tree/") { + return self.parse_tree_url(url); + } + + if url.contains("/ci/") { + return self.parse_commit_url(url); + } + + self.parse_basic_url(url) + } + + fn build_download_urls(&self, parsed: &ParsedRepository) -> Vec { + let mut urls = Vec::new(); + + if let Some(ref branch_or_commit) = parsed.branch_or_commit { + urls.push(format!( + "https://sourceforge.net/p/{}/code/ci/{}/tarball", + parsed.repo, branch_or_commit + )); + } + + urls + } + + async fn get_default_branch( + &self, + _client: &Client, + _parsed: &ParsedRepository, + ) -> Option { + None + } + + fn apply_config(&mut self, config: &ProviderConfig) { + self.credentials = config.credentials.clone(); + } + + fn get_project_name(&self, url: &str) -> String { + if let Some(parsed) = self.parse_url(url) { + return parsed.project_name; + } + + if let Some(filename) = url.split('/').last() { + if filename.ends_with(".tar.gz") { + return filename.trim_end_matches(".tar.gz").to_string(); + } + if filename.ends_with(".tgz") { + return filename.trim_end_matches(".tgz").to_string(); + } + return filename.to_string(); + } + + "sourceforge-project".to_string() + } +} + +impl SourceForgeProvider { + fn parse_commit_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(ci_pos) = parts.iter().position(|&x| x == "ci") { + if ci_pos + 1 < parts.len() && ci_pos >= 3 { + let project = parts[ci_pos - 1].to_string(); + let commit = parts[ci_pos + 1].to_string(); + + return Some( + ParsedRepository::new("sourceforge".to_string(), project) + .with_commit(commit) + .with_host("sourceforge.net".to_string()), + ); + } + } + None + } + + fn parse_tree_url(&self, url: &str) -> Option { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(tree_pos) = parts.iter().position(|&x| x == "tree") { + if tree_pos + 1 < parts.len() { + if let Some(ci_pos) = parts.iter().position(|&x| x == "ci") { + if ci_pos >= 3 { + let project = parts[ci_pos - 1].to_string(); + let branch = parts[tree_pos + 1].to_string(); + + return Some( + ParsedRepository::new("sourceforge".to_string(), project) + .with_branch(branch) + .with_host("sourceforge.net".to_string()), + ); + } + } + } + } + None + } + + fn parse_basic_url(&self, url: &str) -> Option { + if url.contains("/p/") && url.contains("/code") { + let parts: Vec<&str> = url.split('/').collect(); + if let Some(p_pos) = parts.iter().position(|&x| x == "p") { + if p_pos + 1 < parts.len() { + let project = parts[p_pos + 1].to_string(); + return Some( + ParsedRepository::new("sourceforge".to_string(), project) + .with_host("sourceforge.net".to_string()), + ); + } + } + } + None + } +} + +impl Default for SourceForgeProvider { + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_can_handle() { + let provider = SourceForgeProvider::new(); + assert!(provider.can_handle("https://sourceforge.net/p/project/code/")); + assert!(provider.can_handle("https://sf.net/p/project/code/")); + assert!(!provider.can_handle("https://github.com/user/repo")); + } + + #[test] + fn test_parse_basic_url() { + let provider = SourceForgeProvider::new(); + + let parsed = provider + .parse_url("https://sourceforge.net/p/myproject/code/") + .unwrap(); + assert_eq!(parsed.owner, "sourceforge"); + assert_eq!(parsed.repo, "myproject"); + assert_eq!(parsed.project_name, "myproject@main"); + assert_eq!(parsed.branch_or_commit, None); + assert!(!parsed.is_commit); + assert_eq!(parsed.host.as_ref().unwrap(), "sourceforge.net"); + } + + #[test] + fn test_build_download_urls() { + let provider = SourceForgeProvider::new(); + + let parsed = ParsedRepository::new("sourceforge".to_string(), "project".to_string()) + .with_branch("master".to_string()); + + let urls = provider.build_download_urls(&parsed); + assert!( + urls.contains(&"https://sourceforge.net/p/project/code/ci/master/tarball".to_string()) + ); + } +} diff --git a/src/net/stream.rs b/src/net/stream.rs new file mode 100644 index 0000000..c44cb84 --- /dev/null +++ b/src/net/stream.rs @@ -0,0 +1,379 @@ +use super::ProgressHook; +use crate::core::{ + analysis::{FileMetrics, ProjectAnalysis}, + error::{AnalysisError, Result}, + filter::{FilterStats, IntelligentFilter}, + registry::LanguageRegistry, +}; +use flate2::read::GzDecoder; +use futures_util::StreamExt; +use std::io::{Cursor, Read}; +use tar::Archive; +use tokio::sync::mpsc; + +#[cfg(not(target_arch = "wasm32"))] +use tokio::task; + +pub type ProgressCallback = Box) + Send + Sync>; + +pub struct StreamReader { + receiver: mpsc::Receiver>, + current_chunk: Option>, + finished: bool, +} + +impl StreamReader { + #[cfg(not(target_arch = "wasm32"))] + pub fn new( + stream: impl futures_util::Stream> + Send + 'static, + progress_callback: ProgressCallback, + total_size: Option, + ) -> Self { + let (tx, rx) = mpsc::channel(32); + + tokio::spawn(async move { + let mut downloaded = 0u64; + let mut stream = Box::pin(stream); + + while let Some(chunk_result) = stream.next().await { + match chunk_result { + Ok(chunk) => { + downloaded += chunk.len() as u64; + progress_callback(downloaded, total_size); + + if tx.send(Ok(chunk)).await.is_err() { + break; + } + } + Err(e) => { + let _ = tx + .send(Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Stream error: {}", e), + ))) + .await; + break; + } + } + } + }); + + Self { + receiver: rx, + current_chunk: None, + finished: false, + } + } + + #[cfg(target_arch = "wasm32")] + pub fn new( + stream: impl futures_util::Stream> + 'static, + progress_callback: ProgressCallback, + total_size: Option, + ) -> Self { + let (tx, rx) = mpsc::channel(32); + + wasm_bindgen_futures::spawn_local(async move { + let mut downloaded = 0u64; + let mut stream = Box::pin(stream); + + while let Some(chunk_result) = stream.next().await { + match chunk_result { + Ok(chunk) => { + downloaded += chunk.len() as u64; + progress_callback(downloaded, total_size); + + if tx.send(Ok(chunk)).await.is_err() { + break; + } + } + Err(e) => { + let _ = tx + .send(Err(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Stream error: {}", e), + ))) + .await; + break; + } + } + } + }); + + Self { + receiver: rx, + current_chunk: None, + finished: false, + } + } +} + +impl Read for StreamReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if let Some(ref mut cursor) = self.current_chunk { + let read = cursor.read(buf)?; + if read > 0 { + return Ok(read); + } + self.current_chunk = None; + } + + if self.finished { + return Ok(0); + } + + match self.receiver.try_recv() { + Ok(Ok(chunk)) => { + self.current_chunk = Some(Cursor::new(chunk)); + if let Some(ref mut cursor) = self.current_chunk { + cursor.read(buf) + } else { + Ok(0) + } + } + Ok(Err(e)) => { + self.finished = true; + Err(e) + } + Err(mpsc::error::TryRecvError::Empty) => { + #[cfg(not(target_arch = "wasm32"))] + { + match self.receiver.blocking_recv() { + Some(Ok(chunk)) => { + self.current_chunk = Some(Cursor::new(chunk)); + if let Some(ref mut cursor) = self.current_chunk { + cursor.read(buf) + } else { + Ok(0) + } + } + Some(Err(e)) => { + self.finished = true; + Err(e) + } + None => { + self.finished = true; + Ok(0) + } + } + } + #[cfg(target_arch = "wasm32")] + { + Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "Would block in WASM", + )) + } + } + Err(mpsc::error::TryRecvError::Disconnected) => { + self.finished = true; + Ok(0) + } + } + } +} + +pub async fn process_tarball_stream( + stream_reader: StreamReader, + project_analysis: &mut ProjectAnalysis, + filter: &IntelligentFilter, + _progress_hook: &dyn ProgressHook, +) -> Result<()> { + #[cfg(not(target_arch = "wasm32"))] + { + let filter = filter.clone(); + let metrics_result = task::spawn_blocking(move || { + let decoder = GzDecoder::new(stream_reader); + let mut archive = Archive::new(decoder); + + let entries = archive.entries().map_err(|e| { + AnalysisError::archive(format!("Failed to read tar entries: {}", e)) + })?; + + let mut collected_metrics = Vec::new(); + let mut stats = FilterStats::new(); + + for entry in entries { + let entry = entry.map_err(|e| { + AnalysisError::archive(format!("Failed to read tar entry: {}", e)) + })?; + + if let Ok(metrics) = process_tar_entry_sync(entry, &filter, &mut stats) { + collected_metrics.push(metrics); + } + } + + #[cfg(feature = "cli")] + log::info!( + "Filter stats: processed {}/{} files ({:.1}% filtered), saved {}", + stats.processed, + stats.total_entries, + stats.filter_ratio() * 100.0, + stats.format_bytes_saved() + ); + + Ok::, AnalysisError>(collected_metrics) + }) + .await + .map_err(|e| AnalysisError::archive(format!("Task join error: {}", e)))??; + + for metrics in metrics_result { + project_analysis.add_file_metrics(metrics)?; + } + } + + #[cfg(target_arch = "wasm32")] + { + let decoder = GzDecoder::new(stream_reader); + let mut archive = Archive::new(decoder); + + let entries = archive + .entries() + .map_err(|e| AnalysisError::archive(format!("Failed to read tar entries: {}", e)))?; + + let mut stats = FilterStats::new(); + + for entry in entries { + let entry = entry + .map_err(|e| AnalysisError::archive(format!("Failed to read tar entry: {}", e)))?; + + if let Ok(metrics) = process_tar_entry_sync(entry, filter, &mut stats) { + project_analysis.add_file_metrics(metrics)?; + } + } + + web_sys::console::log_1( + &format!( + "Filter stats: processed {}/{} files ({:.1}% filtered), saved {}", + stats.processed, + stats.total_entries, + stats.filter_ratio() * 100.0, + stats.format_bytes_saved() + ) + .into(), + ); + } + + Ok(()) +} + +fn process_tar_entry_sync( + mut entry: tar::Entry<'_, R>, + filter: &IntelligentFilter, + stats: &mut FilterStats, +) -> Result { + let header = entry.header(); + let path = header + .path() + .map_err(|e| AnalysisError::archive(format!("Invalid path in tar entry: {}", e)))?; + + let file_path = path.to_string_lossy().to_string(); + + if !header.entry_type().is_file() || header.size().unwrap_or(0) == 0 { + return Err(AnalysisError::archive("Not a file or empty".to_string())); + } + + let file_size = header.size().unwrap_or(0); + + let should_process = filter.should_process_file(&file_path, file_size); + stats.record_entry(file_size, !should_process); + + if !should_process { + return Err(AnalysisError::archive("File filtered out".to_string())); + } + + let language = LanguageRegistry::detect_by_path(&file_path) + .map(|l| l.name.clone()) + .unwrap_or_else(|| "Text".to_string()); + + let mut content = String::new(); + if entry.read_to_string(&mut content).is_err() { + return Err(AnalysisError::archive( + "Failed to read file content".to_string(), + )); + } + + analyze_file_content(&file_path, &content, &language, file_size) +} + +fn analyze_file_content( + file_path: &str, + content: &str, + language: &str, + file_size: u64, +) -> Result { + let lines: Vec<&str> = content.lines().collect(); + let total_lines = lines.len(); + + let mut code_lines = 0; + let mut comment_lines = 0; + let mut blank_lines = 0; + + let lang_def = LanguageRegistry::get_language(language); + let empty_line_comments = vec![]; + let empty_multi_line_comments = vec![]; + let line_comments = lang_def + .map(|l| &l.line_comments) + .unwrap_or(&empty_line_comments); + let multi_line_comments = lang_def + .map(|l| &l.multi_line_comments) + .unwrap_or(&empty_multi_line_comments); + + let mut in_multi_line_comment = false; + + for line in lines { + let trimmed = line.trim(); + + if trimmed.is_empty() { + blank_lines += 1; + continue; + } + + let mut is_comment = false; + + if !in_multi_line_comment { + for comment_start in line_comments { + if trimmed.starts_with(comment_start) { + is_comment = true; + break; + } + } + + for (start, end) in multi_line_comments { + if trimmed.starts_with(start) { + is_comment = true; + if !trimmed.ends_with(end) { + in_multi_line_comment = true; + } + break; + } + } + } else { + is_comment = true; + for (_, end) in multi_line_comments { + if trimmed.ends_with(end) { + in_multi_line_comment = false; + break; + } + } + } + + if is_comment { + comment_lines += 1; + } else { + code_lines += 1; + } + } + + let metrics = FileMetrics::new( + file_path, + language.to_string(), + total_lines, + code_lines, + comment_lines, + blank_lines, + )? + .with_size_bytes(file_size); + + Ok(metrics) +} diff --git a/src/net/traits.rs b/src/net/traits.rs new file mode 100644 index 0000000..23db293 --- /dev/null +++ b/src/net/traits.rs @@ -0,0 +1,410 @@ +use async_trait::async_trait; +use reqwest::Client; +use std::collections::HashMap; +use std::time::Duration; + +/// Progress hook trait for monitoring download and processing progress +pub trait ProgressHook: Send + Sync { + /// Called when download progress is updated + /// + /// # Arguments + /// * `downloaded` - Number of bytes downloaded so far + /// * `total` - Total size in bytes (if known) + fn on_download_progress(&self, downloaded: u64, total: Option); + + /// Called when processing starts with a status message + /// + /// # Arguments + /// * `message` - Status message describing current operation + fn on_processing_start(&self, message: &str); + + /// Called when processing progress is updated + /// + /// # Arguments + /// * `current` - Current item being processed + /// * `total` - Total items to process + fn on_processing_progress(&self, current: usize, total: usize); +} + +/// No-operation progress hook that ignores all progress updates +pub struct NoOpProgressHook; + +impl ProgressHook for NoOpProgressHook { + fn on_download_progress(&self, _downloaded: u64, _total: Option) {} + fn on_processing_start(&self, _message: &str) {} + fn on_processing_progress(&self, _current: usize, _total: usize) {} +} + +/// Universal configuration for all Git providers +#[derive(Debug, Clone)] +pub struct ProviderConfig { + /// Custom HTTP headers to include in requests + pub headers: HashMap, + + /// Request timeout in seconds (None for default) + pub timeout: Option, + + /// Maximum number of redirects to follow + pub max_redirects: Option, + + /// User agent string to use for requests + pub user_agent: Option, + + /// Whether to accept invalid SSL certificates + pub accept_invalid_certs: bool, + + /// Authentication credentials (varies by provider) + pub credentials: HashMap, + + /// Provider-specific settings + pub provider_settings: HashMap, + + /// Maximum file size to download in bytes + pub max_file_size: Option, + + /// Whether to use compression for requests + pub use_compression: bool, + + /// Custom proxy URL + pub proxy: Option, +} + +impl Default for ProviderConfig { + fn default() -> Self { + Self { + headers: HashMap::new(), + timeout: Some(300), // 5 minutes default + max_redirects: Some(10), + user_agent: Some("bytes-radar/1.0.0".to_string()), + accept_invalid_certs: false, + credentials: HashMap::new(), + provider_settings: HashMap::new(), + max_file_size: Some(100 * 1024 * 1024), // 100MB default + use_compression: true, + proxy: None, + } + } +} + +impl ProviderConfig { + /// Create a new configuration with default values + pub fn new() -> Self { + Self::default() + } + + /// Set a custom header + /// + /// # Arguments + /// * `name` - Header name + /// * `value` - Header value + pub fn with_header(mut self, name: impl Into, value: impl Into) -> Self { + self.headers.insert(name.into(), value.into()); + self + } + + /// Set request timeout in seconds + /// + /// # Arguments + /// * `timeout` - Timeout in seconds + pub fn with_timeout(mut self, timeout: u64) -> Self { + self.timeout = Some(timeout); + self + } + + /// Set user agent string + /// + /// # Arguments + /// * `user_agent` - User agent string + pub fn with_user_agent(mut self, user_agent: impl Into) -> Self { + self.user_agent = Some(user_agent.into()); + self + } + + /// Set whether to accept invalid SSL certificates + /// + /// # Arguments + /// * `accept` - Whether to accept invalid certificates + pub fn with_accept_invalid_certs(mut self, accept: bool) -> Self { + self.accept_invalid_certs = accept; + self + } + + /// Set authentication credentials + /// + /// # Arguments + /// * `key` - Credential key (e.g., "token", "username") + /// * `value` - Credential value + pub fn with_credential(mut self, key: impl Into, value: impl Into) -> Self { + self.credentials.insert(key.into(), value.into()); + self + } + + /// Set provider-specific setting + /// + /// # Arguments + /// * `key` - Setting key + /// * `value` - Setting value + pub fn with_provider_setting( + mut self, + key: impl Into, + value: impl Into, + ) -> Self { + self.provider_settings.insert(key.into(), value.into()); + self + } + + /// Set maximum file size in bytes + /// + /// # Arguments + /// * `size` - Maximum file size in bytes + pub fn with_max_file_size(mut self, size: u64) -> Self { + self.max_file_size = Some(size); + self + } + + /// Set proxy URL + /// + /// # Arguments + /// * `proxy` - Proxy URL + pub fn with_proxy(mut self, proxy: impl Into) -> Self { + self.proxy = Some(proxy.into()); + self + } +} + +/// Parsed repository information from a URL +#[derive(Debug, Clone)] +pub struct ParsedRepository { + /// Repository owner/organization + pub owner: String, + + /// Repository name + pub repo: String, + + /// Branch name or commit hash (if specified) + pub branch_or_commit: Option, + + /// Whether branch_or_commit is a commit hash + pub is_commit: bool, + + /// Generated project name for display + pub project_name: String, + + /// Host name (e.g., "github.com") + pub host: Option, +} + +impl ParsedRepository { + /// Create a new parsed repository with default main branch + /// + /// # Arguments + /// * `owner` - Repository owner + /// * `repo` - Repository name + pub fn new(owner: String, repo: String) -> Self { + let project_name = format!("{}@main", repo); + Self { + owner, + repo, + branch_or_commit: None, + is_commit: false, + project_name, + host: None, + } + } + + /// Set the branch and update project name + /// + /// # Arguments + /// * `branch` - Branch name + pub fn with_branch(mut self, branch: String) -> Self { + self.project_name = format!("{}@{}", self.repo, branch); + self.branch_or_commit = Some(branch); + self.is_commit = false; + self + } + + /// Set the commit hash and update project name + /// + /// # Arguments + /// * `commit` - Commit hash + pub fn with_commit(mut self, commit: String) -> Self { + let short_commit = &commit[..7.min(commit.len())]; + self.project_name = format!("{}@{}", self.repo, short_commit); + self.branch_or_commit = Some(commit); + self.is_commit = true; + self + } + + /// Set the host name + /// + /// # Arguments + /// * `host` - Host name + pub fn with_host(mut self, host: String) -> Self { + self.host = Some(host); + self + } +} + +/// Git provider trait for handling different repository hosting services +#[async_trait] +pub trait GitProvider: Send + Sync { + /// Get the provider name (e.g., "github", "gitlab") + fn name(&self) -> &'static str; + + /// Check if this provider can handle the given URL + /// + /// # Arguments + /// * `url` - URL to check + fn can_handle(&self, url: &str) -> bool; + + /// Parse a URL into repository information + /// + /// # Arguments + /// * `url` - URL to parse + fn parse_url(&self, url: &str) -> Option; + + /// Build download URLs for the parsed repository + /// + /// # Arguments + /// * `parsed` - Parsed repository information + fn build_download_urls(&self, parsed: &ParsedRepository) -> Vec; + + /// Get the default branch for a repository (if supported) + /// + /// # Arguments + /// * `client` - HTTP client to use + /// * `parsed` - Parsed repository information + async fn get_default_branch( + &self, + client: &Client, + parsed: &ParsedRepository, + ) -> Option; + + /// Apply configuration to this provider + /// + /// # Arguments + /// * `config` - Configuration to apply + fn apply_config(&mut self, config: &ProviderConfig); + + /// Get project name from URL + /// + /// # Arguments + /// * `url` - URL to extract project name from + fn get_project_name(&self, url: &str) -> String; + + /// Build HTTP client with provider-specific configuration + /// + /// # Arguments + /// * `config` - Configuration to use + fn build_client( + &self, + config: &ProviderConfig, + ) -> Result> { + let mut builder = Client::builder(); + + // Set user agent + if let Some(ref user_agent) = config.user_agent { + builder = builder.user_agent(user_agent); + } + + // Set timeout (works on both wasm and native) + if let Some(timeout) = config.timeout { + #[cfg(not(target_arch = "wasm32"))] + { + builder = builder.timeout(Duration::from_secs(timeout)); + } + #[cfg(target_arch = "wasm32")] + { + // On WASM, timeout is handled differently + builder = builder.timeout(Duration::from_secs(timeout.min(300))); + // Max 5 minutes for WASM + } + } + + // Set redirects + if let Some(max_redirects) = config.max_redirects { + builder = builder.redirect(reqwest::redirect::Policy::limited(max_redirects as usize)); + } + + // Set SSL verification + #[cfg(not(target_arch = "wasm32"))] + if config.accept_invalid_certs { + builder = builder.danger_accept_invalid_certs(true); + } + + // Set compression + if !config.use_compression { + builder = builder.no_gzip(); + #[cfg(not(target_arch = "wasm32"))] + { + builder = builder.no_brotli(); + builder = builder.no_deflate(); + } + } + + // Set proxy (only on native) + #[cfg(not(target_arch = "wasm32"))] + if let Some(ref proxy) = config.proxy { + let proxy = reqwest::Proxy::all(proxy)?; + builder = builder.proxy(proxy); + } + + // Build default headers + let mut headers = reqwest::header::HeaderMap::new(); + + // Add custom headers + for (name, value) in &config.headers { + let header_name = reqwest::header::HeaderName::from_bytes(name.as_bytes())?; + let header_value = reqwest::header::HeaderValue::from_str(value)?; + headers.insert(header_name, header_value); + } + + // Add provider-specific auth headers + self.add_auth_headers(&mut headers, config)?; + + if !headers.is_empty() { + builder = builder.default_headers(headers); + } + + Ok(builder.build()?) + } + + /// Add authentication headers specific to this provider + /// + /// # Arguments + /// * `headers` - Header map to add to + /// * `config` - Configuration containing credentials + fn add_auth_headers( + &self, + _headers: &mut reqwest::header::HeaderMap, + _config: &ProviderConfig, + ) -> Result<(), Box> { + // Default implementation does nothing + // Providers can override this + Ok(()) + } + + /// Validate configuration for this provider + /// + /// # Arguments + /// * `config` - Configuration to validate + fn validate_config(&self, config: &ProviderConfig) -> Result<(), String> { + // Basic validation + if let Some(timeout) = config.timeout { + if timeout == 0 { + return Err("Timeout cannot be zero".to_string()); + } + if timeout > 3600 { + return Err("Timeout cannot exceed 1 hour".to_string()); + } + } + + if let Some(max_file_size) = config.max_file_size { + if max_file_size > 1024 * 1024 * 1024 { + return Err("Max file size cannot exceed 1GB".to_string()); + } + } + + Ok(()) + } +} diff --git a/src/server/mod.rs b/src/server/mod.rs index c4c6099..2c38dff 100644 --- a/src/server/mod.rs +++ b/src/server/mod.rs @@ -1,4 +1,4 @@ -use crate::core::RemoteAnalyzer; +use crate::net::RemoteAnalyzer; use wasm_bindgen::prelude::*; #[derive(serde::Serialize, serde::Deserialize)] diff --git a/src/worker.rs b/src/worker.rs index 142e89c..6e2a50f 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -1,4 +1,4 @@ -use crate::core::{filter::IntelligentFilter, net::RemoteAnalyzer}; +use crate::{core::filter::IntelligentFilter, net::RemoteAnalyzer}; use wasm_bindgen::prelude::*; #[derive(serde::Serialize, serde::Deserialize)] @@ -58,7 +58,7 @@ pub async fn analyze_url(url: String, options: JsValue) -> Result Date: Thu, 10 Jul 2025 14:09:07 +0800 Subject: [PATCH 02/11] feat: update cli usage --- Cargo.lock | 102 +++++++++ Cargo.toml | 4 +- docs/CLI_USAGE.md | 367 ++++++++++++++++++++++++++++++ examples/github_token_analysis.rs | 7 +- src/cli/args.rs | 260 +++++++++++++++++---- src/cli/mod.rs | 89 ++++++-- src/cli/output.rs | 16 +- src/core/filter.rs | 45 ---- 8 files changed, 780 insertions(+), 110 deletions(-) create mode 100644 docs/CLI_USAGE.md diff --git a/Cargo.lock b/Cargo.lock index 2838bcd..9cf3be8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -153,9 +153,11 @@ dependencies = [ "serde-wasm-bindgen", "serde-xml-rs", "serde_json", + "serde_yaml", "tar", "thiserror 2.0.12", "tokio", + "toml", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", @@ -300,6 +302,12 @@ dependencies = [ "log", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "errno" version = "0.3.13" @@ -441,6 +449,12 @@ version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +[[package]] +name = "hashbrown" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" + [[package]] name = "heck" version = "0.5.0" @@ -654,6 +668,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "indicatif" version = "0.17.11" @@ -1219,6 +1243,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1231,6 +1264,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.34+deprecated" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1446,6 +1492,47 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tower" version = "0.5.2" @@ -1528,6 +1615,12 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a1a07cc7db3810833284e8d372ccdc6da29741639ecc70c9ec107df0fa6154c" +[[package]] +name = "unsafe-libyaml" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" + [[package]] name = "untrusted" version = "0.9.0" @@ -1849,6 +1942,15 @@ version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +[[package]] +name = "winnow" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74c7b26e3480b707944fc872477815d29a8e429d2f93a1ce000f5fa84a15cbcd" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen-rt" version = "0.39.0" diff --git a/Cargo.toml b/Cargo.toml index 89cbb3e..fb32e14 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,7 +23,7 @@ exclude = [ [features] default = ["cli"] -cli = ["colored", "env_logger", "clap", "indicatif"] +cli = ["colored", "env_logger", "clap", "indicatif", "serde_yaml", "toml"] worker = ["wasm-bindgen", "web-sys", "js-sys", "wasm-bindgen-futures", "serde-wasm-bindgen"] [lib] @@ -48,6 +48,8 @@ colored = { version = "2.2", optional = true } env_logger = { version = "0.11", optional = true } clap = { version = "4.5", features = ["derive"], optional = true } indicatif = { version = "0.17", optional = true } +serde_yaml = { version = "0.9", optional = true } +toml = { version = "0.8", optional = true } # WASM dependencies wasm-bindgen = { version = "0.2", optional = true } diff --git a/docs/CLI_USAGE.md b/docs/CLI_USAGE.md new file mode 100644 index 0000000..e23f988 --- /dev/null +++ b/docs/CLI_USAGE.md @@ -0,0 +1,367 @@ +# Bytes Radar CLI Usage Guide + +`bradar` is a professional command-line tool for analyzing code statistics from remote repositories with hyper-fast performance. + +## Table of Contents + +- [Installation](#installation) +- [Basic Usage](#basic-usage) +- [Supported Platforms](#supported-platforms) +- [URL Formats](#url-formats) +- [Command-Line Options](#command-line-options) +- [Output Formats](#output-formats) +- [Advanced Usage Examples](#advanced-usage-examples) +- [Environment Variables](#environment-variables) +- [Performance Tuning](#performance-tuning) +- [Troubleshooting](#troubleshooting) + +## Installation + +### From Source +```bash +git clone https://github.com/zmh-program/bytes-radar.git +cd bytes-radar +cargo build --release +``` + +### Using Cargo +```bash +cargo install bytes-radar +``` + +## Basic Usage + +The simplest way to analyze a repository: + +```bash +bradar user/repo +``` + +This analyzes the default branch of `user/repo` on GitHub and displays results in a human-readable table format. + +## Supported Platforms + +- **GitHub** (github.com, GitHub Enterprise) +- **GitLab** (gitlab.com, self-hosted instances) +- **Bitbucket** (bitbucket.org) +- **Codeberg** (codeberg.org) +- **SourceForge** (sourceforge.net) +- **Gitea** instances +- **Azure DevOps** +- **Direct archive URLs** (tar.gz, tgz, zip) + +## URL Formats + +| Format | Description | Example | +|--------|-------------|---------| +| `user/repo` | GitHub repository (default branch) | `microsoft/vscode` | +| `user/repo@branch` | Specific branch | `torvalds/linux@master` | +| `user/repo@commit` | Specific commit hash | `rust-lang/rust@abc123` | +| Full URL | Complete repository URL | `https://github.com/user/repo` | +| Archive URL | Direct archive link | `https://example.com/project.tar.gz` | + +## Command-Line Options + +### Basic Information + +```bash +bradar --help # Show help information +bradar -v # Show version information +``` + +### Repository Analysis + +```bash +bradar [OPTIONS] +``` + +### Output Options + +| Option | Short | Description | Default | +|--------|-------|-------------|---------| +| `--format` | `-f` | Output format (table, json, csv, xml, yaml, toml) | `table` | +| `--detailed` | | Show detailed file-by-file statistics | `false` | +| `--quiet` | `-q` | Quiet mode - suppress progress and minimize output | `false` | +| `--no-progress` | | Disable progress bar | `false` | +| `--no-color` | | Disable colored output | `false` | + +### Authentication + +| Option | Description | +|--------|-------------| +| `--token` | Authentication token for private repositories | + +### Network Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--timeout` | Request timeout in seconds | `300` | +| `--allow-insecure` | Allow insecure HTTPS connections | `false` | +| `--user-agent` | Custom User-Agent string | | +| `--retry-count` | Number of retry attempts for failed requests | `3` | + +### Filtering Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--aggressive-filter` | Enable aggressive filtering for maximum performance | `false` | +| `--max-file-size` | Maximum file size to process in KB | `1024` | +| `--min-file-size` | Minimum file size to process in bytes | `1` | +| `--include-tests` | Include test directories in analysis | `false` | +| `--include-docs` | Include documentation directories in analysis | `false` | +| `--include-hidden` | Include hidden files and directories | `false` | +| `--exclude-pattern` | Exclude files matching this pattern (glob) | | +| `--include-pattern` | Only include files matching this pattern (glob) | | + +### Language Options + +| Option | Description | +|--------|-------------| +| `--language` | Only analyze files of specific language | +| `--exclude-language` | Exclude specific language from analysis | + +### Analysis Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--ignore-whitespace` | Ignore whitespace-only lines in code analysis | `false` | +| `--count-generated` | Include generated files in analysis | `false` | +| `--max-line-length` | Maximum line length to consider (0 = unlimited) | `0` | + +### Debug and Logging + +| Option | Short | Description | +|--------|-------|-------------| +| `--debug` | `-d` | Enable debug output | +| `--trace` | | Enable trace-level logging | +| `--log-file` | | Write logs to file | + +### Advanced Options + +| Option | Description | Default | +|--------|-------------|---------| +| `--threads` | Number of worker threads (0 = auto) | `0` | +| `--memory-limit` | Memory limit in MB (0 = unlimited) | `0` | +| `--cache-dir` | Directory for caching downloaded files | | +| `--no-cache` | Disable caching of downloaded files | `false` | + +### Experimental Features + +| Option | Description | +|--------|-------------| +| `--experimental-parallel` | Enable experimental parallel processing | +| `--experimental-streaming` | Enable experimental streaming analysis | + +## Output Formats + +### Table (Default) +Human-readable table format with colored output and progress indicators. + +```bash +bradar microsoft/vscode +``` + +### JSON +Machine-readable JSON format for integration with other tools. + +```bash +bradar --format json microsoft/vscode +``` + +### CSV +Comma-separated values format for spreadsheet analysis. + +```bash +bradar --format csv microsoft/vscode +``` + +### XML +XML format for structured data processing. + +```bash +bradar --format xml microsoft/vscode +``` + +### YAML +YAML format for configuration files and human-readable structured data. + +```bash +bradar --format yaml microsoft/vscode +``` + +### TOML +TOML format for configuration management. + +```bash +bradar --format toml microsoft/vscode +``` + +## Advanced Usage Examples + +### Analyzing Private Repositories + +```bash +# Using token parameter +bradar --token ghp_xxxxxxxxxxxxxxxxxxxx private-org/private-repo + +# Using environment variable +export BRADAR_TOKEN=ghp_xxxxxxxxxxxxxxxxxxxx +bradar private-org/private-repo +``` + +### Performance Optimization + +```bash +# For large repositories - aggressive filtering +bradar --aggressive-filter --max-file-size 2048 torvalds/linux + +# Custom thread count and memory limit +bradar --threads 8 --memory-limit 4096 large-org/huge-repo + +# Enable experimental features for better performance +bradar --experimental-parallel --experimental-streaming big-repo +``` + +### Detailed Analysis with Filtering + +```bash +# Include tests and documentation +bradar --include-tests --include-docs --detailed microsoft/typescript + +# Analyze only specific language +bradar --language rust rust-lang/rust + +# Exclude multiple languages +bradar --exclude-language javascript --exclude-language css web-project + +# Custom file patterns +bradar --include-pattern "*.rs" --exclude-pattern "*test*" rust-project +``` + +### Output Customization + +```bash +# Quiet JSON output for scripting +bradar --quiet --format json user/repo | jq '.global_metrics' + +# Detailed CSV for analysis +bradar --format csv --detailed user/repo > analysis.csv + +# Debug mode with log file +bradar --debug --log-file analysis.log --trace user/repo +``` + +### Network Configuration + +```bash +# Custom timeout and retry settings +bradar --timeout 600 --retry-count 5 slow-server/repo + +# Custom User-Agent +bradar --user-agent "MyAnalyzer/1.0" user/repo + +# Allow insecure connections +bradar --allow-insecure https://insecure-server.com/repo.tar.gz +``` + +## Environment Variables + +| Variable | Description | Example | +|----------|-------------|---------| +| `BRADAR_TOKEN` | Default authentication token | `ghp_xxxxxxxxxxxxxxxxxxxx` | + +## Performance Tuning + +### For Large Repositories + +1. **Use aggressive filtering**: `--aggressive-filter` +2. **Reduce max file size**: `--max-file-size 512` +3. **Increase timeout**: `--timeout 600` +4. **Use experimental features**: `--experimental-parallel` + +### For Many Small Files + +1. **Increase thread count**: `--threads 16` +2. **Use streaming**: `--experimental-streaming` +3. **Set memory limit**: `--memory-limit 8192` + +### For Network-Limited Environments + +1. **Enable caching**: `--cache-dir ~/.bradar-cache` +2. **Increase retry count**: `--retry-count 10` +3. **Extend timeout**: `--timeout 900` + +## Troubleshooting + +### Common Issues + +#### Authentication Errors +```bash +# Make sure token has correct permissions +bradar --token ghp_xxxxxxxxxxxxxxxxxxxx --debug private-repo +``` + +#### Network Timeouts +```bash +# Increase timeout and retry count +bradar --timeout 600 --retry-count 5 slow-repo +``` + +#### Memory Issues +```bash +# Set memory limit and use aggressive filtering +bradar --memory-limit 2048 --aggressive-filter large-repo +``` + +#### Slow Analysis +```bash +# Use performance optimizations +bradar --aggressive-filter --experimental-parallel --threads 8 repo +``` + +### Debug Mode + +Enable debug mode for detailed information: + +```bash +bradar --debug --trace --log-file debug.log problematic-repo +``` + +### Getting Help + +```bash +bradar --help # General help +bradar --version # Version information +``` + +## Integration Examples + +### Shell Scripts + +```bash +#!/bin/bash +# Analyze multiple repositories and save to JSON +repos=("user/repo1" "user/repo2" "user/repo3") +for repo in "${repos[@]}"; do + bradar --format json --quiet "$repo" > "${repo//\//_}.json" +done +``` + +### CI/CD Integration + +```yaml +# GitHub Actions example +- name: Analyze Code Statistics + run: | + bradar --format json --quiet ${{ github.repository }} > stats.json + cat stats.json | jq '.global_metrics.total_lines' +``` + +### Monitoring Scripts + +```bash +#!/bin/bash +# Monitor repository growth +bradar --format json --quiet user/repo | \ +jq -r '.global_metrics | "\(.total_files) files, \(.total_lines) lines"' +``` \ No newline at end of file diff --git a/examples/github_token_analysis.rs b/examples/github_token_analysis.rs index 908fa93..12d069d 100644 --- a/examples/github_token_analysis.rs +++ b/examples/github_token_analysis.rs @@ -1,4 +1,5 @@ use bytes_radar::{RemoteAnalyzer, Result}; +use std::collections::HashMap; use std::env; #[tokio::main] @@ -7,7 +8,11 @@ async fn main() -> Result<()> { env::var("GITHUB_TOKEN").expect("Please set GITHUB_TOKEN environment variable"); let mut analyzer = RemoteAnalyzer::new(); - analyzer.set_github_token(&github_token); + + let mut credentials = HashMap::new(); + credentials.insert("token".to_string(), github_token); + analyzer.set_provider_credentials("github", credentials); + analyzer.set_timeout(120); let url = "https://github.com/your-username/your-repo"; diff --git a/src/cli/args.rs b/src/cli/args.rs index 4545cd0..753ff27 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -1,86 +1,262 @@ -use clap::{Parser, ValueEnum}; +use clap::{ArgGroup, Parser, ValueEnum}; #[derive(Parser)] #[command(name = "bradar")] -#[command(about = "A tool for analyzing code statistics from remote repositories")] +#[command(about = "A professional tool for analyzing code statistics from remote repositories")] #[command(version)] #[command(long_about = " -A professional code analysis tool for remote repositories. +bradar - Bytes Radar: A hyper-fast code analysis tool for remote repositories. SUPPORTED PLATFORMS: - - GitHub (github.com) - - GitLab (gitlab.com, self-hosted) - - Bitbucket (bitbucket.org) - - Codeberg (codeberg.org) - - SourceForge (sourceforge.net) - - Direct tar.gz/tgz URLs - -USAGE EXAMPLES: - bradar user/repo # GitHub repo (default branch) - bradar user/repo@master # GitHub repo with specific branch - bradar user/repo@abc123 # GitHub repo with specific commit - bradar https://github.com/user/repo # Full GitHub URL - bradar https://gitlab.com/user/repo # GitLab URL - bradar https://bitbucket.org/user/repo # Bitbucket URL - bradar https://example.com/file.tar.gz # Direct tar.gz URL - bradar -f json user/repo # JSON output format - bradar --token ghp_xxx user/repo # With GitHub token for private repos + • GitHub (github.com, GitHub Enterprise) + • GitLab (gitlab.com, self-hosted instances) + • Bitbucket (bitbucket.org) + • Codeberg (codeberg.org) + • SourceForge (sourceforge.net) + • Gitea instances + • Azure DevOps + • Direct archive URLs (tar.gz, tgz, zip) + +URL FORMATS: + user/repo # GitHub repo (default branch) + user/repo@branch # Specific branch + user/repo@commit-hash # Specific commit + https://github.com/user/repo # Full GitHub URL + https://gitlab.com/user/repo # GitLab URL + https://example.com/archive.tar.gz # Direct archive URL + +EXAMPLES: + bradar microsoft/vscode + bradar torvalds/linux@master + bradar https://github.com/rust-lang/rust + bradar --format json --detailed user/repo + bradar --token ghp_xxx --include-tests private/repo + bradar --aggressive-filter --max-file-size 2048 large/repo ")] +#[command(arg_required_else_help = true)] +#[command(disable_version_flag = true)] +#[command(group( + ArgGroup::new("auth") + .args(&["token"]) + .multiple(false) +))] +#[command(group( + ArgGroup::new("output_control") + .args(&["quiet", "debug"]) + .multiple(false) +))] pub struct Cli { - #[arg(help = "URL to analyze: user/repo, user/repo@branch, or full URL")] + #[arg(help = "Repository URL to analyze (user/repo, user/repo@branch, or full URL)")] pub url: Option, - #[arg(short, long, help = "Output format", value_enum)] - pub format: Option, + // Version + #[arg(short = 'v', long = "version", action = clap::ArgAction::Version, help = "Print version information")] + _version: (), + + // Output Options + #[arg( + short = 'f', + long = "format", + help = "Output format", + value_enum, + default_value = "table" + )] + pub format: OutputFormat, - #[arg(long, help = "Show detailed file-by-file statistics")] + #[arg(long = "detailed", help = "Show detailed file-by-file statistics")] pub detailed: bool, - #[arg(short = 'd', long = "debug", help = "Enable debug output")] - pub debug: bool, + #[arg( + short = 'q', + long = "quiet", + help = "Quiet mode - suppress progress and minimize output" + )] + pub quiet: bool, + + #[arg(long = "no-progress", help = "Disable progress bar")] + pub no_progress: bool, - #[arg(long, help = "GitHub token for private repositories")] + #[arg(long = "no-color", help = "Disable colored output")] + pub no_color: bool, + + // Authentication + #[arg(long = "token", help = "Authentication token for private repositories")] pub token: Option, - #[arg(long, help = "Request timeout in seconds", default_value = "300")] + // Network Options + #[arg( + long = "timeout", + help = "Request timeout in seconds", + default_value = "300", + value_name = "SECONDS" + )] pub timeout: u64, - #[arg(long, help = "Allow insecure HTTP connections")] + #[arg(long = "allow-insecure", help = "Allow insecure HTTPS connections")] pub allow_insecure: bool, - #[arg(long, help = "Disable progress bar")] - pub no_progress: bool, + #[arg( + long = "user-agent", + help = "Custom User-Agent string", + value_name = "STRING" + )] + pub user_agent: Option, - #[arg(long, help = "Quiet mode - minimal output")] - pub quiet: bool, + #[arg( + long = "retry-count", + help = "Number of retry attempts for failed requests", + default_value = "3", + value_name = "COUNT" + )] + pub retry_count: u32, - #[arg(long, help = "Enable aggressive filtering for maximum performance")] + // Filtering Options + #[arg( + long = "aggressive-filter", + help = "Enable aggressive filtering for maximum performance" + )] pub aggressive_filter: bool, #[arg( - long, + long = "max-file-size", help = "Maximum file size to process in KB", - default_value = "1024" + default_value = "1024", + value_name = "KB" )] pub max_file_size: u64, - #[arg(long, help = "Include test directories")] + #[arg(long = "include-tests", help = "Include test directories in analysis")] pub include_tests: bool, - #[arg(long, help = "Include documentation directories")] + #[arg( + long = "include-docs", + help = "Include documentation directories in analysis" + )] pub include_docs: bool, + + #[arg(long = "include-hidden", help = "Include hidden files and directories")] + pub include_hidden: bool, + + #[arg( + long = "exclude-pattern", + help = "Exclude files matching this pattern (glob)", + value_name = "PATTERN" + )] + pub exclude_pattern: Option, + + #[arg( + long = "include-pattern", + help = "Only include files matching this pattern (glob)", + value_name = "PATTERN" + )] + pub include_pattern: Option, + + #[arg( + long = "min-file-size", + help = "Minimum file size to process in bytes", + default_value = "1", + value_name = "BYTES" + )] + pub min_file_size: u64, + + // Language Options + #[arg( + long = "language", + help = "Only analyze files of specific language", + value_name = "LANG" + )] + pub language: Option, + + #[arg( + long = "exclude-language", + help = "Exclude specific language from analysis", + value_name = "LANG" + )] + pub exclude_language: Vec, + + // Analysis Options + #[arg( + long = "ignore-whitespace", + help = "Ignore whitespace-only lines in code analysis" + )] + pub ignore_whitespace: bool, + + #[arg(long = "count-generated", help = "Include generated files in analysis")] + pub count_generated: bool, + + #[arg( + long = "max-line-length", + help = "Maximum line length to consider (0 = unlimited)", + default_value = "0", + value_name = "LENGTH" + )] + pub max_line_length: usize, + + // Debug and Logging + #[arg(short = 'd', long = "debug", help = "Enable debug output")] + pub debug: bool, + + #[arg(long = "trace", help = "Enable trace-level logging")] + pub trace: bool, + + #[arg(long = "log-file", help = "Write logs to file", value_name = "FILE")] + pub log_file: Option, + + // Advanced Options + #[arg( + long = "threads", + help = "Number of worker threads (0 = auto)", + default_value = "0", + value_name = "COUNT" + )] + pub threads: usize, + + #[arg( + long = "memory-limit", + help = "Memory limit in MB (0 = unlimited)", + default_value = "0", + value_name = "MB" + )] + pub memory_limit: usize, + + #[arg( + long = "cache-dir", + help = "Directory for caching downloaded files", + value_name = "DIR" + )] + pub cache_dir: Option, + + #[arg(long = "no-cache", help = "Disable caching of downloaded files")] + pub no_cache: bool, + + // Experimental Features + #[arg( + long = "experimental-parallel", + help = "Enable experimental parallel processing" + )] + pub experimental_parallel: bool, + + #[arg( + long = "experimental-streaming", + help = "Enable experimental streaming analysis" + )] + pub experimental_streaming: bool, } #[derive(Clone, ValueEnum)] pub enum OutputFormat { - #[value(name = "table")] + #[value(name = "table", help = "Human-readable table format")] Table, - #[value(name = "json")] + #[value(name = "json", help = "JSON format")] Json, - #[value(name = "csv")] + #[value(name = "csv", help = "CSV format")] Csv, - #[value(name = "xml")] + #[value(name = "xml", help = "XML format")] Xml, + #[value(name = "yaml", help = "YAML format")] + Yaml, + #[value(name = "toml", help = "TOML format")] + Toml, } impl Default for OutputFormat { diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 92f1892..eb79feb 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -23,10 +23,13 @@ pub use args::{Cli, OutputFormat}; pub async fn run() -> Result<()> { let cli = args::Cli::parse(); - if cli.debug { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("debug")).init(); - } else { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("warn")).init(); + init_logging(&cli)?; + + let mut cli = cli; + if let Some(token) = std::env::var("BRADAR_TOKEN").ok() { + if cli.token.is_none() { + cli.token = Some(token); + } } match &cli.url { @@ -38,11 +41,40 @@ pub async fn run() -> Result<()> { } } +#[cfg(feature = "cli")] +fn init_logging(cli: &Cli) -> Result<()> { + let log_level = if cli.trace { + "trace" + } else if cli.debug { + "debug" + } else { + "warn" + }; + + let mut builder = + env_logger::Builder::from_env(env_logger::Env::default().default_filter_or(log_level)); + + if let Some(log_file) = &cli.log_file { + use std::fs::OpenOptions; + + let target = Box::new( + OpenOptions::new() + .create(true) + .append(true) + .open(log_file) + .map_err(|e| crate::core::error::AnalysisError::file_read(log_file, e))?, + ); + builder.target(env_logger::Target::Pipe(target)); + } + + builder.init(); + Ok(()) +} + #[cfg(feature = "cli")] async fn analyze_remote_archive(url: &str, cli: &Cli) -> Result<()> { - let format = cli.format.as_ref().unwrap_or(&OutputFormat::Table); let should_show_progress = - !cli.no_progress && matches!(format, OutputFormat::Table) && !cli.quiet; + !cli.no_progress && matches!(cli.format, OutputFormat::Table) && !cli.quiet; let processed_url = url_parser::expand_url(url); @@ -68,15 +100,7 @@ async fn analyze_remote_archive(url: &str, cli: &Cli) -> Result<()> { analyzer.set_progress_hook(progress::ProgressBarHook::new(pb)); } - if cli.aggressive_filter { - analyzer.set_aggressive_filtering(true); - } else { - let mut filter = filter::IntelligentFilter::default(); - filter.max_file_size = cli.max_file_size * 1024; - filter.ignore_test_dirs = !cli.include_tests; - filter.ignore_docs_dirs = !cli.include_docs; - analyzer.set_filter(filter); - } + configure_analyzer_filters(&mut analyzer, cli)?; let project_analysis = analyzer.analyze_url(&processed_url).await?; @@ -88,13 +112,38 @@ async fn analyze_remote_archive(url: &str, cli: &Cli) -> Result<()> { progress::show_completion_message(elapsed, cli.quiet); - match format { + output_results(&project_analysis, cli)?; + + Ok(()) +} + +#[cfg(feature = "cli")] +fn configure_analyzer_filters(analyzer: &mut RemoteAnalyzer, cli: &Cli) -> Result<()> { + if cli.aggressive_filter { + analyzer.set_aggressive_filtering(true); + } else { + let mut filter = filter::IntelligentFilter::default(); + filter.max_file_size = cli.max_file_size * 1024; + filter.ignore_test_dirs = !cli.include_tests; + filter.ignore_docs_dirs = !cli.include_docs; + + analyzer.set_filter(filter); + } + + Ok(()) +} + +#[cfg(feature = "cli")] +fn output_results(project_analysis: &analysis::ProjectAnalysis, cli: &Cli) -> Result<()> { + match cli.format { OutputFormat::Table => { - output::print_table_format(&project_analysis, cli.detailed, cli.quiet); + output::print_table_format(project_analysis, cli.detailed, cli.quiet); } - OutputFormat::Json => output::print_json_format(&project_analysis)?, - OutputFormat::Csv => output::print_csv_format(&project_analysis)?, - OutputFormat::Xml => output::print_xml_format(&project_analysis)?, + OutputFormat::Json => output::print_json_format(project_analysis)?, + OutputFormat::Csv => output::print_csv_format(project_analysis)?, + OutputFormat::Xml => output::print_xml_format(project_analysis)?, + OutputFormat::Yaml => output::print_yaml_format(project_analysis)?, + OutputFormat::Toml => output::print_toml_format(project_analysis)?, } Ok(()) diff --git a/src/cli/output.rs b/src/cli/output.rs index c1e4c47..16a524c 100644 --- a/src/cli/output.rs +++ b/src/cli/output.rs @@ -209,7 +209,7 @@ pub fn print_xml_format(project_analysis: &ProjectAnalysis) -> Result<()> { println!(" "); for stats in language_stats { println!(" "); - println!(" {}", xml_escape(&stats.language_name)); + println!(" {}", xml_escape(&stats.language_name)); println!(" {}", stats.file_count); println!(" {}", stats.total_lines); println!(" {}", stats.code_lines); @@ -230,6 +230,20 @@ pub fn print_xml_format(project_analysis: &ProjectAnalysis) -> Result<()> { Ok(()) } +pub fn print_yaml_format(project_analysis: &ProjectAnalysis) -> Result<()> { + let yaml = serde_yaml::to_string(project_analysis) + .map_err(|e| crate::core::error::AnalysisError::invalid_statistics(e.to_string()))?; + println!("{}", yaml); + Ok(()) +} + +pub fn print_toml_format(project_analysis: &ProjectAnalysis) -> Result<()> { + let toml = toml::to_string_pretty(project_analysis) + .map_err(|e| crate::core::error::AnalysisError::invalid_statistics(e.to_string()))?; + println!("{}", toml); + Ok(()) +} + fn xml_escape(text: &str) -> String { text.replace("&", "&") .replace("<", "<") diff --git a/src/core/filter.rs b/src/core/filter.rs index 3855aae..407842d 100644 --- a/src/core/filter.rs +++ b/src/core/filter.rs @@ -317,48 +317,3 @@ impl FilterStats { } } } - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_default_filter() { - let filter = IntelligentFilter::default(); - - assert!(filter.should_process_file("src/main.rs", 1000)); - assert!(!filter.should_process_file(".hidden/file.rs", 1000)); - assert!(!filter.should_process_file("target/debug/main", 1000)); - assert!(!filter.should_process_file("file.exe", 1000)); - assert!(!filter.should_process_file("large_file.rs", 2 * 1024 * 1024)); - } - - #[test] - fn test_aggressive_filter() { - let filter = IntelligentFilter::aggressive(); - - assert!(filter.should_process_file("src/main.rs", 1000)); - assert!(!filter.should_process_file("README.md", 1000)); - assert!(!filter.should_process_file("tests/test.rs", 1000)); - assert!(!filter.should_process_file("docs/guide.rs", 1000)); - } - - #[test] - fn test_build_directory_detection() { - let filter = IntelligentFilter::default(); - - assert!(!filter.should_process_file("target/debug/main.rs", 1000)); - assert!(!filter.should_process_file("build/output/file.rs", 1000)); - assert!(!filter.should_process_file("dist/bundle.js", 1000)); - assert!(filter.should_process_file("src/target_parser.rs", 1000)); - } - - #[test] - fn test_package_directory_detection() { - let filter = IntelligentFilter::default(); - - assert!(!filter.should_process_file("node_modules/package/index.js", 1000)); - assert!(!filter.should_process_file("vendor/package/lib.php", 1000)); - assert!(filter.should_process_file("src/vendor_api.rs", 1000)); - } -} From 20f64b5d6882c46cd43a868aa5c6fb9610018271 Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Thu, 10 Jul 2025 14:50:27 +0800 Subject: [PATCH 03/11] feat: update cli progress style --- src/cli/progress.rs | 9 ++++++++- src/net/mod.rs | 24 ++++++++++++++++-------- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/cli/progress.rs b/src/cli/progress.rs index af2eb4c..e5551ae 100644 --- a/src/cli/progress.rs +++ b/src/cli/progress.rs @@ -10,10 +10,11 @@ pub fn create_progress_bar(show_progress: bool) -> Option { let pb = ProgressBar::new(0); pb.set_style( ProgressStyle::default_spinner() - .template("[{elapsed_precise}] {spinner:.green} {decimal_bytes_per_sec} {msg}") + .template("[{elapsed_precise}] {spinner:.green} {msg} ({decimal_bytes_per_sec})") .unwrap_or_else(|_| ProgressStyle::default_spinner()), ); pb.set_message("Preparing..."); + pb.enable_steady_tick(Duration::from_millis(120)); Some(pb) } @@ -84,6 +85,12 @@ impl ProgressHook for ProgressBarHook { self.progress_bar.set_length(total_size); self.progress_bar.set_position(downloaded); } else { + self.progress_bar.set_style( + ProgressStyle::default_spinner() + .template("[{elapsed_precise}] {spinner:.green} {msg} ({decimal_bytes_per_sec})") + .unwrap_or_else(|_| ProgressStyle::default_spinner()), + ); + self.progress_bar.set_position(downloaded); let formatted = format_bytes(downloaded); self.progress_bar .set_message(format!("Downloaded {}...", formatted)); diff --git a/src/net/mod.rs b/src/net/mod.rs index 27ec794..10504cc 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -6,6 +6,7 @@ use crate::core::{analysis::ProjectAnalysis, error::Result, filter::IntelligentF use providers::*; use reqwest::Client; use std::collections::HashMap; +use std::sync::Arc; use traits::{GitProvider, NoOpProgressHook}; pub use traits::{ParsedRepository, ProgressHook, ProviderConfig}; @@ -35,7 +36,7 @@ pub struct RemoteAnalyzer { providers: Vec>, global_config: ProviderConfig, filter: IntelligentFilter, - progress_hook: Box, + progress_hook: Arc, provider_configs: HashMap, } @@ -46,7 +47,7 @@ impl RemoteAnalyzer { providers: Vec::new(), global_config: ProviderConfig::default(), filter: IntelligentFilter::default(), - progress_hook: Box::new(NoOpProgressHook), + progress_hook: Arc::new(NoOpProgressHook), provider_configs: HashMap::new(), }; @@ -92,7 +93,7 @@ impl RemoteAnalyzer { /// analyzer.set_progress_hook(MyHook); /// ``` pub fn set_progress_hook(&mut self, hook: H) { - self.progress_hook = Box::new(hook); + self.progress_hook = Arc::new(hook); } /// Set global configuration that applies to all providers @@ -430,21 +431,28 @@ impl RemoteAnalyzer { self.progress_hook.on_download_progress(0, total_size); let stream = response.bytes_stream(); + let progress_hook = Arc::clone(&self.progress_hook); let stream_reader = stream::StreamReader::new( stream, - Box::new(|_downloaded, _total| { - // Progress handled by progress hook in stream processing + Box::new(move |downloaded, total| { + progress_hook.on_download_progress(downloaded, total); + log::debug!( + "Downloaded: {} bytes of {} total", + downloaded, + total + .map(|t| t.to_string()) + .unwrap_or_else(|| "unknown".to_string()) + ); }), total_size, ); - self.progress_hook - .on_processing_start("Processing archive..."); + self.progress_hook.on_processing_start("Processing..."); stream::process_tarball_stream( stream_reader, &mut project_analysis, &self.filter, - &*self.progress_hook, + self.progress_hook.as_ref(), ) .await?; From ecd2724c672c680a2e625ebc0cb6e514e195c099 Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Thu, 10 Jul 2025 15:22:13 +0800 Subject: [PATCH 04/11] feat: update cli color --- .github/pull_request_template.md | 8 +- .github/workflows/ci.yml | 104 +-- .github/workflows/release.yml | 70 +- Cargo.lock | 2 +- Cargo.toml | 2 +- README.md | 20 +- docs/CLI_USAGE.md | 126 +-- docs/CONTRIBUTING.md | 8 +- docs/DEPLOYMENT.md | 16 +- examples/README.md | 7 +- package.json | 4 +- pnpm-lock.yaml | 1292 ++++++++++++++++++++++++++++++ server/build.js | 66 +- server/package.json | 2 +- server/tsconfig.json | 2 +- server/types.d.ts | 2 +- server/worker.ts | 201 ++--- src/cli/output.rs | 75 +- src/cli/progress.rs | 4 +- src/languages.json | 825 ++++++++++++++----- tests/README.md | 12 +- 21 files changed, 2348 insertions(+), 500 deletions(-) create mode 100644 pnpm-lock.yaml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 9d1bc06..685fa6d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -23,9 +23,9 @@ Fixes # -- -- -- +- +- +- ## Testing @@ -82,4 +82,4 @@ cargo clippy --all-targets --all-features -- -D warnings ## Additional Notes - \ No newline at end of file + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69c745e..93ef0df 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [ main ] + branches: [main] pull_request: - branches: [ main ] + branches: [main] env: CARGO_TERM_COLOR: always @@ -23,44 +23,44 @@ jobs: experimental: true steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@master - with: - toolchain: ${{ matrix.rust }} - components: rustfmt, clippy - - - name: Cache dependencies - uses: actions/cache@v3 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - - name: Run clippy - run: cargo clippy --all-features - - - name: Run tests - run: cargo test --all-features + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + components: rustfmt, clippy + + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Run clippy + run: cargo clippy --all-features + + - name: Run tests + run: cargo test --all-features security: name: Security Audit runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + - name: Checkout code + uses: actions/checkout@v4 - - name: Install cargo-audit - run: cargo install cargo-audit + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable - - name: Run security audit - run: cargo audit + - name: Install cargo-audit + run: cargo install cargo-audit + + - name: Run security audit + run: cargo audit build: name: Build Release @@ -76,21 +76,21 @@ jobs: target: x86_64-apple-darwin steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} - - - name: Build release - run: cargo build --release --target ${{ matrix.target }} - - - name: Upload binary - uses: actions/upload-artifact@v4 - with: - name: bradar-${{ matrix.target }} - path: | - target/${{ matrix.target }}/release/bradar* - !target/${{ matrix.target }}/release/bradar.d \ No newline at end of file + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} + + - name: Build release + run: cargo build --release --target ${{ matrix.target }} + + - name: Upload binary + uses: actions/upload-artifact@v4 + with: + name: bradar-${{ matrix.target }} + path: | + target/${{ matrix.target }}/release/bradar* + !target/${{ matrix.target }}/release/bradar.d diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c64775c..fb9aed7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Release on: push: tags: - - 'v*' + - "v*" env: CARGO_TERM_COLOR: always @@ -40,50 +40,50 @@ jobs: binary_name: bradar steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ matrix.target }} + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.target }} - - name: Install cross (for Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: cargo install cross + - name: Install cross (for Linux ARM64) + if: matrix.target == 'aarch64-unknown-linux-gnu' + run: cargo install cross - - name: Build release - run: | - if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then - cross build --release --target ${{ matrix.target }} - else - cargo build --release --target ${{ matrix.target }} - fi - shell: bash + - name: Build release + run: | + if [ "${{ matrix.target }}" = "aarch64-unknown-linux-gnu" ]; then + cross build --release --target ${{ matrix.target }} + else + cargo build --release --target ${{ matrix.target }} + fi + shell: bash - - name: Rename binary for release - run: | - mkdir -p release-assets - cp target/${{ matrix.target }}/release/${{ matrix.binary_name }} release-assets/${{ matrix.asset_name }} - shell: bash + - name: Rename binary for release + run: | + mkdir -p release-assets + cp target/${{ matrix.target }}/release/${{ matrix.binary_name }} release-assets/${{ matrix.asset_name }} + shell: bash - - name: Upload Release Asset - uses: softprops/action-gh-release@v2 - with: - files: release-assets/${{ matrix.asset_name }} + - name: Upload Release Asset + uses: softprops/action-gh-release@v2 + with: + files: release-assets/${{ matrix.asset_name }} publish-crate: name: Publish to crates.io needs: build-release runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v4 + - name: Checkout code + uses: actions/checkout@v4 - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@stable - - name: Publish to crates.io - run: cargo publish - env: - CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} \ No newline at end of file + - name: Publish to crates.io + run: cargo publish + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 9cf3be8..76b2793 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -134,7 +134,7 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "bytes-radar" -version = "0.6.0" +version = "1.0.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index fb32e14..d6c3f9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bytes-radar" -version = "0.6.0" +version = "1.0.0" edition = "2021" authors = ["ProgramZmh "] description = "A tool for analyzing code statistics from remote repositories with hyper-fast performance" diff --git a/README.md b/README.md index f627bd2..034aef3 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Deploy to Cloudflare Workers](https://deploy.workers.cloudflare.com/button.svg)](https://deploy.workers.cloudflare.com/?url=https://github.com/zmh-program/bytes-radar) -Hyper-fast **CLOC** *(\*count lines of code)* tool for remote repositories. +Hyper-fast **CLOC** _(\*count lines of code)_ tool for remote repositories. ![Banner](docs/banner.jpg) @@ -117,6 +117,7 @@ See the CLI Options section below for command-line usage. ## Output Formats ### Table (Default) + ```shell $ bradar torvalds/linux Analyzing: https://github.com/torvalds/linux @@ -134,7 +135,7 @@ Analysis completed in 123.76s Code Ratio 80.4% Documentation 8.8% ================================================================================ - Language Files Lines Code Comments Blanks Share% + Language Files Lines Code Comments Blanks % ================================================================================ C 35,586 25,268,107 18,782,347 2,836,806 3,648,954 61.8% C Header 25,845 10,247,647 9,481,722 0 765,925 25.1% @@ -164,6 +165,7 @@ Analysis completed in 123.76s ``` ### JSON Output + ```json { "project_name": "linux@master", @@ -180,13 +182,13 @@ Analysis completed in 123.76s ## Supported Platforms -| Platform | URL Format | Example | -|----------|------------|---------| -| **GitHub** | `user/repo` or full URL | `torvalds/linux` | -| **GitLab** | Full URL | `https://gitlab.com/user/repo` | -| **Bitbucket** | Full URL | `https://bitbucket.org/user/repo` | -| **Codeberg** | Full URL | `https://codeberg.org/user/repo` | -| **Direct** | tar.gz URL | `https://example.com/file.tar.gz` | +| Platform | URL Format | Example | +| ------------- | ----------------------- | --------------------------------- | +| **GitHub** | `user/repo` or full URL | `torvalds/linux` | +| **GitLab** | Full URL | `https://gitlab.com/user/repo` | +| **Bitbucket** | Full URL | `https://bitbucket.org/user/repo` | +| **Codeberg** | Full URL | `https://codeberg.org/user/repo` | +| **Direct** | tar.gz URL | `https://example.com/file.tar.gz` | ## CLI Options diff --git a/docs/CLI_USAGE.md b/docs/CLI_USAGE.md index e23f988..3967a6b 100644 --- a/docs/CLI_USAGE.md +++ b/docs/CLI_USAGE.md @@ -18,6 +18,7 @@ ## Installation ### From Source + ```bash git clone https://github.com/zmh-program/bytes-radar.git cd bytes-radar @@ -25,6 +26,7 @@ cargo build --release ``` ### Using Cargo + ```bash cargo install bytes-radar ``` @@ -52,13 +54,13 @@ This analyzes the default branch of `user/repo` on GitHub and displays results i ## URL Formats -| Format | Description | Example | -|--------|-------------|---------| -| `user/repo` | GitHub repository (default branch) | `microsoft/vscode` | -| `user/repo@branch` | Specific branch | `torvalds/linux@master` | -| `user/repo@commit` | Specific commit hash | `rust-lang/rust@abc123` | -| Full URL | Complete repository URL | `https://github.com/user/repo` | -| Archive URL | Direct archive link | `https://example.com/project.tar.gz` | +| Format | Description | Example | +| ------------------ | ---------------------------------- | ------------------------------------ | +| `user/repo` | GitHub repository (default branch) | `microsoft/vscode` | +| `user/repo@branch` | Specific branch | `torvalds/linux@master` | +| `user/repo@commit` | Specific commit hash | `rust-lang/rust@abc123` | +| Full URL | Complete repository URL | `https://github.com/user/repo` | +| Archive URL | Direct archive link | `https://example.com/project.tar.gz` | ## Command-Line Options @@ -77,84 +79,85 @@ bradar [OPTIONS] ### Output Options -| Option | Short | Description | Default | -|--------|-------|-------------|---------| -| `--format` | `-f` | Output format (table, json, csv, xml, yaml, toml) | `table` | -| `--detailed` | | Show detailed file-by-file statistics | `false` | -| `--quiet` | `-q` | Quiet mode - suppress progress and minimize output | `false` | -| `--no-progress` | | Disable progress bar | `false` | -| `--no-color` | | Disable colored output | `false` | +| Option | Short | Description | Default | +| --------------- | ----- | -------------------------------------------------- | ------- | +| `--format` | `-f` | Output format (table, json, csv, xml, yaml, toml) | `table` | +| `--detailed` | | Show detailed file-by-file statistics | `false` | +| `--quiet` | `-q` | Quiet mode - suppress progress and minimize output | `false` | +| `--no-progress` | | Disable progress bar | `false` | +| `--no-color` | | Disable colored output | `false` | ### Authentication -| Option | Description | -|--------|-------------| +| Option | Description | +| --------- | --------------------------------------------- | | `--token` | Authentication token for private repositories | ### Network Options -| Option | Description | Default | -|--------|-------------|---------| -| `--timeout` | Request timeout in seconds | `300` | -| `--allow-insecure` | Allow insecure HTTPS connections | `false` | -| `--user-agent` | Custom User-Agent string | | -| `--retry-count` | Number of retry attempts for failed requests | `3` | +| Option | Description | Default | +| ------------------ | -------------------------------------------- | ------- | +| `--timeout` | Request timeout in seconds | `300` | +| `--allow-insecure` | Allow insecure HTTPS connections | `false` | +| `--user-agent` | Custom User-Agent string | | +| `--retry-count` | Number of retry attempts for failed requests | `3` | ### Filtering Options -| Option | Description | Default | -|--------|-------------|---------| +| Option | Description | Default | +| --------------------- | --------------------------------------------------- | ------- | | `--aggressive-filter` | Enable aggressive filtering for maximum performance | `false` | -| `--max-file-size` | Maximum file size to process in KB | `1024` | -| `--min-file-size` | Minimum file size to process in bytes | `1` | -| `--include-tests` | Include test directories in analysis | `false` | -| `--include-docs` | Include documentation directories in analysis | `false` | -| `--include-hidden` | Include hidden files and directories | `false` | -| `--exclude-pattern` | Exclude files matching this pattern (glob) | | -| `--include-pattern` | Only include files matching this pattern (glob) | | +| `--max-file-size` | Maximum file size to process in KB | `1024` | +| `--min-file-size` | Minimum file size to process in bytes | `1` | +| `--include-tests` | Include test directories in analysis | `false` | +| `--include-docs` | Include documentation directories in analysis | `false` | +| `--include-hidden` | Include hidden files and directories | `false` | +| `--exclude-pattern` | Exclude files matching this pattern (glob) | | +| `--include-pattern` | Only include files matching this pattern (glob) | | ### Language Options -| Option | Description | -|--------|-------------| -| `--language` | Only analyze files of specific language | +| Option | Description | +| -------------------- | --------------------------------------- | +| `--language` | Only analyze files of specific language | | `--exclude-language` | Exclude specific language from analysis | ### Analysis Options -| Option | Description | Default | -|--------|-------------|---------| -| `--ignore-whitespace` | Ignore whitespace-only lines in code analysis | `false` | -| `--count-generated` | Include generated files in analysis | `false` | -| `--max-line-length` | Maximum line length to consider (0 = unlimited) | `0` | +| Option | Description | Default | +| --------------------- | ----------------------------------------------- | ------- | +| `--ignore-whitespace` | Ignore whitespace-only lines in code analysis | `false` | +| `--count-generated` | Include generated files in analysis | `false` | +| `--max-line-length` | Maximum line length to consider (0 = unlimited) | `0` | ### Debug and Logging -| Option | Short | Description | -|--------|-------|-------------| -| `--debug` | `-d` | Enable debug output | -| `--trace` | | Enable trace-level logging | -| `--log-file` | | Write logs to file | +| Option | Short | Description | +| ------------ | ----- | -------------------------- | +| `--debug` | `-d` | Enable debug output | +| `--trace` | | Enable trace-level logging | +| `--log-file` | | Write logs to file | ### Advanced Options -| Option | Description | Default | -|--------|-------------|---------| -| `--threads` | Number of worker threads (0 = auto) | `0` | -| `--memory-limit` | Memory limit in MB (0 = unlimited) | `0` | -| `--cache-dir` | Directory for caching downloaded files | | -| `--no-cache` | Disable caching of downloaded files | `false` | +| Option | Description | Default | +| ---------------- | -------------------------------------- | ------- | +| `--threads` | Number of worker threads (0 = auto) | `0` | +| `--memory-limit` | Memory limit in MB (0 = unlimited) | `0` | +| `--cache-dir` | Directory for caching downloaded files | | +| `--no-cache` | Disable caching of downloaded files | `false` | ### Experimental Features -| Option | Description | -|--------|-------------| -| `--experimental-parallel` | Enable experimental parallel processing | -| `--experimental-streaming` | Enable experimental streaming analysis | +| Option | Description | +| -------------------------- | --------------------------------------- | +| `--experimental-parallel` | Enable experimental parallel processing | +| `--experimental-streaming` | Enable experimental streaming analysis | ## Output Formats ### Table (Default) + Human-readable table format with colored output and progress indicators. ```bash @@ -162,6 +165,7 @@ bradar microsoft/vscode ``` ### JSON + Machine-readable JSON format for integration with other tools. ```bash @@ -169,6 +173,7 @@ bradar --format json microsoft/vscode ``` ### CSV + Comma-separated values format for spreadsheet analysis. ```bash @@ -176,6 +181,7 @@ bradar --format csv microsoft/vscode ``` ### XML + XML format for structured data processing. ```bash @@ -183,6 +189,7 @@ bradar --format xml microsoft/vscode ``` ### YAML + YAML format for configuration files and human-readable structured data. ```bash @@ -190,6 +197,7 @@ bradar --format yaml microsoft/vscode ``` ### TOML + TOML format for configuration management. ```bash @@ -266,8 +274,8 @@ bradar --allow-insecure https://insecure-server.com/repo.tar.gz ## Environment Variables -| Variable | Description | Example | -|----------|-------------|---------| +| Variable | Description | Example | +| -------------- | ---------------------------- | -------------------------- | | `BRADAR_TOKEN` | Default authentication token | `ghp_xxxxxxxxxxxxxxxxxxxx` | ## Performance Tuning @@ -296,24 +304,28 @@ bradar --allow-insecure https://insecure-server.com/repo.tar.gz ### Common Issues #### Authentication Errors + ```bash # Make sure token has correct permissions bradar --token ghp_xxxxxxxxxxxxxxxxxxxx --debug private-repo ``` #### Network Timeouts + ```bash # Increase timeout and retry count bradar --timeout 600 --retry-count 5 slow-repo ``` #### Memory Issues + ```bash # Set memory limit and use aggressive filtering bradar --memory-limit 2048 --aggressive-filter large-repo ``` #### Slow Analysis + ```bash # Use performance optimizations bradar --aggressive-filter --experimental-parallel --threads 8 repo @@ -364,4 +376,4 @@ done # Monitor repository growth bradar --format json --quiet user/repo | \ jq -r '.global_metrics | "\(.total_files) files, \(.total_lines) lines"' -``` \ No newline at end of file +``` diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 870aa9f..fdbbd8b 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -14,12 +14,14 @@ Thank you for your interest in contributing to bytes-radar! This document provid 1. **Fork the repository** 2. **Clone your fork**: + ```bash git clone https://github.com/YOUR_USERNAME/bytes-radar.git cd bytes-radar ``` 3. **Set up the development environment**: + ```bash cargo build cargo test --all-features @@ -57,6 +59,7 @@ Follow conventional commit format: - `chore:` - Maintenance tasks Examples: + ``` feat: add support for SourceForge repositories fix: handle network timeouts gracefully @@ -190,18 +193,21 @@ bytes-radar/ We welcome contributions in these areas: ### High Priority + - New git platform support - Performance optimizations - Better error messages - More output formats ### Medium Priority + - CLI improvements - Documentation improvements - More language detection - Cross-platform testing ### Low Priority + - Code cleanup - Minor feature additions - Example improvements @@ -253,4 +259,4 @@ By contributing to bytes-radar, you agree that your contributions will be licens --- -Thank you for contributing to bytes-radar! \ No newline at end of file +Thank you for contributing to bytes-radar! diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 5ac8099..58dfb38 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -18,27 +18,32 @@ The server component is automatically built and pushed to the `cf-worker` branch If you prefer to deploy manually: 1. Clone the cf-worker branch which contains the pre-built worker: + ```bash git clone -b cf-worker https://github.com/zmh-program/bytes-radar.git cd bytes-radar/server ``` 2. Install Wrangler CLI: + ```bash pnpm install -g wrangler ``` 3. Authenticate with Cloudflare: + ```bash wrangler login ``` 4. Deploy to staging environment: + ```bash wrangler deploy --env staging ``` 5. Deploy to production: + ```bash wrangler deploy --env production ``` @@ -46,6 +51,7 @@ wrangler deploy --env production ### Environment Configuration The worker supports two environments: + - `staging`: For testing and development (bytes-radar-staging.workers.dev) - `production`: For production use (bytes-radar-prod.workers.dev) @@ -56,6 +62,7 @@ See `server/wrangler.toml` for environment-specific configurations. The Bytes Radar API provides code analysis capabilities through a simple HTTP interface. ### Base URL + ``` https://bradar.zmh.me ``` @@ -63,6 +70,7 @@ https://bradar.zmh.me ### Endpoints #### Analyze Repository + ```http GET /{repository_path} ``` @@ -70,22 +78,26 @@ GET /{repository_path} Analyzes a repository and returns detailed statistics about its codebase. ##### Repository Path Formats + - GitHub repository: `owner/repo` or `owner/repo@branch` - Full GitHub URL: `https://github.com/owner/repo` - GitLab URL: `https://gitlab.com/owner/repo` - Direct archive URL: `https://example.com/archive.tar.gz` ##### Query Parameters + - `ignore_hidden` (boolean, default: true) - Whether to ignore hidden files/directories - `ignore_gitignore` (boolean, default: true) - Whether to respect .gitignore rules - `max_file_size` (number, default: -1) - Maximum file size to analyze in bytes (-1 for no limit) ##### Example Request + ```http GET /zmh-program/bytes-radar ``` ##### Example Response + ```json { "project_name": "bytes-radar@main", @@ -135,6 +147,7 @@ GET /zmh-program/bytes-radar ``` ##### Error Response + ```json { "error": "Error message", @@ -148,11 +161,12 @@ GET /zmh-program/bytes-radar ``` ### Rate Limits and Timeouts + - Request timeout: 20~30 seconds (Free tier) - Rate limits: Based on Cloudflare Workers limits ### Notes + - Large repositories may hit the 20-second timeout limit on the free tier - For analyzing large repositories, consider using the CLI tool or upgrading to Cloudflare Workers Pro - The service automatically tries common branch names (main, master, develop, dev) if not specified - diff --git a/examples/README.md b/examples/README.md index 10a8c48..2550a30 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,6 +13,7 @@ cargo run --example basic_analysis ``` This example analyzes the bytes-radar repository itself and displays: + - Project summary statistics - Language breakdown - Total lines, files, and size information @@ -27,6 +28,7 @@ cargo run --example github_token_analysis ``` Make sure to: + 1. Set the `GITHUB_TOKEN` environment variable 2. Update the repository URL in the example to point to your private repo 3. Ensure your token has access to the repository @@ -40,6 +42,7 @@ cargo run --example compare_repositories ``` This example: + - Analyzes multiple popular repositories - Compares them side by side - Ranks by complexity and documentation ratios @@ -54,6 +57,7 @@ cargo run --example custom_analysis ``` This example shows how to: + - Create FileMetrics manually - Build a ProjectAnalysis step by step - Use different file categories @@ -83,7 +87,8 @@ cargo run --example github_token_analysis ## Customizing Examples Feel free to modify these examples: + - Change repository URLs to analyze your own projects - Adjust timeout values for slower connections - Add error handling for production use -- Experiment with different output formats in the CLI \ No newline at end of file +- Experiment with different output formats in the CLI diff --git a/package.json b/package.json index ca2df87..b3d2a50 100644 --- a/package.json +++ b/package.json @@ -6,12 +6,14 @@ "build": "node build.js", "dev": "wrangler dev", "deploy": "wrangler deploy", - "deploy:prod": "node build.js && wrangler deploy" + "deploy:prod": "node build.js && wrangler deploy", + "prettier": "prettier --write ." }, "dependencies": { "@cloudflare/workers-types": "^4.20240208.0" }, "devDependencies": { + "prettier": "^3.6.2", "typescript": "^5.3.3", "wrangler": "^3.28.1" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml new file mode 100644 index 0000000..befe928 --- /dev/null +++ b/pnpm-lock.yaml @@ -0,0 +1,1292 @@ +lockfileVersion: "9.0" + +settings: + autoInstallPeers: true + excludeLinksFromLockfile: false + +importers: + .: + dependencies: + "@cloudflare/workers-types": + specifier: ^4.20240208.0 + version: 4.20250710.0 + devDependencies: + prettier: + specifier: ^3.6.2 + version: 3.6.2 + typescript: + specifier: ^5.3.3 + version: 5.8.3 + wrangler: + specifier: ^3.28.1 + version: 3.114.11(@cloudflare/workers-types@4.20250710.0) + +packages: + "@cloudflare/kv-asset-handler@0.3.4": + resolution: + { + integrity: sha512-YLPHc8yASwjNkmcDMQMY35yiWjoKAKnhUbPRszBRS0YgH+IXtsMp61j+yTcnCE3oO2DgP0U3iejLC8FTtKDC8Q==, + } + engines: { node: ">=16.13" } + + "@cloudflare/unenv-preset@2.0.2": + resolution: + { + integrity: sha512-nyzYnlZjjV5xT3LizahG1Iu6mnrCaxglJ04rZLpDwlDVDZ7v46lNsfxhV3A/xtfgQuSHmLnc6SVI+KwBpc3Lwg==, + } + peerDependencies: + unenv: 2.0.0-rc.14 + workerd: ^1.20250124.0 + peerDependenciesMeta: + workerd: + optional: true + + "@cloudflare/workerd-darwin-64@1.20250408.0": + resolution: + { + integrity: sha512-bxhIwBWxaNItZLXDNOKY2dCv0FHjDiDkfJFpwv4HvtvU5MKcrivZHVmmfDzLW85rqzfcDOmKbZeMPVfiKxdBZw==, + } + engines: { node: ">=16" } + cpu: [x64] + os: [darwin] + + "@cloudflare/workerd-darwin-arm64@1.20250408.0": + resolution: + { + integrity: sha512-5XZ2Oykr8bSo7zBmERtHh18h5BZYC/6H1YFWVxEj3PtalF3+6SHsO4KZsbGvDml9Pu7sHV277jiZE5eny8Hlyw==, + } + engines: { node: ">=16" } + cpu: [arm64] + os: [darwin] + + "@cloudflare/workerd-linux-64@1.20250408.0": + resolution: + { + integrity: sha512-WbgItXWln6G5d7GvYLWcuOzAVwafysZaWunH3UEfsm95wPuRofpYnlDD861gdWJX10IHSVgMStGESUcs7FLerQ==, + } + engines: { node: ">=16" } + cpu: [x64] + os: [linux] + + "@cloudflare/workerd-linux-arm64@1.20250408.0": + resolution: + { + integrity: sha512-pAhEywPPvr92SLylnQfZEPgXz+9pOG9G9haAPLpEatncZwYiYd9yiR6HYWhKp2erzCoNrOqKg9IlQwU3z1IDiw==, + } + engines: { node: ">=16" } + cpu: [arm64] + os: [linux] + + "@cloudflare/workerd-windows-64@1.20250408.0": + resolution: + { + integrity: sha512-nJ3RjMKGae2aF2rZ/CNeBvQPM+W5V1SUK0FYWG/uomyr7uQ2l4IayHna1ODg/OHHTEgIjwom0Mbn58iXb0WOcQ==, + } + engines: { node: ">=16" } + cpu: [x64] + os: [win32] + + "@cloudflare/workers-types@4.20250710.0": + resolution: + { + integrity: sha512-o055XFgW/ZinLnxDDlm4Q4b2yEmP2x5kekOka/E86blax/RGXj51mBQV/Co3VNog1CePzOA9S/CEOthoIirWRA==, + } + + "@cspotcode/source-map-support@0.8.1": + resolution: + { + integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==, + } + engines: { node: ">=12" } + + "@emnapi/runtime@1.4.4": + resolution: + { + integrity: sha512-hHyapA4A3gPaDCNfiqyZUStTMqIkKRshqPIuDOXv1hcBnD4U3l8cP0T1HMCfGRxQ6V64TGCcoswChANyOAwbQg==, + } + + "@esbuild-plugins/node-globals-polyfill@0.2.3": + resolution: + { + integrity: sha512-r3MIryXDeXDOZh7ih1l/yE9ZLORCd5e8vWg02azWRGj5SPTuoh69A2AIyn0Z31V/kHBfZ4HgWJ+OK3GTTwLmnw==, + } + peerDependencies: + esbuild: "*" + + "@esbuild-plugins/node-modules-polyfill@0.2.2": + resolution: + { + integrity: sha512-LXV7QsWJxRuMYvKbiznh+U1ilIop3g2TeKRzUxOG5X3YITc8JyyTa90BmLwqqv0YnX4v32CSlG+vsziZp9dMvA==, + } + peerDependencies: + esbuild: "*" + + "@esbuild/android-arm64@0.17.19": + resolution: + { + integrity: sha512-KBMWvEZooR7+kzY0BtbTQn0OAYY7CsiydT63pVEaPtVYF0hXbUaOyZog37DKxK7NF3XacBJOpYT4adIJh+avxA==, + } + engines: { node: ">=12" } + cpu: [arm64] + os: [android] + + "@esbuild/android-arm@0.17.19": + resolution: + { + integrity: sha512-rIKddzqhmav7MSmoFCmDIb6e2W57geRsM94gV2l38fzhXMwq7hZoClug9USI2pFRGL06f4IOPHHpFNOkWieR8A==, + } + engines: { node: ">=12" } + cpu: [arm] + os: [android] + + "@esbuild/android-x64@0.17.19": + resolution: + { + integrity: sha512-uUTTc4xGNDT7YSArp/zbtmbhO0uEEK9/ETW29Wk1thYUJBz3IVnvgEiEwEa9IeLyvnpKrWK64Utw2bgUmDveww==, + } + engines: { node: ">=12" } + cpu: [x64] + os: [android] + + "@esbuild/darwin-arm64@0.17.19": + resolution: + { + integrity: sha512-80wEoCfF/hFKM6WE1FyBHc9SfUblloAWx6FJkFWTWiCoht9Mc0ARGEM47e67W9rI09YoUxJL68WHfDRYEAvOhg==, + } + engines: { node: ">=12" } + cpu: [arm64] + os: [darwin] + + "@esbuild/darwin-x64@0.17.19": + resolution: + { + integrity: sha512-IJM4JJsLhRYr9xdtLytPLSH9k/oxR3boaUIYiHkAawtwNOXKE8KoU8tMvryogdcT8AU+Bflmh81Xn6Q0vTZbQw==, + } + engines: { node: ">=12" } + cpu: [x64] + os: [darwin] + + "@esbuild/freebsd-arm64@0.17.19": + resolution: + { + integrity: sha512-pBwbc7DufluUeGdjSU5Si+P3SoMF5DQ/F/UmTSb8HXO80ZEAJmrykPyzo1IfNbAoaqw48YRpv8shwd1NoI0jcQ==, + } + engines: { node: ">=12" } + cpu: [arm64] + os: [freebsd] + + "@esbuild/freebsd-x64@0.17.19": + resolution: + { + integrity: sha512-4lu+n8Wk0XlajEhbEffdy2xy53dpR06SlzvhGByyg36qJw6Kpfk7cp45DR/62aPH9mtJRmIyrXAS5UWBrJT6TQ==, + } + engines: { node: ">=12" } + cpu: [x64] + os: [freebsd] + + "@esbuild/linux-arm64@0.17.19": + resolution: + { + integrity: sha512-ct1Tg3WGwd3P+oZYqic+YZF4snNl2bsnMKRkb3ozHmnM0dGWuxcPTTntAF6bOP0Sp4x0PjSF+4uHQ1xvxfRKqg==, + } + engines: { node: ">=12" } + cpu: [arm64] + os: [linux] + + "@esbuild/linux-arm@0.17.19": + resolution: + { + integrity: sha512-cdmT3KxjlOQ/gZ2cjfrQOtmhG4HJs6hhvm3mWSRDPtZ/lP5oe8FWceS10JaSJC13GBd4eH/haHnqf7hhGNLerA==, + } + engines: { node: ">=12" } + cpu: [arm] + os: [linux] + + "@esbuild/linux-ia32@0.17.19": + resolution: + { + integrity: sha512-w4IRhSy1VbsNxHRQpeGCHEmibqdTUx61Vc38APcsRbuVgK0OPEnQ0YD39Brymn96mOx48Y2laBQGqgZ0j9w6SQ==, + } + engines: { node: ">=12" } + cpu: [ia32] + os: [linux] + + "@esbuild/linux-loong64@0.17.19": + resolution: + { + integrity: sha512-2iAngUbBPMq439a+z//gE+9WBldoMp1s5GWsUSgqHLzLJ9WoZLZhpwWuym0u0u/4XmZ3gpHmzV84PonE+9IIdQ==, + } + engines: { node: ">=12" } + cpu: [loong64] + os: [linux] + + "@esbuild/linux-mips64el@0.17.19": + resolution: + { + integrity: sha512-LKJltc4LVdMKHsrFe4MGNPp0hqDFA1Wpt3jE1gEyM3nKUvOiO//9PheZZHfYRfYl6AwdTH4aTcXSqBerX0ml4A==, + } + engines: { node: ">=12" } + cpu: [mips64el] + os: [linux] + + "@esbuild/linux-ppc64@0.17.19": + resolution: + { + integrity: sha512-/c/DGybs95WXNS8y3Ti/ytqETiW7EU44MEKuCAcpPto3YjQbyK3IQVKfF6nbghD7EcLUGl0NbiL5Rt5DMhn5tg==, + } + engines: { node: ">=12" } + cpu: [ppc64] + os: [linux] + + "@esbuild/linux-riscv64@0.17.19": + resolution: + { + integrity: sha512-FC3nUAWhvFoutlhAkgHf8f5HwFWUL6bYdvLc/TTuxKlvLi3+pPzdZiFKSWz/PF30TB1K19SuCxDTI5KcqASJqA==, + } + engines: { node: ">=12" } + cpu: [riscv64] + os: [linux] + + "@esbuild/linux-s390x@0.17.19": + resolution: + { + integrity: sha512-IbFsFbxMWLuKEbH+7sTkKzL6NJmG2vRyy6K7JJo55w+8xDk7RElYn6xvXtDW8HCfoKBFK69f3pgBJSUSQPr+4Q==, + } + engines: { node: ">=12" } + cpu: [s390x] + os: [linux] + + "@esbuild/linux-x64@0.17.19": + resolution: + { + integrity: sha512-68ngA9lg2H6zkZcyp22tsVt38mlhWde8l3eJLWkyLrp4HwMUr3c1s/M2t7+kHIhvMjglIBrFpncX1SzMckomGw==, + } + engines: { node: ">=12" } + cpu: [x64] + os: [linux] + + "@esbuild/netbsd-x64@0.17.19": + resolution: + { + integrity: sha512-CwFq42rXCR8TYIjIfpXCbRX0rp1jo6cPIUPSaWwzbVI4aOfX96OXY8M6KNmtPcg7QjYeDmN+DD0Wp3LaBOLf4Q==, + } + engines: { node: ">=12" } + cpu: [x64] + os: [netbsd] + + "@esbuild/openbsd-x64@0.17.19": + resolution: + { + integrity: sha512-cnq5brJYrSZ2CF6c35eCmviIN3k3RczmHz8eYaVlNasVqsNY+JKohZU5MKmaOI+KkllCdzOKKdPs762VCPC20g==, + } + engines: { node: ">=12" } + cpu: [x64] + os: [openbsd] + + "@esbuild/sunos-x64@0.17.19": + resolution: + { + integrity: sha512-vCRT7yP3zX+bKWFeP/zdS6SqdWB8OIpaRq/mbXQxTGHnIxspRtigpkUcDMlSCOejlHowLqII7K2JKevwyRP2rg==, + } + engines: { node: ">=12" } + cpu: [x64] + os: [sunos] + + "@esbuild/win32-arm64@0.17.19": + resolution: + { + integrity: sha512-yYx+8jwowUstVdorcMdNlzklLYhPxjniHWFKgRqH7IFlUEa0Umu3KuYplf1HUZZ422e3NU9F4LGb+4O0Kdcaag==, + } + engines: { node: ">=12" } + cpu: [arm64] + os: [win32] + + "@esbuild/win32-ia32@0.17.19": + resolution: + { + integrity: sha512-eggDKanJszUtCdlVs0RB+h35wNlb5v4TWEkq4vZcmVt5u/HiDZrTXe2bWFQUez3RgNHwx/x4sk5++4NSSicKkw==, + } + engines: { node: ">=12" } + cpu: [ia32] + os: [win32] + + "@esbuild/win32-x64@0.17.19": + resolution: + { + integrity: sha512-lAhycmKnVOuRYNtRtatQR1LPQf2oYCkRGkSFnseDAKPl8lu5SOsK/e1sXe5a0Pc5kHIHe6P2I/ilntNv2xf3cA==, + } + engines: { node: ">=12" } + cpu: [x64] + os: [win32] + + "@fastify/busboy@2.1.1": + resolution: + { + integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==, + } + engines: { node: ">=14" } + + "@img/sharp-darwin-arm64@0.33.5": + resolution: + { + integrity: sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [arm64] + os: [darwin] + + "@img/sharp-darwin-x64@0.33.5": + resolution: + { + integrity: sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [x64] + os: [darwin] + + "@img/sharp-libvips-darwin-arm64@1.0.4": + resolution: + { + integrity: sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==, + } + cpu: [arm64] + os: [darwin] + + "@img/sharp-libvips-darwin-x64@1.0.4": + resolution: + { + integrity: sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==, + } + cpu: [x64] + os: [darwin] + + "@img/sharp-libvips-linux-arm64@1.0.4": + resolution: + { + integrity: sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==, + } + cpu: [arm64] + os: [linux] + + "@img/sharp-libvips-linux-arm@1.0.5": + resolution: + { + integrity: sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==, + } + cpu: [arm] + os: [linux] + + "@img/sharp-libvips-linux-s390x@1.0.4": + resolution: + { + integrity: sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==, + } + cpu: [s390x] + os: [linux] + + "@img/sharp-libvips-linux-x64@1.0.4": + resolution: + { + integrity: sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==, + } + cpu: [x64] + os: [linux] + + "@img/sharp-libvips-linuxmusl-arm64@1.0.4": + resolution: + { + integrity: sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==, + } + cpu: [arm64] + os: [linux] + + "@img/sharp-libvips-linuxmusl-x64@1.0.4": + resolution: + { + integrity: sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==, + } + cpu: [x64] + os: [linux] + + "@img/sharp-linux-arm64@0.33.5": + resolution: + { + integrity: sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [arm64] + os: [linux] + + "@img/sharp-linux-arm@0.33.5": + resolution: + { + integrity: sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [arm] + os: [linux] + + "@img/sharp-linux-s390x@0.33.5": + resolution: + { + integrity: sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [s390x] + os: [linux] + + "@img/sharp-linux-x64@0.33.5": + resolution: + { + integrity: sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [x64] + os: [linux] + + "@img/sharp-linuxmusl-arm64@0.33.5": + resolution: + { + integrity: sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [arm64] + os: [linux] + + "@img/sharp-linuxmusl-x64@0.33.5": + resolution: + { + integrity: sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [x64] + os: [linux] + + "@img/sharp-wasm32@0.33.5": + resolution: + { + integrity: sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [wasm32] + + "@img/sharp-win32-ia32@0.33.5": + resolution: + { + integrity: sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [ia32] + os: [win32] + + "@img/sharp-win32-x64@0.33.5": + resolution: + { + integrity: sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + cpu: [x64] + os: [win32] + + "@jridgewell/resolve-uri@3.1.2": + resolution: + { + integrity: sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==, + } + engines: { node: ">=6.0.0" } + + "@jridgewell/sourcemap-codec@1.5.4": + resolution: + { + integrity: sha512-VT2+G1VQs/9oz078bLrYbecdZKs912zQlkelYpuf+SXF+QvZDYJlbx/LSx+meSAwdDFnF8FVXW92AVjjkVmgFw==, + } + + "@jridgewell/trace-mapping@0.3.9": + resolution: + { + integrity: sha512-3Belt6tdc8bPgAtbcmdtNJlirVoTmEb5e2gC94PnkwEW9jI6CAHUeoG85tjWP5WquqfavoMtMwiG4P926ZKKuQ==, + } + + acorn-walk@8.3.2: + resolution: + { + integrity: sha512-cjkyv4OtNCIeqhHrfS81QWXoCBPExR/J62oyEqepVw8WaQeSqpW2uhuLPh1m9eWhDuOo/jUXVTlifvesOWp/4A==, + } + engines: { node: ">=0.4.0" } + + acorn@8.14.0: + resolution: + { + integrity: sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==, + } + engines: { node: ">=0.4.0" } + hasBin: true + + as-table@1.0.55: + resolution: + { + integrity: sha512-xvsWESUJn0JN421Xb9MQw6AsMHRCUknCe0Wjlxvjud80mU4E6hQf1A6NzQKcYNmYw62MfzEtXc+badstZP3JpQ==, + } + + blake3-wasm@2.1.5: + resolution: + { + integrity: sha512-F1+K8EbfOZE49dtoPtmxUQrpXaBIl3ICvasLh+nJta0xkz+9kF/7uet9fLnwKqhDrmj6g+6K3Tw9yQPUg2ka5g==, + } + + color-convert@2.0.1: + resolution: + { + integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==, + } + engines: { node: ">=7.0.0" } + + color-name@1.1.4: + resolution: + { + integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==, + } + + color-string@1.9.1: + resolution: + { + integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==, + } + + color@4.2.3: + resolution: + { + integrity: sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==, + } + engines: { node: ">=12.5.0" } + + cookie@0.7.2: + resolution: + { + integrity: sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==, + } + engines: { node: ">= 0.6" } + + data-uri-to-buffer@2.0.2: + resolution: + { + integrity: sha512-ND9qDTLc6diwj+Xe5cdAgVTbLVdXbtxTJRXRhli8Mowuaan+0EJOtdqJ0QCHNSSPyoXGx9HX2/VMnKeC34AChA==, + } + + defu@6.1.4: + resolution: + { + integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==, + } + + detect-libc@2.0.4: + resolution: + { + integrity: sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==, + } + engines: { node: ">=8" } + + esbuild@0.17.19: + resolution: + { + integrity: sha512-XQ0jAPFkK/u3LcVRcvVHQcTIqD6E2H1fvZMA5dQPSOWb3suUbWbfbRf94pjc0bNzRYLfIrDRQXr7X+LHIm5oHw==, + } + engines: { node: ">=12" } + hasBin: true + + escape-string-regexp@4.0.0: + resolution: + { + integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==, + } + engines: { node: ">=10" } + + estree-walker@0.6.1: + resolution: + { + integrity: sha512-SqmZANLWS0mnatqbSfRP5g8OXZC12Fgg1IwNtLsyHDzJizORW4khDfjPqJZsemPWBB2uqykUah5YpQ6epsqC/w==, + } + + exit-hook@2.2.1: + resolution: + { + integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==, + } + engines: { node: ">=6" } + + exsolve@1.0.7: + resolution: + { + integrity: sha512-VO5fQUzZtI6C+vx4w/4BWJpg3s/5l+6pRQEHzFRM8WFi4XffSP1Z+4qi7GbjWbvRQEbdIco5mIMq+zX4rPuLrw==, + } + + fsevents@2.3.3: + resolution: + { + integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==, + } + engines: { node: ^8.16.0 || ^10.6.0 || >=11.0.0 } + os: [darwin] + + get-source@2.0.12: + resolution: + { + integrity: sha512-X5+4+iD+HoSeEED+uwrQ07BOQr0kEDFMVqqpBuI+RaZBpBpHCuXxo70bjar6f0b0u/DQJsJ7ssurpP0V60Az+w==, + } + + glob-to-regexp@0.4.1: + resolution: + { + integrity: sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==, + } + + is-arrayish@0.3.2: + resolution: + { + integrity: sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==, + } + + magic-string@0.25.9: + resolution: + { + integrity: sha512-RmF0AsMzgt25qzqqLc1+MbHmhdx0ojF2Fvs4XnOqz2ZOBXzzkEwc/dJQZCYHAn7v1jbVOjAZfK8msRn4BxO4VQ==, + } + + mime@3.0.0: + resolution: + { + integrity: sha512-jSCU7/VB1loIWBZe14aEYHU/+1UMEHoaO7qxCOVJOw9GgH72VAWppxNcjU+x9a2k3GSIBXNKxXQFqRvvZ7vr3A==, + } + engines: { node: ">=10.0.0" } + hasBin: true + + miniflare@3.20250408.2: + resolution: + { + integrity: sha512-uTs7cGWFErgJTKtBdmtctwhuoxniuCQqDT8+xaEiJdEC8d+HsaZVYfZwIX2NuSmdAiHMe7NtbdZYjFMbIXtJsQ==, + } + engines: { node: ">=16.13" } + hasBin: true + + mustache@4.2.0: + resolution: + { + integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==, + } + hasBin: true + + ohash@2.0.11: + resolution: + { + integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==, + } + + path-to-regexp@6.3.0: + resolution: + { + integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==, + } + + pathe@2.0.3: + resolution: + { + integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==, + } + + prettier@3.6.2: + resolution: + { + integrity: sha512-I7AIg5boAr5R0FFtJ6rCfD+LFsWHp81dolrFD8S79U9tb8Az2nGrJncnMSnys+bpQJfRUzqs9hnA81OAA3hCuQ==, + } + engines: { node: ">=14" } + hasBin: true + + printable-characters@1.0.42: + resolution: + { + integrity: sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==, + } + + rollup-plugin-inject@3.0.2: + resolution: + { + integrity: sha512-ptg9PQwzs3orn4jkgXJ74bfs5vYz1NCZlSQMBUA0wKcGp5i5pA1AO3fOUEte8enhGUC+iapTCzEWw2jEFFUO/w==, + } + deprecated: This package has been deprecated and is no longer maintained. Please use @rollup/plugin-inject. + + rollup-plugin-node-polyfills@0.2.1: + resolution: + { + integrity: sha512-4kCrKPTJ6sK4/gLL/U5QzVT8cxJcofO0OU74tnB19F40cmuAKSzH5/siithxlofFEjwvw1YAhPmbvGNA6jEroA==, + } + + rollup-pluginutils@2.8.2: + resolution: + { + integrity: sha512-EEp9NhnUkwY8aif6bxgovPHMoMoNr2FulJziTndpt5H9RdwC47GSGuII9XxpSdzVGM0GWrNPHV6ie1LTNJPaLQ==, + } + + semver@7.7.2: + resolution: + { + integrity: sha512-RF0Fw+rO5AMf9MAyaRXI4AV0Ulj5lMHqVxxdSgiVbixSCXoEmmX/jk0CuJw4+3SqroYO9VoUh+HcuJivvtJemA==, + } + engines: { node: ">=10" } + hasBin: true + + sharp@0.33.5: + resolution: + { + integrity: sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==, + } + engines: { node: ^18.17.0 || ^20.3.0 || >=21.0.0 } + + simple-swizzle@0.2.2: + resolution: + { + integrity: sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==, + } + + source-map@0.6.1: + resolution: + { + integrity: sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==, + } + engines: { node: ">=0.10.0" } + + sourcemap-codec@1.4.8: + resolution: + { + integrity: sha512-9NykojV5Uih4lgo5So5dtw+f0JgJX30KCNI8gwhz2J9A15wD0Ml6tjHKwf6fTSa6fAdVBdZeNOs9eJ71qCk8vA==, + } + deprecated: Please use @jridgewell/sourcemap-codec instead + + stacktracey@2.1.8: + resolution: + { + integrity: sha512-Kpij9riA+UNg7TnphqjH7/CzctQ/owJGNbFkfEeve4Z4uxT5+JapVLFXcsurIfN34gnTWZNJ/f7NMG0E8JDzTw==, + } + + stoppable@1.1.0: + resolution: + { + integrity: sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==, + } + engines: { node: ">=4", npm: ">=6" } + + tslib@2.8.1: + resolution: + { + integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==, + } + + typescript@5.8.3: + resolution: + { + integrity: sha512-p1diW6TqL9L07nNxvRMM7hMMw4c5XOo/1ibL4aAIGmSAt9slTE1Xgw5KWuof2uTOvCg9BY7ZRi+GaF+7sfgPeQ==, + } + engines: { node: ">=14.17" } + hasBin: true + + ufo@1.6.1: + resolution: + { + integrity: sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA==, + } + + undici@5.29.0: + resolution: + { + integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==, + } + engines: { node: ">=14.0" } + + unenv@2.0.0-rc.14: + resolution: + { + integrity: sha512-od496pShMen7nOy5VmVJCnq8rptd45vh6Nx/r2iPbrba6pa6p+tS2ywuIHRZ/OBvSbQZB0kWvpO9XBNVFXHD3Q==, + } + + workerd@1.20250408.0: + resolution: + { + integrity: sha512-bBUX+UsvpzAqiWFNeZrlZmDGddiGZdBBbftZJz2wE6iUg/cIAJeVQYTtS/3ahaicguoLBz4nJiDo8luqM9fx1A==, + } + engines: { node: ">=16" } + hasBin: true + + wrangler@3.114.11: + resolution: + { + integrity: sha512-g0KhNj0AzlDXrW/XNzOrbfjBnDHbmf3a+3+UR67BbKnS8EUC9yTPrKCRymERdFKfiSvlPs34tHGAzOTw62Jb4g==, + } + engines: { node: ">=16.17.0" } + hasBin: true + peerDependencies: + "@cloudflare/workers-types": ^4.20250408.0 + peerDependenciesMeta: + "@cloudflare/workers-types": + optional: true + + ws@8.18.0: + resolution: + { + integrity: sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==, + } + engines: { node: ">=10.0.0" } + peerDependencies: + bufferutil: ^4.0.1 + utf-8-validate: ">=5.0.2" + peerDependenciesMeta: + bufferutil: + optional: true + utf-8-validate: + optional: true + + youch@3.3.4: + resolution: + { + integrity: sha512-UeVBXie8cA35DS6+nBkls68xaBBXCye0CNznrhszZjTbRVnJKQuNsyLKBTTL4ln1o1rh2PKtv35twV7irj5SEg==, + } + + zod@3.22.3: + resolution: + { + integrity: sha512-EjIevzuJRiRPbVH4mGc8nApb/lVLKVpmUhAaR5R5doKGfAnGJ6Gr3CViAVjP+4FWSxCsybeWQdcgCtbX+7oZug==, + } + +snapshots: + "@cloudflare/kv-asset-handler@0.3.4": + dependencies: + mime: 3.0.0 + + "@cloudflare/unenv-preset@2.0.2(unenv@2.0.0-rc.14)(workerd@1.20250408.0)": + dependencies: + unenv: 2.0.0-rc.14 + optionalDependencies: + workerd: 1.20250408.0 + + "@cloudflare/workerd-darwin-64@1.20250408.0": + optional: true + + "@cloudflare/workerd-darwin-arm64@1.20250408.0": + optional: true + + "@cloudflare/workerd-linux-64@1.20250408.0": + optional: true + + "@cloudflare/workerd-linux-arm64@1.20250408.0": + optional: true + + "@cloudflare/workerd-windows-64@1.20250408.0": + optional: true + + "@cloudflare/workers-types@4.20250710.0": {} + + "@cspotcode/source-map-support@0.8.1": + dependencies: + "@jridgewell/trace-mapping": 0.3.9 + + "@emnapi/runtime@1.4.4": + dependencies: + tslib: 2.8.1 + optional: true + + "@esbuild-plugins/node-globals-polyfill@0.2.3(esbuild@0.17.19)": + dependencies: + esbuild: 0.17.19 + + "@esbuild-plugins/node-modules-polyfill@0.2.2(esbuild@0.17.19)": + dependencies: + esbuild: 0.17.19 + escape-string-regexp: 4.0.0 + rollup-plugin-node-polyfills: 0.2.1 + + "@esbuild/android-arm64@0.17.19": + optional: true + + "@esbuild/android-arm@0.17.19": + optional: true + + "@esbuild/android-x64@0.17.19": + optional: true + + "@esbuild/darwin-arm64@0.17.19": + optional: true + + "@esbuild/darwin-x64@0.17.19": + optional: true + + "@esbuild/freebsd-arm64@0.17.19": + optional: true + + "@esbuild/freebsd-x64@0.17.19": + optional: true + + "@esbuild/linux-arm64@0.17.19": + optional: true + + "@esbuild/linux-arm@0.17.19": + optional: true + + "@esbuild/linux-ia32@0.17.19": + optional: true + + "@esbuild/linux-loong64@0.17.19": + optional: true + + "@esbuild/linux-mips64el@0.17.19": + optional: true + + "@esbuild/linux-ppc64@0.17.19": + optional: true + + "@esbuild/linux-riscv64@0.17.19": + optional: true + + "@esbuild/linux-s390x@0.17.19": + optional: true + + "@esbuild/linux-x64@0.17.19": + optional: true + + "@esbuild/netbsd-x64@0.17.19": + optional: true + + "@esbuild/openbsd-x64@0.17.19": + optional: true + + "@esbuild/sunos-x64@0.17.19": + optional: true + + "@esbuild/win32-arm64@0.17.19": + optional: true + + "@esbuild/win32-ia32@0.17.19": + optional: true + + "@esbuild/win32-x64@0.17.19": + optional: true + + "@fastify/busboy@2.1.1": {} + + "@img/sharp-darwin-arm64@0.33.5": + optionalDependencies: + "@img/sharp-libvips-darwin-arm64": 1.0.4 + optional: true + + "@img/sharp-darwin-x64@0.33.5": + optionalDependencies: + "@img/sharp-libvips-darwin-x64": 1.0.4 + optional: true + + "@img/sharp-libvips-darwin-arm64@1.0.4": + optional: true + + "@img/sharp-libvips-darwin-x64@1.0.4": + optional: true + + "@img/sharp-libvips-linux-arm64@1.0.4": + optional: true + + "@img/sharp-libvips-linux-arm@1.0.5": + optional: true + + "@img/sharp-libvips-linux-s390x@1.0.4": + optional: true + + "@img/sharp-libvips-linux-x64@1.0.4": + optional: true + + "@img/sharp-libvips-linuxmusl-arm64@1.0.4": + optional: true + + "@img/sharp-libvips-linuxmusl-x64@1.0.4": + optional: true + + "@img/sharp-linux-arm64@0.33.5": + optionalDependencies: + "@img/sharp-libvips-linux-arm64": 1.0.4 + optional: true + + "@img/sharp-linux-arm@0.33.5": + optionalDependencies: + "@img/sharp-libvips-linux-arm": 1.0.5 + optional: true + + "@img/sharp-linux-s390x@0.33.5": + optionalDependencies: + "@img/sharp-libvips-linux-s390x": 1.0.4 + optional: true + + "@img/sharp-linux-x64@0.33.5": + optionalDependencies: + "@img/sharp-libvips-linux-x64": 1.0.4 + optional: true + + "@img/sharp-linuxmusl-arm64@0.33.5": + optionalDependencies: + "@img/sharp-libvips-linuxmusl-arm64": 1.0.4 + optional: true + + "@img/sharp-linuxmusl-x64@0.33.5": + optionalDependencies: + "@img/sharp-libvips-linuxmusl-x64": 1.0.4 + optional: true + + "@img/sharp-wasm32@0.33.5": + dependencies: + "@emnapi/runtime": 1.4.4 + optional: true + + "@img/sharp-win32-ia32@0.33.5": + optional: true + + "@img/sharp-win32-x64@0.33.5": + optional: true + + "@jridgewell/resolve-uri@3.1.2": {} + + "@jridgewell/sourcemap-codec@1.5.4": {} + + "@jridgewell/trace-mapping@0.3.9": + dependencies: + "@jridgewell/resolve-uri": 3.1.2 + "@jridgewell/sourcemap-codec": 1.5.4 + + acorn-walk@8.3.2: {} + + acorn@8.14.0: {} + + as-table@1.0.55: + dependencies: + printable-characters: 1.0.42 + + blake3-wasm@2.1.5: {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + optional: true + + color-name@1.1.4: + optional: true + + color-string@1.9.1: + dependencies: + color-name: 1.1.4 + simple-swizzle: 0.2.2 + optional: true + + color@4.2.3: + dependencies: + color-convert: 2.0.1 + color-string: 1.9.1 + optional: true + + cookie@0.7.2: {} + + data-uri-to-buffer@2.0.2: {} + + defu@6.1.4: {} + + detect-libc@2.0.4: + optional: true + + esbuild@0.17.19: + optionalDependencies: + "@esbuild/android-arm": 0.17.19 + "@esbuild/android-arm64": 0.17.19 + "@esbuild/android-x64": 0.17.19 + "@esbuild/darwin-arm64": 0.17.19 + "@esbuild/darwin-x64": 0.17.19 + "@esbuild/freebsd-arm64": 0.17.19 + "@esbuild/freebsd-x64": 0.17.19 + "@esbuild/linux-arm": 0.17.19 + "@esbuild/linux-arm64": 0.17.19 + "@esbuild/linux-ia32": 0.17.19 + "@esbuild/linux-loong64": 0.17.19 + "@esbuild/linux-mips64el": 0.17.19 + "@esbuild/linux-ppc64": 0.17.19 + "@esbuild/linux-riscv64": 0.17.19 + "@esbuild/linux-s390x": 0.17.19 + "@esbuild/linux-x64": 0.17.19 + "@esbuild/netbsd-x64": 0.17.19 + "@esbuild/openbsd-x64": 0.17.19 + "@esbuild/sunos-x64": 0.17.19 + "@esbuild/win32-arm64": 0.17.19 + "@esbuild/win32-ia32": 0.17.19 + "@esbuild/win32-x64": 0.17.19 + + escape-string-regexp@4.0.0: {} + + estree-walker@0.6.1: {} + + exit-hook@2.2.1: {} + + exsolve@1.0.7: {} + + fsevents@2.3.3: + optional: true + + get-source@2.0.12: + dependencies: + data-uri-to-buffer: 2.0.2 + source-map: 0.6.1 + + glob-to-regexp@0.4.1: {} + + is-arrayish@0.3.2: + optional: true + + magic-string@0.25.9: + dependencies: + sourcemap-codec: 1.4.8 + + mime@3.0.0: {} + + miniflare@3.20250408.2: + dependencies: + "@cspotcode/source-map-support": 0.8.1 + acorn: 8.14.0 + acorn-walk: 8.3.2 + exit-hook: 2.2.1 + glob-to-regexp: 0.4.1 + stoppable: 1.1.0 + undici: 5.29.0 + workerd: 1.20250408.0 + ws: 8.18.0 + youch: 3.3.4 + zod: 3.22.3 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + mustache@4.2.0: {} + + ohash@2.0.11: {} + + path-to-regexp@6.3.0: {} + + pathe@2.0.3: {} + + prettier@3.6.2: {} + + printable-characters@1.0.42: {} + + rollup-plugin-inject@3.0.2: + dependencies: + estree-walker: 0.6.1 + magic-string: 0.25.9 + rollup-pluginutils: 2.8.2 + + rollup-plugin-node-polyfills@0.2.1: + dependencies: + rollup-plugin-inject: 3.0.2 + + rollup-pluginutils@2.8.2: + dependencies: + estree-walker: 0.6.1 + + semver@7.7.2: + optional: true + + sharp@0.33.5: + dependencies: + color: 4.2.3 + detect-libc: 2.0.4 + semver: 7.7.2 + optionalDependencies: + "@img/sharp-darwin-arm64": 0.33.5 + "@img/sharp-darwin-x64": 0.33.5 + "@img/sharp-libvips-darwin-arm64": 1.0.4 + "@img/sharp-libvips-darwin-x64": 1.0.4 + "@img/sharp-libvips-linux-arm": 1.0.5 + "@img/sharp-libvips-linux-arm64": 1.0.4 + "@img/sharp-libvips-linux-s390x": 1.0.4 + "@img/sharp-libvips-linux-x64": 1.0.4 + "@img/sharp-libvips-linuxmusl-arm64": 1.0.4 + "@img/sharp-libvips-linuxmusl-x64": 1.0.4 + "@img/sharp-linux-arm": 0.33.5 + "@img/sharp-linux-arm64": 0.33.5 + "@img/sharp-linux-s390x": 0.33.5 + "@img/sharp-linux-x64": 0.33.5 + "@img/sharp-linuxmusl-arm64": 0.33.5 + "@img/sharp-linuxmusl-x64": 0.33.5 + "@img/sharp-wasm32": 0.33.5 + "@img/sharp-win32-ia32": 0.33.5 + "@img/sharp-win32-x64": 0.33.5 + optional: true + + simple-swizzle@0.2.2: + dependencies: + is-arrayish: 0.3.2 + optional: true + + source-map@0.6.1: {} + + sourcemap-codec@1.4.8: {} + + stacktracey@2.1.8: + dependencies: + as-table: 1.0.55 + get-source: 2.0.12 + + stoppable@1.1.0: {} + + tslib@2.8.1: + optional: true + + typescript@5.8.3: {} + + ufo@1.6.1: {} + + undici@5.29.0: + dependencies: + "@fastify/busboy": 2.1.1 + + unenv@2.0.0-rc.14: + dependencies: + defu: 6.1.4 + exsolve: 1.0.7 + ohash: 2.0.11 + pathe: 2.0.3 + ufo: 1.6.1 + + workerd@1.20250408.0: + optionalDependencies: + "@cloudflare/workerd-darwin-64": 1.20250408.0 + "@cloudflare/workerd-darwin-arm64": 1.20250408.0 + "@cloudflare/workerd-linux-64": 1.20250408.0 + "@cloudflare/workerd-linux-arm64": 1.20250408.0 + "@cloudflare/workerd-windows-64": 1.20250408.0 + + wrangler@3.114.11(@cloudflare/workers-types@4.20250710.0): + dependencies: + "@cloudflare/kv-asset-handler": 0.3.4 + "@cloudflare/unenv-preset": 2.0.2(unenv@2.0.0-rc.14)(workerd@1.20250408.0) + "@esbuild-plugins/node-globals-polyfill": 0.2.3(esbuild@0.17.19) + "@esbuild-plugins/node-modules-polyfill": 0.2.2(esbuild@0.17.19) + blake3-wasm: 2.1.5 + esbuild: 0.17.19 + miniflare: 3.20250408.2 + path-to-regexp: 6.3.0 + unenv: 2.0.0-rc.14 + workerd: 1.20250408.0 + optionalDependencies: + "@cloudflare/workers-types": 4.20250710.0 + fsevents: 2.3.3 + sharp: 0.33.5 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + ws@8.18.0: {} + + youch@3.3.4: + dependencies: + cookie: 0.7.2 + mustache: 4.2.0 + stacktracey: 2.1.8 + + zod@3.22.3: {} diff --git a/server/build.js b/server/build.js index c1e14c6..f887b4f 100644 --- a/server/build.js +++ b/server/build.js @@ -1,44 +1,53 @@ -const { execSync } = require('child_process'); -const fs = require('fs'); -const path = require('path'); +const { execSync } = require("child_process"); +const fs = require("fs"); +const path = require("path"); // Install Rust if not installed -console.log('Installing Rust toolchain...'); +console.log("Installing Rust toolchain..."); try { - execSync('rustc --version'); - console.log('Rust already installed'); + execSync("rustc --version"); + console.log("Rust already installed"); } catch { - console.log('Installing Rust...'); - execSync('curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y', { - stdio: 'inherit' - }); + console.log("Installing Rust..."); + execSync( + 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y', + { + stdio: "inherit", + }, + ); // Add cargo to PATH process.env.PATH = `${process.env.HOME}/.cargo/bin:${process.env.PATH}`; } // Install wasm-pack if not installed -console.log('Installing wasm-pack...'); +console.log("Installing wasm-pack..."); try { - execSync('wasm-pack --version'); - console.log('wasm-pack already installed'); + execSync("wasm-pack --version"); + console.log("wasm-pack already installed"); } catch { - console.log('Installing wasm-pack...'); - execSync('curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh', { - stdio: 'inherit' - }); + console.log("Installing wasm-pack..."); + execSync( + "curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh", + { + stdio: "inherit", + }, + ); } -if (!fs.existsSync('server/pkg')) { - fs.mkdirSync('server/pkg', { recursive: true }); +if (!fs.existsSync("server/pkg")) { + fs.mkdirSync("server/pkg", { recursive: true }); } -console.log('Building WebAssembly module...'); -execSync('wasm-pack build --target web --out-dir server/pkg --no-default-features --features worker', { - stdio: 'inherit', - cwd: process.cwd(), -}); +console.log("Building WebAssembly module..."); +execSync( + "wasm-pack build --target web --out-dir server/pkg --no-default-features --features worker", + { + stdio: "inherit", + cwd: process.cwd(), + }, +); -console.log('Generating TypeScript types...'); +console.log("Generating TypeScript types..."); const typesContent = ` export interface AnalyzeOptions { ignore_hidden: boolean; @@ -49,6 +58,9 @@ export interface AnalyzeOptions { export function analyze_url(url: string, options: AnalyzeOptions): Promise; `; -fs.writeFileSync(path.join(process.cwd(), 'server/pkg', 'bytes_radar.d.ts'), typesContent); +fs.writeFileSync( + path.join(process.cwd(), "server/pkg", "bytes_radar.d.ts"), + typesContent, +); -console.log('Build complete!'); +console.log("Build complete!"); diff --git a/server/package.json b/server/package.json index 75117f5..0232f10 100644 --- a/server/package.json +++ b/server/package.json @@ -16,4 +16,4 @@ "typescript": "^5.3.3", "wrangler": "^3.28.1" } -} \ No newline at end of file +} diff --git a/server/tsconfig.json b/server/tsconfig.json index 3f6ae45..a8b16ad 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -17,4 +17,4 @@ }, "include": ["./**/*"], "exclude": ["node_modules"] -} \ No newline at end of file +} diff --git a/server/types.d.ts b/server/types.d.ts index 3c4b72b..f0213b0 100644 --- a/server/types.d.ts +++ b/server/types.d.ts @@ -11,4 +11,4 @@ declare module "*.wasm?module" { declare module "*.wasm?url" { const content: string; export default content; -} \ No newline at end of file +} diff --git a/server/worker.ts b/server/worker.ts index 11108e8..722a6ab 100644 --- a/server/worker.ts +++ b/server/worker.ts @@ -1,5 +1,5 @@ -import type { AnalyzeOptions } from './pkg/bytes_radar'; -import wasmBinary from './pkg/bytes_radar_bg.wasm'; +import type { AnalyzeOptions } from "./pkg/bytes_radar"; +import wasmBinary from "./pkg/bytes_radar_bg.wasm"; export interface Env { BYTES_RADAR: DurableObjectNamespace; @@ -18,14 +18,18 @@ export class BytesRadar { this.env = env; } - private log(level: 'debug' | 'info' | 'warn' | 'error', message: string, data?: any) { - const logLevel = this.env.LOG_LEVEL || 'info'; - const environment = this.env.ENVIRONMENT || 'development'; - + private log( + level: "debug" | "info" | "warn" | "error", + message: string, + data?: any, + ) { + const logLevel = this.env.LOG_LEVEL || "info"; + const environment = this.env.ENVIRONMENT || "development"; + const levels = { debug: 0, info: 1, warn: 2, error: 3 }; const currentLevel = levels[logLevel as keyof typeof levels] || 1; const messageLevel = levels[level]; - + if (messageLevel >= currentLevel) { const timestamp = new Date().toISOString(); const logEntry = { @@ -33,7 +37,7 @@ export class BytesRadar { level: level.toUpperCase(), environment, message, - ...(data && { data }) + ...(data && { data }), }; const fn = { @@ -41,12 +45,12 @@ export class BytesRadar { info: console.info, warn: console.warn, error: console.error, - } - - if (environment === 'production') { + }; + + if (environment === "production") { fn[level](JSON.stringify(logEntry)); } else { - fn[level](`[${level.toUpperCase()}] ${message}`, data ? data : ''); + fn[level](`[${level.toUpperCase()}] ${message}`, data ? data : ""); } } } @@ -54,13 +58,13 @@ export class BytesRadar { private async initializeWasm() { if (!this.wasmInitialized) { try { - this.wasmModule = await import('./pkg/bytes_radar'); + this.wasmModule = await import("./pkg/bytes_radar"); await this.wasmModule.default(wasmBinary); this.wasmInitialized = true; - this.log('info', 'WebAssembly module initialized successfully'); + this.log("info", "WebAssembly module initialized successfully"); } catch (error) { - this.log('error', 'Failed to initialize WebAssembly module', { - error: error instanceof Error ? error.message : String(error) + this.log("error", "Failed to initialize WebAssembly module", { + error: error instanceof Error ? error.message : String(error), }); throw error; } @@ -69,7 +73,7 @@ export class BytesRadar { async fetch(request: Request) { const url = new URL(request.url); - if (url.pathname === '/favicon.ico') { + if (url.pathname === "/favicon.ico") { return new Response(null, { status: 404 }); } @@ -78,138 +82,155 @@ export class BytesRadar { timestamp: new Date().toISOString(), wasm_initialized: this.wasmInitialized, }; - + try { await this.initializeWasm(); debugInfo.wasm_initialized = true; - - const pathParts = url.pathname.split('/').filter(Boolean); - const targetUrl = pathParts.join('/'); - + + const pathParts = url.pathname.split("/").filter(Boolean); + const targetUrl = pathParts.join("/"); + if (!targetUrl) { - debugInfo.error = 'Missing repository path'; + debugInfo.error = "Missing repository path"; debugInfo.duration_ms = performance.now() - startTime; - return new Response(JSON.stringify({ - error: 'Missing repository path', - usage: [ - "/[user/repo", - "/user/repo@master", - "/github.com/user/repo", - "/gitlab.com/user/repo", - "http://example.com/example-asset.tar.gz", - ], - debug_info: debugInfo - }), { - status: 400, - headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - } - }); + return new Response( + JSON.stringify({ + error: "Missing repository path", + usage: [ + "/[user/repo", + "/user/repo@master", + "/github.com/user/repo", + "/gitlab.com/user/repo", + "http://example.com/example-asset.tar.gz", + ], + debug_info: debugInfo, + }), + { + status: 400, + headers: { + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, + }, + ); } - const maxSizeParam = url.searchParams.get('max_size'); - const ignoreHiddenParam = url.searchParams.get('ignore_hidden'); - const ignoreGitignoreParam = url.searchParams.get('ignore_gitignore'); + const maxSizeParam = url.searchParams.get("max_size"); + const ignoreHiddenParam = url.searchParams.get("ignore_hidden"); + const ignoreGitignoreParam = url.searchParams.get("ignore_gitignore"); const options: AnalyzeOptions = { - ignore_hidden: ignoreHiddenParam === 'false' ? false : true, - ignore_gitignore: ignoreGitignoreParam === 'false' ? false : true, - max_file_size: maxSizeParam === '-1' ? -1 : - maxSizeParam ? parseInt(maxSizeParam) : - -1, + ignore_hidden: ignoreHiddenParam === "false" ? false : true, + ignore_gitignore: ignoreGitignoreParam === "false" ? false : true, + max_file_size: + maxSizeParam === "-1" + ? -1 + : maxSizeParam + ? parseInt(maxSizeParam) + : -1, }; debugInfo.target_url = targetUrl; debugInfo.options = options; - - this.log('info', 'Starting analysis', { url: targetUrl, options }); - + + this.log("info", "Starting analysis", { url: targetUrl, options }); + const analysisStartTime = performance.now(); const result = await this.wasmModule.analyze_url(targetUrl, options); const analysisEndTime = performance.now(); - + debugInfo.analysis_duration_ms = analysisEndTime - analysisStartTime; debugInfo.total_duration_ms = analysisEndTime - startTime; - + if (result && result.wasm_debug_info) { Object.assign(debugInfo, result.wasm_debug_info); delete result.wasm_debug_info; } - + const response = { ...result, - debug_info: debugInfo + debug_info: debugInfo, }; - - this.log('info', 'Analysis completed successfully', debugInfo); - + + this.log("info", "Analysis completed successfully", debugInfo); + return new Response(JSON.stringify(response), { headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", }, }); } catch (error: unknown) { let errorMessage = error instanceof Error ? error.message : String(error); const errorStack = error instanceof Error ? error.stack : undefined; - let errorType = 'UnknownError'; - - if (error && typeof error === 'object' && 'error' in error) { + let errorType = "UnknownError"; + + if (error && typeof error === "object" && "error" in error) { const wasmError = error as any; errorMessage = wasmError.error || errorMessage; - errorType = wasmError.error_type || 'WASMError'; - + errorType = wasmError.error_type || "WASMError"; + if (wasmError.wasm_debug_info) { Object.assign(debugInfo, wasmError.wasm_debug_info); delete wasmError.wasm_debug_info; } } - + debugInfo.error = errorMessage; debugInfo.error_type = errorType; debugInfo.error_stack = errorStack; debugInfo.duration_ms = performance.now() - startTime; - - if (errorMessage.includes('URL parsing error')) { - debugInfo.error_category = 'URL_PARSING'; - debugInfo.suggested_fix = 'Please check the URL format. Use formats like: user/repo, user/repo@branch, or full GitHub URLs'; - } else if (errorMessage.includes('network') || errorMessage.includes('download')) { - debugInfo.error_category = 'NETWORK'; - debugInfo.suggested_fix = 'Check your internet connection and ensure the repository is accessible'; - } else if (errorMessage.includes('branch')) { - debugInfo.error_category = 'BRANCH_ACCESS'; - debugInfo.suggested_fix = 'The repository may not have the expected default branches (main, master, develop, dev)'; + + if (errorMessage.includes("URL parsing error")) { + debugInfo.error_category = "URL_PARSING"; + debugInfo.suggested_fix = + "Please check the URL format. Use formats like: user/repo, user/repo@branch, or full GitHub URLs"; + } else if ( + errorMessage.includes("network") || + errorMessage.includes("download") + ) { + debugInfo.error_category = "NETWORK"; + debugInfo.suggested_fix = + "Check your internet connection and ensure the repository is accessible"; + } else if (errorMessage.includes("branch")) { + debugInfo.error_category = "BRANCH_ACCESS"; + debugInfo.suggested_fix = + "The repository may not have the expected default branches (main, master, develop, dev)"; } else { - debugInfo.error_category = 'UNKNOWN'; - debugInfo.suggested_fix = 'Please check the error details and try again'; + debugInfo.error_category = "UNKNOWN"; + debugInfo.suggested_fix = + "Please check the error details and try again"; } - - this.log('error', 'Error in BytesRadar fetch', debugInfo); - + + this.log("error", "Error in BytesRadar fetch", debugInfo); + const errorResponse: any = { error: errorMessage, error_type: errorType, error_category: debugInfo.error_category, suggested_fix: debugInfo.suggested_fix, - debug_info: debugInfo + debug_info: debugInfo, }; - - return new Response(JSON.stringify(errorResponse), { + + return new Response(JSON.stringify(errorResponse), { status: 500, headers: { - 'Content-Type': 'application/json', - 'Access-Control-Allow-Origin': '*', - } + "Content-Type": "application/json", + "Access-Control-Allow-Origin": "*", + }, }); } } } export default { - async fetch(request: Request, env: Env, ctx: ExecutionContext): Promise { - const id = env.BYTES_RADAR.idFromName('default'); + async fetch( + request: Request, + env: Env, + ctx: ExecutionContext, + ): Promise { + const id = env.BYTES_RADAR.idFromName("default"); const obj = env.BYTES_RADAR.get(id); return obj.fetch(request); }, -}; \ No newline at end of file +}; diff --git a/src/cli/output.rs b/src/cli/output.rs index 16a524c..8f76b19 100644 --- a/src/cli/output.rs +++ b/src/cli/output.rs @@ -2,6 +2,23 @@ use super::progress::format_number; use crate::core::{analysis::ProjectAnalysis, error::Result}; use colored::Colorize; +fn get_percentage_color(percentage: f64) -> colored::ColoredString { + let percentage_str = format!("{:.1}%", percentage); + if percentage >= 50.0 { + percentage_str.red() + } else if percentage >= 10.0 { + percentage_str.yellow() + } else if percentage >= 1.0 { + percentage_str.white() + } else { + percentage_str.dimmed() + } +} + +fn color_number(num: usize) -> colored::ColoredString { + format_number(num).bright_white() +} + pub fn print_table_format(project_analysis: &ProjectAnalysis, detailed: bool, quiet: bool) { let summary = project_analysis.get_summary(); let language_stats = project_analysis.get_language_statistics(); @@ -14,59 +31,59 @@ pub fn print_table_format(project_analysis: &ProjectAnalysis, detailed: bool, qu println!( " {:<56} {}", "Total Files", - format_number(summary.total_files) + color_number(summary.total_files) ); println!( " {:<56} {}", "Total Lines", - format_number(summary.total_lines) + color_number(summary.total_lines) ); println!( " {:<56} {}", "Code Lines", - format_number(summary.total_code_lines) + color_number(summary.total_code_lines) ); println!( " {:<56} {}", "Comment Lines", - format_number(summary.total_comment_lines) + color_number(summary.total_comment_lines) ); println!( " {:<56} {}", "Blank Lines", - format_number(summary.total_blank_lines) + color_number(summary.total_blank_lines) ); println!( " {:<56} {}", "Languages", - format_number(summary.language_count) + color_number(summary.language_count) ); if let Some(ref primary) = summary.primary_language { println!(" {:<56} {}", "Primary Language", primary); } println!( - " {:<56} {:.1}%", + " {:<56} {}", "Code Ratio", - summary.overall_complexity_ratio * 100.0 + get_percentage_color(summary.overall_complexity_ratio * 100.0) ); println!( - " {:<56} {:.1}%", + " {:<56} {}", "Documentation", - summary.overall_documentation_ratio * 100.0 + get_percentage_color(summary.overall_documentation_ratio * 100.0) ); if !language_stats.is_empty() && !quiet { println!("{}", "=".repeat(80)); println!( - " {:<20} {:>8} {:>12} {:>8} {:>10} {:>8} {:>8}", + " {:<20} {:>8} {:>12} {:>8} {:>10} {:>8} {:>7}", "Language".bold(), "Files", "Lines", "Code", "Comments", "Blanks", - "Share%" + "%" ); println!("{}", "=".repeat(80)); @@ -78,27 +95,27 @@ pub fn print_table_format(project_analysis: &ProjectAnalysis, detailed: bool, qu }; println!( - " {:<20} {:>8} {:>12} {:>8} {:>10} {:>8} {:>7.1}%", + " {:<20} {:>8} {:>12} {:>8} {:>10} {:>8} {:>7}", stats.language_name, - format_number(stats.file_count), - format_number(stats.total_lines), - format_number(stats.code_lines), - format_number(stats.comment_lines), - format_number(stats.blank_lines), - share_percentage + color_number(stats.file_count), + color_number(stats.total_lines), + color_number(stats.code_lines), + color_number(stats.comment_lines), + color_number(stats.blank_lines), + get_percentage_color(share_percentage) ); } println!("{}", "=".repeat(80)); println!( - " {:<20} {:>8} {:>12} {:>8} {:>10} {:>8} {:>7.1}%", + " {:<20} {:>8} {:>12} {:>8} {:>10} {:>8} {:>7}", "Total".bold(), - format_number(summary.total_files), - format_number(summary.total_lines), - format_number(summary.total_code_lines), - format_number(summary.total_comment_lines), - format_number(summary.total_blank_lines), - 100.0 + color_number(summary.total_files), + color_number(summary.total_lines), + color_number(summary.total_code_lines), + color_number(summary.total_comment_lines), + color_number(summary.total_blank_lines), + "%" ); } @@ -114,9 +131,9 @@ pub fn print_table_format(project_analysis: &ProjectAnalysis, detailed: bool, qu println!( " {:<50} {:>6} lines ({} code, {} comments)", file.file_path, - format_number(file.total_lines), - format_number(file.code_lines), - format_number(file.comment_lines) + color_number(file.total_lines), + color_number(file.code_lines), + color_number(file.comment_lines) ); } } diff --git a/src/cli/progress.rs b/src/cli/progress.rs index e5551ae..5a476ad 100644 --- a/src/cli/progress.rs +++ b/src/cli/progress.rs @@ -87,7 +87,9 @@ impl ProgressHook for ProgressBarHook { } else { self.progress_bar.set_style( ProgressStyle::default_spinner() - .template("[{elapsed_precise}] {spinner:.green} {msg} ({decimal_bytes_per_sec})") + .template( + "[{elapsed_precise}] {spinner:.green} {msg} ({decimal_bytes_per_sec})", + ) .unwrap_or_else(|_| ProgressStyle::default_spinner()), ); self.progress_bar.set_position(downloaded); diff --git a/src/languages.json b/src/languages.json index 3df3db4..e913624 100644 --- a/src/languages.json +++ b/src/languages.json @@ -59,7 +59,10 @@ "Asn1": { "name": "ASN.1", "line_comment": ["--"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "multi_line_comments": [["/*", "*/"]], "extensions": ["asn1"] }, @@ -70,7 +73,10 @@ }, "AspNet": { "name": "ASP.NET", - "multi_line_comments": [[""], ["<%--", "-->"]], + "multi_line_comments": [ + [""], + ["<%--", "-->"] + ], "extensions": [ "asax", "ascx", @@ -83,7 +89,10 @@ }, "Assembly": { "line_comment": [";"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["asm"] }, "AssemblyGAS": { @@ -95,20 +104,21 @@ }, "Astro": { "line_comment": ["//"], - "multi_line_comments": [["/*", "*/"], [""]], + "multi_line_comments": [ + ["/*", "*/"], + [""] + ], "extensions": ["astro"] }, "Ats": { "name": "ATS", "line_comment": ["//"], - "multi_line_comments": [["(*", "*)"], ["/*", "*/"]], + "multi_line_comments": [ + ["(*", "*)"], + ["/*", "*/"] + ], "quotes": [["\\\"", "\\\""]], - "extensions": [ - "dats", - "hats", - "sats", - "atxt" - ] + "extensions": ["dats", "hats", "sats", "atxt"] }, "Autoconf": { "line_comment": ["#", "dnl"], @@ -116,7 +126,10 @@ }, "Autoit": { "line_comment": [";"], - "multi_line_comments": [["#comments-start", "#comments-end"], ["#cs", "#ce"]], + "multi_line_comments": [ + ["#comments-start", "#comments-end"], + ["#cs", "#ce"] + ], "extensions": ["au3"] }, "AutoHotKey": { @@ -145,7 +158,10 @@ "name": "BASH", "shebangs": ["#!/bin/bash"], "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "env": ["bash"], "extensions": ["bash"] }, @@ -155,8 +171,14 @@ }, "Bazel": { "line_comment": ["#"], - "doc_quotes": [["\\\"\\\"\\\"", "\\\"\\\"\\\""], ["'''", "'''"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "doc_quotes": [ + ["\\\"\\\"\\\"", "\\\"\\\"\\\""], + ["'''", "'''"] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["bzl", "bazel", "bzlmod"], "filenames": ["build", "workspace", "module"] }, @@ -165,23 +187,32 @@ "quotes": [["\\\"", "\\\""]], "extensions": ["bean", "beancount"] }, - "Bicep" : { + "Bicep": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["'", "'"], ["'''", "'''"]], + "quotes": [ + ["'", "'"], + ["'''", "'''"] + ], "extensions": ["bicep", "bicepparam"] }, "Bitbake": { "name": "Bitbake", "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["bb", "bbclass", "bbappend", "inc"] }, "Bqn": { "name": "BQN", "line_comment": ["#"], "extensions": ["bqn"], - "quotes": [["\\\"", "\\\""], ["'", "'"]] + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ] }, "BrightScript": { "quotes": [["\\\"", "\\\""]], @@ -212,27 +243,45 @@ "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], "nested": true, - "quotes": [["\\\"", "\\\""],["\\\"\\\"\\\"", "\\\"\\\"\\\""]], - "verbatim_quotes": [["#\\\"", "\\\"#"],["##\\\"", "\\\"##"],["###\\\"", "\\\"###"], - ["#'", "'#"],["##'", "'##"],["###'", "'###"]], + "quotes": [ + ["\\\"", "\\\""], + ["\\\"\\\"\\\"", "\\\"\\\"\\\""] + ], + "verbatim_quotes": [ + ["#\\\"", "\\\"#"], + ["##\\\"", "\\\"##"], + ["###\\\"", "\\\"###"], + ["#'", "'#"], + ["##'", "'##"], + ["###'", "'###"] + ], "extensions": ["cj"] }, "Cassius": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["cassius"] }, "Ceylon": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["\\\"\\\"\\\"", "\\\"\\\"\\\""]], + "quotes": [ + ["\\\"", "\\\""], + ["\\\"\\\"\\\"", "\\\"\\\"\\\""] + ], "extensions": ["ceylon"] }, "Chapel": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["chpl"] }, "CHeader": { @@ -288,7 +337,10 @@ "CoffeeScript": { "line_comment": ["#"], "multi_line_comments": [["###", "###"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["coffee", "cjsx"] }, "Cogent": { @@ -297,7 +349,10 @@ }, "ColdFusion": { "multi_line_comments": [[""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["cfm"] }, "ColdFusionScript": { @@ -330,7 +385,10 @@ "Crystal": { "line_comment": ["#"], "shebangs": ["#!/usr/bin/crystal"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "env": ["crystal"], "extensions": ["cr"] }, @@ -353,7 +411,10 @@ "name": "CSS", "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "mime": ["text/css"], "extensions": ["css"] }, @@ -377,15 +438,24 @@ }, "Cython": { "line_comment": ["#"], - "doc_quotes": [["\\\"\\\"\\\"", "\\\"\\\"\\\""], ["'''", "'''"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "doc_quotes": [ + ["\\\"\\\"\\\"", "\\\"\\\"\\\""], + ["'''", "'''"] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "env": ["cython"], "extensions": ["pyx", "pxd", "pxi"] }, "D": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "nested_comments": [["/+", "+/"]], "extensions": ["d"] }, @@ -419,18 +489,24 @@ "quotes": [["\\\"", "\\\""]], "extensions": ["dts", "dtsi"] }, - "Dhall":{ + "Dhall": { "nested": true, "line_comment": ["--"], "multi_line_comments": [["{-", "-}"]], - "quotes": [["\\\"", "\\\""], ["''", "''"]], + "quotes": [ + ["\\\"", "\\\""], + ["''", "''"] + ], "extensions": ["dhall"] }, "Dockerfile": { "line_comment": ["#"], "extensions": ["dockerfile", "dockerignore"], "filenames": ["dockerfile"], - "quotes": [["\\\"", "\\\""], ["'", "'"]] + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ] }, "DotNetResource": { "name": ".NET Resource", @@ -444,7 +520,11 @@ "multi_line_comments": [["/*", "*/"]], "nested": true, "extensions": ["dm", "dme"], - "quotes": [["\\\"", "\\\""], ["{\\\"", "\\\"}"], ["'", "'"]] + "quotes": [ + ["\\\"", "\\\""], + ["{\\\"", "\\\"}"], + ["'", "'"] + ] }, "Dust": { "name": "Dust.js", @@ -453,19 +533,29 @@ }, "Ebuild": { "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["ebuild", "eclass"] }, "EdgeQL": { "name": "EdgeQL", "line_comment": ["#"], - "quotes": [["'", "'"], ["\\\"", "\\\""], ["$", "$"]], + "quotes": [ + ["'", "'"], + ["\\\"", "\\\""], + ["$", "$"] + ], "extensions": ["edgeql"] }, "ESDL": { "name": "EdgeDB Schema Definition", "line_comment": ["#"], - "quotes": [["'", "'"], ["\\\"", "\\\""]], + "quotes": [ + ["'", "'"], + ["\\\"", "\\\""] + ], "extensions": ["esdl"] }, "Edn": { @@ -503,7 +593,10 @@ }, "Elvish": { "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "env": ["elvish"], "extensions": ["elv"] }, @@ -514,7 +607,11 @@ }, "Emojicode": { "line_comment": ["💭"], - "multi_line_comments": [["💭🔜", "🔚💭"], ["📗", "📗"], ["📘", "📘"]], + "multi_line_comments": [ + ["💭🔜", "🔚💭"], + ["📗", "📗"], + ["📘", "📘"] + ], "quotes": [["❌🔤", "❌🔤"]], "extensions": ["emojic", "🍇"] }, @@ -523,16 +620,16 @@ "extensions": ["erl", "hrl"] }, "Factor": { - "line_comment": ["!", "#!"], - "multi_line_comments": [["/*", "*/"]], - "extensions": ["factor"] + "line_comment": ["!", "#!"], + "multi_line_comments": [["/*", "*/"]], + "extensions": ["factor"] }, "FEN": { "name": "FEN", "blank": true, "extensions": ["fen"] }, - "Fennel" : { + "Fennel": { "line_comment": [";", ";;"], "quotes": [["\\\"", "\\\""]], "extensions": ["fnl"] @@ -540,7 +637,10 @@ "Fish": { "shebangs": ["#!/bin/fish"], "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "env": ["fish"], "extensions": ["fish"] }, @@ -576,7 +676,10 @@ "FortranLegacy": { "name": "FORTRAN Legacy", "line_comment": ["c", "C", "!", "*"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["f", "for", "ftn", "f77", "pfo"] }, "FortranModern": { @@ -637,16 +740,30 @@ "GlimmerJs": { "name": "Glimmer JS", "line_comment": ["//"], - "multi_line_comments": [["/*", "*/"], [""]], - "quotes": [["\\\"", "\\\""], ["'", "'"], ["`", "`"]], + "multi_line_comments": [ + ["/*", "*/"], + [""] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"], + ["`", "`"] + ], "important_syntax": [""]], - "quotes": [["\\\"", "\\\""], ["'", "'"], ["`", "`"]], + "multi_line_comments": [ + ["/*", "*/"], + [""] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"], + ["`", "`"] + ], "important_syntax": [""], ["{{/*", "*/}}"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "multi_line_comments": [ + [""], + ["{{/*", "*/}}"] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["gohtml"] }, "Graphql": { "name": "GraphQL", - "quotes": [["\\\"", "\\\""], ["\\\"\\\"\\\"", "\\\"\\\"\\\""]], + "quotes": [ + ["\\\"", "\\\""], + ["\\\"\\\"\\\"", "\\\"\\\"\\\""] + ], "line_comment": ["#"], "extensions": ["gql", "graphql"] }, @@ -696,20 +838,32 @@ }, "Haml": { "line_comment": ["-#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["haml"] }, "Hamlet": { "multi_line_comments": [[""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["hamlet"] }, "Happy": { "extensions": ["y", "ly"] }, "Handlebars": { - "multi_line_comments": [[""], ["{{!", "}}"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "multi_line_comments": [ + [""], + ["{{!", "}}"] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["hbs", "handlebars"] }, "Haskell": { @@ -721,7 +875,10 @@ "Haxe": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["hx"] }, "Hcl": { @@ -758,12 +915,15 @@ "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], "quotes": [["\\\"", "\\\""]], - "extensions": ["HC", "hc","ZC","zc"] + "extensions": ["HC", "hc", "ZC", "zc"] }, "Html": { "name": "HTML", "multi_line_comments": [[""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "kind": "html", "important_syntax": [""], ["{% comment %}", "{% endcomment %}"]] + "multi_line_comments": [ + [""], + ["{% comment %}", "{% endcomment %}"] + ] }, "LinguaFranca": { "name": "Lingua Franca", @@ -1003,12 +1218,18 @@ "LiveScript": { "line_comment": ["#"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["ls"] }, "LLVM": { "line_comment": [";"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["ll"] }, "Logtalk": { @@ -1027,13 +1248,19 @@ "Lua": { "line_comment": ["--"], "multi_line_comments": [["--[[", "]]"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["lua", "luau"] }, "Lucius": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["lucius"] }, "M4": { @@ -1077,7 +1304,10 @@ }, "Meson": { "line_comment": ["#"], - "quotes": [["'", "'"], ["'''", "'''"]], + "quotes": [ + ["'", "'"], + ["'''", "'''"] + ], "filenames": ["meson.build", "meson_options.txt"] }, "Metal": { @@ -1109,8 +1339,14 @@ }, "Mojo": { "line_comment": ["#"], - "doc_quotes": [["\\\"\\\"\\\"", "\\\"\\\"\\\""], ["'''", "'''"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "doc_quotes": [ + ["\\\"\\\"\\\"", "\\\"\\\"\\\""], + ["'''", "'''"] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["mojo", "🔥"] }, "MonkeyC": { @@ -1127,18 +1363,27 @@ }, "MoonScript": { "line_comment": ["--"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["moon"] }, "MsBuild": { "name": "MSBuild", "multi_line_comments": [[""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["csproj", "vbproj", "fsproj", "props", "targets"] }, "Mustache": { "multi_line_comments": [["{{!", "}}"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["mustache"] }, "Nextflow": { @@ -1149,7 +1394,10 @@ }, "Nim": { "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["\\\"\\\"\\\"", "\\\"\\\"\\\""]], + "quotes": [ + ["\\\"", "\\\""], + ["\\\"\\\"\\\"", "\\\"\\\"\\\""] + ], "extensions": ["nim"] }, "Nix": { @@ -1162,13 +1410,19 @@ "name": "Not Quite Perl", "line_comment": ["#"], "multi_line_comments": [["=begin", "=end"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["nqp"] }, "NuGetConfig": { "name": "NuGet Config", "multi_line_comments": [[""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "filenames": ["nuget.config", "packages.config", "nugetdefaults.config"] }, "Nushell": { @@ -1202,19 +1456,28 @@ "extensions": ["odin"], "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]] + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ] }, "OpenScad": { "name": "OpenSCAD", "extensions": ["scad"], "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]] + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ] }, "OpenPolicyAgent": { "name": "Open Policy Agent", "line_comment": ["#"], - "quotes": [["\\\"","\\\""], ["`", "`"]], + "quotes": [ + ["\\\"", "\\\""], + ["`", "`"] + ], "extensions": ["rego"] }, "OpenCL": { @@ -1246,18 +1509,27 @@ "PacmanMakepkg": { "name": "Pacman's makepkg", "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "filenames": ["pkgbuild"] }, "Pan": { "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["pan", "tpl"] }, "Pascal": { "nested": true, "line_comment": ["//"], - "multi_line_comments": [["{", "}"], ["(*", "*)"]], + "multi_line_comments": [ + ["{", "}"], + ["(*", "*)"] + ], "quotes": [["'", "'"]], "extensions": ["pas"] }, @@ -1265,27 +1537,45 @@ "shebangs": ["#!/usr/bin/perl"], "line_comment": ["#"], "multi_line_comments": [["=pod", "=cut"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["pl", "pm"] }, "Pest": { "line_comment": ["//"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["pest"] }, "Phix": { "line_comment": ["--", "//", "#!"], - "multi_line_comments": [["/*", "*/"], ["--/*", "--*/"]], + "multi_line_comments": [ + ["/*", "*/"], + ["--/*", "--*/"] + ], "nested": true, - "quotes": [["\\\"", "\\\""], ["'", "'"]], - "verbatim_quotes": [["\\\"\\\"\\\"", "\\\"\\\"\\\""], ["`", "`"]], - "extensions": ["e","exw"] + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], + "verbatim_quotes": [ + ["\\\"\\\"\\\"", "\\\"\\\"\\\""], + ["`", "`"] + ], + "extensions": ["e", "exw"] }, "Php": { "name": "PHP", "line_comment": ["#", "//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["php"] }, "PlantUml": { @@ -1296,9 +1586,9 @@ "extensions": ["puml"] }, "Po": { - "name": "PO File", - "line_comment": ["#"], - "extensions": ["po", "pot"] + "name": "PO File", + "line_comment": ["#"], + "extensions": ["po", "pot"] }, "Poke": { "multi_line_comments": [["/*", "*/"]], @@ -1306,7 +1596,10 @@ }, "Polly": { "multi_line_comments": [[""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["polly"] }, "Pony": { @@ -1320,7 +1613,10 @@ "name": "PostCSS", "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["pcss", "sss"] }, "PowerShell": { @@ -1364,7 +1660,7 @@ "line_comment": ["//"], "extensions": ["proto"] }, - "Pug" : { + "Pug": { "line_comment": ["//", "//-"], "quotes": [ ["#{\\\"", "\\\"}"], @@ -1375,7 +1671,10 @@ }, "Puppet": { "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["pp"] }, "PureScript": { @@ -1387,21 +1686,33 @@ "Pyret": { "line_comment": ["#"], "multi_line_comments": [["#|", "|#"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["arr"], "nested": true }, "Python": { "line_comment": ["#"], - "doc_quotes": [["\\\"\\\"\\\"", "\\\"\\\"\\\""], ["'''", "'''"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "doc_quotes": [ + ["\\\"\\\"\\\"", "\\\"\\\"\\\""], + ["'''", "'''"] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "env": ["python", "python2", "python3"], "mime": ["text/x-python"], "extensions": ["py", "pyw", "pyi"] }, "PRQL": { "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "mime": ["application/prql"], "extensions": ["prql"] }, @@ -1423,7 +1734,10 @@ "name": "QML", "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["qml"] }, "R": { @@ -1440,7 +1754,10 @@ "Rakefile": { "line_comment": ["#"], "multi_line_comments": [["=begin", "=end"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "filenames": ["rakefile"], "extensions": ["rake"] }, @@ -1454,7 +1771,10 @@ ["#`｢", "｣"] ], "nested": true, - "quotes": [["\\\"", "\\\""] , ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "verbatim_quotes": [["｢", "｣"]], "doc_quotes": [ ["#|{", "}"], @@ -1483,7 +1803,11 @@ }, "Razor": { "line_comment": ["//"], - "multi_line_comments": [[""], ["@*", "*@"], ["/*", "*/"]], + "multi_line_comments": [ + [""], + ["@*", "*@"], + ["/*", "*/"] + ], "quotes": [["\\\"", "\\\""]], "verbatim_quotes": [["@\\\"", "\\\""]], "extensions": ["cshtml", "razor"] @@ -1499,7 +1823,11 @@ "Renpy": { "name": "Ren'Py", "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"], ["`", "`"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"], + ["`", "`"] + ], "extensions": ["rpy"] }, "ReScript": { @@ -1537,7 +1865,10 @@ "Ruby": { "line_comment": ["#"], "multi_line_comments": [["=begin", "=end"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "env": ["ruby"], "extensions": ["rb"] }, @@ -1545,7 +1876,10 @@ "name": "Ruby HTML", "multi_line_comments": [[""]], "important_syntax": [""], ["${", "}"]], + "quotes": [ + ["\\\"", "\\\""], + ["$[", "]"], + ["$<", ">"], + ["${", "}"] + ], "extensions": ["str"] }, "Stylus": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["styl"] }, "Svelte": { "multi_line_comments": [[""]], "important_syntax": [""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "mime": ["image/svg+xml"], "extensions": ["svg"] }, @@ -1743,19 +2119,35 @@ "Tcl": { "name": "TCL", "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["tcl"] }, "Tera": { - "multi_line_comments": [[""], ["{#", "#}"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "multi_line_comments": [ + [""], + ["{#", "#}"] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["tera"] }, "Templ": { "name": "Templ", "line_comment": ["//"], - "multi_line_comments": [[""], ["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"], ["`", "`"]], + "multi_line_comments": [ + [""], + ["/*", "*/"] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"], + ["`", "`"] + ], "important_syntax": ["templ", "script", "css"], "extensions": ["templ", "tmpl"] }, @@ -1773,7 +2165,10 @@ "Thrift": { "line_comment": ["#", "//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["thrift"] }, "Toml": { @@ -1791,7 +2186,11 @@ "name": "TSX", "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"], ["`", "`"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"], + ["`", "`"] + ], "extensions": ["tsx"] }, "Ttcn": { @@ -1803,14 +2202,24 @@ }, "Twig": { "name": "Twig", - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["twig"], - "multi_line_comments": [[""], ["{#", "#}"]] + "multi_line_comments": [ + [""], + ["{#", "#}"] + ] }, "TypeScript": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"], ["`", "`"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"], + ["`", "`"] + ], "extensions": ["ts", "mts", "cts"] }, "Typst": { @@ -1905,7 +2314,10 @@ "line_comment": ["##"], "multi_line_comments": [["#*", "*#"]], "extensions": ["vm"], - "quotes": [["'", "'"], ["\\\"", "\\\""]] + "quotes": [ + ["'", "'"], + ["\\\"", "\\\""] + ] }, "Verilog": { "line_comment": ["//"], @@ -1938,7 +2350,10 @@ "VisualStudioProject": { "name": "Visual Studio Project", "multi_line_comments": [[""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["vcproj", "vcxproj"] }, "VisualStudioSolution": { @@ -1949,26 +2364,42 @@ "VimScript": { "name": "Vim Script", "line_comment": ["\\\""], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["vim"] }, "Vue": { "name": "Vue", "line_comment": ["//"], - "multi_line_comments": [[""], ["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"], ["`", "`"]], + "multi_line_comments": [ + [""], + ["/*", "*/"] + ], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"], + ["`", "`"] + ], "important_syntax": [""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["xaml"] }, "XcodeConfig": { "name": "Xcode Config", "line_comment": ["//"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["xcconfig"] }, "Xml": { "name": "XML", "multi_line_comments": [[""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["xml"] }, "XSL": { "name": "XSL", "multi_line_comments": [[""]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["xsl", "xslt"] }, "Xtend": { "line_comment": ["//"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"], ["'''", "'''"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"], + ["'''", "'''"] + ], "extensions": ["xtend"] }, "Yaml": { "name": "YAML", "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["yaml", "yml"] }, "ZenCode": { "line_comment": ["//", "#"], "multi_line_comments": [["/*", "*/"]], - "quotes": [["\\\"", "\\\""], ["'", "'"]], - "verbatim_quotes": [["@\\\"", "\\\""], ["@'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], + "verbatim_quotes": [ + ["@\\\"", "\\\""], + ["@'", "'"] + ], "extensions": ["zs"] }, "Zig": { @@ -2037,7 +2493,10 @@ "Zsh": { "shebangs": ["#!/bin/zsh"], "line_comment": ["#"], - "quotes": [["\\\"", "\\\""], ["'", "'"]], + "quotes": [ + ["\\\"", "\\\""], + ["'", "'"] + ], "extensions": ["zsh"] }, "GdShader": { @@ -2046,4 +2505,4 @@ "multi_line_comments": [["/*", "*/"]], "extensions": ["gdshader"] } -} \ No newline at end of file +} diff --git a/tests/README.md b/tests/README.md index 521dad4..15368e5 100644 --- a/tests/README.md +++ b/tests/README.md @@ -48,7 +48,7 @@ cargo test # Integration tests only cargo test integration_tests -# Analysis tests only +# Analysis tests only cargo test analysis_tests # CLI tests only @@ -79,16 +79,19 @@ cargo tarpaulin ## Test Categories ### Unit Tests + - Test individual functions and methods in isolation - Focus on specific functionality and edge cases - Fast execution, no external dependencies -### Integration Tests +### Integration Tests + - Test interaction between multiple components - Test complete workflows and use cases - May include mock network operations ### Network Tests + - Test actual network operations (marked with `#[ignore]` by default) - Require internet connection - May be flaky due to network conditions @@ -108,8 +111,9 @@ cargo test network --ignored ## Test Data Tests use predefined test data and mock repositories where possible to ensure: + - Consistent results across environments -- Fast test execution +- Fast test execution - No dependency on external services ## Contributing @@ -127,4 +131,4 @@ When adding new tests: - Tests should complete quickly (< 1 second for unit tests) - Network tests may take longer but should have reasonable timeouts -- Use `cargo test --release` for performance-sensitive tests \ No newline at end of file +- Use `cargo test --release` for performance-sensitive tests From 42099cdceefbe265facfddec7e6fadbafc90f7a4 Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Thu, 10 Jul 2025 15:41:35 +0800 Subject: [PATCH 05/11] feat: update cli param and cleanup code --- package.json | 2 +- src/cli/args.rs | 3 ++- src/cli/mod.rs | 12 +++++++----- src/cli/output.rs | 2 +- src/core/filter.rs | 6 ++++++ src/core/registry.rs | 18 ++++-------------- src/net/mod.rs | 2 +- src/net/providers/archive.rs | 4 ++-- src/net/providers/azure_devops.rs | 2 +- src/net/providers/bitbucket.rs | 2 +- src/net/providers/codeberg.rs | 2 +- src/net/providers/gitea.rs | 2 +- src/net/providers/github.rs | 2 +- src/net/providers/gitlab.rs | 2 +- src/net/providers/sourceforge.rs | 2 +- src/worker.rs | 8 +++++--- 16 files changed, 36 insertions(+), 35 deletions(-) diff --git a/package.json b/package.json index b3d2a50..87d7fcb 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "build": "node build.js", "dev": "wrangler dev", "deploy": "wrangler deploy", - "deploy:prod": "node build.js && wrangler deploy", + "deploy:prod": "node server/build.js && wrangler deploy", "prettier": "prettier --write ." }, "dependencies": { diff --git a/src/cli/args.rs b/src/cli/args.rs index 753ff27..9864978 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -45,13 +45,14 @@ EXAMPLES: .args(&["quiet", "debug"]) .multiple(false) ))] +#[non_exhaustive] pub struct Cli { #[arg(help = "Repository URL to analyze (user/repo, user/repo@branch, or full URL)")] pub url: Option, // Version #[arg(short = 'v', long = "version", action = clap::ArgAction::Version, help = "Print version information")] - _version: (), + version: (), // Output Options #[arg( diff --git a/src/cli/mod.rs b/src/cli/mod.rs index eb79feb..dcc7685 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -26,7 +26,7 @@ pub async fn run() -> Result<()> { init_logging(&cli)?; let mut cli = cli; - if let Some(token) = std::env::var("BRADAR_TOKEN").ok() { + if let Ok(token) = std::env::var("BRADAR_TOKEN") { if cli.token.is_none() { cli.token = Some(token); } @@ -122,10 +122,12 @@ fn configure_analyzer_filters(analyzer: &mut RemoteAnalyzer, cli: &Cli) -> Resul if cli.aggressive_filter { analyzer.set_aggressive_filtering(true); } else { - let mut filter = filter::IntelligentFilter::default(); - filter.max_file_size = cli.max_file_size * 1024; - filter.ignore_test_dirs = !cli.include_tests; - filter.ignore_docs_dirs = !cli.include_docs; + let filter = filter::IntelligentFilter { + max_file_size: cli.max_file_size * 1024, + ignore_test_dirs: !cli.include_tests, + ignore_docs_dirs: !cli.include_docs, + ..filter::IntelligentFilter::default() + }; analyzer.set_filter(filter); } diff --git a/src/cli/output.rs b/src/cli/output.rs index 8f76b19..5f42667 100644 --- a/src/cli/output.rs +++ b/src/cli/output.rs @@ -260,7 +260,6 @@ pub fn print_toml_format(project_analysis: &ProjectAnalysis) -> Result<()> { println!("{}", toml); Ok(()) } - fn xml_escape(text: &str) -> String { text.replace("&", "&") .replace("<", "<") @@ -268,3 +267,4 @@ fn xml_escape(text: &str) -> String { .replace("\"", """) .replace("'", "'") } + diff --git a/src/core/filter.rs b/src/core/filter.rs index 407842d..62b7ff1 100644 --- a/src/core/filter.rs +++ b/src/core/filter.rs @@ -272,6 +272,12 @@ pub struct FilterStats { pub bytes_saved: u64, } +impl Default for FilterStats { + fn default() -> Self { + Self::new() + } +} + impl FilterStats { pub fn new() -> Self { Self { diff --git a/src/core/registry.rs b/src/core/registry.rs index a1e255d..cd52042 100644 --- a/src/core/registry.rs +++ b/src/core/registry.rs @@ -4,8 +4,9 @@ use std::collections::HashMap; use std::fmt::{Display, Formatter}; use std::path::Path; -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub enum LanguageType { + #[default] Programming, Markup, Data, @@ -14,12 +15,6 @@ pub enum LanguageType { Other, } -impl Default for LanguageType { - fn default() -> Self { - LanguageType::Programming - } -} - impl Display for LanguageType { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { @@ -33,18 +28,13 @@ impl Display for LanguageType { } } -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] pub enum LineCommentPosition { + #[default] Any, Start, } -impl Default for LineCommentPosition { - fn default() -> Self { - LineCommentPosition::Any - } -} - #[derive(Debug, Clone, Serialize, Deserialize)] pub struct LanguageDefinition { #[serde(default)] diff --git a/src/net/mod.rs b/src/net/mod.rs index 10504cc..7c40d71 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -507,7 +507,7 @@ impl RemoteAnalyzer { fn extract_project_name_from_url(&self, url: &str) -> String { let url_path = url.trim_end_matches('/'); - if let Some(filename) = url_path.split('/').last() { + if let Some(filename) = url_path.split('/').next_back() { if filename.ends_with(".tar.gz") { return filename.trim_end_matches(".tar.gz").to_string(); } diff --git a/src/net/providers/archive.rs b/src/net/providers/archive.rs index c7ed80e..6088cc1 100644 --- a/src/net/providers/archive.rs +++ b/src/net/providers/archive.rs @@ -37,7 +37,7 @@ impl GitProvider for ArchiveProvider { return None; } - let filename = url.split('/').last()?; + let filename = url.split('/').next_back()?; let name = self.extract_name_from_filename(filename); Some( @@ -63,7 +63,7 @@ impl GitProvider for ArchiveProvider { } fn get_project_name(&self, url: &str) -> String { - if let Some(filename) = url.split('/').last() { + if let Some(filename) = url.split('/').next_back() { return self.extract_name_from_filename(filename); } diff --git a/src/net/providers/azure_devops.rs b/src/net/providers/azure_devops.rs index 784cbee..9126979 100644 --- a/src/net/providers/azure_devops.rs +++ b/src/net/providers/azure_devops.rs @@ -82,7 +82,7 @@ impl GitProvider for AzureDevOpsProvider { return parsed.project_name; } - if let Some(filename) = url.split('/').last() { + if let Some(filename) = url.split('/').next_back() { if filename.ends_with(".zip") { return filename.trim_end_matches(".zip").to_string(); } diff --git a/src/net/providers/bitbucket.rs b/src/net/providers/bitbucket.rs index aa62c25..2b980c0 100644 --- a/src/net/providers/bitbucket.rs +++ b/src/net/providers/bitbucket.rs @@ -73,7 +73,7 @@ impl GitProvider for BitbucketProvider { return parsed.project_name; } - if let Some(filename) = url.split('/').last() { + if let Some(filename) = url.split('/').next_back() { if filename.ends_with(".tar.gz") { return filename.trim_end_matches(".tar.gz").to_string(); } diff --git a/src/net/providers/codeberg.rs b/src/net/providers/codeberg.rs index 586ecb5..cc50fc7 100644 --- a/src/net/providers/codeberg.rs +++ b/src/net/providers/codeberg.rs @@ -73,7 +73,7 @@ impl GitProvider for CodebergProvider { return parsed.project_name; } - if let Some(filename) = url.split('/').last() { + if let Some(filename) = url.split('/').next_back() { if filename.ends_with(".tar.gz") { return filename.trim_end_matches(".tar.gz").to_string(); } diff --git a/src/net/providers/gitea.rs b/src/net/providers/gitea.rs index b0fe9dc..5f254f4 100644 --- a/src/net/providers/gitea.rs +++ b/src/net/providers/gitea.rs @@ -78,7 +78,7 @@ impl GitProvider for GiteaProvider { return parsed.project_name; } - if let Some(filename) = url.split('/').last() { + if let Some(filename) = url.split('/').next_back() { if filename.ends_with(".tar.gz") { return filename.trim_end_matches(".tar.gz").to_string(); } diff --git a/src/net/providers/github.rs b/src/net/providers/github.rs index fa6a90f..631079a 100644 --- a/src/net/providers/github.rs +++ b/src/net/providers/github.rs @@ -148,7 +148,7 @@ impl GitProvider for GitHubProvider { return parsed.project_name; } - if let Some(filename) = url.split('/').last() { + if let Some(filename) = url.split('/').next_back() { if filename.ends_with(".tar.gz") { return filename.trim_end_matches(".tar.gz").to_string(); } diff --git a/src/net/providers/gitlab.rs b/src/net/providers/gitlab.rs index 72e3d73..3bf5d7c 100644 --- a/src/net/providers/gitlab.rs +++ b/src/net/providers/gitlab.rs @@ -91,7 +91,7 @@ impl GitProvider for GitLabProvider { return parsed.project_name; } - if let Some(filename) = url.split('/').last() { + if let Some(filename) = url.split('/').next_back() { if filename.ends_with(".tar.gz") { return filename.trim_end_matches(".tar.gz").to_string(); } diff --git a/src/net/providers/sourceforge.rs b/src/net/providers/sourceforge.rs index be7cb2c..4760214 100644 --- a/src/net/providers/sourceforge.rs +++ b/src/net/providers/sourceforge.rs @@ -73,7 +73,7 @@ impl GitProvider for SourceForgeProvider { return parsed.project_name; } - if let Some(filename) = url.split('/').last() { + if let Some(filename) = url.split('/').next_back() { if filename.ends_with(".tar.gz") { return filename.trim_end_matches(".tar.gz").to_string(); } diff --git a/src/worker.rs b/src/worker.rs index 6e2a50f..00efe08 100644 --- a/src/worker.rs +++ b/src/worker.rs @@ -23,9 +23,11 @@ pub async fn analyze_url(url: String, options: JsValue) -> Result Date: Thu, 10 Jul 2025 15:45:11 +0800 Subject: [PATCH 06/11] chore: update clippy lint --- examples/compare_repositories.rs | 6 +++--- src/cli/output.rs | 1 - tests/cli_tests.rs | 16 ---------------- tests/integration_tests.rs | 4 ---- 4 files changed, 3 insertions(+), 24 deletions(-) diff --git a/examples/compare_repositories.rs b/examples/compare_repositories.rs index bc139f5..96d0d17 100644 --- a/examples/compare_repositories.rs +++ b/examples/compare_repositories.rs @@ -47,7 +47,7 @@ async fn main() -> Result<()> { println!("{}", "-".repeat(75)); for (repo, summary) in &results { - let repo_name = repo.split('/').last().unwrap_or(repo); + let repo_name = repo.split('/').next_back().unwrap_or(repo); let primary_lang = summary.primary_language.as_deref().unwrap_or("Unknown"); println!( @@ -70,7 +70,7 @@ async fn main() -> Result<()> { }); for (i, (repo, summary)) in complexity_ranking.iter().enumerate() { - let repo_name = repo.split('/').last().unwrap_or(repo); + let repo_name = repo.split('/').next_back().unwrap_or(repo); println!( "{}. {}: {:.1}% code ratio", i + 1, @@ -89,7 +89,7 @@ async fn main() -> Result<()> { }); for (i, (repo, summary)) in doc_ranking.iter().enumerate() { - let repo_name = repo.split('/').last().unwrap_or(repo); + let repo_name = repo.split('/').next_back().unwrap_or(repo); println!( "{}. {}: {:.1}% documentation ratio", i + 1, diff --git a/src/cli/output.rs b/src/cli/output.rs index 5f42667..24a7d46 100644 --- a/src/cli/output.rs +++ b/src/cli/output.rs @@ -267,4 +267,3 @@ fn xml_escape(text: &str) -> String { .replace("\"", """) .replace("'", "'") } - diff --git a/tests/cli_tests.rs b/tests/cli_tests.rs index 34418b8..961a30e 100644 --- a/tests/cli_tests.rs +++ b/tests/cli_tests.rs @@ -11,8 +11,6 @@ mod cli_tests { analyzer.set_timeout(120); analyzer.set_timeout(60); analyzer.set_timeout(300); - - assert!(true); } #[test] @@ -21,8 +19,6 @@ mod cli_tests { analyzer.set_allow_insecure(true); analyzer.set_allow_insecure(false); - - assert!(true); } #[test] @@ -39,8 +35,6 @@ mod cli_tests { "token_with_different_format".to_string(), ); analyzer.set_provider_credentials("github", credentials2); - - assert!(true); } #[test] @@ -56,8 +50,6 @@ mod cli_tests { analyzer.set_timeout(60); analyzer.set_allow_insecure(false); - - assert!(true); } #[tokio::test] @@ -96,16 +88,12 @@ mod cli_tests { analyzer.set_provider_credentials("github", credentials2); analyzer.set_allow_insecure(true); - - assert!(true); } #[test] fn test_default_analyzer_creation() { let _analyzer1 = RemoteAnalyzer::new(); let _analyzer2 = RemoteAnalyzer::default(); - - assert!(true); } #[test] @@ -121,8 +109,6 @@ mod cli_tests { analyzer2.set_timeout(120); analyzer2.set_allow_insecure(true); - - assert!(true); } #[test] @@ -139,7 +125,5 @@ mod cli_tests { let mut long_credentials = std::collections::HashMap::new(); long_credentials.insert("token".to_string(), "a".repeat(1000)); analyzer.set_provider_credentials("github", long_credentials); - - assert!(true); } } diff --git a/tests/integration_tests.rs b/tests/integration_tests.rs index 26d383d..c80ca1e 100644 --- a/tests/integration_tests.rs +++ b/tests/integration_tests.rs @@ -143,8 +143,6 @@ mod integration_tests { let _analyzer = RemoteAnalyzer::new(); let _default_analyzer = RemoteAnalyzer::default(); - - assert!(true); } #[test] @@ -157,8 +155,6 @@ mod integration_tests { let mut credentials = std::collections::HashMap::new(); credentials.insert("token".to_string(), "test-token".to_string()); analyzer.set_provider_credentials("github", credentials); - - assert!(true); } #[tokio::test] From aa3a9292714a3b74e879d926986f1744c75607fe Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Thu, 10 Jul 2025 15:56:39 +0800 Subject: [PATCH 07/11] fix: wasi build error --- Cargo.lock | 184 ++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- src/cli/args.rs | 2 +- src/net/providers/gitlab.rs | 6 +- src/net/traits.rs | 15 +-- src/server/mod.rs | 34 ------- 6 files changed, 191 insertions(+), 52 deletions(-) delete mode 100644 src/server/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 76b2793..46c9e33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,6 +253,22 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "crc32fast" version = "1.4.2" @@ -318,6 +334,12 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "filetime" version = "0.2.25" @@ -346,6 +368,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -537,6 +574,22 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.14" @@ -837,6 +890,23 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "native-tls" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -864,6 +934,50 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad" +[[package]] +name = "openssl" +version = "0.10.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" + +[[package]] +name = "openssl-sys" +version = "0.9.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -882,6 +996,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "portable-atomic" version = "1.11.1" @@ -1076,9 +1196,11 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -1089,6 +1211,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -1188,6 +1311,38 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.219" @@ -1374,6 +1529,19 @@ dependencies = [ "xattr", ] +[[package]] +name = "tempfile" +version = "3.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +dependencies = [ + "fastrand", + "getrandom 0.3.3", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -1469,6 +1637,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.2" @@ -1650,6 +1828,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "want" version = "0.3.1" diff --git a/Cargo.toml b/Cargo.toml index d6c3f9d..0e3edce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ reqwest = { version = "0.12", features = ["stream", "rustls-tls", "json"], defau [target.'cfg(target_arch = "wasm32")'.dependencies] tokio = { version = "1.46", default-features = false, features = ["macros", "rt", "sync"] } -reqwest = { version = "0.12", features = ["stream", "json"], default-features = false } +reqwest = { version = "0.12", features = ["stream", "json", "default-tls"], default-features = false } [profile.release] opt-level = 3 diff --git a/src/cli/args.rs b/src/cli/args.rs index 9864978..8eb6c0f 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -51,7 +51,7 @@ pub struct Cli { pub url: Option, // Version - #[arg(short = 'v', long = "version", action = clap::ArgAction::Version, help = "Print version information")] + #[arg(short = 'v', long = "version", action = clap::ArgAction::Version, help = "Current version information")] version: (), // Output Options diff --git a/src/net/providers/gitlab.rs b/src/net/providers/gitlab.rs index 3bf5d7c..7806933 100644 --- a/src/net/providers/gitlab.rs +++ b/src/net/providers/gitlab.rs @@ -162,11 +162,7 @@ impl GitLabProvider { } fn build_project_path(&self, owner: &str, repo: &str) -> String { - if owner.contains("/") { - format!("{}/{}", owner, repo) - } else { - format!("{}/{}", owner, repo) - } + format!("{}/{}", owner, repo) } } diff --git a/src/net/traits.rs b/src/net/traits.rs index 23db293..a115159 100644 --- a/src/net/traits.rs +++ b/src/net/traits.rs @@ -313,15 +313,10 @@ pub trait GitProvider: Send + Sync { { builder = builder.timeout(Duration::from_secs(timeout)); } - #[cfg(target_arch = "wasm32")] - { - // On WASM, timeout is handled differently - builder = builder.timeout(Duration::from_secs(timeout.min(300))); - // Max 5 minutes for WASM - } } // Set redirects + #[cfg(not(target_arch = "wasm32"))] if let Some(max_redirects) = config.max_redirects { builder = builder.redirect(reqwest::redirect::Policy::limited(max_redirects as usize)); } @@ -333,13 +328,11 @@ pub trait GitProvider: Send + Sync { } // Set compression + #[cfg(not(target_arch = "wasm32"))] if !config.use_compression { builder = builder.no_gzip(); - #[cfg(not(target_arch = "wasm32"))] - { - builder = builder.no_brotli(); - builder = builder.no_deflate(); - } + builder = builder.no_brotli(); + builder = builder.no_deflate(); } // Set proxy (only on native) diff --git a/src/server/mod.rs b/src/server/mod.rs deleted file mode 100644 index 2c38dff..0000000 --- a/src/server/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -use crate::net::RemoteAnalyzer; -use wasm_bindgen::prelude::*; - -#[derive(serde::Serialize, serde::Deserialize)] -pub struct AnalysisOptions { - pub ignore_hidden: bool, - pub ignore_gitignore: bool, - pub max_file_size: i64, -} - -#[cfg(feature = "worker")] -#[wasm_bindgen] -pub async fn analyze_url_server(url: &str, options: JsValue) -> Result { - let _opts: AnalysisOptions = serde_wasm_bindgen::from_value(options) - .map_err(|e| JsValue::from_str(&format!("Failed to parse options: {}", e)))?; - - let mut analyzer = RemoteAnalyzer::new(); - analyzer.set_timeout(300); - - match analyzer.analyze_url(url).await { - Ok(analysis) => { - web_sys::console::log_1(&format!("Server: Successfully analyzed project: {}", analysis.project_name).into()); - serde_wasm_bindgen::to_value(&analysis).map_err(|e| { - JsValue::from_str(&format!("Failed to serialize analysis result: {}", e)) - }) - } - Err(e) => { - web_sys::console::log_1(&format!("Server: Analysis failed: {}", e).into()); - Err(JsValue::from_str(&format!("Analysis failed: {}", e))) - } - } -} - - \ No newline at end of file From 8350e7c8432ff4c22f5d7108862d91a418947e16 Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Thu, 10 Jul 2025 16:26:33 +0800 Subject: [PATCH 08/11] chore: update docs --- README.md | 243 ++++++++++++++++----------------------------- examples/README.md | 94 ------------------ package.json | 2 +- tests/README.md | 134 ------------------------- 4 files changed, 85 insertions(+), 388 deletions(-) delete mode 100644 examples/README.md delete mode 100644 tests/README.md diff --git a/README.md b/README.md index 034aef3..57e5815 100644 --- a/README.md +++ b/README.md @@ -13,51 +13,42 @@ Hyper-fast **CLOC** _(\*count lines of code)_ tool for remote repositories. ## Features -- **Asynchronous Repository Processing**: Non-blocking HTTP client with async streaming request processing for efficient remote repository fetching and decompression, optimized for **low memory usage** and **serverless environments** (always `<32MiB` runtime memory usage for large files) -- **Multi-Platform URL Resolution**: Features intelligent URL parsing engine that normalizes different Git hosting platform APIs (GitHub, GitLab, Bitbucket, Codeberg) into unified archive endpoints with branch/commit resolution -- **Streaming Archive Analysis**: Processes tar.gz archives directly in memory using streaming decompression without temporary file extraction, reducing I/O overhead and memory footprint -- **Language Detection Engine**: Implements rule-based file extension and content analysis system supporting 150+ programming languages with configurable pattern matching and statistical computation (use tokei [languages map](https://github.com/XAMPPRocky/tokei/blob/master/languages.json)) -- **Real-time Progress Monitoring**: Features bandwidth-aware progress tracking with download speed calculation, ETA estimation, and adaptive UI rendering for terminal environments -- **Structured Data Serialization**: Provides multiple output format engines (Table, JSON, CSV, XML) with schema validation and type-safe serialization for integration with external tools -- **Authentication Layer**: Implements OAuth token management with secure credential handling for accessing private repositories across different hosting platforms -- **Cross-Platform Binary Distribution**: Supports native compilation targets for Linux, macOS, and Windows with platform-specific optimizations and dependency management +- Efficient remote repository analysis with async streaming and in-memory decompression, optimized for low memory usage (always <12MB runtime mem) +- Unified URL parsing for GitHub, GitLab, Bitbucket, Codeberg, SourceForge, Gitea and Azure DevOps +- Rule-based language detection supporting 150+ programming languages with [Tokei's Language Rules](https://github.com/XAMPPRocky/tokei/blob/master/languages.json) +- Real-time progress tracking with download speed, ETA and adaptive terminal UI +- Multiple output formats (Table, JSON, CSV, XML, YAML, TOML) with schema validation +- OAuth token management for private repository access +- Native binaries for Linux, macOS and Windows +- Experimental parallel processing and streaming analysis -## Installation - -### From Cargo (Recommended) - -```bash -cargo install bytes-radar -``` +## Supported Platforms -### From Releases +| Platform | URL Format | Example | +| ------------------ | ----------------------------------------- | ------------------------------------------- | +| **GitHub** | `user/repo`, `user/repo@branch`, full URL | `microsoft/vscode`, `torvalds/linux@master` | +| **GitLab** | Full URL | `https://gitlab.com/user/repo` | +| **Bitbucket** | Full URL | `https://bitbucket.org/user/repo` | +| **Codeberg** | Full URL | `https://codeberg.org/user/repo` | +| **SourceForge** | Full URL | `https://sourceforge.net/user/repo` | +| **Gitea** | Full URL | `https://gitea.example.com/user/repo` | +| **Azure DevOps** | Full URL | `https://dev.azure.com/org/project` | +| **Direct Archive** | tar.gz, tgz, zip URL | `https://example.com/archive.tar.gz` | -Download the latest binary from [GitHub Releases](https://github.com/zmh-program/bytes-radar/releases) +## Installation -### From Source +Download the latest binary from **[GitHub Releases](https://github.com/zmh-program/bytes-radar/releases)** or install via Cargo: ```bash -git clone https://github.com/zmh-program/bytes-radar.git -cd bytes-radar -cargo build --release +cargo install bytes-radar ``` ## Usage -```bash -bradar [OPTIONS] -``` - -### Examples - -#### Basic Repository Analysis - -Analyze GitHub repositories using shorthand notation: +#### Basic Repo Analysis ```bash -bradar torvalds/linux -bradar microsoft/typescript -bradar rust-lang/cargo +bradar torvalds/linux # or https://github.com/torvalds/linux ``` #### Branch and Commit Targeting @@ -65,9 +56,8 @@ bradar rust-lang/cargo Specify particular branches or commit hashes for analysis: ```bash -bradar microsoft/vscode@main -bradar kubernetes/kubernetes@release-1.28 -bradar rust-lang/rust@abc1234567 +bradar microsoft/vscode@main # or https://github.com/microsoft/vscode/tree/main +bradar kubernetes/kubernetes@release-1.28 # or https://github.com/kubernetes/kubernetes/tree/release-1.28 ``` #### Multi-Platform Repository Support @@ -86,8 +76,6 @@ Generate analysis results in structured data formats: ```bash bradar -f json torvalds/linux -bradar -f csv microsoft/typescript -bradar -f xml rust-lang/cargo ``` #### Private Repository Access @@ -108,132 +96,71 @@ bradar --quiet --no-progress user/repo bradar --timeout 600 --detailed large-org/massive-repo ``` -## Usage Environments - -### CLI - -See the CLI Options section below for command-line usage. - -## Output Formats - -### Table (Default) - -```shell -$ bradar torvalds/linux -Analyzing: https://github.com/torvalds/linux -Analysis completed in 123.76s - -================================================================================ - Project linux@main - Total Files 89,639 - Total Lines 40,876,027 - Code Lines 32,848,710 - Comment Lines 2,877,885 - Blank Lines 5,149,432 - Languages 51 - Primary Language C - Code Ratio 80.4% - Documentation 8.8% -================================================================================ - Language Files Lines Code Comments Blanks % -================================================================================ - C 35,586 25,268,107 18,782,347 2,836,806 3,648,954 61.8% - C Header 25,845 10,247,647 9,481,722 0 765,925 25.1% - Device Tree 5,789 1,831,396 1,589,630 0 241,766 4.5% - ReStructuredText 3,785 782,387 593,628 0 188,759 1.9% - JSON 961 572,657 572,655 0 2 1.4% - Text 5,100 566,733 499,590 0 67,143 1.4% - YAML 4,862 548,408 458,948 0 89,460 1.3% - GNU Style Assembly 1,343 373,956 326,745 0 47,211 0.9% - Shell 960 189,965 155,974 0 33,991 0.5% - Plain Text 1,298 128,205 105,235 0 22,970 0.3% - Python 293 89,285 69,449 5,770 14,066 0.2% - Makefile 3,115 82,692 57,091 13,109 12,492 0.2% - SVG 82 53,409 53,316 0 93 0.1% - Perl 58 43,986 33,264 4,406 6,316 0.1% - Rust 158 39,561 19,032 16,697 3,832 0.1% - XML 24 22,193 20,971 0 1,222 0.1% - PO File 7 6,711 5,605 0 1,106 0.0% - Happy 10 6,078 5,352 0 726 0.0% - Assembly 11 5,361 4,427 0 934 0.0% - Lex 10 2,996 2,277 347 372 0.0% - AWK 12 2,611 1,777 487 347 0.0% - C++ 7 2,267 1,932 0 335 0.0% - ... -================================================================================ - Total 89,639 40,876,027 32,848,710 2,877,885 5,149,432 100.0% -``` - -### JSON Output - -```json -{ - "project_name": "linux@master", - "summary": { - "total_files": 75823, - "total_lines": 28691744, - "code_lines": 22453891, - "comment_lines": 3891234, - "blank_lines": 2346619 - }, - "language_statistics": [...] -} -``` - -## Supported Platforms - -| Platform | URL Format | Example | -| ------------- | ----------------------- | --------------------------------- | -| **GitHub** | `user/repo` or full URL | `torvalds/linux` | -| **GitLab** | Full URL | `https://gitlab.com/user/repo` | -| **Bitbucket** | Full URL | `https://bitbucket.org/user/repo` | -| **Codeberg** | Full URL | `https://codeberg.org/user/repo` | -| **Direct** | tar.gz URL | `https://example.com/file.tar.gz` | - ## CLI Options ```bash bradar [OPTIONS] ARGUMENTS: - URL to analyze: user/repo, user/repo@branch, or full URL + Repository URL to analyze (user/repo, user/repo@branch, or full URL) OPTIONS: - -f, --format Output format [table|json|csv|xml] - --detailed Show detailed file-by-file statistics - -d, --debug Enable debug output - --token GitHub token for private repositories - --timeout Request timeout in seconds [default: 300] - --allow-insecure Allow insecure HTTP connections - --no-progress Disable progress bar - --quiet Quiet mode - minimal output - -h, --help Print help - -V, --version Print version + # Output Options + -f, --format Output format [table|json|csv|xml|yaml|toml] + --detailed Show detailed file-by-file statistics + -q, --quiet Quiet mode - suppress progress and minimize output + --no-progress Disable progress bar + --no-color Disable colored output + + # Authentication + --token Authentication token for private repositories + + # Network Options + --timeout Request timeout in seconds [default: 300] + --allow-insecure Allow insecure HTTPS connections + --user-agent Custom User-Agent string + --retry-count Number of retry attempts for failed requests [default: 3] + + # Filtering Options + --aggressive-filter Enable aggressive filtering for maximum performance + --max-file-size Maximum file size to process in KB [default: 1024] + --min-file-size Minimum file size to process in bytes [default: 1] + --include-tests Include test directories in analysis + --include-docs Include documentation directories in analysis + --include-hidden Include hidden files and directories + --exclude-pattern Exclude files matching this pattern (glob) + --include-pattern Only include files matching this pattern (glob) + + # Language Options + --language Only analyze files of specific language + --exclude-language Exclude specific language from analysis + + # Analysis Options + --ignore-whitespace Ignore whitespace-only lines in code analysis + --count-generated Include generated files in analysis + --max-line-length Maximum line length to consider (0 = unlimited) [default: 0] + + # Debug and Logging + -d, --debug Enable debug output + --trace Enable trace-level logging + --log-file Write logs to file + + # Advanced Options + --threads Number of worker threads (0 = auto) [default: 0] + --memory-limit Memory limit in MB (0 = unlimited) [default: 0] + --cache-dir Directory for caching downloaded files + --no-cache Disable caching of downloaded files + + # Experimental Features + --experimental-parallel Enable experimental parallel processing + --experimental-streaming Enable experimental streaming analysis + + # General + -v, --version Print version + -h, --help Print help ``` -## Contributing - -We welcome contributions! Please see [CONTRIBUTING.md](docs/CONTRIBUTING.md) for guidelines. - -### Development Setup - -```bash -# Clone the repository -git clone https://github.com/zmh-program/bytes-radar.git -cd bytes-radar - -# Install dependencies -cargo build - -# Run tests -cargo test --all-features - -# Format code -cargo fmt - -# Lint code -cargo clippy --all-targets --all-features -``` +See [CLI USAGE GUIDE](docs/CLI_USAGE.md) for more detailed usage examples and advanced configurations. ## Deployment @@ -244,10 +171,8 @@ cargo clippy --all-targets --all-features > [!TIP] > The Free Tier of Cloudflare Workers has a **20s request timeout limit** (wall time). Analysis of large repositories may fail due to this limitation. Consider upgrading to Cloudflare Workers Pro or using alternative deployment methods for processing large repositories. -For detailed deployment instructions and API documentation, see [DEPLOYMENT.md](docs/DEPLOYMENT.md). +For detailed deployment instructions and API documentation, see [DEPLOYMENT GUIDE](docs/DEPLOYMENT.md). -## Usage Environments - -### CLI +## Contributing -See the CLI Options section below for command-line usage. +We welcome contributions! Please see [CONTRIBUTING GUIDE](docs/CONTRIBUTING.md) for guidelines. diff --git a/examples/README.md b/examples/README.md deleted file mode 100644 index 2550a30..0000000 --- a/examples/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# Examples - -This directory contains examples demonstrating how to use the bytes-radar library. - -## Available Examples - -### 1. Basic Analysis (`basic_analysis.rs`) - -Demonstrates basic usage of the RemoteAnalyzer to analyze a public repository. - -```bash -cargo run --example basic_analysis -``` - -This example analyzes the bytes-radar repository itself and displays: - -- Project summary statistics -- Language breakdown -- Total lines, files, and size information - -### 2. GitHub Token Analysis (`github_token_analysis.rs`) - -Shows how to use GitHub authentication for private repositories. - -```bash -export GITHUB_TOKEN=your_github_token_here -cargo run --example github_token_analysis -``` - -Make sure to: - -1. Set the `GITHUB_TOKEN` environment variable -2. Update the repository URL in the example to point to your private repo -3. Ensure your token has access to the repository - -### 3. Compare Repositories (`compare_repositories.rs`) - -Compares multiple repositories and ranks them by various metrics. - -```bash -cargo run --example compare_repositories -``` - -This example: - -- Analyzes multiple popular repositories -- Compares them side by side -- Ranks by complexity and documentation ratios -- Demonstrates batch processing - -### 4. Custom Analysis (`custom_analysis.rs`) - -Demonstrates manual construction of project analysis without remote fetching. - -```bash -cargo run --example custom_analysis -``` - -This example shows how to: - -- Create FileMetrics manually -- Build a ProjectAnalysis step by step -- Use different file categories -- Display detailed file-by-file information - -## Running All Examples - -To run all examples in sequence: - -```bash -cargo run --example basic_analysis -cargo run --example custom_analysis -cargo run --example compare_repositories - -# For github_token_analysis, set your token first: -export GITHUB_TOKEN=your_token -cargo run --example github_token_analysis -``` - -## Notes - -- Examples that analyze remote repositories require an internet connection -- GitHub token examples require valid authentication -- Some examples may take time to complete due to network requests -- Adjust timeout values if you experience connection issues - -## Customizing Examples - -Feel free to modify these examples: - -- Change repository URLs to analyze your own projects -- Adjust timeout values for slower connections -- Add error handling for production use -- Experiment with different output formats in the CLI diff --git a/package.json b/package.json index 87d7fcb..0d850c1 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "dev": "wrangler dev", "deploy": "wrangler deploy", "deploy:prod": "node server/build.js && wrangler deploy", - "prettier": "prettier --write ." + "prettier": "prettier --write . && cargo fmt" }, "dependencies": { "@cloudflare/workers-types": "^4.20240208.0" diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index 15368e5..0000000 --- a/tests/README.md +++ /dev/null @@ -1,134 +0,0 @@ -# Tests - -This directory contains comprehensive tests for the bytes-radar library. - -## Test Structure - -### Integration Tests (`integration_tests.rs`) - -Tests the main API functionality and integration between components: - -- **File Metrics Creation**: Tests creating and validating file metrics -- **Project Analysis**: Tests building project analysis from multiple files -- **Language Statistics**: Tests aggregation and calculation of language-specific statistics -- **Remote Analyzer**: Tests basic analyzer configuration and usage -- **Data Validation**: Tests input validation and error handling - -### Analysis Tests (`analysis_tests.rs`) - -Focuses on the core analysis functionality: - -- **Language Analysis**: Tests language-specific analysis and aggregation -- **Aggregate Metrics**: Tests metric accumulation and calculation -- **Merging Operations**: Tests combining analysis from multiple sources -- **Statistics Calculation**: Tests ratio calculations and derived metrics -- **Edge Cases**: Tests boundary conditions and error scenarios - -### CLI Tests (`cli_tests.rs`) - -Tests the command-line interface and remote operations: - -- **Analyzer Configuration**: Tests timeout, authentication, and security settings -- **URL Handling**: Tests parsing and validation of repository URLs -- **Error Handling**: Tests behavior with invalid inputs -- **Network Operations**: Tests timeout and connection handling -- **Configuration Persistence**: Tests setting and updating analyzer parameters - -## Running Tests - -### Run All Tests - -```bash -cargo test -``` - -### Run Specific Test Modules - -```bash -# Integration tests only -cargo test integration_tests - -# Analysis tests only -cargo test analysis_tests - -# CLI tests only -cargo test cli_tests -``` - -### Run Individual Tests - -```bash -# Run a specific test -cargo test test_file_metrics_creation - -# Run tests matching a pattern -cargo test file_metrics - -# Run with output -cargo test -- --nocapture -``` - -### Test Coverage - -```bash -# Run tests with coverage (requires cargo-tarpaulin) -cargo install cargo-tarpaulin -cargo tarpaulin -``` - -## Test Categories - -### Unit Tests - -- Test individual functions and methods in isolation -- Focus on specific functionality and edge cases -- Fast execution, no external dependencies - -### Integration Tests - -- Test interaction between multiple components -- Test complete workflows and use cases -- May include mock network operations - -### Network Tests - -- Test actual network operations (marked with `#[ignore]` by default) -- Require internet connection -- May be flaky due to network conditions - -## Running Network Tests - -Some tests require internet connectivity and are ignored by default: - -```bash -# Run all tests including network tests -cargo test -- --ignored - -# Run only network tests -cargo test network --ignored -``` - -## Test Data - -Tests use predefined test data and mock repositories where possible to ensure: - -- Consistent results across environments -- Fast test execution -- No dependency on external services - -## Contributing - -When adding new tests: - -1. **Unit tests** should go in the same file as the code they test (using `#[cfg(test)]`) -2. **Integration tests** should go in the appropriate test file in this directory -3. **Network tests** should be marked with `#[ignore]` and documented -4. Include both positive and negative test cases -5. Test edge cases and error conditions -6. Use descriptive test names that explain what is being tested - -## Test Performance - -- Tests should complete quickly (< 1 second for unit tests) -- Network tests may take longer but should have reasonable timeouts -- Use `cargo test --release` for performance-sensitive tests From e051bb1c93646471bae5f4970b74f369ef6f45d7 Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Thu, 10 Jul 2025 16:31:31 +0800 Subject: [PATCH 09/11] chore: update pr rules --- .github/pull_request_template.md | 75 +------------------------------- 1 file changed, 1 insertion(+), 74 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 685fa6d..4c4bd85 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,7 +1,3 @@ -## Description - - - ## Type of Change @@ -13,73 +9,4 @@ - [ ] Performance improvement - [ ] Code refactoring (no functional changes) -## Related Issues - - - -Fixes # - -## Changes Made - - - -- -- -- - -## Testing - - - -- [ ] All existing tests pass -- [ ] New tests added for new functionality -- [ ] Manual testing performed -- [ ] Tested on multiple platforms (Linux/macOS/Windows) - -### Test Commands - -```bash -# Commands used to test the changes -cargo test --all-features -cargo fmt --all -- --check -cargo clippy --all-targets --all-features -- -D warnings -``` - -## Performance Impact - - - -- [ ] No performance impact -- [ ] Performance improved -- [ ] Performance impact acceptable for the feature -- [ ] Performance benchmarks included - -## Breaking Changes - - - -## Documentation - -- [ ] Documentation updated (if applicable) -- [ ] README updated (if applicable) -- [ ] Help text updated (if applicable) -- [ ] Examples updated (if applicable) - -## Checklist - -- [ ] Code follows the project's style guidelines -- [ ] Self-review of the code completed -- [ ] Code is commented, particularly in hard-to-understand areas -- [ ] Corresponding changes to documentation made -- [ ] Changes generate no new warnings -- [ ] New tests added that prove the fix is effective or the feature works -- [ ] All new and existing tests pass locally -- [ ] Any dependent changes have been merged - -## Screenshots - - - -## Additional Notes - - +## Description From ba3c3f027444ef062eeb76fceea8ba3f5c16334d Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Thu, 10 Jul 2025 16:40:44 +0800 Subject: [PATCH 10/11] chore: update cli colors --- src/cli/output.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/output.rs b/src/cli/output.rs index 24a7d46..b7106b6 100644 --- a/src/cli/output.rs +++ b/src/cli/output.rs @@ -5,7 +5,7 @@ use colored::Colorize; fn get_percentage_color(percentage: f64) -> colored::ColoredString { let percentage_str = format!("{:.1}%", percentage); if percentage >= 50.0 { - percentage_str.red() + percentage_str.bright_green() } else if percentage >= 10.0 { percentage_str.yellow() } else if percentage >= 1.0 { @@ -64,12 +64,12 @@ pub fn print_table_format(project_analysis: &ProjectAnalysis, detailed: bool, qu println!( " {:<56} {}", "Code Ratio", - get_percentage_color(summary.overall_complexity_ratio * 100.0) + format!("{:.1}%", summary.overall_complexity_ratio * 100.0).bold() ); println!( " {:<56} {}", "Documentation", - get_percentage_color(summary.overall_documentation_ratio * 100.0) + format!("{:.1}%", summary.overall_documentation_ratio * 100.0).bold() ); if !language_stats.is_empty() && !quiet { From 84e023feac3b183bbe429d55187eb8f5ad9b3154 Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Thu, 10 Jul 2025 16:44:50 +0800 Subject: [PATCH 11/11] refactor: worker structure --- .gitignore | 4 ++-- README.md | 2 +- docs/CONTRIBUTING.md | 13 ------------- docs/DEPLOYMENT.md | 25 ++++++------------------- package.json | 2 +- {server => worker}/build.js | 8 ++++---- {server => worker}/package.json | 0 {server => worker}/tsconfig.json | 0 {server => worker}/types.d.ts | 0 {server => worker}/worker.ts | 0 wrangler.toml | 4 ++-- 11 files changed, 16 insertions(+), 42 deletions(-) rename {server => worker}/build.js (86%) rename {server => worker}/package.json (100%) rename {server => worker}/tsconfig.json (100%) rename {server => worker}/types.d.ts (100%) rename {server => worker}/worker.ts (100%) diff --git a/.gitignore b/.gitignore index 09e7d70..d2c2404 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,5 @@ pkg/ node_modules/ wasm/node_modules/ -server/.wrangler/ -server/node_modules/ +worker/.wrangler/ +worker/node_modules/ diff --git a/README.md b/README.md index 57e5815..4e15111 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ See [CLI USAGE GUIDE](docs/CLI_USAGE.md) for more detailed usage examples and ad [![Deploy to Cloudflare Workers](https://deploy.workers.cloudflare.com/button.svg)](https://deploy.workers.cloudflare.com/?url=https://github.com/zmh-program/bytes-radar) > [!TIP] -> The Free Tier of Cloudflare Workers has a **20s request timeout limit** (wall time). Analysis of large repositories may fail due to this limitation. Consider upgrading to Cloudflare Workers Pro or using alternative deployment methods for processing large repositories. +> The Free Tier of Cloudflare Workers has a **20s request timeout limit** (wall time). Analysis of large repositories may fail due to this limitation. Consider upgrading to Cloudflare Workers Pro or using alternative methods for processing large repositories. For detailed deployment instructions and API documentation, see [DEPLOYMENT GUIDE](docs/DEPLOYMENT.md). diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index fdbbd8b..38c11f4 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -175,19 +175,6 @@ Releases are handled by maintainers: 3. Create git tag 4. GitHub Actions builds and publishes -## Project Structure - -``` -bytes-radar/ -├── .github/ # GitHub workflows and templates -├── src/ -│ ├── cli/ # Command-line interface -│ ├── core/ # Core analysis logic -│ └── lib.rs # Library entry point -├── tests/ # Integration tests -└── examples/ # Usage examples -``` - ## Areas for Contribution We welcome contributions in these areas: diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 58dfb38..bcd492c 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -5,11 +5,9 @@ [![Deploy to Cloudflare Workers](https://deploy.workers.cloudflare.com/button.svg)](https://deploy.workers.cloudflare.com/?url=https://github.com/zmh-program/bytes-radar) > [!TIP] -> The Free Tier of Cloudflare Workers has a **20s request timeout limit**. Analysis of large repositories may fail due to this limitation. Consider upgrading to Cloudflare Workers Pro or using alternative deployment methods for processing large repositories. +> The Free Tier of Cloudflare Workers has a **20s request timeout limit**. Analysis of large repositories may fail due to this limitation. Consider upgrading to Cloudflare Workers Pro or using alternative methods for processing large repositories. -The server component is automatically built and pushed to the `cf-worker` branch whenever changes are made to the server code. You can deploy to Cloudflare Workers with one click using the button above. This will: - -1. Fork the repository to your GitHub account (using the pre-built worker from cf-worker branch) +1. Fork the repository to your GitHub account 2. Connect it to your Cloudflare Workers account 3. Deploy the worker to your chosen environment @@ -17,11 +15,11 @@ The server component is automatically built and pushed to the `cf-worker` branch If you prefer to deploy manually: -1. Clone the cf-worker branch which contains the pre-built worker: +1. Clone the repository ```bash -git clone -b cf-worker https://github.com/zmh-program/bytes-radar.git -cd bytes-radar/server +git clone https://github.com/zmh-program/bytes-radar.git +cd bytes-radar/worker ``` 2. Install Wrangler CLI: @@ -55,7 +53,7 @@ The worker supports two environments: - `staging`: For testing and development (bytes-radar-staging.workers.dev) - `production`: For production use (bytes-radar-prod.workers.dev) -See `server/wrangler.toml` for environment-specific configurations. +See `worker/wrangler.toml` for environment-specific configurations. ## API Documentation @@ -159,14 +157,3 @@ GET /zmh-program/bytes-radar } } ``` - -### Rate Limits and Timeouts - -- Request timeout: 20~30 seconds (Free tier) -- Rate limits: Based on Cloudflare Workers limits - -### Notes - -- Large repositories may hit the 20-second timeout limit on the free tier -- For analyzing large repositories, consider using the CLI tool or upgrading to Cloudflare Workers Pro -- The service automatically tries common branch names (main, master, develop, dev) if not specified diff --git a/package.json b/package.json index 0d850c1..2236d62 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "build": "node build.js", "dev": "wrangler dev", "deploy": "wrangler deploy", - "deploy:prod": "node server/build.js && wrangler deploy", + "deploy:prod": "node worker/build.js && wrangler deploy", "prettier": "prettier --write . && cargo fmt" }, "dependencies": { diff --git a/server/build.js b/worker/build.js similarity index 86% rename from server/build.js rename to worker/build.js index f887b4f..30d7b27 100644 --- a/server/build.js +++ b/worker/build.js @@ -34,13 +34,13 @@ try { ); } -if (!fs.existsSync("server/pkg")) { - fs.mkdirSync("server/pkg", { recursive: true }); +if (!fs.existsSync("worker/pkg")) { + fs.mkdirSync("worker/pkg", { recursive: true }); } console.log("Building WebAssembly module..."); execSync( - "wasm-pack build --target web --out-dir server/pkg --no-default-features --features worker", + "wasm-pack build --target web --out-dir worker/pkg --no-default-features --features worker", { stdio: "inherit", cwd: process.cwd(), @@ -59,7 +59,7 @@ export function analyze_url(url: string, options: AnalyzeOptions): Promise; `; fs.writeFileSync( - path.join(process.cwd(), "server/pkg", "bytes_radar.d.ts"), + path.join(process.cwd(), "worker/pkg", "bytes_radar.d.ts"), typesContent, ); diff --git a/server/package.json b/worker/package.json similarity index 100% rename from server/package.json rename to worker/package.json diff --git a/server/tsconfig.json b/worker/tsconfig.json similarity index 100% rename from server/tsconfig.json rename to worker/tsconfig.json diff --git a/server/types.d.ts b/worker/types.d.ts similarity index 100% rename from server/types.d.ts rename to worker/types.d.ts diff --git a/server/worker.ts b/worker/worker.ts similarity index 100% rename from server/worker.ts rename to worker/worker.ts diff --git a/wrangler.toml b/wrangler.toml index cbd70c7..1509fc5 100644 --- a/wrangler.toml +++ b/wrangler.toml @@ -1,9 +1,9 @@ name = "bradar" -main = "server/worker.ts" +main = "worker/worker.ts" compatibility_date = "2024-01-01" [build] -command = "node server/build.js" +command = "node worker/build.js" [durable_objects] bindings = [