From d268cfbb2b6a0ca097f1089d97761550dca84d71 Mon Sep 17 00:00:00 2001 From: Firas al-Khalil Date: Tue, 8 Apr 2025 10:16:47 +0200 Subject: [PATCH 1/5] chore(lint): address rustc 1.86 new warnings --- src/config.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.rs b/src/config.rs index 2b68042..4b1a2ea 100644 --- a/src/config.rs +++ b/src/config.rs @@ -23,7 +23,7 @@ pub fn run(command: &ConfigCommand, config: &Path) -> Result<()> { ); } ConfigCommand::Default => println!("{}", toml::to_string(&BuildCommand::default())?), - }; + } Ok(()) } @@ -44,7 +44,7 @@ pub fn current(config: &Path, command: Option<&BuildCommand>) -> Result { debug!("Skipping cli args + config file merger."); } - }; + } debug!("from_both = {:?}", from_file); // Figment is screwing with me, and it's overriding config coming // from Env::prefixed("TSDL_"). From 43a98b01060d6b45d2977920236667c918a0ff43 Mon Sep 17 00:00:00 2001 From: Firas al-Khalil Date: Tue, 8 Apr 2025 10:18:08 +0200 Subject: [PATCH 2/5] fix(tree_sitter): avoid needless iteration --- src/tree_sitter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tree_sitter.rs b/src/tree_sitter.rs index 665c811..c5d8c12 100644 --- a/src/tree_sitter.rs +++ b/src/tree_sitter.rs @@ -38,7 +38,7 @@ fn parse_refs(stdout: &str) -> HashMap { for line in stdout.lines() { let ref_line = line.split('\t').map(str::trim).collect::>(); let (sha1, full_ref) = (ref_line[0], ref_line[1]); - if let Some(tag) = full_ref.split('/').last() { + if let Some(tag) = full_ref.split('/').next_back() { trace!("insert {tag} -> {sha1}"); refs.insert(tag.to_string(), sha1.to_string()); } From 6b0d5cc33c12fb0723e5d6c25c763a533ff82604 Mon Sep 17 00:00:00 2001 From: Firas al-Khalil Date: Wed, 2 Apr 2025 10:01:52 +0200 Subject: [PATCH 3/5] chore(cliff): ignore tests --- cliff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/cliff.toml b/cliff.toml index b22ad6a..8865f1c 100644 --- a/cliff.toml +++ b/cliff.toml @@ -80,6 +80,7 @@ commit_parsers = [ { message = "^perf", group = "Performance" }, { message = "^doc", group = "Documentation" }, { message = "^refactor", skip = true }, + { message = "^test", skip = true }, { message = "^style", skip = true }, { message = "^chore", skip = true }, { message = ".*", group = "Other" }, From d9bd34ebcaddb3b36c4dbe3637a174464b45fee1 Mon Sep 17 00:00:00 2001 From: Firas al-Khalil Date: Mon, 7 Apr 2025 17:46:06 +0200 Subject: [PATCH 4/5] chore(build): sort --- build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.rs b/build.rs index d72cb77..b05cd12 100644 --- a/build.rs +++ b/build.rs @@ -61,13 +61,13 @@ fn write_tsdl_consts(meta: &serde_json::Map, out_dir: let tsdl = meta.get("tsdl").unwrap(); let tsdl_build_dir = tsdl.get("build-dir").unwrap().as_str().unwrap(); let tsdl_config_file = tsdl.get("config").unwrap().as_str().unwrap(); + let tsdl_consts = Path::new(&out_dir).join("tsdl_consts.rs"); let tsdl_fresh = tsdl.get("fresh").unwrap().as_bool().unwrap(); let tsdl_from = tsdl.get("from").unwrap().as_str().unwrap(); let tsdl_out_dir = tsdl.get("out").unwrap().as_str().unwrap(); let tsdl_prefix = tsdl.get("prefix").unwrap().as_str().unwrap(); let tsdl_ref = tsdl.get("ref").unwrap().as_str().unwrap(); let tsdl_show_config = tsdl.get("show-config").unwrap().as_bool().unwrap(); - let tsdl_consts = Path::new(&out_dir).join("tsdl_consts.rs"); fs::write( tsdl_consts, formatdoc!( From a5e63232f93d71c6a1c53662254038182a9d331e Mon Sep 17 00:00:00 2001 From: Firas al-Khalil Date: Mon, 7 Apr 2025 20:00:16 +0200 Subject: [PATCH 5/5] feat(target): use --target [native, wasm, all] PR: https://github.com/stackmystack/tsdl/pull/21 --- .github/workflows/test.yml | 5 +++ CHANGELOG.md | 5 +++ src/args.rs | 38 +++++++++++++++++- src/build.rs | 9 ++++- src/parser.rs | 81 +++++++++++++++++++++++++++----------- tests/cmd/build.rs | 44 +++++++++++++++++++-- 6 files changed, 153 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9601c0..5189cb8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,4 +33,9 @@ jobs: with: tool: cargo-nextest,just + - name: Install emcc + uses: mymindstorm/setup-emsdk@v14 + - name: Verify emcc + run: emcc -v + - run: just test diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c582e0..6ab2f9d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +- Add support for `--target`: + - `native` for native shared libraries, `dylib` or `so` (default). + - `wasm` for web assembly. + - `all` for all of the above. + ## [1.4.0] - 2025-03-22 ### Features diff --git a/src/args.rs b/src/args.rs index bda5944..40e1765 100644 --- a/src/args.rs +++ b/src/args.rs @@ -107,6 +107,35 @@ impl Command { } } +#[derive(clap::ValueEnum, Clone, Copy, Debug, Deserialize, Diff, PartialEq, Eq, Serialize)] +#[diff(attr( + #[derive(Debug, PartialEq)] +))] +#[serde(rename_all = "kebab-case")] +pub enum Target { + Native, + Wasm, + All, +} + +impl Default for Target { + fn default() -> Self { + Self::Native + } +} + +impl Target { + #[must_use] + pub fn native(&self) -> bool { + matches!(self, Self::All | Self::Native) + } + + #[must_use] + pub fn wasm(&self) -> bool { + matches!(self, Self::All | Self::Wasm) + } +} + #[allow(clippy::struct_excessive_bools)] #[derive(clap::Args, Clone, Debug, Deserialize, Diff, PartialEq, Eq, Serialize)] #[diff(attr( @@ -153,6 +182,10 @@ pub struct BuildCommand { #[serde(default)] pub show_config: bool, + /// Build target. + #[arg(short, long, value_enum, default_value_t = Target::default())] + pub target: Target, + #[command(flatten)] #[serde(default)] pub tree_sitter: TreeSitter, @@ -161,14 +194,15 @@ pub struct BuildCommand { impl Default for BuildCommand { fn default() -> Self { Self { - languages: None, - parsers: None, build_dir: PathBuf::from(TSDL_BUILD_DIR), fresh: TSDL_FRESH, + languages: None, ncpus: num_cpus::get(), out_dir: PathBuf::from(TSDL_OUT_DIR), + parsers: None, prefix: String::from(TSDL_PREFIX), show_config: TSDL_SHOW_CONFIG, + target: Target::default(), tree_sitter: TreeSitter::default(), } } diff --git a/src/build.rs b/src/build.rs index 805c4bf..aad2dbb 100644 --- a/src/build.rs +++ b/src/build.rs @@ -10,7 +10,7 @@ use tokio::time; use url::Url; use crate::{ - args::{BuildCommand, ParserConfig}, + args::{BuildCommand, ParserConfig, Target}, config, consts::TSDL_FROM, display::{Handle, Progress, ProgressState, TICK_CHARS}, @@ -61,6 +61,7 @@ fn build(command: &BuildCommand, progress: Progress) -> Result<()> { command.build_dir.clone(), command.out_dir.clone(), &command.prefix, + command.target, )?; create_dir_all(&command.out_dir) .with_context(|| format!("Creating output dir {}", &command.out_dir.display()))?; @@ -79,6 +80,7 @@ async fn update_screen(progress: Arc>) { } } +#[allow(clippy::too_many_arguments)] fn collect_languages( ts_cli: PathBuf, progress: Arc>, @@ -87,12 +89,14 @@ fn collect_languages( build_dir: PathBuf, out_dir: PathBuf, prefix: &str, + target: Target, ) -> Result, error::LanguageCollection> { let (res, errs) = unique_languages( ts_cli, build_dir, out_dir, prefix, + target, requested_languages, defined_parsers, progress, @@ -112,11 +116,13 @@ type Languages = ( ); #[allow(clippy::needless_pass_by_value)] +#[allow(clippy::too_many_arguments)] fn unique_languages( ts_cli: PathBuf, build_dir: PathBuf, out_dir: PathBuf, prefix: &str, + target: Target, requested_languages: Option<&Vec>, defined_parsers: Option<&BTreeMap>, progress: Arc>, @@ -147,6 +153,7 @@ fn unique_languages( out_dir.canon().unwrap(), prefix.into(), repo, + target, ts_cli.clone(), ) }) diff --git a/src/parser.rs b/src/parser.rs index 462ec5b..3abe729 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -11,6 +11,7 @@ use tracing::warn; use url::Url; use crate::{ + args::Target, display::{Handle, ProgressHandle}, error, git::{clone_fast, Ref}, @@ -19,6 +20,7 @@ use crate::{ }; pub const NUM_STEPS: usize = 3; +pub const WASM_EXTENSION: &str = "wasm"; pub async fn build_languages(languages: Vec) -> Result<()> { let buffer = if languages.is_empty() { @@ -57,6 +59,7 @@ pub struct Language { out_dir: PathBuf, prefix: String, repo: Url, + target: Target, ts_cli: Arc, } @@ -72,6 +75,7 @@ impl Language { out_dir: PathBuf, prefix: String, repo: Url, + target: Target, ts_cli: Arc, ) -> Self { Language { @@ -83,6 +87,7 @@ impl Language { out_dir, prefix, repo, + target, ts_cli, } } @@ -124,7 +129,24 @@ impl Language { } else { warn!("I don't know how to generate parsers when a script/cmd is specified (it's typescript's fault)"); } - self.build(&dir).await?; + + if self.target.native() { + self.handle.msg(format!( + "Building {} native parser: {}", + self.git_ref, + dir.file_name().unwrap().to_str().unwrap(), + )); + self.build(&dir, DLL_EXTENSION).await?; + } + + if self.target.wasm() { + self.handle.msg(format!( + "Building {} wasm parser: {}", + self.git_ref, + dir.file_name().unwrap().to_str().unwrap(), + )); + self.build(&dir, WASM_EXTENSION).await?; + } self.handle.msg(format!( "Copying {} parser: {}", self.git_ref, @@ -134,13 +156,25 @@ impl Language { Ok(()) } - async fn build(&self, dir: &Path) -> Result<()> { + async fn build(&self, dir: &Path, ext: &str) -> Result<()> { + let effective_name = dir + .file_name() + .map(|n| { + n.to_string_lossy() + .strip_prefix("tree-sitter-") + .map_or_else(|| n.to_string_lossy().to_string(), str::to_string) + }) + .unwrap(); self.build_script .as_ref() .map_or_else( || { let mut cmd = Command::new(&*self.ts_cli); cmd.arg("build"); + if ext == WASM_EXTENSION { + cmd.arg("--wasm"); + } + cmd.args(["--output", &format!("{effective_name}.{ext}")]); cmd }, |script| Command::from_str(script), @@ -197,16 +231,21 @@ impl Language { .collect() } - async fn copy(&self, dir: impl Into) -> Result<()> { - let dir = dir.into(); - let prefix = &self.prefix; - let dll = self.find_dll_files(&dir).await?; - let name = Self::extract_parser_name(&dll); - let dst = self - .out_dir - .clone() - .join(format!("{prefix}{name}.{DLL_EXTENSION}")); + async fn copy(&self, dir: &Path) -> Result<()> { + if self.target.native() { + self.do_copy(dir, DLL_EXTENSION).await?; + } + if self.target.wasm() { + self.do_copy(dir, WASM_EXTENSION).await?; + } + Ok(()) + } + async fn do_copy(&self, dir: &Path, ext: &str) -> Result<()> { + let dll = self.find_dll_files(dir, ext).await?; + let name = Self::extract_parser_name(&dll, ext); + let prefix = &self.prefix; + let dst = self.out_dir.clone().join(format!("{prefix}{name}.{ext}")); fs::copy(&dll, &dst) .await .with_context(|| format!("cp {} {}", &dll.display(), dst.display())) @@ -248,37 +287,35 @@ impl Language { .and(Ok(())) } - async fn find_dll_files(&self, dir: &Path) -> Result { + async fn find_dll_files(&self, dir: &Path, ext: &str) -> Result { let mut files = fs::read_dir(&dir).await.unwrap(); let mut dlls = Vec::with_capacity(1); while let Ok(Some(entry)) = files.next_entry().await { let file_name = entry.file_name(); let name = file_name.as_os_str().to_str().unwrap(); - if entry.file_type().await.unwrap().is_file() - && name.ends_with(&format!(".{DLL_EXTENSION}")) - { + if entry.file_type().await.unwrap().is_file() && name.ends_with(&format!(".{ext}")) { dlls.push(dir.join(name)); } } // Error handling for no DLLs or too many DLLs match dlls.len() { 0 => Err(self - .create_copy_error(dir, format!("Couldn't find any {DLL_EXTENSION} file")) + .create_copy_error(dir, format!("Couldn't find any {ext} file")) .into()), n if n > 1 => Err(self - .create_copy_error(dir, format!("Found many {DLL_EXTENSION} files: {dlls:?}")) + .create_copy_error(dir, format!("Found many {ext} files: {dlls:?}")) .into()), _ => Ok(dlls[0].clone()), } } - fn extract_parser_name(dll_path: &Path) -> String { + fn extract_parser_name(dll_path: &Path, ext: &str) -> String { let mut name = dll_path .file_name() .and_then(|n| n.to_str()) .map(String::from) .unwrap(); - if name == format!("parser.{DLL_EXTENSION}") { + if name == format!("parser.{ext}") { name = dll_path .parent() .and_then(|p| p.file_name()) @@ -289,10 +326,8 @@ impl Language { if name.starts_with("libtree-sitter-") { name = name.trim_start_matches("libtree-sitter-").to_string(); } - if name.ends_with(&format!(".{DLL_EXTENSION}")) { - name = name - .trim_end_matches(&format!(".{DLL_EXTENSION}")) - .to_string(); + if name.ends_with(&format!(".{ext}")) { + name = name.trim_end_matches(&format!(".{ext}")).to_string(); } name } diff --git a/tests/cmd/build.rs b/tests/cmd/build.rs index a15e081..115515a 100644 --- a/tests/cmd/build.rs +++ b/tests/cmd/build.rs @@ -6,9 +6,12 @@ use indoc::{formatdoc, indoc}; use predicates::{self as p}; use rstest::*; -use tsdl::consts::{ - TREE_SITTER_PLATFORM, TREE_SITTER_VERSION, TSDL_BUILD_DIR, TSDL_CONFIG_FILE, TSDL_OUT_DIR, - TSDL_PREFIX, +use tsdl::{ + consts::{ + TREE_SITTER_PLATFORM, TREE_SITTER_VERSION, TSDL_BUILD_DIR, TSDL_CONFIG_FILE, TSDL_OUT_DIR, + TSDL_PREFIX, + }, + parser::WASM_EXTENSION, }; use crate::cmd::Sandbox; @@ -238,3 +241,38 @@ fn multi_parsers_cmd() { dylib.assert(p::path::exists()).assert(p::path::is_file()); } } + +#[rstest] +#[case::default(None, &[DLL_EXTENSION])] +#[case::all(Some("all"), &[DLL_EXTENSION, WASM_EXTENSION])] +#[case::native(Some("native"), &[DLL_EXTENSION])] +#[case::wasm(Some("wasm"), &[WASM_EXTENSION])] +fn build_target(#[case] target: Option<&str>, #[case] exts: &[&str]) { + use std::fmt::Write as _; + + let languages = [("json", "0.21.0")]; + let mut config = String::new(); + writeln!(config, "[parsers]").unwrap(); + for (lang, ver) in languages { + writeln!(config, " {lang} = \"{ver}\"").unwrap(); + } + if let Some(target) = target { + config = format!("target = \"{target}\"\n{config}"); + } + let mut sandbox = Sandbox::new(); + sandbox + .tmp + .child(TSDL_CONFIG_FILE) + .write_str(&config) + .unwrap(); + sandbox.cmd.args(["build"]).assert().success(); + for (lang, _) in languages { + for ext in exts { + let dylib = sandbox + .tmp + .child(TSDL_OUT_DIR) + .child(format!("{TSDL_PREFIX}{lang}.{ext}")); + dylib.assert(p::path::exists()).assert(p::path::is_file()); + } + } +}