diff --git a/crates/analysis/Cargo.toml b/crates/analysis/Cargo.toml index a252f8c..1d7ce6f 100644 --- a/crates/analysis/Cargo.toml +++ b/crates/analysis/Cargo.toml @@ -8,9 +8,9 @@ license.workspace = true zerosyntax-schema.workspace = true zerosyntax-syntax.workspace = true rowan.workspace = true +serde.workspace = true [dev-dependencies] -serde.workspace = true toml.workspace = true criterion.workspace = true diff --git a/crates/analysis/src/index.rs b/crates/analysis/src/index.rs index 4ccaebb..c168b20 100644 --- a/crates/analysis/src/index.rs +++ b/crates/analysis/src/index.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; +use serde::{Deserialize, Serialize}; use zerosyntax_schema::{RefKind, ValueType}; use zerosyntax_syntax::ast::{Block, Field, Module}; use zerosyntax_syntax::{Parse, SyntaxKind, SyntaxNode, SyntaxToken}; @@ -14,20 +15,20 @@ use zerosyntax_syntax::{Parse, SyntaxKind, SyntaxNode, SyntaxToken}; use crate::model::{scope_schema, ScopeSchema}; use crate::{Analyzer, Span}; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum AssetKind { Audio, Texture, } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FileAsset { pub kind: AssetKind, pub name: String, } /// Model data discovered from a W3D asset. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ModelAsset { pub name: String, pub members: Vec, @@ -41,7 +42,7 @@ pub struct Location { } /// A named definition discovered in a document. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Definition { pub name: String, pub kind: RefKind, @@ -49,7 +50,7 @@ pub struct Definition { } /// A place where a definition is *referenced* (a Reference-typed field value). -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReferenceSite { pub name: String, pub kind: RefKind, diff --git a/crates/analysis/src/lib.rs b/crates/analysis/src/lib.rs index 9e57208..46cf5ec 100644 --- a/crates/analysis/src/lib.rs +++ b/crates/analysis/src/lib.rs @@ -8,6 +8,7 @@ use std::collections::{HashMap, HashSet}; +use serde::{Deserialize, Serialize}; use zerosyntax_schema::{BlockType, ModuleType, RefKind, Schema, ValueSet}; use zerosyntax_syntax::{parse, Edit, OpenerOracle, Parse, Strategy}; @@ -25,7 +26,7 @@ pub use diagnostics::{Diagnostic, Severity}; pub use index::WorkspaceIndex; /// A half-open byte range `[start, end)` into the source text. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct Span { pub start: u32, pub end: u32, diff --git a/crates/server/src/backend.rs b/crates/server/src/backend.rs index c1e7226..d7dcc31 100644 --- a/crates/server/src/backend.rs +++ b/crates/server/src/backend.rs @@ -26,10 +26,15 @@ use zerosyntax_analysis::{actions, completion, diagnostics, format, outline, sem use zerosyntax_syntax::{Edit, Parse}; use crate::convert::{self, PositionEnc}; -use crate::scan::{collect_scan_paths, load_sibling_str_keys, read_lossy, scan_files}; +use crate::scan::{ + clear_index_cache, index_cache_path, load_sibling_str_keys, read_lossy, scan_with_cache, +}; #[cfg(test)] use crate::scan::{parse_w3d_models, scan_big, scan_roots}; +const CLEAR_INDEX_CACHE_COMMAND: &str = "zerosyntax.clearIndexCache"; +const REBUILD_INDEX_CACHE_COMMAND: &str = "zerosyntax.rebuildIndexCache"; + /// An open document: its text (as both a rope for position math and a string /// for the parser) and the parse of that exact text. `did_open`/`did_change` /// are the only places a new parse is produced for an open document; @@ -629,14 +634,11 @@ impl Backend { let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<(usize, usize)>(); let scan_analyzer = analyzer.clone(); let handle = tokio::task::spawn_blocking(move || { - let workspace_paths = collect_scan_paths(&roots); - let base_paths = collect_scan_paths(&base_roots); - let total = workspace_paths.len() + base_paths.len(); let mut done = 0; let mut last_percent = u32::MAX; // Throttle to whole-percent changes (plus the final count) so a // 10k-file scan sends ~100 notifications, not 10k. - let mut progress = |_done_in_batch: usize, _batch_total: usize| { + let mut progress = |_done_in_batch: usize, total: usize| { done += 1; let percent = (done * 100 / total.max(1)) as u32; if percent != last_percent || done == total { @@ -644,16 +646,13 @@ impl Backend { let _ = tx.send((done, total)); } }; - let scanned = scan_files(&scan_analyzer, &workspace_paths, &mut progress); - let base_scanned = scan_files(&scan_analyzer, &base_paths, &mut progress); - (scanned, base_scanned) + scan_with_cache(&scan_analyzer, &roots, &base_roots, &mut progress) }); while let Some((done, total)) = rx.recv().await { self.report_scan_progress(&progress_token, done, total) .await; } - let (scanned, base_scanned) = handle.await.unwrap_or_default(); - + let scanned = handle.await.unwrap_or_default(); if replace_analyzer { if let Ok(mut current) = self.analyzer.write() { *current = analyzer.clone(); @@ -665,29 +664,28 @@ impl Backend { } } - let base_ini_count = base_scanned + let base_ini_count = scanned .iter() - .filter(|(_, _, _, _, _, _, models, assets, _)| models.is_empty() && assets.is_empty()) + .filter(|(is_base, (_, _, _, _, _, _, models, assets, _))| { + *is_base && models.is_empty() && assets.is_empty() + }) .count(); self.base_indexed_count .store(base_ini_count, Ordering::Relaxed); self.scan_finished.store(true, Ordering::Relaxed); - let ini_total = base_ini_count - + scanned - .iter() - .filter(|(_, _, _, _, _, _, models, assets, _)| { - models.is_empty() && assets.is_empty() - }) - .count(); - let model_total: usize = base_scanned + let ini_total = scanned + .iter() + .filter(|(_, (_, _, _, _, _, _, models, assets, _))| { + models.is_empty() && assets.is_empty() + }) + .count(); + let model_total: usize = scanned .iter() - .chain(scanned.iter()) - .map(|(_, _, _, _, _, _, models, _, _)| models.len()) + .map(|(_, (_, _, _, _, _, _, models, _, _))| models.len()) .sum(); - let (audio_total, texture_total) = base_scanned + let (audio_total, texture_total) = scanned .iter() - .chain(scanned.iter()) - .flat_map(|(_, _, _, _, _, _, _, assets, _)| assets) + .flat_map(|(_, (_, _, _, _, _, _, _, assets, _))| assets) .fold((0, 0), |(audio, texture), asset| match asset.kind { zerosyntax_analysis::index::AssetKind::Audio => (audio + 1, texture), zerosyntax_analysis::index::AssetKind::Texture => (audio, texture + 1), @@ -702,8 +700,8 @@ impl Backend { let mut replacement = WorkspaceIndex::new(); replacement.set_model_member_strictness(model_member_strictness); self.virtual_files.clear(); - for (uri, defs, refs, tags, object_models, object_parents, models, assets, text) in - base_scanned.into_iter().chain(scanned) + for (_, (uri, defs, refs, tags, object_models, object_parents, models, assets, text)) in + scanned { if let Some(text) = text { self.virtual_files.insert(uri.clone(), text); @@ -848,6 +846,22 @@ impl Backend { .map(|text| text.to_string())) } + pub async fn index_cache_path(&self) -> Result { + let roots = self + .roots + .lock() + .map(|roots| roots.clone()) + .unwrap_or_default(); + let base_roots = self + .settings + .lock() + .map(|settings| settings.base_ini_roots.clone()) + .unwrap_or_default(); + Ok(index_cache_path(&roots, &base_roots) + .to_string_lossy() + .into_owned()) + } + /// The (kind, name, span) under the cursor — a reference-typed value token /// or a definition's name token. The shared entry point for /// find-references and rename, which work from either end of an edge. @@ -1019,6 +1033,13 @@ impl LanguageServer for Backend { ..Default::default() }, )), + execute_command_provider: Some(ExecuteCommandOptions { + commands: vec![ + CLEAR_INDEX_CACHE_COMMAND.into(), + REBUILD_INDEX_CACHE_COMMAND.into(), + ], + work_done_progress_options: Default::default(), + }), ..Default::default() }, }) @@ -1073,6 +1094,53 @@ impl LanguageServer for Backend { .await; } + async fn execute_command( + &self, + params: ExecuteCommandParams, + ) -> Result> { + if params.command != CLEAR_INDEX_CACHE_COMMAND + && params.command != REBUILD_INDEX_CACHE_COMMAND + { + return Ok(None); + } + let roots = self + .roots + .lock() + .map(|roots| roots.clone()) + .unwrap_or_default(); + let base_roots = self + .settings + .lock() + .map(|settings| settings.base_ini_roots.clone()) + .unwrap_or_default(); + let cleared = clear_index_cache(&roots, &base_roots).map_err(|error| { + tracing::warn!(%error, "could not clear asset index cache"); + tower_lsp::jsonrpc::Error::internal_error() + })?; + if params.command == REBUILD_INDEX_CACHE_COMMAND { + self.scan_finished.store(false, Ordering::Relaxed); + self.base_indexed_count.store(0, Ordering::Relaxed); + self.scan_workspace(self.analyzer(), false).await; + let open: Vec = self.docs.iter().map(|entry| entry.key().clone()).collect(); + for uri in open { + self.refresh(&uri, None).await; + } + self.client + .show_message(MessageType::INFO, "ZeroSyntax index cache rebuilt.") + .await; + return Ok(Some( + serde_json::json!({ "rebuilt": true, "cleared": cleared }), + )); + } + let message = if cleared { + "ZeroSyntax index cache cleared. Restart the language server to rebuild it." + } else { + "ZeroSyntax index cache is already clear." + }; + self.client.show_message(MessageType::INFO, message).await; + Ok(Some(serde_json::json!({ "cleared": cleared }))) + } + async fn shutdown(&self) -> Result<()> { Ok(()) } diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index ffafb35..f9b3e9a 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -29,6 +29,7 @@ async fn main() -> std::process::ExitCode { let stdout = tokio::io::stdout(); let (service, socket) = LspService::build(Backend::new) .custom_method("zerosyntax/readVirtualFile", Backend::read_virtual_file) + .custom_method("zerosyntax/indexCachePath", Backend::index_cache_path) .finish(); Server::new(stdin, stdout, socket).serve(service).await; std::process::ExitCode::SUCCESS diff --git a/crates/server/src/scan.rs b/crates/server/src/scan.rs index 0b2e557..5128e0f 100644 --- a/crates/server/src/scan.rs +++ b/crates/server/src/scan.rs @@ -1,10 +1,14 @@ //! Shared filesystem, BIG archive, and W3D workspace scanning. +use std::collections::{hash_map::DefaultHasher, HashMap, HashSet}; +use std::hash::{Hash, Hasher}; use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use std::sync::Arc; +use std::time::UNIX_EPOCH; use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; use tower_lsp::lsp_types::Url; use zerosyntax_analysis::index::{ definitions_in, module_tags_in, object_models_in, object_parents_in, references_in, AssetKind, @@ -24,6 +28,141 @@ pub(crate) type ScanEntry = ( Option>, ); +const INDEX_CACHE_VERSION: u32 = 2; + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +struct Fingerprint { + len: u64, + modified_secs: u64, + modified_nanos: u32, +} + +#[derive(Serialize, Deserialize)] +struct CachedEntry { + file: String, + definitions: Vec, + references: Vec, + tags: Vec<(String, String)>, + object_models: Vec<(String, Vec)>, + object_parents: Vec<(String, String)>, + models: Vec, + assets: Vec, + text: Option, +} + +impl From<&ScanEntry> for CachedEntry { + fn from(entry: &ScanEntry) -> Self { + Self { + file: entry.0.clone(), + definitions: entry.1.clone(), + references: entry.2.clone(), + tags: entry.3.clone(), + object_models: entry.4.clone(), + object_parents: entry.5.clone(), + models: entry.6.clone(), + assets: entry.7.clone(), + text: entry.8.as_deref().map(str::to_owned), + } + } +} + +impl From for ScanEntry { + fn from(entry: CachedEntry) -> Self { + ( + entry.file, + entry.definitions, + entry.references, + entry.tags, + entry.object_models, + entry.object_parents, + entry.models, + entry.assets, + entry.text.map(Arc::from), + ) + } +} + +#[derive(Serialize, Deserialize)] +struct CachedFile { + fingerprint: Fingerprint, + entries: Vec, +} + +#[derive(Serialize, Deserialize)] +struct IndexCache { + version: u32, + schema_hash: u64, + files: HashMap, +} + +fn cache_dir() -> PathBuf { + #[cfg(windows)] + if let Some(path) = std::env::var_os("LOCALAPPDATA") { + return PathBuf::from(path).join("zerosyntax"); + } + #[cfg(not(windows))] + if let Some(path) = std::env::var_os("XDG_CACHE_HOME") { + return PathBuf::from(path).join("zerosyntax"); + } + std::env::temp_dir().join("zerosyntax") +} + +fn path_key(path: &Path) -> String { + let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let key = path.to_string_lossy().replace('\\', "/"); + if cfg!(windows) { + key.to_ascii_lowercase() + } else { + key + } +} + +fn schema_hash() -> u64 { + let mut hasher = DefaultHasher::new(); + zerosyntax_schema::EMBEDDED_SCHEMA_JSON.hash(&mut hasher); + hasher.finish() +} + +fn fingerprint(path: &Path) -> Option { + let metadata = std::fs::metadata(path).ok()?; + let modified = metadata.modified().ok()?.duration_since(UNIX_EPOCH).ok()?; + Some(Fingerprint { + len: metadata.len(), + modified_secs: modified.as_secs(), + modified_nanos: modified.subsec_nanos(), + }) +} + +pub(crate) fn index_cache_path(workspace_roots: &[PathBuf], base_roots: &[PathBuf]) -> PathBuf { + let mut roots: Vec<_> = workspace_roots + .iter() + .map(|root| format!("workspace:{}", path_key(root))) + .chain( + base_roots + .iter() + .map(|root| format!("base:{}", path_key(root))), + ) + .collect(); + roots.sort_unstable(); + let mut hasher = DefaultHasher::new(); + roots.hash(&mut hasher); + cache_dir().join(format!( + "index-v{INDEX_CACHE_VERSION}-{:016x}.json", + hasher.finish() + )) +} + +pub(crate) fn clear_index_cache( + workspace_roots: &[PathBuf], + base_roots: &[PathBuf], +) -> std::io::Result { + match std::fs::remove_file(index_cache_path(workspace_roots, base_roots)) { + Ok(()) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + struct BigEntry { name: String, offset: u64, @@ -378,6 +517,7 @@ fn collect_paths(roots: &[PathBuf], checked: bool) -> Result> { } /// Best-effort indexing used by the interactive server. +#[cfg(test)] pub(crate) fn scan_files( analyzer: &Analyzer, paths: &[PathBuf], @@ -393,6 +533,79 @@ pub(crate) fn scan_files( out } +/// Scan workspace and base roots, reusing unchanged files from the persistent +/// asset index cache. Base entries stay first so workspace definitions retain +/// their existing override order. +pub(crate) fn scan_with_cache( + analyzer: &Analyzer, + workspace_roots: &[PathBuf], + base_roots: &[PathBuf], + progress: &mut impl FnMut(usize, usize), +) -> Vec<(bool, ScanEntry)> { + let cache_path = index_cache_path(workspace_roots, base_roots); + let expected_schema_hash = schema_hash(); + let mut cache = std::fs::read(&cache_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()) + .filter(|cache| { + cache.version == INDEX_CACHE_VERSION && cache.schema_hash == expected_schema_hash + }) + .unwrap_or(IndexCache { + version: INDEX_CACHE_VERSION, + schema_hash: expected_schema_hash, + files: HashMap::new(), + }); + + let mut seen = HashSet::new(); + let mut paths = Vec::new(); + for (roots, is_base) in [(base_roots, true), (workspace_roots, false)] { + for path in collect_scan_paths(roots) { + let key = path_key(&path); + if seen.insert(key.clone()) { + if let Some(fingerprint) = fingerprint(&path) { + paths.push((path, key, fingerprint, is_base)); + } + } + } + } + + let mut next = HashMap::with_capacity(paths.len()); + let mut scanned = Vec::new(); + let total = paths.len(); + for (done, (path, key, fingerprint, is_base)) in paths.into_iter().enumerate() { + let entries = match cache.files.remove(&key) { + Some(cached) if cached.fingerprint == fingerprint => { + cached.entries.into_iter().map(ScanEntry::from).collect() + } + _ => scan_path(analyzer, &path).unwrap_or_default(), + }; + next.insert( + key, + CachedFile { + fingerprint, + entries: entries.iter().map(CachedEntry::from).collect(), + }, + ); + scanned.extend(entries.into_iter().map(|entry| (is_base, entry))); + progress(done + 1, total); + } + let cache = IndexCache { + version: INDEX_CACHE_VERSION, + schema_hash: expected_schema_hash, + files: next, + }; + if let Some(parent) = cache_path.parent() { + if let Err(error) = std::fs::create_dir_all(parent).and_then(|()| { + serde_json::to_vec(&cache) + .map_err(std::io::Error::other) + .and_then(|bytes| std::fs::write(&cache_path, bytes)) + }) { + tracing::warn!(%error, path = %cache_path.display(), "could not write asset index cache"); + } + } + scanned +} + pub(crate) fn scan_files_checked(analyzer: &Analyzer, paths: &[PathBuf]) -> Result> { let mut out = Vec::new(); for path in paths { diff --git a/editors/vscode/package.json b/editors/vscode/package.json index 8e4a1fb..4a303d5 100644 --- a/editors/vscode/package.json +++ b/editors/vscode/package.json @@ -19,10 +19,25 @@ ], "main": "./out/extension.js", "activationEvents": [ - "onLanguage:generals-ini" + "onLanguage:generals-ini", + "onCommand:zerosyntax.clearIndexCache", + "onCommand:zerosyntax.rebuildIndexCache", + "onCommand:zerosyntax.openIndexCacheLocation" ], "contributes": { "commands": [ + { + "command": "zerosyntax.clearIndexCache", + "title": "ZeroSyntax: Clear Index Cache" + }, + { + "command": "zerosyntax.rebuildIndexCache", + "title": "ZeroSyntax: Rebuild Index Cache" + }, + { + "command": "zerosyntax.openIndexCacheLocation", + "title": "ZeroSyntax: Open Index Cache Location" + }, { "command": "zerosyntax.selectSchema", "title": "ZeroSyntax: Select Custom Schema" diff --git a/editors/vscode/src/extension.ts b/editors/vscode/src/extension.ts index 0ae2547..c6d0366 100644 --- a/editors/vscode/src/extension.ts +++ b/editors/vscode/src/extension.ts @@ -73,6 +73,18 @@ export function activate(context: vscode.ExtensionContext) { }); context.subscriptions.push( + vscode.commands.registerCommand("zerosyntax.openIndexCacheLocation", async () => { + const cachePath = await client?.sendRequest("zerosyntax/indexCachePath"); + if (!cachePath) { + return; + } + const cacheUri = vscode.Uri.file(cachePath); + if (fs.existsSync(cachePath)) { + await vscode.commands.executeCommand("revealFileInOS", cacheUri); + } else { + vscode.window.showInformationMessage(`ZeroSyntax index cache will be created at ${cachePath}.`); + } + }), vscode.commands.registerCommand("zerosyntax.selectSchema", async () => { const selected = await vscode.window.showOpenDialog({ canSelectMany: false,