From c04a03664b0b9740aa7c3e68199e3f4702fae802 Mon Sep 17 00:00:00 2001 From: gpwclark Date: Tue, 12 Aug 2025 01:59:26 +0000 Subject: [PATCH 01/12] hmmmm fuse --- Cargo.lock | 53 +++- Cargo.toml | 1 + builtins/Cargo.toml | 2 + builtins/src/lib.rs | 1 + slosh-fuse/Cargo.toml | 19 ++ slosh-fuse/README.md | 63 ++++ slosh-fuse/examples/dynamic-env-file.sl | 38 +++ slosh-fuse/examples/dynamic-env-generator.sl | 109 +++++++ slosh-fuse/examples/dynamic-file-demo.sl | 37 +++ slosh-fuse/examples/slosh-fuse/Cargo.toml | 19 ++ slosh-fuse/examples/slosh-fuse/README.md | 63 ++++ .../slosh-fuse/src/bin/slosh-fuse-server.rs | 44 +++ slosh-fuse/examples/slosh-fuse/src/eval_fs.rs | 282 ++++++++++++++++++ .../examples/slosh-fuse/src/file_mapping.rs | 95 ++++++ slosh-fuse/examples/slosh-fuse/src/lib.rs | 37 +++ slosh-fuse/examples/test-fuse-systemd.sl | 60 ++++ slosh-fuse/src/bin/slosh-fuse-server.rs | 44 +++ slosh-fuse/src/eval_fs.rs | 282 ++++++++++++++++++ slosh-fuse/src/file_mapping.rs | 95 ++++++ slosh-fuse/src/lib.rs | 37 +++ slosh_lib/src/lib.rs | 2 + 21 files changed, 1380 insertions(+), 3 deletions(-) create mode 100644 slosh-fuse/Cargo.toml create mode 100644 slosh-fuse/README.md create mode 100644 slosh-fuse/examples/dynamic-env-file.sl create mode 100644 slosh-fuse/examples/dynamic-env-generator.sl create mode 100644 slosh-fuse/examples/dynamic-file-demo.sl create mode 100644 slosh-fuse/examples/slosh-fuse/Cargo.toml create mode 100644 slosh-fuse/examples/slosh-fuse/README.md create mode 100644 slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs create mode 100644 slosh-fuse/examples/slosh-fuse/src/eval_fs.rs create mode 100644 slosh-fuse/examples/slosh-fuse/src/file_mapping.rs create mode 100644 slosh-fuse/examples/slosh-fuse/src/lib.rs create mode 100644 slosh-fuse/examples/test-fuse-systemd.sl create mode 100644 slosh-fuse/src/bin/slosh-fuse-server.rs create mode 100644 slosh-fuse/src/eval_fs.rs create mode 100644 slosh-fuse/src/file_mapping.rs create mode 100644 slosh-fuse/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index bec2a79cd0..89a4898ac7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,6 +228,7 @@ dependencies = [ "bridge_types", "compile_state", "glob", + "lazy_static", "rand", "same-file", "shell", @@ -237,6 +238,7 @@ dependencies = [ "trybuild", "unicode-segmentation", "unicode_reader", + "uuid", "walkdir", ] @@ -731,7 +733,19 @@ checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", +] + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi 0.14.2+wasi-0.2.4", ] [[package]] @@ -1405,7 +1419,7 @@ checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", "log", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -1837,6 +1851,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "rand" version = "0.8.5" @@ -1864,7 +1884,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.15", ] [[package]] @@ -2694,6 +2714,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +dependencies = [ + "getrandom 0.3.3", +] + [[package]] name = "version_check" version = "0.9.5" @@ -2753,6 +2782,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.99" @@ -2948,6 +2986,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags 2.8.0", +] + [[package]] name = "write16" version = "1.0.0" diff --git a/Cargo.toml b/Cargo.toml index 21fee4fbd7..25086d7b9d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ members = [ "bridge_adapters", "slosh_test", "slosh_test_lib", + "slosh-fuse", ] exclude = ["legacy"] diff --git a/builtins/Cargo.toml b/builtins/Cargo.toml index 3a07807a26..5d955e308f 100644 --- a/builtins/Cargo.toml +++ b/builtins/Cargo.toml @@ -20,6 +20,8 @@ rand = { workspace = true } walkdir = { workspace = true } same-file = { workspace = true } glob = { workspace = true } +lazy_static = "1.4" +uuid = { version = "1.0", features = ["v4"] } [dev-dependencies] trybuild = { workspace = true } diff --git a/builtins/src/lib.rs b/builtins/src/lib.rs index 10a96ca282..0db369a143 100644 --- a/builtins/src/lib.rs +++ b/builtins/src/lib.rs @@ -13,6 +13,7 @@ pub mod collections; pub mod conversions; pub mod fs_meta; pub mod fs_temp; +pub mod fuse; pub mod io; pub mod math; pub mod print; diff --git a/slosh-fuse/Cargo.toml b/slosh-fuse/Cargo.toml new file mode 100644 index 0000000000..086bcaabe4 --- /dev/null +++ b/slosh-fuse/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "slosh-fuse" +version.workspace = true +edition = "2024" + +[dependencies] +fuser = "0.14" +libc = "0.2" +log = "0.4" +env_logger = "0.11" + +# Local slosh dependencies +vm = { path = "../vm" } +compile_state = { path = "../compile_state" } +slosh_lib = { path = "../slosh_lib" } + +[dependencies.nix] +version = "0.27" +features = ["signal", "process", "fs"] \ No newline at end of file diff --git a/slosh-fuse/README.md b/slosh-fuse/README.md new file mode 100644 index 0000000000..9a7d4c43b1 --- /dev/null +++ b/slosh-fuse/README.md @@ -0,0 +1,63 @@ +# Slosh FUSE Integration + +This module provides FUSE (Filesystem in Userspace) integration for slosh, allowing you to create dynamic files whose contents are generated by evaluating slosh expressions. + +## Use Case + +The primary use case is creating dynamic configuration files for services like systemd, where environment files need to contain values from slosh variables, environment variables, or computed values. + +For example, a systemd service file at `~/.config/systemd/user/my-service.env` could dynamically generate content like: + +```bash +HOST=meow +CUSTOM_TYPE=value-of-global +``` + +Where `meow` is the hostname and `value-of-global` comes from a slosh global variable. + +## Architecture + +The implementation consists of: + +1. **slosh-fuse crate**: Implements the FUSE filesystem that serves dynamic files +2. **Builtin functions**: Slosh functions to mount/unmount filesystems and register files +3. **Expression evaluation**: Files contain slosh expressions that are evaluated on read + +## Usage + +```lisp +;; Mount a dynamic filesystem +(def mount-id (mount-eval-fs "/tmp/dynamic")) + +;; Register a file with dynamic content +(register-eval-file mount-id "config.env" + "(str \"HOST=\" (hostname) \"\\n\" + \"TYPE=\" *my-global-var* \"\\n\")") + +;; The file /tmp/dynamic/config.env will now return evaluated content when read + +;; Unmount when done +(unmount-eval-fs mount-id) +``` + +## Implementation Status + +The current implementation provides: +- Basic FUSE filesystem structure +- File registration and mapping system +- Builtin functions for mounting and file management +- Example scripts demonstrating the concept + +Full FUSE integration with expression evaluation requires: +- Process forking and IPC for evaluation +- FUSE mount/unmount handling +- Security considerations for file access + +## Alternative Approach + +For simpler use cases, the `dynamic-env-generator.sl` example shows how to achieve similar functionality without FUSE by: +- Registering file paths with generator functions +- Periodically regenerating files when configuration changes +- Using standard file I/O operations + +This approach is simpler to implement and doesn't require FUSE support. \ No newline at end of file diff --git a/slosh-fuse/examples/dynamic-env-file.sl b/slosh-fuse/examples/dynamic-env-file.sl new file mode 100644 index 0000000000..69e40d7260 --- /dev/null +++ b/slosh-fuse/examples/dynamic-env-file.sl @@ -0,0 +1,38 @@ +#!/usr/bin/env slosh + +;; Dynamic Environment File Generator +;; This demonstrates generating dynamic env files for systemd + +(def *custom-type* "production") +(def *app-version* "1.0.0") + +(defn generate-env-file [path] + "Generate a dynamic environment file at the given path" + (let [content (str "# Generated by slosh at " (current-time-millis) "\n" + "HOST=" (or (env "HOSTNAME") "localhost") "\n" + "CUSTOM_TYPE=" *custom-type* "\n" + "APP_VERSION=" *app-version* "\n" + "USER=" (env "USER") "\n" + "PATH=" (env "PATH") "\n")] + (fs-write path content) + (println "Generated" path "with content:") + (println content))) + +;; Example usage +(def env-file-path "/tmp/my-service.env") +(generate-env-file env-file-path) + +;; Function to create a background process that updates the file periodically +(defn start-dynamic-env-updater [path interval-ms] + "Start a background process that updates the env file every interval-ms milliseconds" + (fork + (loop [] + (generate-env-file path) + (sleep interval-ms) + (recur)))) + +;; Start updating the file every 5 seconds +(def updater-proc (start-dynamic-env-updater env-file-path 5000)) +(println "Started background updater process:" updater-proc) +(println "File will be updated every 5 seconds") +(println "Kill the process with: (kill" (pid updater-proc) "9)") \ No newline at end of file diff --git a/slosh-fuse/examples/dynamic-env-generator.sl b/slosh-fuse/examples/dynamic-env-generator.sl new file mode 100644 index 0000000000..158ed1820c --- /dev/null +++ b/slosh-fuse/examples/dynamic-env-generator.sl @@ -0,0 +1,109 @@ +#!/usr/bin/env slosh + +;; Dynamic Environment File Generator +;; A practical implementation that generates env files based on slosh expressions + +(def *env-expressions* (hash)) + +(defn register-env-file [path expr-fn] + "Register an environment file path with a function that generates its content" + (hash-set! *env-expressions* path expr-fn)) + +(defn generate-env-file [path] + "Generate the environment file by evaluating its registered expression" + (if-let [expr-fn (hash-get *env-expressions* path)] + (let [content (expr-fn) + file (open path :create :truncate)] + (write-string file content) + (close file) + (println "Generated" path)) + (println "No expression registered for" path))) + +(defn generate-all-env-files [] + "Generate all registered environment files" + (doseq [[path _] *env-expressions*] + (generate-env-file path))) + +;; Example: Register systemd environment files + +;; Define some global configuration +(def *app-config* + {:environment "production" + :version "1.0.0" + :debug false}) + +;; Register my-service.env +(register-env-file "/tmp/my-service.env" + (fn [] + (str "# Generated by slosh at " (current-time-millis) "\n" + "HOST=" (or (env "HOSTNAME") "meow") "\n" + "CUSTOM_TYPE=" (:environment *app-config*) "\n" + "APP_VERSION=" (:version *app-config*) "\n" + "DEBUG=" (if (:debug *app-config*) "true" "false") "\n" + "USER=" (env "USER") "\n" + "PATH=" (env "PATH") "\n"))) + +;; Register database.env +(register-env-file "/tmp/database.env" + (fn [] + (str "# Database configuration\n" + "DB_HOST=" (if (= (:environment *app-config*) "production") + "prod-db.example.com" + "localhost") "\n" + "DB_PORT=5432\n" + "DB_NAME=" (:environment *app-config*) "_db\n" + "DB_USER=" (env "USER") "_app\n" + "DB_POOL_SIZE=" (if (= (:environment *app-config*) "production") "20" "5") "\n"))) + +;; Generate the files +(println "Generating environment files with current configuration:") +(println "*app-config*:" *app-config*) +(generate-all-env-files) + +;; Show the generated content +(println "\n--- Generated my-service.env ---") +(let [file (open "/tmp/my-service.env" :read)] + (println (read-string file 10000)) + (close file)) + +(println "--- Generated database.env ---") +(let [file (open "/tmp/database.env" :read)] + (println (read-string file 10000)) + (close file)) + +;; Demonstrate dynamic updates +(println "\n--- Updating configuration to development ---") +(set! *app-config* (assoc *app-config* + :environment "development" + :debug true)) + +(generate-all-env-files) + +(println "\n--- Updated my-service.env ---") +(let [file (open "/tmp/my-service.env" :read)] + (println (read-string file 10000)) + (close file)) + +(println "--- Updated database.env ---") +(let [file (open "/tmp/database.env" :read)] + (println (read-string file 10000)) + (close file)) + +;; Create a watcher function that regenerates files when config changes +(defn watch-and-regenerate [check-interval-ms] + "Watch for config changes and regenerate files" + (let [last-config (atom *app-config*)] + (fork + (loop [] + (when (not= @last-config *app-config*) + (println "Configuration changed, regenerating files...") + (generate-all-env-files) + (reset! last-config *app-config*)) + (sleep check-interval-ms) + (recur))))) + +(println "\n--- Starting config watcher ---") +(def watcher (watch-and-regenerate 1000)) +(println "Watcher started, will check for config changes every second") +(println "Try changing *app-config* and the files will auto-regenerate!") +(println "Stop watcher with: (kill" (pid watcher) "9)") \ No newline at end of file diff --git a/slosh-fuse/examples/dynamic-file-demo.sl b/slosh-fuse/examples/dynamic-file-demo.sl new file mode 100644 index 0000000000..7928cda429 --- /dev/null +++ b/slosh-fuse/examples/dynamic-file-demo.sl @@ -0,0 +1,37 @@ +#!/usr/bin/env slosh + +;; Dynamic File Demo +;; This demonstrates how dynamic files would work with the FUSE integration + +;; Simulate mounting a dynamic filesystem +(def mount-id (mount-eval-fs "/tmp/slosh-dynamic")) + +;; Register a systemd-style environment file +;; This would dynamically generate content based on slosh expressions +(register-eval-file mount-id "my-service.env" + "(str \"HOST=\" (or (env \"HOSTNAME\") \"localhost\") \"\\n\" + \"CUSTOM_TYPE=\" (or *custom-type* \"default\") \"\\n\" + \"TIMESTAMP=\" (current-time-millis) \"\\n\")") + +;; Register another dynamic file showing system info +(register-eval-file mount-id "system-info.txt" + "(str \"User: \" (env \"USER\") \"\\n\" + \"Home: \" (env \"HOME\") \"\\n\" + \"Shell: \" (env \"SHELL\") \"\\n\" + \"PWD: \" (env \"PWD\") \"\\n\")") + +;; List registered files +(println "Registered files:") +(doseq [file (list-eval-files mount-id)] + (println " " file)) + +;; Demonstrate what the content would look like +(println "\nFile contents would be:") +(println "--- my-service.env ---") +(println (eval-file-expression mount-id "my-service.env")) +(println "\n--- system-info.txt ---") +(println (eval-file-expression mount-id "system-info.txt")) + +;; Clean up +(unmount-eval-fs mount-id) +(println "\nFilesystem unmounted.") \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/Cargo.toml b/slosh-fuse/examples/slosh-fuse/Cargo.toml new file mode 100644 index 0000000000..086bcaabe4 --- /dev/null +++ b/slosh-fuse/examples/slosh-fuse/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "slosh-fuse" +version.workspace = true +edition = "2024" + +[dependencies] +fuser = "0.14" +libc = "0.2" +log = "0.4" +env_logger = "0.11" + +# Local slosh dependencies +vm = { path = "../vm" } +compile_state = { path = "../compile_state" } +slosh_lib = { path = "../slosh_lib" } + +[dependencies.nix] +version = "0.27" +features = ["signal", "process", "fs"] \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/README.md b/slosh-fuse/examples/slosh-fuse/README.md new file mode 100644 index 0000000000..9a7d4c43b1 --- /dev/null +++ b/slosh-fuse/examples/slosh-fuse/README.md @@ -0,0 +1,63 @@ +# Slosh FUSE Integration + +This module provides FUSE (Filesystem in Userspace) integration for slosh, allowing you to create dynamic files whose contents are generated by evaluating slosh expressions. + +## Use Case + +The primary use case is creating dynamic configuration files for services like systemd, where environment files need to contain values from slosh variables, environment variables, or computed values. + +For example, a systemd service file at `~/.config/systemd/user/my-service.env` could dynamically generate content like: + +```bash +HOST=meow +CUSTOM_TYPE=value-of-global +``` + +Where `meow` is the hostname and `value-of-global` comes from a slosh global variable. + +## Architecture + +The implementation consists of: + +1. **slosh-fuse crate**: Implements the FUSE filesystem that serves dynamic files +2. **Builtin functions**: Slosh functions to mount/unmount filesystems and register files +3. **Expression evaluation**: Files contain slosh expressions that are evaluated on read + +## Usage + +```lisp +;; Mount a dynamic filesystem +(def mount-id (mount-eval-fs "/tmp/dynamic")) + +;; Register a file with dynamic content +(register-eval-file mount-id "config.env" + "(str \"HOST=\" (hostname) \"\\n\" + \"TYPE=\" *my-global-var* \"\\n\")") + +;; The file /tmp/dynamic/config.env will now return evaluated content when read + +;; Unmount when done +(unmount-eval-fs mount-id) +``` + +## Implementation Status + +The current implementation provides: +- Basic FUSE filesystem structure +- File registration and mapping system +- Builtin functions for mounting and file management +- Example scripts demonstrating the concept + +Full FUSE integration with expression evaluation requires: +- Process forking and IPC for evaluation +- FUSE mount/unmount handling +- Security considerations for file access + +## Alternative Approach + +For simpler use cases, the `dynamic-env-generator.sl` example shows how to achieve similar functionality without FUSE by: +- Registering file paths with generator functions +- Periodically regenerating files when configuration changes +- Using standard file I/O operations + +This approach is simpler to implement and doesn't require FUSE support. \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs b/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs new file mode 100644 index 0000000000..9accd99ca2 --- /dev/null +++ b/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs @@ -0,0 +1,44 @@ +use fuser::{MountOption, Session}; +use slosh_fuse::{EvalFs, FileMapping}; +use std::sync::{Arc, Mutex}; +use std::env; +use std::os::unix::io::{RawFd, FromRawFd}; + +fn main() { + env_logger::init(); + + let args: Vec = env::args().collect(); + if args.len() < 4 { + eprintln!("Usage: {} ", args[0]); + std::process::exit(1); + } + + let mount_point = &args[1]; + let read_fd: RawFd = args[2].parse().expect("Invalid read fd"); + let write_fd: RawFd = args[3].parse().expect("Invalid write fd"); + + let file_mapping = Arc::new(Mutex::new(FileMapping::new())); + + // Pre-populate with test data if provided + if args.len() > 4 { + let mut mapping = file_mapping.lock().unwrap(); + for i in (4..args.len()).step_by(2) { + if i + 1 < args.len() { + mapping.register(&args[i], args[i + 1].clone()); + } + } + } + + let fs = EvalFs::new(file_mapping, read_fd, write_fd); + + let options = vec![ + MountOption::FSName("slosh-eval".to_string()), + MountOption::AllowOther, + MountOption::DefaultPermissions, + ]; + + let session = Session::new(fs, &mount_point, &options).expect("Failed to mount"); + + // Run until unmounted + session.run().expect("Failed to run session"); +} \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs b/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs new file mode 100644 index 0000000000..2bf4f43b0a --- /dev/null +++ b/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs @@ -0,0 +1,282 @@ +use fuser::{ + FileAttr, FileType, Filesystem, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, + Request, FUSE_ROOT_ID, +}; +use libc::{ENOENT, ENOTDIR}; +use std::collections::HashMap; +use std::ffi::OsStr; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::os::unix::io::{RawFd, FromRawFd, IntoRawFd}; +use std::io::{Read, Write}; + +use crate::file_mapping::FileMapping; + +const TTL: Duration = Duration::from_secs(1); + +pub struct EvalFs { + file_mapping: Arc>, + inode_to_path: HashMap, + path_to_inode: HashMap, + next_inode: u64, + eval_pipe_read: RawFd, + eval_pipe_write: RawFd, +} + +impl EvalFs { + pub fn new(file_mapping: Arc>, eval_pipe_read: RawFd, eval_pipe_write: RawFd) -> Self { + let mut fs = Self { + file_mapping, + inode_to_path: HashMap::new(), + path_to_inode: HashMap::new(), + next_inode: 2, // 1 is reserved for root + eval_pipe_read, + eval_pipe_write, + }; + + // Pre-populate inodes for registered files + let mapping = fs.file_mapping.lock().unwrap(); + for path in mapping.list_files() { + fs.allocate_inode(&path); + } + drop(mapping); + + fs + } + + fn allocate_inode(&mut self, path: &str) -> u64 { + if let Some(&inode) = self.path_to_inode.get(path) { + return inode; + } + + let inode = self.next_inode; + self.next_inode += 1; + self.inode_to_path.insert(inode, path.to_string()); + self.path_to_inode.insert(path.to_string(), inode); + inode + } + + fn get_inode_for_path(&self, path: &str) -> Option { + self.path_to_inode.get(path).copied() + } + + fn get_path_for_inode(&self, inode: u64) -> Option<&str> { + self.inode_to_path.get(&inode).map(|s| s.as_str()) + } + + fn evaluate_expression(&self, expression: &str) -> String { + // Send expression to parent process for evaluation + let msg = format!("{}\n", expression); + let mut pipe_write = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_write) }; + if let Err(e) = pipe_write.write_all(msg.as_bytes()) { + log::error!("Failed to send expression for evaluation: {}", e); + // Prevent closing the fd when File is dropped + let _ = pipe_write.into_raw_fd(); + return format!("ERROR: {}", e); + } + // Prevent closing the fd when File is dropped + let _ = pipe_write.into_raw_fd(); + + // Read response + let mut pipe_read = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_read) }; + let mut response = String::new(); + let mut buffer = [0u8; 4096]; + + loop { + match pipe_read.read(&mut buffer) { + Ok(0) => break, + Ok(n) => { + response.push_str(&String::from_utf8_lossy(&buffer[..n])); + if response.ends_with('\n') { + response.pop(); // Remove trailing newline + break; + } + } + Err(e) => { + log::error!("Failed to read evaluation result: {}", e); + // Prevent closing the fd when File is dropped + let _ = pipe_read.into_raw_fd(); + return format!("ERROR: {}", e); + } + } + } + // Prevent closing the fd when File is dropped + let _ = pipe_read.into_raw_fd(); + + response + } + + fn file_attr(inode: u64, size: u64) -> FileAttr { + let now = SystemTime::now(); + FileAttr { + ino: inode, + size, + blocks: 1, + atime: now, + mtime: now, + ctime: now, + crtime: now, + kind: FileType::RegularFile, + perm: 0o644, + nlink: 1, + uid: unsafe { libc::getuid() }, + gid: unsafe { libc::getgid() }, + rdev: 0, + blksize: 512, + flags: 0, + } + } + + fn dir_attr(inode: u64) -> FileAttr { + let now = SystemTime::now(); + FileAttr { + ino: inode, + size: 0, + blocks: 0, + atime: now, + mtime: now, + ctime: now, + crtime: now, + kind: FileType::Directory, + perm: 0o755, + nlink: 2, + uid: unsafe { libc::getuid() }, + gid: unsafe { libc::getgid() }, + rdev: 0, + blksize: 512, + flags: 0, + } + } +} + +impl Filesystem for EvalFs { + fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: ReplyEntry) { + if parent != FUSE_ROOT_ID { + reply.error(ENOENT); + return; + } + + let name_str = name.to_string_lossy(); + let mapping = self.file_mapping.lock().unwrap(); + + if mapping.get(&name_str).is_some() { + let inode = self.allocate_inode(&name_str); + reply.entry(&TTL, &Self::file_attr(inode, 0), 0); + } else { + reply.error(ENOENT); + } + } + + fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) { + if ino == FUSE_ROOT_ID { + reply.attr(&TTL, &Self::dir_attr(FUSE_ROOT_ID)); + } else if let Some(path) = self.get_path_for_inode(ino) { + let mapping = self.file_mapping.lock().unwrap(); + if let Some(entry) = mapping.get(path) { + // Get content to determine size + let content = if entry.is_cache_valid() { + entry.cached_value.as_ref().unwrap().clone() + } else { + drop(mapping); + self.evaluate_expression(&entry.expression) + }; + reply.attr(&TTL, &Self::file_attr(ino, content.len() as u64)); + } else { + reply.error(ENOENT); + } + } else { + reply.error(ENOENT); + } + } + + fn read( + &mut self, + _req: &Request, + ino: u64, + _fh: u64, + offset: i64, + size: u32, + _flags: i32, + _lock: Option, + reply: ReplyData, + ) { + if let Some(path) = self.get_path_for_inode(ino) { + let mut mapping = self.file_mapping.lock().unwrap(); + if let Some(entry) = mapping.get_mut(path) { + let content = if entry.is_cache_valid() { + entry.cached_value.as_ref().unwrap().clone() + } else { + let expr = entry.expression.clone(); + drop(mapping); + let result = self.evaluate_expression(&expr); + + // Update cache if caching is enabled + let mut mapping = self.file_mapping.lock().unwrap(); + if let Some(entry) = mapping.get_mut(path) { + if entry.cache_ttl_secs > 0 { + entry.update_cache(result.clone()); + } + } + result + }; + + let data = content.as_bytes(); + let start = offset as usize; + let end = (start + size as usize).min(data.len()); + + if start < data.len() { + reply.data(&data[start..end]); + } else { + reply.data(&[]); + } + } else { + reply.error(ENOENT); + } + } else { + reply.error(ENOENT); + } + } + + fn readdir( + &mut self, + _req: &Request, + ino: u64, + _fh: u64, + offset: i64, + mut reply: ReplyDirectory, + ) { + if ino != FUSE_ROOT_ID { + reply.error(ENOTDIR); + return; + } + + let entries = vec![ + (FUSE_ROOT_ID, FileType::Directory, "."), + (FUSE_ROOT_ID, FileType::Directory, ".."), + ]; + + for (i, (inode, file_type, name)) in entries.iter().enumerate() { + if offset <= i as i64 { + if reply.add(*inode, (i + 1) as i64, *file_type, name) { + break; + } + } + } + + let mapping = self.file_mapping.lock().unwrap(); + let files = mapping.list_files(); + drop(mapping); + + for (i, path) in files.iter().enumerate() { + let index = i + entries.len(); + if offset <= index as i64 { + let inode = self.allocate_inode(path); + if reply.add(inode, (index + 1) as i64, FileType::RegularFile, path) { + break; + } + } + } + + reply.ok(); + } +} \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs b/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs new file mode 100644 index 0000000000..10b42bb31e --- /dev/null +++ b/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs @@ -0,0 +1,95 @@ +use std::collections::HashMap; +use std::time::SystemTime; + +#[derive(Clone)] +pub struct FileEntry { + pub expression: String, + pub cached_value: Option, + pub cache_time: Option, + pub cache_ttl_secs: u64, +} + +impl FileEntry { + pub fn new(expression: String) -> Self { + Self { + expression, + cached_value: None, + cache_time: None, + cache_ttl_secs: 0, // No caching by default + } + } + + pub fn is_cache_valid(&self) -> bool { + if self.cache_ttl_secs == 0 { + return false; + } + + if let (Some(cached_time), Some(_)) = (self.cache_time, &self.cached_value) { + if let Ok(elapsed) = cached_time.elapsed() { + return elapsed.as_secs() < self.cache_ttl_secs; + } + } + false + } + + pub fn update_cache(&mut self, value: String) { + self.cached_value = Some(value); + self.cache_time = Some(SystemTime::now()); + } +} + +pub struct FileMapping { + files: HashMap, +} + +impl FileMapping { + pub fn new() -> Self { + Self { + files: HashMap::new(), + } + } + + pub fn register(&mut self, path: &str, expression: String) { + let normalized_path = if path.starts_with('/') { + path[1..].to_string() + } else { + path.to_string() + }; + + self.files.insert(normalized_path, FileEntry::new(expression)); + } + + pub fn get(&self, path: &str) -> Option<&FileEntry> { + let normalized_path = if path.starts_with('/') { + &path[1..] + } else { + path + }; + + self.files.get(normalized_path) + } + + pub fn get_mut(&mut self, path: &str) -> Option<&mut FileEntry> { + let normalized_path = if path.starts_with('/') { + path[1..].to_string() + } else { + path.to_string() + }; + + self.files.get_mut(&normalized_path) + } + + pub fn remove(&mut self, path: &str) -> Option { + let normalized_path = if path.starts_with('/') { + path[1..].to_string() + } else { + path.to_string() + }; + + self.files.remove(&normalized_path) + } + + pub fn list_files(&self) -> Vec { + self.files.keys().cloned().collect() + } +} \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/lib.rs b/slosh-fuse/examples/slosh-fuse/src/lib.rs new file mode 100644 index 0000000000..312ba5b37a --- /dev/null +++ b/slosh-fuse/examples/slosh-fuse/src/lib.rs @@ -0,0 +1,37 @@ +pub mod eval_fs; +pub mod file_mapping; + +pub use eval_fs::EvalFs; +pub use file_mapping::{FileMapping, FileEntry}; + +use std::sync::{Arc, Mutex}; +use std::path::PathBuf; + +pub struct FuseMount { + pub mount_point: PathBuf, + pub file_mapping: Arc>, + pub process_handle: Option, +} + +impl FuseMount { + pub fn new(mount_point: PathBuf) -> Self { + Self { + mount_point, + file_mapping: Arc::new(Mutex::new(FileMapping::new())), + process_handle: None, + } + } + + pub fn register_file(&self, path: &str, expression: String) { + let mut mapping = self.file_mapping.lock().unwrap(); + mapping.register(path, expression); + } + + pub fn unmount(&mut self) -> Result<(), std::io::Error> { + if let Some(mut handle) = self.process_handle.take() { + handle.kill()?; + handle.wait()?; + } + Ok(()) + } +} \ No newline at end of file diff --git a/slosh-fuse/examples/test-fuse-systemd.sl b/slosh-fuse/examples/test-fuse-systemd.sl new file mode 100644 index 0000000000..9319def331 --- /dev/null +++ b/slosh-fuse/examples/test-fuse-systemd.sl @@ -0,0 +1,60 @@ +#!/usr/bin/env slosh + +;; Test FUSE dynamic files for systemd environment +;; This demonstrates how to create dynamic environment files + +;; Set some global variables that will be used in the env file +(def *custom-type* "production") +(def *app-version* "1.2.3") + +;; Mount the dynamic filesystem +(println "Mounting dynamic filesystem...") +(def mount-id (mount-eval-fs "/tmp/slosh-systemd-test")) + +;; Register a systemd-style environment file with dynamic content +(println "Registering my-service.env...") +(register-eval-file mount-id "my-service.env" + "(str \"# Generated by slosh\\n\" + \"HOST=\" (or (env \"HOSTNAME\") \"meow\") \"\\n\" + \"CUSTOM_TYPE=\" *custom-type* \"\\n\" + \"APP_VERSION=\" *app-version* \"\\n\" + \"USER=\" (env \"USER\") \"\\n\" + \"HOME=\" (env \"HOME\") \"\\n\")") + +;; Register another file for database config +(register-eval-file mount-id "database.env" + "(str \"DB_HOST=localhost\\n\" + \"DB_PORT=5432\\n\" + \"DB_NAME=\" *custom-type* \"_db\\n\" + \"DB_USER=\" (env \"USER\") \"_app\\n\")") + +;; List all registered files +(println "\nRegistered files:") +(doseq [file (list-eval-files mount-id)] + (println " -" file)) + +;; Show what the content would look like when evaluated +(println "\n--- Testing file evaluation ---") +(println "my-service.env would contain:") +(println (eval-file-expression mount-id "my-service.env")) + +(println "\ndatabase.env would contain:") +(println (eval-file-expression mount-id "database.env")) + +;; Change a global variable and show how it affects the file +(println "\n--- Changing *custom-type* to 'development' ---") +(set! *custom-type* "development") + +(println "my-service.env now contains:") +(println (eval-file-expression mount-id "my-service.env")) + +(println "\ndatabase.env now contains:") +(println (eval-file-expression mount-id "database.env")) + +;; Clean up +(unmount-eval-fs mount-id) +(println "\nFilesystem unmounted.") +(println "\nIn a real implementation, these files would be accessible at:") +(println " /tmp/slosh-systemd-test/my-service.env") +(println " /tmp/slosh-systemd-test/database.env") +(println "And systemd could read them directly!") \ No newline at end of file diff --git a/slosh-fuse/src/bin/slosh-fuse-server.rs b/slosh-fuse/src/bin/slosh-fuse-server.rs new file mode 100644 index 0000000000..9accd99ca2 --- /dev/null +++ b/slosh-fuse/src/bin/slosh-fuse-server.rs @@ -0,0 +1,44 @@ +use fuser::{MountOption, Session}; +use slosh_fuse::{EvalFs, FileMapping}; +use std::sync::{Arc, Mutex}; +use std::env; +use std::os::unix::io::{RawFd, FromRawFd}; + +fn main() { + env_logger::init(); + + let args: Vec = env::args().collect(); + if args.len() < 4 { + eprintln!("Usage: {} ", args[0]); + std::process::exit(1); + } + + let mount_point = &args[1]; + let read_fd: RawFd = args[2].parse().expect("Invalid read fd"); + let write_fd: RawFd = args[3].parse().expect("Invalid write fd"); + + let file_mapping = Arc::new(Mutex::new(FileMapping::new())); + + // Pre-populate with test data if provided + if args.len() > 4 { + let mut mapping = file_mapping.lock().unwrap(); + for i in (4..args.len()).step_by(2) { + if i + 1 < args.len() { + mapping.register(&args[i], args[i + 1].clone()); + } + } + } + + let fs = EvalFs::new(file_mapping, read_fd, write_fd); + + let options = vec![ + MountOption::FSName("slosh-eval".to_string()), + MountOption::AllowOther, + MountOption::DefaultPermissions, + ]; + + let session = Session::new(fs, &mount_point, &options).expect("Failed to mount"); + + // Run until unmounted + session.run().expect("Failed to run session"); +} \ No newline at end of file diff --git a/slosh-fuse/src/eval_fs.rs b/slosh-fuse/src/eval_fs.rs new file mode 100644 index 0000000000..2bf4f43b0a --- /dev/null +++ b/slosh-fuse/src/eval_fs.rs @@ -0,0 +1,282 @@ +use fuser::{ + FileAttr, FileType, Filesystem, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, + Request, FUSE_ROOT_ID, +}; +use libc::{ENOENT, ENOTDIR}; +use std::collections::HashMap; +use std::ffi::OsStr; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::os::unix::io::{RawFd, FromRawFd, IntoRawFd}; +use std::io::{Read, Write}; + +use crate::file_mapping::FileMapping; + +const TTL: Duration = Duration::from_secs(1); + +pub struct EvalFs { + file_mapping: Arc>, + inode_to_path: HashMap, + path_to_inode: HashMap, + next_inode: u64, + eval_pipe_read: RawFd, + eval_pipe_write: RawFd, +} + +impl EvalFs { + pub fn new(file_mapping: Arc>, eval_pipe_read: RawFd, eval_pipe_write: RawFd) -> Self { + let mut fs = Self { + file_mapping, + inode_to_path: HashMap::new(), + path_to_inode: HashMap::new(), + next_inode: 2, // 1 is reserved for root + eval_pipe_read, + eval_pipe_write, + }; + + // Pre-populate inodes for registered files + let mapping = fs.file_mapping.lock().unwrap(); + for path in mapping.list_files() { + fs.allocate_inode(&path); + } + drop(mapping); + + fs + } + + fn allocate_inode(&mut self, path: &str) -> u64 { + if let Some(&inode) = self.path_to_inode.get(path) { + return inode; + } + + let inode = self.next_inode; + self.next_inode += 1; + self.inode_to_path.insert(inode, path.to_string()); + self.path_to_inode.insert(path.to_string(), inode); + inode + } + + fn get_inode_for_path(&self, path: &str) -> Option { + self.path_to_inode.get(path).copied() + } + + fn get_path_for_inode(&self, inode: u64) -> Option<&str> { + self.inode_to_path.get(&inode).map(|s| s.as_str()) + } + + fn evaluate_expression(&self, expression: &str) -> String { + // Send expression to parent process for evaluation + let msg = format!("{}\n", expression); + let mut pipe_write = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_write) }; + if let Err(e) = pipe_write.write_all(msg.as_bytes()) { + log::error!("Failed to send expression for evaluation: {}", e); + // Prevent closing the fd when File is dropped + let _ = pipe_write.into_raw_fd(); + return format!("ERROR: {}", e); + } + // Prevent closing the fd when File is dropped + let _ = pipe_write.into_raw_fd(); + + // Read response + let mut pipe_read = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_read) }; + let mut response = String::new(); + let mut buffer = [0u8; 4096]; + + loop { + match pipe_read.read(&mut buffer) { + Ok(0) => break, + Ok(n) => { + response.push_str(&String::from_utf8_lossy(&buffer[..n])); + if response.ends_with('\n') { + response.pop(); // Remove trailing newline + break; + } + } + Err(e) => { + log::error!("Failed to read evaluation result: {}", e); + // Prevent closing the fd when File is dropped + let _ = pipe_read.into_raw_fd(); + return format!("ERROR: {}", e); + } + } + } + // Prevent closing the fd when File is dropped + let _ = pipe_read.into_raw_fd(); + + response + } + + fn file_attr(inode: u64, size: u64) -> FileAttr { + let now = SystemTime::now(); + FileAttr { + ino: inode, + size, + blocks: 1, + atime: now, + mtime: now, + ctime: now, + crtime: now, + kind: FileType::RegularFile, + perm: 0o644, + nlink: 1, + uid: unsafe { libc::getuid() }, + gid: unsafe { libc::getgid() }, + rdev: 0, + blksize: 512, + flags: 0, + } + } + + fn dir_attr(inode: u64) -> FileAttr { + let now = SystemTime::now(); + FileAttr { + ino: inode, + size: 0, + blocks: 0, + atime: now, + mtime: now, + ctime: now, + crtime: now, + kind: FileType::Directory, + perm: 0o755, + nlink: 2, + uid: unsafe { libc::getuid() }, + gid: unsafe { libc::getgid() }, + rdev: 0, + blksize: 512, + flags: 0, + } + } +} + +impl Filesystem for EvalFs { + fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: ReplyEntry) { + if parent != FUSE_ROOT_ID { + reply.error(ENOENT); + return; + } + + let name_str = name.to_string_lossy(); + let mapping = self.file_mapping.lock().unwrap(); + + if mapping.get(&name_str).is_some() { + let inode = self.allocate_inode(&name_str); + reply.entry(&TTL, &Self::file_attr(inode, 0), 0); + } else { + reply.error(ENOENT); + } + } + + fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) { + if ino == FUSE_ROOT_ID { + reply.attr(&TTL, &Self::dir_attr(FUSE_ROOT_ID)); + } else if let Some(path) = self.get_path_for_inode(ino) { + let mapping = self.file_mapping.lock().unwrap(); + if let Some(entry) = mapping.get(path) { + // Get content to determine size + let content = if entry.is_cache_valid() { + entry.cached_value.as_ref().unwrap().clone() + } else { + drop(mapping); + self.evaluate_expression(&entry.expression) + }; + reply.attr(&TTL, &Self::file_attr(ino, content.len() as u64)); + } else { + reply.error(ENOENT); + } + } else { + reply.error(ENOENT); + } + } + + fn read( + &mut self, + _req: &Request, + ino: u64, + _fh: u64, + offset: i64, + size: u32, + _flags: i32, + _lock: Option, + reply: ReplyData, + ) { + if let Some(path) = self.get_path_for_inode(ino) { + let mut mapping = self.file_mapping.lock().unwrap(); + if let Some(entry) = mapping.get_mut(path) { + let content = if entry.is_cache_valid() { + entry.cached_value.as_ref().unwrap().clone() + } else { + let expr = entry.expression.clone(); + drop(mapping); + let result = self.evaluate_expression(&expr); + + // Update cache if caching is enabled + let mut mapping = self.file_mapping.lock().unwrap(); + if let Some(entry) = mapping.get_mut(path) { + if entry.cache_ttl_secs > 0 { + entry.update_cache(result.clone()); + } + } + result + }; + + let data = content.as_bytes(); + let start = offset as usize; + let end = (start + size as usize).min(data.len()); + + if start < data.len() { + reply.data(&data[start..end]); + } else { + reply.data(&[]); + } + } else { + reply.error(ENOENT); + } + } else { + reply.error(ENOENT); + } + } + + fn readdir( + &mut self, + _req: &Request, + ino: u64, + _fh: u64, + offset: i64, + mut reply: ReplyDirectory, + ) { + if ino != FUSE_ROOT_ID { + reply.error(ENOTDIR); + return; + } + + let entries = vec![ + (FUSE_ROOT_ID, FileType::Directory, "."), + (FUSE_ROOT_ID, FileType::Directory, ".."), + ]; + + for (i, (inode, file_type, name)) in entries.iter().enumerate() { + if offset <= i as i64 { + if reply.add(*inode, (i + 1) as i64, *file_type, name) { + break; + } + } + } + + let mapping = self.file_mapping.lock().unwrap(); + let files = mapping.list_files(); + drop(mapping); + + for (i, path) in files.iter().enumerate() { + let index = i + entries.len(); + if offset <= index as i64 { + let inode = self.allocate_inode(path); + if reply.add(inode, (index + 1) as i64, FileType::RegularFile, path) { + break; + } + } + } + + reply.ok(); + } +} \ No newline at end of file diff --git a/slosh-fuse/src/file_mapping.rs b/slosh-fuse/src/file_mapping.rs new file mode 100644 index 0000000000..10b42bb31e --- /dev/null +++ b/slosh-fuse/src/file_mapping.rs @@ -0,0 +1,95 @@ +use std::collections::HashMap; +use std::time::SystemTime; + +#[derive(Clone)] +pub struct FileEntry { + pub expression: String, + pub cached_value: Option, + pub cache_time: Option, + pub cache_ttl_secs: u64, +} + +impl FileEntry { + pub fn new(expression: String) -> Self { + Self { + expression, + cached_value: None, + cache_time: None, + cache_ttl_secs: 0, // No caching by default + } + } + + pub fn is_cache_valid(&self) -> bool { + if self.cache_ttl_secs == 0 { + return false; + } + + if let (Some(cached_time), Some(_)) = (self.cache_time, &self.cached_value) { + if let Ok(elapsed) = cached_time.elapsed() { + return elapsed.as_secs() < self.cache_ttl_secs; + } + } + false + } + + pub fn update_cache(&mut self, value: String) { + self.cached_value = Some(value); + self.cache_time = Some(SystemTime::now()); + } +} + +pub struct FileMapping { + files: HashMap, +} + +impl FileMapping { + pub fn new() -> Self { + Self { + files: HashMap::new(), + } + } + + pub fn register(&mut self, path: &str, expression: String) { + let normalized_path = if path.starts_with('/') { + path[1..].to_string() + } else { + path.to_string() + }; + + self.files.insert(normalized_path, FileEntry::new(expression)); + } + + pub fn get(&self, path: &str) -> Option<&FileEntry> { + let normalized_path = if path.starts_with('/') { + &path[1..] + } else { + path + }; + + self.files.get(normalized_path) + } + + pub fn get_mut(&mut self, path: &str) -> Option<&mut FileEntry> { + let normalized_path = if path.starts_with('/') { + path[1..].to_string() + } else { + path.to_string() + }; + + self.files.get_mut(&normalized_path) + } + + pub fn remove(&mut self, path: &str) -> Option { + let normalized_path = if path.starts_with('/') { + path[1..].to_string() + } else { + path.to_string() + }; + + self.files.remove(&normalized_path) + } + + pub fn list_files(&self) -> Vec { + self.files.keys().cloned().collect() + } +} \ No newline at end of file diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs new file mode 100644 index 0000000000..312ba5b37a --- /dev/null +++ b/slosh-fuse/src/lib.rs @@ -0,0 +1,37 @@ +pub mod eval_fs; +pub mod file_mapping; + +pub use eval_fs::EvalFs; +pub use file_mapping::{FileMapping, FileEntry}; + +use std::sync::{Arc, Mutex}; +use std::path::PathBuf; + +pub struct FuseMount { + pub mount_point: PathBuf, + pub file_mapping: Arc>, + pub process_handle: Option, +} + +impl FuseMount { + pub fn new(mount_point: PathBuf) -> Self { + Self { + mount_point, + file_mapping: Arc::new(Mutex::new(FileMapping::new())), + process_handle: None, + } + } + + pub fn register_file(&self, path: &str, expression: String) { + let mut mapping = self.file_mapping.lock().unwrap(); + mapping.register(path, expression); + } + + pub fn unmount(&mut self) -> Result<(), std::io::Error> { + if let Some(mut handle) = self.process_handle.take() { + handle.kill()?; + handle.wait()?; + } + Ok(()) + } +} \ No newline at end of file diff --git a/slosh_lib/src/lib.rs b/slosh_lib/src/lib.rs index 34cd5f88e8..7d23fe2ada 100644 --- a/slosh_lib/src/lib.rs +++ b/slosh_lib/src/lib.rs @@ -24,6 +24,7 @@ use builtins::collections::setup_collection_builtins; use builtins::conversions::add_conv_builtins; use builtins::fs_meta::add_fs_meta_builtins; use builtins::fs_temp::add_fs_temp_builtins; +use builtins::fuse::add_fuse_builtins; use builtins::io::add_io_builtins; use builtins::math::add_math_builtins; use builtins::print::{add_print_builtins, display_value}; @@ -438,6 +439,7 @@ pub fn set_builtins(env: &mut SloshVm) { add_conv_builtins(env); add_fs_meta_builtins(env); add_fs_temp_builtins(env); + add_fuse_builtins(env); add_rand_builtins(env); add_math_builtins(env); add_doc_builtins(env); From 79cbf344e52d88cfeb78492866e5654e91a3a40d Mon Sep 17 00:00:00 2001 From: gpwclark Date: Tue, 12 Aug 2025 01:59:44 +0000 Subject: [PATCH 02/12] now it's starting a server? --- Cargo.lock | 52 +++++- Cargo.toml | 1 + builtins/src/fuse.rs | 204 ++++++++++++++++++++++++ builtins/src/string.rs | 3 + doc/mdbook-slosh-eval/Cargo.lock | 13 ++ slosh-fuse/Cargo.toml | 6 +- slosh-fuse/src/bin/slosh-fuse-server.rs | 4 +- slosh-fuse/src/eval_fs.rs | 44 +++-- 8 files changed, 306 insertions(+), 21 deletions(-) create mode 100644 builtins/src/fuse.rs diff --git a/Cargo.lock b/Cargo.lock index 89a4898ac7..4ad8a860bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -651,6 +651,21 @@ dependencies = [ "libc", ] +[[package]] +name = "fuser" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e697f6f62c20b6fad1ba0f84ae909f25971cf16e735273524e3977c94604cf8" +dependencies = [ + "libc", + "log", + "memchr", + "page_size", + "pkg-config", + "smallvec", + "zerocopy", +] + [[package]] name = "futf" version = "0.1.5" @@ -1429,6 +1444,17 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +dependencies = [ + "bitflags 2.8.0", + "cfg-if", + "libc", +] + [[package]] name = "nix" version = "0.29.0" @@ -1550,6 +1576,16 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking_lot" version = "0.11.2" @@ -2082,7 +2118,7 @@ dependencies = [ "cfg-if", "chrono", "glob", - "nix", + "nix 0.29.0", "sl-liner", ] @@ -2173,6 +2209,20 @@ dependencies = [ "unicode_reader", ] +[[package]] +name = "slosh-fuse" +version = "0.11.1" +dependencies = [ + "compile_state", + "env_logger", + "fuser", + "libc", + "log", + "nix 0.27.1", + "slosh_lib", + "slvm", +] + [[package]] name = "slosh_lib" version = "0.11.1" diff --git a/Cargo.toml b/Cargo.toml index 25086d7b9d..8ae2a6b2ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,7 @@ debug = true version = "0.11.1" [workspace.dependencies] +slosh_lib = { path = "slosh_lib" } bridge_adapters = { path = "bridge_adapters" } bridge_macros = { path = "bridge_macros" } bridge_types = { path = "bridge_types" } diff --git a/builtins/src/fuse.rs b/builtins/src/fuse.rs new file mode 100644 index 0000000000..ebab7687c0 --- /dev/null +++ b/builtins/src/fuse.rs @@ -0,0 +1,204 @@ +use bridge_adapters::add_builtin; +use compile_state::state::SloshVm; +use slvm::{VMError, VMResult, Value}; +use std::collections::HashMap; +use std::sync::Mutex; +use std::path::PathBuf; + +// Simple registry to track mount points and their expressions +lazy_static::lazy_static! { + static ref FUSE_REGISTRY: Mutex> = Mutex::new(HashMap::new()); +} + +struct FuseInfo { + mount_point: PathBuf, + expressions: HashMap, +} + +fn mount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 1 { + return Err(VMError::new_vm( + "mount-eval-fs: requires one argument (mount-path)".to_string(), + )); + } + + let mount_path = registers[0].get_string(vm)?; + let mount_point = PathBuf::from(mount_path); + + // Create mount point if it doesn't exist + if !mount_point.exists() { + std::fs::create_dir_all(&mount_point) + .map_err(|e| VMError::new_vm(format!("Failed to create mount point: {}", e)))?; + } + + let mount_id = format!("mount-{}", uuid::Uuid::new_v4()); + + let info = FuseInfo { + mount_point: mount_point.clone(), + expressions: HashMap::new(), + }; + + FUSE_REGISTRY.lock().unwrap().insert(mount_id.clone(), info); + + // For now, return the mount ID. The actual FUSE mounting will be done separately + Ok(vm.alloc_string(mount_id)) +} + +fn register_eval_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 3 { + return Err(VMError::new_vm( + "register-eval-file: requires three arguments (mount-id path expression)".to_string(), + )); + } + + let mount_id = registers[0].get_string(vm)?; + let path = registers[1].get_string(vm)?; + let expression = registers[2].get_string(vm)?; + + let mut registry = FUSE_REGISTRY.lock().unwrap(); + if let Some(info) = registry.get_mut(mount_id) { + info.expressions.insert(path.to_string(), expression.to_string()); + Ok(Value::True) + } else { + Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) + } +} + +fn unmount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 1 { + return Err(VMError::new_vm( + "unmount-eval-fs: requires one argument (mount-id)".to_string(), + )); + } + + let mount_id = registers[0].get_string(vm)?; + + let mut registry = FUSE_REGISTRY.lock().unwrap(); + if registry.remove(mount_id).is_some() { + Ok(Value::True) + } else { + Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) + } +} + +fn list_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 1 { + return Err(VMError::new_vm( + "list-eval-files: requires one argument (mount-id)".to_string(), + )); + } + + let mount_id = registers[0].get_string(vm)?; + + let registry = FUSE_REGISTRY.lock().unwrap(); + if let Some(info) = registry.get(mount_id) { + let files: Vec = info.expressions.keys() + .map(|k| vm.alloc_string(k.clone())) + .collect(); + Ok(vm.alloc_vector(files)) + } else { + Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) + } +} + +// For testing - evaluate an expression for a file +fn eval_file_expression(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 2 { + return Err(VMError::new_vm( + "eval-file-expression: requires two arguments (mount-id path)".to_string(), + )); + } + + let mount_id = registers[0].get_string(vm)?; + let path = registers[1].get_string(vm)?; + + let registry = FUSE_REGISTRY.lock().unwrap(); + if let Some(info) = registry.get(mount_id) { + if let Some(expression) = info.expressions.get(path) { + // For now, just return the expression itself + // In the real implementation, this would evaluate the expression + Ok(vm.alloc_string(format!("Would evaluate: {}", expression))) + } else { + Err(VMError::new_vm(format!("File not registered: {}", path))) + } + } else { + Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) + } +} + +pub fn add_fuse_builtins(env: &mut SloshVm) { + add_builtin( + env, + "mount-eval-fs", + mount_eval_fs, + r#"Usage: (mount-eval-fs path) + +Mount an evaluation filesystem at the specified path. Returns a mount ID. + +Section: fuse + +Example: +(def mount-id (mount-eval-fs "/tmp/eval-fs")) +"#, + ); + + add_builtin( + env, + "register-eval-file", + register_eval_file, + r#"Usage: (register-eval-file mount-id path expression) + +Register a file with a slosh expression that will be evaluated when the file is read. + +Section: fuse + +Example: +(register-eval-file mount-id "config.env" "(str \"HOST=\" (hostname))") +"#, + ); + + add_builtin( + env, + "unmount-eval-fs", + unmount_eval_fs, + r#"Usage: (unmount-eval-fs mount-id) + +Unmount an evaluation filesystem. + +Section: fuse + +Example: +(unmount-eval-fs mount-id) +"#, + ); + + add_builtin( + env, + "list-eval-files", + list_eval_files, + r#"Usage: (list-eval-files mount-id) + +List all files registered in a mount. + +Section: fuse + +Example: +(list-eval-files mount-id) +"#, + ); + + add_builtin( + env, + "eval-file-expression", + eval_file_expression, + r#"Usage: (eval-file-expression mount-id path) + +Test function to evaluate the expression for a file. For debugging. + +Section: fuse + +Example: +(eval-file-expression mount-id "config.env") +"#, + ); +} \ No newline at end of file diff --git a/builtins/src/string.rs b/builtins/src/string.rs index c2dfed1a4e..51a8d23545 100644 --- a/builtins/src/string.rs +++ b/builtins/src/string.rs @@ -225,6 +225,9 @@ fn str_empty(vm: &mut SloshVm, registers: &[Value]) -> VMResult { Ok(Value::False) } } + Value::Nil => { + Ok(Value::True) + } _ => Err(VMError::new_vm(format!( "str-empty?: takes a string, got a {}", string.display_type(vm) diff --git a/doc/mdbook-slosh-eval/Cargo.lock b/doc/mdbook-slosh-eval/Cargo.lock index 164e633cad..6a341f1043 100644 --- a/doc/mdbook-slosh-eval/Cargo.lock +++ b/doc/mdbook-slosh-eval/Cargo.lock @@ -202,6 +202,7 @@ dependencies = [ "bridge_types", "compile_state", "glob", + "lazy_static", "rand", "same-file", "shell", @@ -210,6 +211,7 @@ dependencies = [ "static_assertions", "unicode-segmentation", "unicode_reader", + "uuid", "walkdir", ] @@ -2381,6 +2383,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" +dependencies = [ + "getrandom 0.3.1", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "version_check" version = "0.9.5" diff --git a/slosh-fuse/Cargo.toml b/slosh-fuse/Cargo.toml index 086bcaabe4..e777572371 100644 --- a/slosh-fuse/Cargo.toml +++ b/slosh-fuse/Cargo.toml @@ -10,9 +10,9 @@ log = "0.4" env_logger = "0.11" # Local slosh dependencies -vm = { path = "../vm" } -compile_state = { path = "../compile_state" } -slosh_lib = { path = "../slosh_lib" } +slosh_lib.workspace = true +slvm.workspace = true +compile_state.workspace = true [dependencies.nix] version = "0.27" diff --git a/slosh-fuse/src/bin/slosh-fuse-server.rs b/slosh-fuse/src/bin/slosh-fuse-server.rs index 9accd99ca2..d64f5a91bc 100644 --- a/slosh-fuse/src/bin/slosh-fuse-server.rs +++ b/slosh-fuse/src/bin/slosh-fuse-server.rs @@ -2,7 +2,7 @@ use fuser::{MountOption, Session}; use slosh_fuse::{EvalFs, FileMapping}; use std::sync::{Arc, Mutex}; use std::env; -use std::os::unix::io::{RawFd, FromRawFd}; +use std::os::unix::io::RawFd; fn main() { env_logger::init(); @@ -37,7 +37,7 @@ fn main() { MountOption::DefaultPermissions, ]; - let session = Session::new(fs, &mount_point, &options).expect("Failed to mount"); + let mut session = Session::new(fs, mount_point.as_ref(), &options).expect("Failed to mount"); // Run until unmounted session.run().expect("Failed to run session"); diff --git a/slosh-fuse/src/eval_fs.rs b/slosh-fuse/src/eval_fs.rs index 2bf4f43b0a..0c2cfbf1af 100644 --- a/slosh-fuse/src/eval_fs.rs +++ b/slosh-fuse/src/eval_fs.rs @@ -6,7 +6,7 @@ use libc::{ENOENT, ENOTDIR}; use std::collections::HashMap; use std::ffi::OsStr; use std::sync::{Arc, Mutex}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, SystemTime}; use std::os::unix::io::{RawFd, FromRawFd, IntoRawFd}; use std::io::{Read, Write}; @@ -35,11 +35,14 @@ impl EvalFs { }; // Pre-populate inodes for registered files - let mapping = fs.file_mapping.lock().unwrap(); - for path in mapping.list_files() { - fs.allocate_inode(&path); + { + let mapping = fs.file_mapping.lock().unwrap(); + let paths = mapping.list_files(); + drop(mapping); + for path in paths { + fs.allocate_inode(&path); + } } - drop(mapping); fs } @@ -157,9 +160,13 @@ impl Filesystem for EvalFs { } let name_str = name.to_string_lossy(); - let mapping = self.file_mapping.lock().unwrap(); - if mapping.get(&name_str).is_some() { + let found = { + let mapping = self.file_mapping.lock().unwrap(); + mapping.get(&name_str).is_some() + }; + + if found { let inode = self.allocate_inode(&name_str); reply.entry(&TTL, &Self::file_attr(inode, 0), 0); } else { @@ -171,15 +178,22 @@ impl Filesystem for EvalFs { if ino == FUSE_ROOT_ID { reply.attr(&TTL, &Self::dir_attr(FUSE_ROOT_ID)); } else if let Some(path) = self.get_path_for_inode(ino) { - let mapping = self.file_mapping.lock().unwrap(); - if let Some(entry) = mapping.get(path) { - // Get content to determine size - let content = if entry.is_cache_valid() { - entry.cached_value.as_ref().unwrap().clone() + let (is_valid, expression) = { + let mapping = self.file_mapping.lock().unwrap(); + if let Some(entry) = mapping.get(path) { + if entry.is_cache_valid() { + let content = entry.cached_value.as_ref().unwrap().clone(); + reply.attr(&TTL, &Self::file_attr(ino, content.len() as u64)); + return; + } + (true, entry.expression.clone()) } else { - drop(mapping); - self.evaluate_expression(&entry.expression) - }; + (false, String::new()) + } + }; + + if is_valid { + let content = self.evaluate_expression(&expression); reply.attr(&TTL, &Self::file_attr(ino, content.len() as u64)); } else { reply.error(ENOENT); From 2e72fe4a9cace8d270599cffdce5c68364128164 Mon Sep 17 00:00:00 2001 From: gpwclark Date: Fri, 15 Aug 2025 00:07:39 +0000 Subject: [PATCH 03/12] more fleshed out, needs review --- builtins/src/fuse.rs | 72 ++++++++++++++----- slosh-fuse/README.md | 2 +- slosh-fuse/examples/dynamic-env-file.sl | 2 +- slosh-fuse/examples/dynamic-env-generator.sl | 10 +-- slosh-fuse/examples/dynamic-file-demo.sl | 2 +- slosh-fuse/examples/slosh-fuse/README.md | 2 +- .../slosh-fuse/src/bin/slosh-fuse-server.rs | 16 ++--- slosh-fuse/examples/slosh-fuse/src/eval_fs.rs | 50 ++++++------- .../examples/slosh-fuse/src/file_mapping.rs | 24 +++---- slosh-fuse/examples/slosh-fuse/src/lib.rs | 4 +- slosh-fuse/examples/test-fuse-systemd.sl | 2 +- slosh-fuse/src/bin/slosh-fuse-server.rs | 16 ++--- slosh-fuse/src/eval_fs.rs | 54 +++++++------- slosh-fuse/src/file_mapping.rs | 24 +++---- slosh-fuse/src/lib.rs | 4 +- 15 files changed, 160 insertions(+), 124 deletions(-) diff --git a/builtins/src/fuse.rs b/builtins/src/fuse.rs index ebab7687c0..c0654e4a34 100644 --- a/builtins/src/fuse.rs +++ b/builtins/src/fuse.rs @@ -21,25 +21,25 @@ fn mount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { "mount-eval-fs: requires one argument (mount-path)".to_string(), )); } - + let mount_path = registers[0].get_string(vm)?; let mount_point = PathBuf::from(mount_path); - + // Create mount point if it doesn't exist if !mount_point.exists() { std::fs::create_dir_all(&mount_point) .map_err(|e| VMError::new_vm(format!("Failed to create mount point: {}", e)))?; } - + let mount_id = format!("mount-{}", uuid::Uuid::new_v4()); - + let info = FuseInfo { mount_point: mount_point.clone(), expressions: HashMap::new(), }; - + FUSE_REGISTRY.lock().unwrap().insert(mount_id.clone(), info); - + // For now, return the mount ID. The actual FUSE mounting will be done separately Ok(vm.alloc_string(mount_id)) } @@ -50,11 +50,11 @@ fn register_eval_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult "register-eval-file: requires three arguments (mount-id path expression)".to_string(), )); } - + let mount_id = registers[0].get_string(vm)?; let path = registers[1].get_string(vm)?; let expression = registers[2].get_string(vm)?; - + let mut registry = FUSE_REGISTRY.lock().unwrap(); if let Some(info) = registry.get_mut(mount_id) { info.expressions.insert(path.to_string(), expression.to_string()); @@ -70,9 +70,9 @@ fn unmount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { "unmount-eval-fs: requires one argument (mount-id)".to_string(), )); } - + let mount_id = registers[0].get_string(vm)?; - + let mut registry = FUSE_REGISTRY.lock().unwrap(); if registry.remove(mount_id).is_some() { Ok(Value::True) @@ -87,9 +87,9 @@ fn list_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { "list-eval-files: requires one argument (mount-id)".to_string(), )); } - + let mount_id = registers[0].get_string(vm)?; - + let registry = FUSE_REGISTRY.lock().unwrap(); if let Some(info) = registry.get(mount_id) { let files: Vec = info.expressions.keys() @@ -101,6 +101,25 @@ fn list_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { } } +fn list_mounts(vm: &mut SloshVm, _registers: &[Value]) -> VMResult { + let registry = FUSE_REGISTRY.lock().unwrap(); + let mut mounts = Vec::new(); + + for (mount_id, info) in registry.iter() { + let mount_info = vec![ + vm.alloc_string("mount-id"), + vm.alloc_string(mount_id.clone()), + vm.alloc_string("mount-point"), + vm.alloc_string(info.mount_point.display().to_string()), + vm.alloc_string("file-count"), + vm.alloc_int(info.expressions.len() as i64), + ]; + mounts.push(vm.alloc_vector(mount_info)); + } + + Ok(vm.alloc_vector(mounts)) +} + // For testing - evaluate an expression for a file fn eval_file_expression(vm: &mut SloshVm, registers: &[Value]) -> VMResult { if registers.len() != 2 { @@ -108,10 +127,10 @@ fn eval_file_expression(vm: &mut SloshVm, registers: &[Value]) -> VMResult "mount-point" "file-count" ] +"#, + ); + add_builtin( env, "eval-file-expression", diff --git a/slosh-fuse/README.md b/slosh-fuse/README.md index 9a7d4c43b1..99a26fd5c6 100644 --- a/slosh-fuse/README.md +++ b/slosh-fuse/README.md @@ -44,7 +44,7 @@ The implementation consists of: The current implementation provides: - Basic FUSE filesystem structure -- File registration and mapping system +- File registration and mapping system - Builtin functions for mounting and file management - Example scripts demonstrating the concept diff --git a/slosh-fuse/examples/dynamic-env-file.sl b/slosh-fuse/examples/dynamic-env-file.sl index 69e40d7260..fd09f59061 100644 --- a/slosh-fuse/examples/dynamic-env-file.sl +++ b/slosh-fuse/examples/dynamic-env-file.sl @@ -25,7 +25,7 @@ ;; Function to create a background process that updates the file periodically (defn start-dynamic-env-updater [path interval-ms] "Start a background process that updates the env file every interval-ms milliseconds" - (fork + (fork (loop [] (generate-env-file path) (sleep interval-ms) diff --git a/slosh-fuse/examples/dynamic-env-generator.sl b/slosh-fuse/examples/dynamic-env-generator.sl index 158ed1820c..ee32e6af0e 100644 --- a/slosh-fuse/examples/dynamic-env-generator.sl +++ b/slosh-fuse/examples/dynamic-env-generator.sl @@ -27,7 +27,7 @@ ;; Example: Register systemd environment files ;; Define some global configuration -(def *app-config* +(def *app-config* {:environment "production" :version "1.0.0" :debug false}) @@ -47,8 +47,8 @@ (register-env-file "/tmp/database.env" (fn [] (str "# Database configuration\n" - "DB_HOST=" (if (= (:environment *app-config*) "production") - "prod-db.example.com" + "DB_HOST=" (if (= (:environment *app-config*) "production") + "prod-db.example.com" "localhost") "\n" "DB_PORT=5432\n" "DB_NAME=" (:environment *app-config*) "_db\n" @@ -66,14 +66,14 @@ (println (read-string file 10000)) (close file)) -(println "--- Generated database.env ---") +(println "--- Generated database.env ---") (let [file (open "/tmp/database.env" :read)] (println (read-string file 10000)) (close file)) ;; Demonstrate dynamic updates (println "\n--- Updating configuration to development ---") -(set! *app-config* (assoc *app-config* +(set! *app-config* (assoc *app-config* :environment "development" :debug true)) diff --git a/slosh-fuse/examples/dynamic-file-demo.sl b/slosh-fuse/examples/dynamic-file-demo.sl index 7928cda429..b9eddf1977 100644 --- a/slosh-fuse/examples/dynamic-file-demo.sl +++ b/slosh-fuse/examples/dynamic-file-demo.sl @@ -8,7 +8,7 @@ ;; Register a systemd-style environment file ;; This would dynamically generate content based on slosh expressions -(register-eval-file mount-id "my-service.env" +(register-eval-file mount-id "my-service.env" "(str \"HOST=\" (or (env \"HOSTNAME\") \"localhost\") \"\\n\" \"CUSTOM_TYPE=\" (or *custom-type* \"default\") \"\\n\" \"TIMESTAMP=\" (current-time-millis) \"\\n\")") diff --git a/slosh-fuse/examples/slosh-fuse/README.md b/slosh-fuse/examples/slosh-fuse/README.md index 9a7d4c43b1..99a26fd5c6 100644 --- a/slosh-fuse/examples/slosh-fuse/README.md +++ b/slosh-fuse/examples/slosh-fuse/README.md @@ -44,7 +44,7 @@ The implementation consists of: The current implementation provides: - Basic FUSE filesystem structure -- File registration and mapping system +- File registration and mapping system - Builtin functions for mounting and file management - Example scripts demonstrating the concept diff --git a/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs b/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs index 9accd99ca2..e546353baa 100644 --- a/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs +++ b/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs @@ -6,19 +6,19 @@ use std::os::unix::io::{RawFd, FromRawFd}; fn main() { env_logger::init(); - + let args: Vec = env::args().collect(); if args.len() < 4 { eprintln!("Usage: {} ", args[0]); std::process::exit(1); } - + let mount_point = &args[1]; let read_fd: RawFd = args[2].parse().expect("Invalid read fd"); let write_fd: RawFd = args[3].parse().expect("Invalid write fd"); - + let file_mapping = Arc::new(Mutex::new(FileMapping::new())); - + // Pre-populate with test data if provided if args.len() > 4 { let mut mapping = file_mapping.lock().unwrap(); @@ -28,17 +28,17 @@ fn main() { } } } - + let fs = EvalFs::new(file_mapping, read_fd, write_fd); - + let options = vec![ MountOption::FSName("slosh-eval".to_string()), MountOption::AllowOther, MountOption::DefaultPermissions, ]; - + let session = Session::new(fs, &mount_point, &options).expect("Failed to mount"); - + // Run until unmounted session.run().expect("Failed to run session"); } \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs b/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs index 2bf4f43b0a..e06147ec80 100644 --- a/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs +++ b/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs @@ -33,37 +33,37 @@ impl EvalFs { eval_pipe_read, eval_pipe_write, }; - + // Pre-populate inodes for registered files let mapping = fs.file_mapping.lock().unwrap(); for path in mapping.list_files() { fs.allocate_inode(&path); } drop(mapping); - + fs } - + fn allocate_inode(&mut self, path: &str) -> u64 { if let Some(&inode) = self.path_to_inode.get(path) { return inode; } - + let inode = self.next_inode; self.next_inode += 1; self.inode_to_path.insert(inode, path.to_string()); self.path_to_inode.insert(path.to_string(), inode); inode } - + fn get_inode_for_path(&self, path: &str) -> Option { self.path_to_inode.get(path).copied() } - + fn get_path_for_inode(&self, inode: u64) -> Option<&str> { self.inode_to_path.get(&inode).map(|s| s.as_str()) } - + fn evaluate_expression(&self, expression: &str) -> String { // Send expression to parent process for evaluation let msg = format!("{}\n", expression); @@ -76,12 +76,12 @@ impl EvalFs { } // Prevent closing the fd when File is dropped let _ = pipe_write.into_raw_fd(); - + // Read response let mut pipe_read = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_read) }; let mut response = String::new(); let mut buffer = [0u8; 4096]; - + loop { match pipe_read.read(&mut buffer) { Ok(0) => break, @@ -102,10 +102,10 @@ impl EvalFs { } // Prevent closing the fd when File is dropped let _ = pipe_read.into_raw_fd(); - + response } - + fn file_attr(inode: u64, size: u64) -> FileAttr { let now = SystemTime::now(); FileAttr { @@ -126,7 +126,7 @@ impl EvalFs { flags: 0, } } - + fn dir_attr(inode: u64) -> FileAttr { let now = SystemTime::now(); FileAttr { @@ -155,10 +155,10 @@ impl Filesystem for EvalFs { reply.error(ENOENT); return; } - + let name_str = name.to_string_lossy(); let mapping = self.file_mapping.lock().unwrap(); - + if mapping.get(&name_str).is_some() { let inode = self.allocate_inode(&name_str); reply.entry(&TTL, &Self::file_attr(inode, 0), 0); @@ -166,7 +166,7 @@ impl Filesystem for EvalFs { reply.error(ENOENT); } } - + fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) { if ino == FUSE_ROOT_ID { reply.attr(&TTL, &Self::dir_attr(FUSE_ROOT_ID)); @@ -188,7 +188,7 @@ impl Filesystem for EvalFs { reply.error(ENOENT); } } - + fn read( &mut self, _req: &Request, @@ -209,7 +209,7 @@ impl Filesystem for EvalFs { let expr = entry.expression.clone(); drop(mapping); let result = self.evaluate_expression(&expr); - + // Update cache if caching is enabled let mut mapping = self.file_mapping.lock().unwrap(); if let Some(entry) = mapping.get_mut(path) { @@ -219,11 +219,11 @@ impl Filesystem for EvalFs { } result }; - + let data = content.as_bytes(); let start = offset as usize; let end = (start + size as usize).min(data.len()); - + if start < data.len() { reply.data(&data[start..end]); } else { @@ -236,7 +236,7 @@ impl Filesystem for EvalFs { reply.error(ENOENT); } } - + fn readdir( &mut self, _req: &Request, @@ -249,12 +249,12 @@ impl Filesystem for EvalFs { reply.error(ENOTDIR); return; } - + let entries = vec![ (FUSE_ROOT_ID, FileType::Directory, "."), (FUSE_ROOT_ID, FileType::Directory, ".."), ]; - + for (i, (inode, file_type, name)) in entries.iter().enumerate() { if offset <= i as i64 { if reply.add(*inode, (i + 1) as i64, *file_type, name) { @@ -262,11 +262,11 @@ impl Filesystem for EvalFs { } } } - + let mapping = self.file_mapping.lock().unwrap(); let files = mapping.list_files(); drop(mapping); - + for (i, path) in files.iter().enumerate() { let index = i + entries.len(); if offset <= index as i64 { @@ -276,7 +276,7 @@ impl Filesystem for EvalFs { } } } - + reply.ok(); } } \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs b/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs index 10b42bb31e..8ebbe040b8 100644 --- a/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs +++ b/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs @@ -18,12 +18,12 @@ impl FileEntry { cache_ttl_secs: 0, // No caching by default } } - + pub fn is_cache_valid(&self) -> bool { if self.cache_ttl_secs == 0 { return false; } - + if let (Some(cached_time), Some(_)) = (self.cache_time, &self.cached_value) { if let Ok(elapsed) = cached_time.elapsed() { return elapsed.as_secs() < self.cache_ttl_secs; @@ -31,7 +31,7 @@ impl FileEntry { } false } - + pub fn update_cache(&mut self, value: String) { self.cached_value = Some(value); self.cache_time = Some(SystemTime::now()); @@ -48,47 +48,47 @@ impl FileMapping { files: HashMap::new(), } } - + pub fn register(&mut self, path: &str, expression: String) { let normalized_path = if path.starts_with('/') { path[1..].to_string() } else { path.to_string() }; - + self.files.insert(normalized_path, FileEntry::new(expression)); } - + pub fn get(&self, path: &str) -> Option<&FileEntry> { let normalized_path = if path.starts_with('/') { &path[1..] } else { path }; - + self.files.get(normalized_path) } - + pub fn get_mut(&mut self, path: &str) -> Option<&mut FileEntry> { let normalized_path = if path.starts_with('/') { path[1..].to_string() } else { path.to_string() }; - + self.files.get_mut(&normalized_path) } - + pub fn remove(&mut self, path: &str) -> Option { let normalized_path = if path.starts_with('/') { path[1..].to_string() } else { path.to_string() }; - + self.files.remove(&normalized_path) } - + pub fn list_files(&self) -> Vec { self.files.keys().cloned().collect() } diff --git a/slosh-fuse/examples/slosh-fuse/src/lib.rs b/slosh-fuse/examples/slosh-fuse/src/lib.rs index 312ba5b37a..53a0d1322c 100644 --- a/slosh-fuse/examples/slosh-fuse/src/lib.rs +++ b/slosh-fuse/examples/slosh-fuse/src/lib.rs @@ -21,12 +21,12 @@ impl FuseMount { process_handle: None, } } - + pub fn register_file(&self, path: &str, expression: String) { let mut mapping = self.file_mapping.lock().unwrap(); mapping.register(path, expression); } - + pub fn unmount(&mut self) -> Result<(), std::io::Error> { if let Some(mut handle) = self.process_handle.take() { handle.kill()?; diff --git a/slosh-fuse/examples/test-fuse-systemd.sl b/slosh-fuse/examples/test-fuse-systemd.sl index 9319def331..7a5615acd7 100644 --- a/slosh-fuse/examples/test-fuse-systemd.sl +++ b/slosh-fuse/examples/test-fuse-systemd.sl @@ -16,7 +16,7 @@ (register-eval-file mount-id "my-service.env" "(str \"# Generated by slosh\\n\" \"HOST=\" (or (env \"HOSTNAME\") \"meow\") \"\\n\" - \"CUSTOM_TYPE=\" *custom-type* \"\\n\" + \"CUSTOM_TYPE=\" *custom-type* \"\\n\" \"APP_VERSION=\" *app-version* \"\\n\" \"USER=\" (env \"USER\") \"\\n\" \"HOME=\" (env \"HOME\") \"\\n\")") diff --git a/slosh-fuse/src/bin/slosh-fuse-server.rs b/slosh-fuse/src/bin/slosh-fuse-server.rs index d64f5a91bc..30f6da7cd7 100644 --- a/slosh-fuse/src/bin/slosh-fuse-server.rs +++ b/slosh-fuse/src/bin/slosh-fuse-server.rs @@ -6,19 +6,19 @@ use std::os::unix::io::RawFd; fn main() { env_logger::init(); - + let args: Vec = env::args().collect(); if args.len() < 4 { eprintln!("Usage: {} ", args[0]); std::process::exit(1); } - + let mount_point = &args[1]; let read_fd: RawFd = args[2].parse().expect("Invalid read fd"); let write_fd: RawFd = args[3].parse().expect("Invalid write fd"); - + let file_mapping = Arc::new(Mutex::new(FileMapping::new())); - + // Pre-populate with test data if provided if args.len() > 4 { let mut mapping = file_mapping.lock().unwrap(); @@ -28,17 +28,17 @@ fn main() { } } } - + let fs = EvalFs::new(file_mapping, read_fd, write_fd); - + let options = vec![ MountOption::FSName("slosh-eval".to_string()), MountOption::AllowOther, MountOption::DefaultPermissions, ]; - + let mut session = Session::new(fs, mount_point.as_ref(), &options).expect("Failed to mount"); - + // Run until unmounted session.run().expect("Failed to run session"); } \ No newline at end of file diff --git a/slosh-fuse/src/eval_fs.rs b/slosh-fuse/src/eval_fs.rs index 0c2cfbf1af..f140c24f4a 100644 --- a/slosh-fuse/src/eval_fs.rs +++ b/slosh-fuse/src/eval_fs.rs @@ -33,7 +33,7 @@ impl EvalFs { eval_pipe_read, eval_pipe_write, }; - + // Pre-populate inodes for registered files { let mapping = fs.file_mapping.lock().unwrap(); @@ -43,30 +43,30 @@ impl EvalFs { fs.allocate_inode(&path); } } - + fs } - + fn allocate_inode(&mut self, path: &str) -> u64 { if let Some(&inode) = self.path_to_inode.get(path) { return inode; } - + let inode = self.next_inode; self.next_inode += 1; self.inode_to_path.insert(inode, path.to_string()); self.path_to_inode.insert(path.to_string(), inode); inode } - + fn get_inode_for_path(&self, path: &str) -> Option { self.path_to_inode.get(path).copied() } - + fn get_path_for_inode(&self, inode: u64) -> Option<&str> { self.inode_to_path.get(&inode).map(|s| s.as_str()) } - + fn evaluate_expression(&self, expression: &str) -> String { // Send expression to parent process for evaluation let msg = format!("{}\n", expression); @@ -79,12 +79,12 @@ impl EvalFs { } // Prevent closing the fd when File is dropped let _ = pipe_write.into_raw_fd(); - + // Read response let mut pipe_read = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_read) }; let mut response = String::new(); let mut buffer = [0u8; 4096]; - + loop { match pipe_read.read(&mut buffer) { Ok(0) => break, @@ -105,10 +105,10 @@ impl EvalFs { } // Prevent closing the fd when File is dropped let _ = pipe_read.into_raw_fd(); - + response } - + fn file_attr(inode: u64, size: u64) -> FileAttr { let now = SystemTime::now(); FileAttr { @@ -129,7 +129,7 @@ impl EvalFs { flags: 0, } } - + fn dir_attr(inode: u64) -> FileAttr { let now = SystemTime::now(); FileAttr { @@ -158,14 +158,14 @@ impl Filesystem for EvalFs { reply.error(ENOENT); return; } - + let name_str = name.to_string_lossy(); - + let found = { let mapping = self.file_mapping.lock().unwrap(); mapping.get(&name_str).is_some() }; - + if found { let inode = self.allocate_inode(&name_str); reply.entry(&TTL, &Self::file_attr(inode, 0), 0); @@ -173,7 +173,7 @@ impl Filesystem for EvalFs { reply.error(ENOENT); } } - + fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) { if ino == FUSE_ROOT_ID { reply.attr(&TTL, &Self::dir_attr(FUSE_ROOT_ID)); @@ -191,7 +191,7 @@ impl Filesystem for EvalFs { (false, String::new()) } }; - + if is_valid { let content = self.evaluate_expression(&expression); reply.attr(&TTL, &Self::file_attr(ino, content.len() as u64)); @@ -202,7 +202,7 @@ impl Filesystem for EvalFs { reply.error(ENOENT); } } - + fn read( &mut self, _req: &Request, @@ -223,7 +223,7 @@ impl Filesystem for EvalFs { let expr = entry.expression.clone(); drop(mapping); let result = self.evaluate_expression(&expr); - + // Update cache if caching is enabled let mut mapping = self.file_mapping.lock().unwrap(); if let Some(entry) = mapping.get_mut(path) { @@ -233,11 +233,11 @@ impl Filesystem for EvalFs { } result }; - + let data = content.as_bytes(); let start = offset as usize; let end = (start + size as usize).min(data.len()); - + if start < data.len() { reply.data(&data[start..end]); } else { @@ -250,7 +250,7 @@ impl Filesystem for EvalFs { reply.error(ENOENT); } } - + fn readdir( &mut self, _req: &Request, @@ -263,12 +263,12 @@ impl Filesystem for EvalFs { reply.error(ENOTDIR); return; } - + let entries = vec![ (FUSE_ROOT_ID, FileType::Directory, "."), (FUSE_ROOT_ID, FileType::Directory, ".."), ]; - + for (i, (inode, file_type, name)) in entries.iter().enumerate() { if offset <= i as i64 { if reply.add(*inode, (i + 1) as i64, *file_type, name) { @@ -276,11 +276,11 @@ impl Filesystem for EvalFs { } } } - + let mapping = self.file_mapping.lock().unwrap(); let files = mapping.list_files(); drop(mapping); - + for (i, path) in files.iter().enumerate() { let index = i + entries.len(); if offset <= index as i64 { @@ -290,7 +290,7 @@ impl Filesystem for EvalFs { } } } - + reply.ok(); } } \ No newline at end of file diff --git a/slosh-fuse/src/file_mapping.rs b/slosh-fuse/src/file_mapping.rs index 10b42bb31e..8ebbe040b8 100644 --- a/slosh-fuse/src/file_mapping.rs +++ b/slosh-fuse/src/file_mapping.rs @@ -18,12 +18,12 @@ impl FileEntry { cache_ttl_secs: 0, // No caching by default } } - + pub fn is_cache_valid(&self) -> bool { if self.cache_ttl_secs == 0 { return false; } - + if let (Some(cached_time), Some(_)) = (self.cache_time, &self.cached_value) { if let Ok(elapsed) = cached_time.elapsed() { return elapsed.as_secs() < self.cache_ttl_secs; @@ -31,7 +31,7 @@ impl FileEntry { } false } - + pub fn update_cache(&mut self, value: String) { self.cached_value = Some(value); self.cache_time = Some(SystemTime::now()); @@ -48,47 +48,47 @@ impl FileMapping { files: HashMap::new(), } } - + pub fn register(&mut self, path: &str, expression: String) { let normalized_path = if path.starts_with('/') { path[1..].to_string() } else { path.to_string() }; - + self.files.insert(normalized_path, FileEntry::new(expression)); } - + pub fn get(&self, path: &str) -> Option<&FileEntry> { let normalized_path = if path.starts_with('/') { &path[1..] } else { path }; - + self.files.get(normalized_path) } - + pub fn get_mut(&mut self, path: &str) -> Option<&mut FileEntry> { let normalized_path = if path.starts_with('/') { path[1..].to_string() } else { path.to_string() }; - + self.files.get_mut(&normalized_path) } - + pub fn remove(&mut self, path: &str) -> Option { let normalized_path = if path.starts_with('/') { path[1..].to_string() } else { path.to_string() }; - + self.files.remove(&normalized_path) } - + pub fn list_files(&self) -> Vec { self.files.keys().cloned().collect() } diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs index 312ba5b37a..53a0d1322c 100644 --- a/slosh-fuse/src/lib.rs +++ b/slosh-fuse/src/lib.rs @@ -21,12 +21,12 @@ impl FuseMount { process_handle: None, } } - + pub fn register_file(&self, path: &str, expression: String) { let mut mapping = self.file_mapping.lock().unwrap(); mapping.register(path, expression); } - + pub fn unmount(&mut self) -> Result<(), std::io::Error> { if let Some(mut handle) = self.process_handle.take() { handle.kill()?; From 1631373a82ad5e0a064b66283808227c65e0ea82 Mon Sep 17 00:00:00 2001 From: gpwclark Date: Wed, 20 Aug 2025 13:09:59 +0000 Subject: [PATCH 04/12] again --- Cargo.lock | 10 +- builtins/Cargo.toml | 2 - builtins/src/fuse.rs | 240 ---------------------------- builtins/src/lib.rs | 1 - slosh-fuse/Cargo.toml | 12 +- slosh-fuse/src/lib.rs | 8 +- slosh/Cargo.toml | 6 +- slosh/src/main.rs | 3 + slosh_lib/Cargo.toml | 8 + slosh_lib/src/fuse_bindings.rs | 275 +++++++++++++++++++++++++++++++++ slosh_lib/src/lib.rs | 15 +- 11 files changed, 322 insertions(+), 258 deletions(-) delete mode 100644 builtins/src/fuse.rs create mode 100644 slosh_lib/src/fuse_bindings.rs diff --git a/Cargo.lock b/Cargo.lock index 4ad8a860bb..86dde9de70 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -228,7 +228,6 @@ dependencies = [ "bridge_types", "compile_state", "glob", - "lazy_static", "rand", "same-file", "shell", @@ -238,7 +237,6 @@ dependencies = [ "trybuild", "unicode-segmentation", "unicode_reader", - "uuid", "walkdir", ] @@ -2201,6 +2199,7 @@ dependencies = [ "shell", "sl-compiler", "sl-liner", + "slosh-fuse", "slosh_lib", "slvm", "temp-env", @@ -2213,14 +2212,12 @@ dependencies = [ name = "slosh-fuse" version = "0.11.1" dependencies = [ - "compile_state", "env_logger", "fuser", "libc", "log", "nix 0.27.1", - "slosh_lib", - "slvm", + "uuid", ] [[package]] @@ -2237,15 +2234,18 @@ dependencies = [ "criterion", "glob", "iai-callgrind", + "nix 0.27.1", "shell", "sl-compiler", "sl-liner", + "slosh-fuse", "slvm", "static_assertions", "temp-env", "tempfile", "unicode-width 0.2.0", "unicode_reader", + "uuid", ] [[package]] diff --git a/builtins/Cargo.toml b/builtins/Cargo.toml index 5d955e308f..3a07807a26 100644 --- a/builtins/Cargo.toml +++ b/builtins/Cargo.toml @@ -20,8 +20,6 @@ rand = { workspace = true } walkdir = { workspace = true } same-file = { workspace = true } glob = { workspace = true } -lazy_static = "1.4" -uuid = { version = "1.0", features = ["v4"] } [dev-dependencies] trybuild = { workspace = true } diff --git a/builtins/src/fuse.rs b/builtins/src/fuse.rs deleted file mode 100644 index c0654e4a34..0000000000 --- a/builtins/src/fuse.rs +++ /dev/null @@ -1,240 +0,0 @@ -use bridge_adapters::add_builtin; -use compile_state::state::SloshVm; -use slvm::{VMError, VMResult, Value}; -use std::collections::HashMap; -use std::sync::Mutex; -use std::path::PathBuf; - -// Simple registry to track mount points and their expressions -lazy_static::lazy_static! { - static ref FUSE_REGISTRY: Mutex> = Mutex::new(HashMap::new()); -} - -struct FuseInfo { - mount_point: PathBuf, - expressions: HashMap, -} - -fn mount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 1 { - return Err(VMError::new_vm( - "mount-eval-fs: requires one argument (mount-path)".to_string(), - )); - } - - let mount_path = registers[0].get_string(vm)?; - let mount_point = PathBuf::from(mount_path); - - // Create mount point if it doesn't exist - if !mount_point.exists() { - std::fs::create_dir_all(&mount_point) - .map_err(|e| VMError::new_vm(format!("Failed to create mount point: {}", e)))?; - } - - let mount_id = format!("mount-{}", uuid::Uuid::new_v4()); - - let info = FuseInfo { - mount_point: mount_point.clone(), - expressions: HashMap::new(), - }; - - FUSE_REGISTRY.lock().unwrap().insert(mount_id.clone(), info); - - // For now, return the mount ID. The actual FUSE mounting will be done separately - Ok(vm.alloc_string(mount_id)) -} - -fn register_eval_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 3 { - return Err(VMError::new_vm( - "register-eval-file: requires three arguments (mount-id path expression)".to_string(), - )); - } - - let mount_id = registers[0].get_string(vm)?; - let path = registers[1].get_string(vm)?; - let expression = registers[2].get_string(vm)?; - - let mut registry = FUSE_REGISTRY.lock().unwrap(); - if let Some(info) = registry.get_mut(mount_id) { - info.expressions.insert(path.to_string(), expression.to_string()); - Ok(Value::True) - } else { - Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) - } -} - -fn unmount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 1 { - return Err(VMError::new_vm( - "unmount-eval-fs: requires one argument (mount-id)".to_string(), - )); - } - - let mount_id = registers[0].get_string(vm)?; - - let mut registry = FUSE_REGISTRY.lock().unwrap(); - if registry.remove(mount_id).is_some() { - Ok(Value::True) - } else { - Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) - } -} - -fn list_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 1 { - return Err(VMError::new_vm( - "list-eval-files: requires one argument (mount-id)".to_string(), - )); - } - - let mount_id = registers[0].get_string(vm)?; - - let registry = FUSE_REGISTRY.lock().unwrap(); - if let Some(info) = registry.get(mount_id) { - let files: Vec = info.expressions.keys() - .map(|k| vm.alloc_string(k.clone())) - .collect(); - Ok(vm.alloc_vector(files)) - } else { - Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) - } -} - -fn list_mounts(vm: &mut SloshVm, _registers: &[Value]) -> VMResult { - let registry = FUSE_REGISTRY.lock().unwrap(); - let mut mounts = Vec::new(); - - for (mount_id, info) in registry.iter() { - let mount_info = vec![ - vm.alloc_string("mount-id"), - vm.alloc_string(mount_id.clone()), - vm.alloc_string("mount-point"), - vm.alloc_string(info.mount_point.display().to_string()), - vm.alloc_string("file-count"), - vm.alloc_int(info.expressions.len() as i64), - ]; - mounts.push(vm.alloc_vector(mount_info)); - } - - Ok(vm.alloc_vector(mounts)) -} - -// For testing - evaluate an expression for a file -fn eval_file_expression(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 2 { - return Err(VMError::new_vm( - "eval-file-expression: requires two arguments (mount-id path)".to_string(), - )); - } - - let mount_id = registers[0].get_string(vm)?; - let path = registers[1].get_string(vm)?; - - let registry = FUSE_REGISTRY.lock().unwrap(); - if let Some(info) = registry.get(mount_id) { - if let Some(expression) = info.expressions.get(path) { - // For now, just return the expression itself - // In the real implementation, this would evaluate the expression - Ok(vm.alloc_string(format!("Would evaluate: {}", expression))) - } else { - Err(VMError::new_vm(format!("File not registered: {}", path))) - } - } else { - Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) - } -} - -pub fn add_fuse_builtins(env: &mut SloshVm) { - add_builtin( - env, - "mount-eval-fs", - mount_eval_fs, - r#"Usage: (mount-eval-fs path) - -Mount an evaluation filesystem at the specified path. Returns a mount ID. - -Section: fuse - -Example: -(def mount-id (mount-eval-fs "/tmp/eval-fs")) -"#, - ); - - add_builtin( - env, - "register-eval-file", - register_eval_file, - r#"Usage: (register-eval-file mount-id path expression) - -Register a file with a slosh expression that will be evaluated when the file is read. - -Section: fuse - -Example: -(register-eval-file mount-id "config.env" "(str \"HOST=\" (hostname))") -"#, - ); - - add_builtin( - env, - "unmount-eval-fs", - unmount_eval_fs, - r#"Usage: (unmount-eval-fs mount-id) - -Unmount an evaluation filesystem. - -Section: fuse - -Example: -(unmount-eval-fs mount-id) -"#, - ); - - add_builtin( - env, - "list-eval-files", - list_eval_files, - r#"Usage: (list-eval-files mount-id) - -List all files registered in a mount. - -Section: fuse - -Example: -(list-eval-files mount-id) -"#, - ); - - add_builtin( - env, - "list-mounts", - list_mounts, - r#"Usage: (list-mounts) - -List all active FUSE mounts with their mount IDs, mount points, and file counts. - -Section: fuse - -Example: -(list-mounts) -; Returns a list of vectors, each containing: -; ["mount-id" "mount-point" "file-count" ] -"#, - ); - - add_builtin( - env, - "eval-file-expression", - eval_file_expression, - r#"Usage: (eval-file-expression mount-id path) - -Test function to evaluate the expression for a file. For debugging. - -Section: fuse - -Example: -(eval-file-expression mount-id "config.env") -"#, - ); -} \ No newline at end of file diff --git a/builtins/src/lib.rs b/builtins/src/lib.rs index 0db369a143..10a96ca282 100644 --- a/builtins/src/lib.rs +++ b/builtins/src/lib.rs @@ -13,7 +13,6 @@ pub mod collections; pub mod conversions; pub mod fs_meta; pub mod fs_temp; -pub mod fuse; pub mod io; pub mod math; pub mod print; diff --git a/slosh-fuse/Cargo.toml b/slosh-fuse/Cargo.toml index e777572371..91aad1ccc3 100644 --- a/slosh-fuse/Cargo.toml +++ b/slosh-fuse/Cargo.toml @@ -1,18 +1,20 @@ [package] name = "slosh-fuse" version.workspace = true -edition = "2024" +edition = "2021" + +[[bin]] +name = "slosh-fuse-server" +path = "src/bin/slosh-fuse-server.rs" [dependencies] fuser = "0.14" libc = "0.2" log = "0.4" env_logger = "0.11" +uuid = { version = "1.0", features = ["v4"] } -# Local slosh dependencies -slosh_lib.workspace = true -slvm.workspace = true -compile_state.workspace = true +# No slosh dependencies to avoid circular dependency [dependencies.nix] version = "0.27" diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs index 53a0d1322c..1f959f5d7b 100644 --- a/slosh-fuse/src/lib.rs +++ b/slosh-fuse/src/lib.rs @@ -11,6 +11,7 @@ pub struct FuseMount { pub mount_point: PathBuf, pub file_mapping: Arc>, pub process_handle: Option, + pub comm_write_fd: Option, } impl FuseMount { @@ -19,12 +20,15 @@ impl FuseMount { mount_point, file_mapping: Arc::new(Mutex::new(FileMapping::new())), process_handle: None, + comm_write_fd: None, } } pub fn register_file(&self, path: &str, expression: String) { - let mut mapping = self.file_mapping.lock().unwrap(); - mapping.register(path, expression); + if let Ok(mut mapping) = self.file_mapping.lock() { + mapping.register(path, expression); + } + // Silently ignore lock poisoning - the file just won't be registered } pub fn unmount(&mut self) -> Result<(), std::io::Error> { diff --git a/slosh/Cargo.toml b/slosh/Cargo.toml index c5fb10f9f3..25e1e8517b 100644 --- a/slosh/Cargo.toml +++ b/slosh/Cargo.toml @@ -7,12 +7,13 @@ edition = "2024" [features] default = [] lisp-test = ["dep:regex", "dep:lazy_static", "dep:mdbook", "dep:env_logger"] +fuse = ["dep:slosh-fuse", "slosh_lib/fuse"] [dependencies] unicode_reader = { workspace = true } sl-compiler = { workspace = true } bridge_adapters = { path = "../bridge_adapters" } -slosh_lib = { path = "../slosh_lib" } +slosh_lib = { path = "../slosh_lib", features = [] } sl-liner = { git = "https://github.com/sl-sh-dev/sl-liner.git" } slvm = { workspace = true } builtins = { path = "../builtins" } @@ -21,6 +22,9 @@ shell = { path = "../shell" } unicode-width = { workspace = true } glob = { workspace = true } +# Optional dependencies +slosh-fuse = { path = "../slosh-fuse", optional = true } + # lisp-test dependencies regex = { workspace = true, optional = true } lazy_static = { workspace = true, optional = true } diff --git a/slosh/src/main.rs b/slosh/src/main.rs index 810a034b61..32efa70a42 100644 --- a/slosh/src/main.rs +++ b/slosh/src/main.rs @@ -24,6 +24,9 @@ fn version(vm: &mut SloshVm, registers: &[Value]) -> VMResult { } fn main() { + #[cfg(feature = "fuse")] + eprintln!("FUSE feature is enabled in slosh binary!"); + let exit_code = run(modify_vm); process::exit(exit_code) } diff --git a/slosh_lib/Cargo.toml b/slosh_lib/Cargo.toml index e06f66bc64..cf3843ab3a 100644 --- a/slosh_lib/Cargo.toml +++ b/slosh_lib/Cargo.toml @@ -4,6 +4,9 @@ version.workspace = true edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +fuse = ["dep:slosh-fuse", "dep:nix", "dep:uuid"] + [dependencies] unicode_reader = { workspace = true } sl-compiler = { workspace = true } @@ -19,6 +22,11 @@ unicode-width = { workspace = true } glob = { workspace = true } static_assertions = { workspace = true } +# Optional dependencies +slosh-fuse = { path = "../slosh-fuse", optional = true } +nix = { version = "0.27", features = ["signal", "process", "fs", "mount"], optional = true } +uuid = { version = "1.0", features = ["v4"], optional = true } + [dev-dependencies] tempfile = { workspace = true } temp-env = { workspace = true } diff --git a/slosh_lib/src/fuse_bindings.rs b/slosh_lib/src/fuse_bindings.rs new file mode 100644 index 0000000000..7dd7831ae5 --- /dev/null +++ b/slosh_lib/src/fuse_bindings.rs @@ -0,0 +1,275 @@ +use bridge_adapters::add_builtin; +use compile_state::state::SloshVm; +use slvm::{VMError, VMResult, Value}; +use std::collections::HashMap; +use std::sync::Mutex; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use nix::unistd::pipe; + +use slosh_fuse::FuseMount; + +// Registry for active mounts - only created when first mount is created +use std::sync::OnceLock; +static MOUNT_REGISTRY: OnceLock>> = OnceLock::new(); + +fn get_registry() -> &'static Mutex> { + MOUNT_REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn mount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 1 { + return Err(VMError::new_vm( + "mount-eval-fs: requires one argument (mount-path)".to_string(), + )); + } + + let mount_path = registers[0].get_string(vm)?; + let mount_point = PathBuf::from(mount_path); + + // Create mount point if it doesn't exist + if !mount_point.exists() { + std::fs::create_dir_all(&mount_point) + .map_err(|e| VMError::new_vm(format!("Failed to create mount point: {}", e)))?; + } + + // Create communication pipes + let (_comm_read, comm_write) = pipe() + .map_err(|e| VMError::new_vm(format!("Failed to create communication pipe: {}", e)))?; + let (eval_read, eval_write) = pipe() + .map_err(|e| VMError::new_vm(format!("Failed to create eval pipe: {}", e)))?; + + let mount_id = format!("mount-{}", uuid::Uuid::new_v4()); + let mut mount = FuseMount::new(mount_point.clone()); + + // Build the server binary name + let server_name = "slosh-fuse-server"; + + // Try multiple possible locations for the server binary + let possible_paths = vec![ + // Same directory as current executable + std::env::current_exe() + .ok() + .and_then(|p| p.parent().map(|p| p.join(server_name))), + // In target/debug or target/release + std::env::current_dir() + .ok() + .map(|p| p.join("target").join(if cfg!(debug_assertions) { "debug" } else { "release" }).join(server_name)), + // System PATH + Some(PathBuf::from(server_name)), + ]; + + let server_path = possible_paths + .into_iter() + .flatten() + .find(|p| p.exists()) + .ok_or_else(|| VMError::new_vm(format!("Could not find slosh-fuse-server binary. Make sure to build it with 'cargo build --bin slosh-fuse-server'")))?; + + let child = Command::new(&server_path) + .arg(mount_point.to_str().unwrap()) + .arg(eval_read.to_string()) + .arg(eval_write.to_string()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) // Let errors go to stderr for debugging + .spawn() + .map_err(|e| VMError::new_vm(format!("Failed to spawn FUSE server at {:?}: {}", server_path, e)))?; + + mount.process_handle = Some(child); + mount.comm_write_fd = Some(comm_write); + + // Store mount in registry + get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))? + .insert(mount_id.clone(), mount); + + Ok(vm.alloc_string(mount_id)) +} + +fn register_eval_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 3 { + return Err(VMError::new_vm( + "register-eval-file: requires three arguments (mount-id path expression)".to_string(), + )); + } + + let mount_id = registers[0].get_string(vm)?; + let path = registers[1].get_string(vm)?; + let expression = registers[2].get_string(vm)?; + + let registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(mount) = registry.get(mount_id) { + mount.register_file(&path, expression.to_string()); + Ok(Value::True) + } else { + Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) + } +} + +fn unmount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 1 { + return Err(VMError::new_vm( + "unmount-eval-fs: requires one argument (mount-id)".to_string(), + )); + } + + let mount_id = registers[0].get_string(vm)?; + + let mut registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(mut mount) = registry.remove(mount_id) { + mount.unmount() + .map_err(|e| VMError::new_vm(format!("Failed to unmount: {}", e)))?; + + // Also unmount the FUSE filesystem + if let Err(e) = nix::mount::umount(&mount.mount_point) { + // It's okay if unmount fails - the process termination should handle it + eprintln!("Warning: umount failed: {}", e); + } + Ok(Value::True) + } else { + Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) + } +} + +fn list_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 1 { + return Err(VMError::new_vm( + "list-eval-files: requires one argument (mount-id)".to_string(), + )); + } + + let mount_id = registers[0].get_string(vm)?; + + let registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(mount) = registry.get(mount_id) { + let mapping = mount.file_mapping + .lock() + .map_err(|e| VMError::new_vm(format!("File mapping lock poisoned: {}", e)))?; + + let files: Vec = mapping.list_files() + .into_iter() + .map(|f| vm.alloc_string(f)) + .collect(); + + Ok(vm.alloc_vector(files)) + } else { + Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) + } +} + +fn list_mounts(vm: &mut SloshVm, _registers: &[Value]) -> VMResult { + let registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + let mut mounts = Vec::new(); + + for (mount_id, mount) in registry.iter() { + let file_count = mount.file_mapping + .lock() + .map_err(|e| VMError::new_vm(format!("File mapping lock poisoned: {}", e)))? + .list_files() + .len(); + + let mount_info = vec![ + vm.alloc_string("mount-id".to_string()), + vm.alloc_string(mount_id.clone()), + vm.alloc_string("mount-point".to_string()), + vm.alloc_string(mount.mount_point.display().to_string()), + vm.alloc_string("file-count".to_string()), + Value::from(file_count as i64), + ]; + mounts.push(vm.alloc_vector(mount_info)); + } + + Ok(vm.alloc_vector(mounts)) +} + +pub fn add_fuse_builtins(env: &mut SloshVm) { + add_builtin( + env, + "mount-eval-fs", + mount_eval_fs, + r#"Usage: (mount-eval-fs path) + +Mount an evaluation filesystem at the specified path. Returns a mount ID. + +Section: fuse + +Example: +(def mount-id (mount-eval-fs "/tmp/eval-fs")) +"#, + ); + + add_builtin( + env, + "register-eval-file", + register_eval_file, + r#"Usage: (register-eval-file mount-id path expression) + +Register a file with a slosh expression that will be evaluated when the file is read. + +Section: fuse + +Example: +(register-eval-file mount-id "config.env" "(str \"HOST=\" (hostname))") +"#, + ); + + add_builtin( + env, + "unmount-eval-fs", + unmount_eval_fs, + r#"Usage: (unmount-eval-fs mount-id) + +Unmount an evaluation filesystem. + +Section: fuse + +Example: +(unmount-eval-fs mount-id) +"#, + ); + + add_builtin( + env, + "list-eval-files", + list_eval_files, + r#"Usage: (list-eval-files mount-id) + +List all files registered in a mount. + +Section: fuse + +Example: +(list-eval-files mount-id) +"#, + ); + + add_builtin( + env, + "list-mounts", + list_mounts, + r#"Usage: (list-mounts) + +List all active FUSE mounts with their mount IDs, mount points, and file counts. + +Section: fuse + +Example: +(list-mounts) +; Returns a list of vectors, each containing: +; ["mount-id" "mount-point" "file-count" ] +"#, + ); +} \ No newline at end of file diff --git a/slosh_lib/src/lib.rs b/slosh_lib/src/lib.rs index 7d23fe2ada..b6d74bdb93 100644 --- a/slosh_lib/src/lib.rs +++ b/slosh_lib/src/lib.rs @@ -24,7 +24,6 @@ use builtins::collections::setup_collection_builtins; use builtins::conversions::add_conv_builtins; use builtins::fs_meta::add_fs_meta_builtins; use builtins::fs_temp::add_fs_temp_builtins; -use builtins::fuse::add_fuse_builtins; use builtins::io::add_io_builtins; use builtins::math::add_math_builtins; use builtins::print::{add_print_builtins, display_value}; @@ -439,12 +438,20 @@ pub fn set_builtins(env: &mut SloshVm) { add_conv_builtins(env); add_fs_meta_builtins(env); add_fs_temp_builtins(env); - add_fuse_builtins(env); add_rand_builtins(env); add_math_builtins(env); add_doc_builtins(env); } +#[cfg(feature = "fuse")] +mod fuse_bindings; + +/// Add FUSE builtins if the feature is enabled +#[cfg(feature = "fuse")] +pub fn add_fuse_builtins(env: &mut SloshVm) { + fuse_bindings::add_fuse_builtins(env); +} + /// Loads the user's sloshrc file, has side-effects, and sets some important /// constants in the environment. pub fn set_environment(env: &mut SloshVm) { @@ -520,6 +527,8 @@ Section: shell"# pub fn set_builtins_and_shell_builtins(env: &mut SloshVm) { set_builtins(env); set_shell_builtins(env); + #[cfg(feature = "fuse")] + add_fuse_builtins(env); } pub fn set_shell_builtins(env: &mut SloshVm) { @@ -641,6 +650,8 @@ pub fn run_slosh(modify_vm: impl FnOnce(&mut SloshVm)) -> i32 { set_builtins(&mut env); modify_vm(&mut env); set_shell_builtins(&mut env); + #[cfg(feature = "fuse")] + add_fuse_builtins(&mut env); env.unpause_gc(); }); if config.command.is_none() && config.script.is_none() { From 7f06b6858a2a77e31546cac07e8a2b9cee30b7de Mon Sep 17 00:00:00 2001 From: gpwclark Date: Sat, 23 Aug 2025 00:15:49 +0000 Subject: [PATCH 05/12] hangs the shell --- Cargo.lock | 1 + slosh-fuse/Cargo.toml | 1 + slosh-fuse/src/lib.rs | 1 + slosh-fuse/src/proc_subst.rs | 152 ++++++++++++++ slosh_lib/src/lib.rs | 4 + slosh_lib/src/proc_subst_bindings.rs | 288 +++++++++++++++++++++++++++ 6 files changed, 447 insertions(+) create mode 100644 slosh-fuse/src/proc_subst.rs create mode 100644 slosh_lib/src/proc_subst_bindings.rs diff --git a/Cargo.lock b/Cargo.lock index 86dde9de70..b5e5d1bed4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2217,6 +2217,7 @@ dependencies = [ "libc", "log", "nix 0.27.1", + "tempfile", "uuid", ] diff --git a/slosh-fuse/Cargo.toml b/slosh-fuse/Cargo.toml index 91aad1ccc3..9733c59ca8 100644 --- a/slosh-fuse/Cargo.toml +++ b/slosh-fuse/Cargo.toml @@ -13,6 +13,7 @@ libc = "0.2" log = "0.4" env_logger = "0.11" uuid = { version = "1.0", features = ["v4"] } +tempfile = "3.0" # No slosh dependencies to avoid circular dependency diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs index 1f959f5d7b..c23d6f7811 100644 --- a/slosh-fuse/src/lib.rs +++ b/slosh-fuse/src/lib.rs @@ -1,5 +1,6 @@ pub mod eval_fs; pub mod file_mapping; +pub mod proc_subst; pub use eval_fs::EvalFs; pub use file_mapping::{FileMapping, FileEntry}; diff --git a/slosh-fuse/src/proc_subst.rs b/slosh-fuse/src/proc_subst.rs new file mode 100644 index 0000000000..bb6bf6fc2b --- /dev/null +++ b/slosh-fuse/src/proc_subst.rs @@ -0,0 +1,152 @@ +use std::collections::HashMap; +use std::fs; +use std::io::Write; +use std::os::unix::fs::symlink; +use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::sync::{Arc, Mutex}; + +/// Process substitution implementation that creates /dev/fd/* symlinks +pub struct ProcSubst { + /// Map of virtual paths to their backing processes + processes: Arc>>, + /// Base directory for symlinks (e.g., /tmp/slosh-proc-subst-XXXXX) + base_dir: PathBuf, +} + +struct ProcInfo { + process: Child, + fd: i32, + symlink_path: PathBuf, +} + +impl ProcSubst { + pub fn new() -> Result { + let base_dir = tempfile::tempdir()?.into_path(); + Ok(Self { + processes: Arc::new(Mutex::new(HashMap::new())), + base_dir, + }) + } + + /// Create a virtual file backed by a process + pub fn create_file(&self, name: &str, command: &str) -> Result { + // Create the symlink path + let symlink_path = self.base_dir.join(name); + + // Ensure parent directory exists + if let Some(parent) = symlink_path.parent() { + fs::create_dir_all(parent)?; + } + + // Spawn the process + let mut child = Command::new("sh") + .arg("-c") + .arg(command) + .stdout(Stdio::piped()) + .spawn()?; + + // Get the file descriptor + let stdout = child.stdout.take() + .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "No stdout"))?; + let fd = stdout.as_raw_fd(); + + // Create symlink to /dev/fd/N + let fd_path = format!("/dev/fd/{}", fd); + symlink(&fd_path, &symlink_path)?; + + // Store process info (leak the fd to keep it open) + std::mem::forget(stdout); + + let info = ProcInfo { + process: child, + fd, + symlink_path: symlink_path.clone(), + }; + + self.processes.lock().unwrap() + .insert(name.to_string(), info); + + Ok(symlink_path) + } + + /// Remove a virtual file + pub fn remove_file(&self, name: &str) -> Result<(), std::io::Error> { + let mut processes = self.processes.lock().unwrap(); + + if let Some(mut info) = processes.remove(name) { + // Remove the symlink + let _ = fs::remove_file(&info.symlink_path); + + // Kill the process + let _ = info.process.kill(); + let _ = info.process.wait(); + + // Close the file descriptor + unsafe { + libc::close(info.fd); + } + } + + Ok(()) + } + + /// List all virtual files + pub fn list_files(&self) -> Vec { + self.processes.lock().unwrap() + .keys() + .cloned() + .collect() + } + + /// Get the path to a virtual file + pub fn get_file_path(&self, name: &str) -> Option { + self.processes.lock().unwrap() + .get(name) + .map(|info| info.symlink_path.clone()) + } +} + +impl Drop for ProcSubst { + fn drop(&mut self) { + // Clean up all processes + let mut processes = self.processes.lock().unwrap(); + for (_, mut info) in processes.drain() { + let _ = fs::remove_file(&info.symlink_path); + let _ = info.process.kill(); + let _ = info.process.wait(); + unsafe { + libc::close(info.fd); + } + } + + // Remove base directory + let _ = fs::remove_dir_all(&self.base_dir); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + use std::io::Read; + + #[test] + fn test_proc_subst() { + let ps = ProcSubst::new().unwrap(); + + // Create a virtual file + let path = ps.create_file("test.txt", "echo 'Hello, World!'").unwrap(); + + // Read from it + let mut file = File::open(&path).unwrap(); + let mut contents = String::new(); + file.read_to_string(&mut contents).unwrap(); + + assert_eq!(contents.trim(), "Hello, World!"); + + // Clean up + ps.remove_file("test.txt").unwrap(); + } +} \ No newline at end of file diff --git a/slosh_lib/src/lib.rs b/slosh_lib/src/lib.rs index b6d74bdb93..d916193d03 100644 --- a/slosh_lib/src/lib.rs +++ b/slosh_lib/src/lib.rs @@ -446,10 +446,14 @@ pub fn set_builtins(env: &mut SloshVm) { #[cfg(feature = "fuse")] mod fuse_bindings; +#[cfg(feature = "fuse")] +mod proc_subst_bindings; + /// Add FUSE builtins if the feature is enabled #[cfg(feature = "fuse")] pub fn add_fuse_builtins(env: &mut SloshVm) { fuse_bindings::add_fuse_builtins(env); + proc_subst_bindings::add_proc_subst_builtins(env); } /// Loads the user's sloshrc file, has side-effects, and sets some important diff --git a/slosh_lib/src/proc_subst_bindings.rs b/slosh_lib/src/proc_subst_bindings.rs new file mode 100644 index 0000000000..b36c690018 --- /dev/null +++ b/slosh_lib/src/proc_subst_bindings.rs @@ -0,0 +1,288 @@ +use bridge_adapters::add_builtin; +use compile_state::state::SloshVm; +use slvm::{VMError, VMResult, Value}; +use std::collections::HashMap; +use std::sync::Mutex; + +use slosh_fuse::proc_subst::ProcSubst; + +// Registry for process substitution instances +use std::sync::OnceLock; +static PROC_SUBST_REGISTRY: OnceLock>> = OnceLock::new(); + +fn get_registry() -> &'static Mutex> { + PROC_SUBST_REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) +} + +fn create_proc_subst(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 0 { + return Err(VMError::new_vm( + "create-proc-subst: requires no arguments".to_string(), + )); + } + + let ps = ProcSubst::new() + .map_err(|e| VMError::new_vm(format!("Failed to create proc subst: {}", e)))?; + + let ps_id = format!("proc-subst-{}", uuid::Uuid::new_v4()); + + get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))? + .insert(ps_id.clone(), ps); + + Ok(vm.alloc_string(ps_id)) +} + +fn proc_subst_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 3 { + return Err(VMError::new_vm( + "proc-subst-file: requires three arguments (ps-id name command)".to_string(), + )); + } + + let ps_id = registers[0].get_string(vm)?; + let name = registers[1].get_string(vm)?; + let command = registers[2].get_string(vm)?; + + let registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(ps) = registry.get(ps_id) { + let path = ps.create_file(&name, &command) + .map_err(|e| VMError::new_vm(format!("Failed to create file: {}", e)))?; + + Ok(vm.alloc_string(path.display().to_string())) + } else { + Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) + } +} + +fn proc_subst_eval_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 3 { + return Err(VMError::new_vm( + "proc-subst-eval-file: requires three arguments (ps-id name expression)".to_string(), + )); + } + + let ps_id = registers[0].get_string(vm)?; + let name = registers[1].get_string(vm)?; + let expression = registers[2].get_string(vm)?; + + // Build slosh command to evaluate the expression + let command = format!("slosh -c 'pr {}'", expression); + + let registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(ps) = registry.get(ps_id) { + let path = ps.create_file(&name, &command) + .map_err(|e| VMError::new_vm(format!("Failed to create file: {}", e)))?; + + Ok(vm.alloc_string(path.display().to_string())) + } else { + Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) + } +} + +fn proc_subst_remove(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 2 { + return Err(VMError::new_vm( + "proc-subst-remove: requires two arguments (ps-id name)".to_string(), + )); + } + + let ps_id = registers[0].get_string(vm)?; + let name = registers[1].get_string(vm)?; + + let registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(ps) = registry.get(ps_id) { + ps.remove_file(&name) + .map_err(|e| VMError::new_vm(format!("Failed to remove file: {}", e)))?; + Ok(Value::True) + } else { + Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) + } +} + +fn proc_subst_list(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 1 { + return Err(VMError::new_vm( + "proc-subst-list: requires one argument (ps-id)".to_string(), + )); + } + + let ps_id = registers[0].get_string(vm)?; + + let registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(ps) = registry.get(ps_id) { + let files: Vec = ps.list_files() + .into_iter() + .map(|f| vm.alloc_string(f)) + .collect(); + + Ok(vm.alloc_vector(files)) + } else { + Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) + } +} + +fn proc_subst_path(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 2 { + return Err(VMError::new_vm( + "proc-subst-path: requires two arguments (ps-id name)".to_string(), + )); + } + + let ps_id = registers[0].get_string(vm)?; + let name = registers[1].get_string(vm)?; + + let registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(ps) = registry.get(ps_id) { + if let Some(path) = ps.get_file_path(&name) { + Ok(vm.alloc_string(path.display().to_string())) + } else { + Err(VMError::new_vm(format!("File not found: {}", name))) + } + } else { + Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) + } +} + +fn destroy_proc_subst(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 1 { + return Err(VMError::new_vm( + "destroy-proc-subst: requires one argument (ps-id)".to_string(), + )); + } + + let ps_id = registers[0].get_string(vm)?; + + let mut registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if registry.remove(ps_id).is_some() { + Ok(Value::True) + } else { + Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) + } +} + +pub fn add_proc_subst_builtins(env: &mut SloshVm) { + add_builtin( + env, + "create-proc-subst", + create_proc_subst, + r#"Usage: (create-proc-subst) + +Create a new process substitution context. Returns an ID. + +Section: proc-subst + +Example: +(def ps-id (create-proc-subst)) +"#, + ); + + add_builtin( + env, + "proc-subst-file", + proc_subst_file, + r#"Usage: (proc-subst-file ps-id name command) + +Create a virtual file backed by a command. Returns the file path. + +Section: proc-subst + +Example: +(proc-subst-file ps-id "date.txt" "date +%Y-%m-%d") +"#, + ); + + add_builtin( + env, + "proc-subst-eval-file", + proc_subst_eval_file, + r#"Usage: (proc-subst-eval-file ps-id name expression) + +Create a virtual file backed by a slosh expression. Returns the file path. + +Section: proc-subst + +Example: +(proc-subst-eval-file ps-id "config.env" "(str \"HOST=\" (hostname))") +"#, + ); + + add_builtin( + env, + "proc-subst-remove", + proc_subst_remove, + r#"Usage: (proc-subst-remove ps-id name) + +Remove a virtual file. + +Section: proc-subst + +Example: +(proc-subst-remove ps-id "date.txt") +"#, + ); + + add_builtin( + env, + "proc-subst-list", + proc_subst_list, + r#"Usage: (proc-subst-list ps-id) + +List all virtual files in a proc-subst context. + +Section: proc-subst + +Example: +(proc-subst-list ps-id) +"#, + ); + + add_builtin( + env, + "proc-subst-path", + proc_subst_path, + r#"Usage: (proc-subst-path ps-id name) + +Get the path to a virtual file. + +Section: proc-subst + +Example: +(proc-subst-path ps-id "date.txt") +"#, + ); + + add_builtin( + env, + "destroy-proc-subst", + destroy_proc_subst, + r#"Usage: (destroy-proc-subst ps-id) + +Destroy a process substitution context and clean up all files. + +Section: proc-subst + +Example: +(destroy-proc-subst ps-id) +"#, + ); +} \ No newline at end of file From d41e37cb01f64cb8069542a28e8748f3cb236837 Mon Sep 17 00:00:00 2001 From: gpwclark Date: Sat, 7 Mar 2026 22:05:25 -0500 Subject: [PATCH 06/12] poc --- Cargo.lock | 17 +- Cargo.toml | 16 ++ slosh-fuse/Cargo.toml | 8 +- slosh-fuse/README.md | 75 ++++--- slosh-fuse/examples/dynamic-file-demo.sl | 31 ++- slosh-fuse/examples/test-fuse-systemd.sl | 62 +++--- slosh-fuse/src/bin/slosh-fuse-server.rs | 79 +++++-- slosh-fuse/src/eval_fs.rs | 116 ++-------- slosh-fuse/src/file_mapping.rs | 272 +++++++++++++++++++---- slosh-fuse/src/lib.rs | 62 +++++- slosh-fuse/src/proc_subst.rs | 29 ++- slosh-fuse/test-fuse.sh | 191 ++++++++++++++++ slosh_lib/Cargo.toml | 3 +- slosh_lib/src/fuse_bindings.rs | 156 ++++++++----- 14 files changed, 765 insertions(+), 352 deletions(-) create mode 100755 slosh-fuse/test-fuse.sh diff --git a/Cargo.lock b/Cargo.lock index b5e5d1bed4..0a4afd1a46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1442,17 +1442,6 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" -[[package]] -name = "nix" -version = "0.27.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" -dependencies = [ - "bitflags 2.8.0", - "cfg-if", - "libc", -] - [[package]] name = "nix" version = "0.29.0" @@ -2116,7 +2105,7 @@ dependencies = [ "cfg-if", "chrono", "glob", - "nix 0.29.0", + "nix", "sl-liner", ] @@ -2212,13 +2201,12 @@ dependencies = [ name = "slosh-fuse" version = "0.11.1" dependencies = [ + "base64", "env_logger", "fuser", "libc", "log", - "nix 0.27.1", "tempfile", - "uuid", ] [[package]] @@ -2235,7 +2223,6 @@ dependencies = [ "criterion", "glob", "iai-callgrind", - "nix 0.27.1", "shell", "sl-compiler", "sl-liner", diff --git a/Cargo.toml b/Cargo.toml index 8ae2a6b2ab..cc2b1dc3ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,22 @@ members = [ "slosh-fuse", ] +# slosh-fuse requires libfuse (Linux only), exclude from default builds +default-members = [ + "slosh", + "slosh_lib", + "compiler", + "vm", + "builtins", + "compile_state", + "shell", + "bridge_macros", + "bridge_types", + "bridge_adapters", + "slosh_test", + "slosh_test_lib", +] + exclude = ["legacy"] [profile.release] diff --git a/slosh-fuse/Cargo.toml b/slosh-fuse/Cargo.toml index 9733c59ca8..ef179feff0 100644 --- a/slosh-fuse/Cargo.toml +++ b/slosh-fuse/Cargo.toml @@ -12,11 +12,5 @@ fuser = "0.14" libc = "0.2" log = "0.4" env_logger = "0.11" -uuid = { version = "1.0", features = ["v4"] } +base64 = "0.21" tempfile = "3.0" - -# No slosh dependencies to avoid circular dependency - -[dependencies.nix] -version = "0.27" -features = ["signal", "process", "fs"] \ No newline at end of file diff --git a/slosh-fuse/README.md b/slosh-fuse/README.md index 99a26fd5c6..a2dac25856 100644 --- a/slosh-fuse/README.md +++ b/slosh-fuse/README.md @@ -1,27 +1,25 @@ # Slosh FUSE Integration -This module provides FUSE (Filesystem in Userspace) integration for slosh, allowing you to create dynamic files whose contents are generated by evaluating slosh expressions. +FUSE (Filesystem in Userspace) integration for slosh. Creates virtual files in a mounted directory whose contents come from slosh expressions evaluated at registration time. ## Use Case -The primary use case is creating dynamic configuration files for services like systemd, where environment files need to contain values from slosh variables, environment variables, or computed values. - -For example, a systemd service file at `~/.config/systemd/user/my-service.env` could dynamically generate content like: +Dynamic configuration files for services like systemd, where environment files need values from slosh variables or computed values: ```bash -HOST=meow -CUSTOM_TYPE=value-of-global +cat /tmp/dynamic/config.env +# => HOST=meow +# => CUSTOM_TYPE=production ``` -Where `meow` is the hostname and `value-of-global` comes from a slosh global variable. - ## Architecture -The implementation consists of: +Content is **evaluated at registration time** in the parent slosh process, then sent as static bytes to a child FUSE server process via stdin (base64-encoded, tab-delimited protocol). This avoids the complexity of IPC-based expression evaluation since the Slosh VM is `thread_local!` and can't be accessed from child processes. -1. **slosh-fuse crate**: Implements the FUSE filesystem that serves dynamic files -2. **Builtin functions**: Slosh functions to mount/unmount filesystems and register files -3. **Expression evaluation**: Files contain slosh expressions that are evaluated on read +Components: +1. **slosh-fuse crate** - FUSE filesystem serving static file content, plus the `FuseMount` parent-side handle +2. **slosh-fuse-server binary** - Child process that mounts FUSE and reads `REGISTER`/`REMOVE` commands from stdin +3. **Builtin functions** (in slosh_lib) - `mount-eval-fs`, `register-eval-file`, `unmount-eval-fs`, `list-eval-files`, `list-mounts` ## Usage @@ -29,35 +27,48 @@ The implementation consists of: ;; Mount a dynamic filesystem (def mount-id (mount-eval-fs "/tmp/dynamic")) -;; Register a file with dynamic content +;; Register a file with content (evaluated now, served statically) (register-eval-file mount-id "config.env" - "(str \"HOST=\" (hostname) \"\\n\" - \"TYPE=\" *my-global-var* \"\\n\")") + (str "HOST=" (hostname) "\n" "TYPE=" *my-global-var* "\n")) -;; The file /tmp/dynamic/config.env will now return evaluated content when read +;; The file /tmp/dynamic/config.env is now readable + +;; To update content, re-register: +(register-eval-file mount-id "config.env" + (str "HOST=" (hostname) "\n" "TYPE=" *my-global-var* "\n")) ;; Unmount when done (unmount-eval-fs mount-id) ``` -## Implementation Status +## Building + +Requires Linux with libfuse3 (FUSE doesn't work natively on macOS). + +```bash +# On Linux +cargo build -p slosh-fuse +cargo build -p slosh_lib --features fuse + +# On macOS, use an Apple Container +container build -t slosh-fuse -f Containerfile . +container run --name slosh-fuse slosh-fuse bash /src/slosh-fuse/test-fuse.sh +``` + +## Testing -The current implementation provides: -- Basic FUSE filesystem structure -- File registration and mapping system -- Builtin functions for mounting and file management -- Example scripts demonstrating the concept +```bash +# Standalone FUSE server test (inside container or on Linux) +bash slosh-fuse/test-fuse.sh +``` -Full FUSE integration with expression evaluation requires: -- Process forking and IPC for evaluation -- FUSE mount/unmount handling -- Security considerations for file access +## Protocol -## Alternative Approach +The parent communicates with the FUSE server child via stdin using tab-delimited lines: -For simpler use cases, the `dynamic-env-generator.sl` example shows how to achieve similar functionality without FUSE by: -- Registering file paths with generator functions -- Periodically regenerating files when configuration changes -- Using standard file I/O operations +``` +REGISTER\t\t\n +REMOVE\t\n +``` -This approach is simpler to implement and doesn't require FUSE support. \ No newline at end of file +The server uses `AutoUnmount` so the filesystem is automatically unmounted when the server process exits. diff --git a/slosh-fuse/examples/dynamic-file-demo.sl b/slosh-fuse/examples/dynamic-file-demo.sl index b9eddf1977..ed0f53cb8e 100644 --- a/slosh-fuse/examples/dynamic-file-demo.sl +++ b/slosh-fuse/examples/dynamic-file-demo.sl @@ -1,37 +1,32 @@ #!/usr/bin/env slosh ;; Dynamic File Demo -;; This demonstrates how dynamic files would work with the FUSE integration +;; Demonstrates FUSE integration with static content evaluated at registration time -;; Simulate mounting a dynamic filesystem +;; Mount a dynamic filesystem (def mount-id (mount-eval-fs "/tmp/slosh-dynamic")) ;; Register a systemd-style environment file -;; This would dynamically generate content based on slosh expressions +;; Content is evaluated NOW and sent to the FUSE server (register-eval-file mount-id "my-service.env" - "(str \"HOST=\" (or (env \"HOSTNAME\") \"localhost\") \"\\n\" - \"CUSTOM_TYPE=\" (or *custom-type* \"default\") \"\\n\" - \"TIMESTAMP=\" (current-time-millis) \"\\n\")") + (str "HOST=" (or (env "HOSTNAME") "localhost") "\n" + "CUSTOM_TYPE=" (or *custom-type* "default") "\n")) -;; Register another dynamic file showing system info +;; Register another file showing system info (register-eval-file mount-id "system-info.txt" - "(str \"User: \" (env \"USER\") \"\\n\" - \"Home: \" (env \"HOME\") \"\\n\" - \"Shell: \" (env \"SHELL\") \"\\n\" - \"PWD: \" (env \"PWD\") \"\\n\")") + (str "User: " (env "USER") "\n" + "Home: " (env "HOME") "\n" + "Shell: " (env "SHELL") "\n")) ;; List registered files (println "Registered files:") (doseq [file (list-eval-files mount-id)] (println " " file)) -;; Demonstrate what the content would look like -(println "\nFile contents would be:") -(println "--- my-service.env ---") -(println (eval-file-expression mount-id "my-service.env")) -(println "\n--- system-info.txt ---") -(println (eval-file-expression mount-id "system-info.txt")) +;; Files are now readable at the mount point: +;; cat /tmp/slosh-dynamic/my-service.env +;; cat /tmp/slosh-dynamic/system-info.txt ;; Clean up (unmount-eval-fs mount-id) -(println "\nFilesystem unmounted.") \ No newline at end of file +(println "Filesystem unmounted.") diff --git a/slosh-fuse/examples/test-fuse-systemd.sl b/slosh-fuse/examples/test-fuse-systemd.sl index 7a5615acd7..e4748bfaf8 100644 --- a/slosh-fuse/examples/test-fuse-systemd.sl +++ b/slosh-fuse/examples/test-fuse-systemd.sl @@ -1,7 +1,7 @@ #!/usr/bin/env slosh ;; Test FUSE dynamic files for systemd environment -;; This demonstrates how to create dynamic environment files +;; Content is evaluated at registration time and served statically by the FUSE server ;; Set some global variables that will be used in the env file (def *custom-type* "production") @@ -11,50 +11,54 @@ (println "Mounting dynamic filesystem...") (def mount-id (mount-eval-fs "/tmp/slosh-systemd-test")) -;; Register a systemd-style environment file with dynamic content +;; Register a systemd-style environment file with content evaluated now (println "Registering my-service.env...") (register-eval-file mount-id "my-service.env" - "(str \"# Generated by slosh\\n\" - \"HOST=\" (or (env \"HOSTNAME\") \"meow\") \"\\n\" - \"CUSTOM_TYPE=\" *custom-type* \"\\n\" - \"APP_VERSION=\" *app-version* \"\\n\" - \"USER=\" (env \"USER\") \"\\n\" - \"HOME=\" (env \"HOME\") \"\\n\")") + (str "# Generated by slosh\n" + "HOST=" (or (env "HOSTNAME") "meow") "\n" + "CUSTOM_TYPE=" *custom-type* "\n" + "APP_VERSION=" *app-version* "\n" + "USER=" (env "USER") "\n" + "HOME=" (env "HOME") "\n")) ;; Register another file for database config (register-eval-file mount-id "database.env" - "(str \"DB_HOST=localhost\\n\" - \"DB_PORT=5432\\n\" - \"DB_NAME=\" *custom-type* \"_db\\n\" - \"DB_USER=\" (env \"USER\") \"_app\\n\")") + (str "DB_HOST=localhost\n" + "DB_PORT=5432\n" + "DB_NAME=" *custom-type* "_db\n" + "DB_USER=" (env "USER") "_app\n")) ;; List all registered files (println "\nRegistered files:") (doseq [file (list-eval-files mount-id)] (println " -" file)) -;; Show what the content would look like when evaluated -(println "\n--- Testing file evaluation ---") -(println "my-service.env would contain:") -(println (eval-file-expression mount-id "my-service.env")) - -(println "\ndatabase.env would contain:") -(println (eval-file-expression mount-id "database.env")) +;; Files are now readable: +(println "\nFiles are now accessible at:") +(println " /tmp/slosh-systemd-test/my-service.env") +(println " /tmp/slosh-systemd-test/database.env") +(println "And systemd could read them directly!") -;; Change a global variable and show how it affects the file -(println "\n--- Changing *custom-type* to 'development' ---") +;; To update content after changing a global, re-register: +(println "\n--- Changing *custom-type* to 'development' and re-registering ---") (set! *custom-type* "development") -(println "my-service.env now contains:") -(println (eval-file-expression mount-id "my-service.env")) +(register-eval-file mount-id "my-service.env" + (str "# Generated by slosh\n" + "HOST=" (or (env "HOSTNAME") "meow") "\n" + "CUSTOM_TYPE=" *custom-type* "\n" + "APP_VERSION=" *app-version* "\n" + "USER=" (env "USER") "\n" + "HOME=" (env "HOME") "\n")) + +(register-eval-file mount-id "database.env" + (str "DB_HOST=localhost\n" + "DB_PORT=5432\n" + "DB_NAME=" *custom-type* "_db\n" + "DB_USER=" (env "USER") "_app\n")) -(println "\ndatabase.env now contains:") -(println (eval-file-expression mount-id "database.env")) +(println "Files re-registered with new content.") ;; Clean up (unmount-eval-fs mount-id) (println "\nFilesystem unmounted.") -(println "\nIn a real implementation, these files would be accessible at:") -(println " /tmp/slosh-systemd-test/my-service.env") -(println " /tmp/slosh-systemd-test/database.env") -(println "And systemd could read them directly!") \ No newline at end of file diff --git a/slosh-fuse/src/bin/slosh-fuse-server.rs b/slosh-fuse/src/bin/slosh-fuse-server.rs index 30f6da7cd7..03d7b930ce 100644 --- a/slosh-fuse/src/bin/slosh-fuse-server.rs +++ b/slosh-fuse/src/bin/slosh-fuse-server.rs @@ -1,44 +1,85 @@ +use base64::Engine; use fuser::{MountOption, Session}; use slosh_fuse::{EvalFs, FileMapping}; +use std::io::BufRead; +use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::env; -use std::os::unix::io::RawFd; fn main() { env_logger::init(); let args: Vec = env::args().collect(); - if args.len() < 4 { - eprintln!("Usage: {} ", args[0]); + if args.len() < 2 { + eprintln!("Usage: {} ", args[0]); std::process::exit(1); } let mount_point = &args[1]; - let read_fd: RawFd = args[2].parse().expect("Invalid read fd"); - let write_fd: RawFd = args[3].parse().expect("Invalid write fd"); - let file_mapping = Arc::new(Mutex::new(FileMapping::new())); - // Pre-populate with test data if provided - if args.len() > 4 { - let mut mapping = file_mapping.lock().unwrap(); - for i in (4..args.len()).step_by(2) { - if i + 1 < args.len() { - mapping.register(&args[i], args[i + 1].clone()); + // Spawn a background thread to read commands from stdin + let mapping_for_reader = Arc::clone(&file_mapping); + std::thread::spawn(move || { + let stdin = std::io::stdin(); + let reader = stdin.lock(); + for line in reader.lines() { + let line = match line { + Ok(l) => l, + Err(e) => { + log::error!("Error reading stdin: {}", e); + break; + } + }; + + let parts: Vec<&str> = line.split('\t').collect(); + match parts.first().copied() { + Some("REGISTER") if parts.len() == 3 => { + let path = parts[1]; + let b64_content = parts[2]; + match base64::engine::general_purpose::STANDARD.decode(b64_content) { + Ok(content) => { + let mut mapping = mapping_for_reader.lock().unwrap(); + mapping.register(path, content); + log::info!("Registered file: {}", path); + } + Err(e) => { + log::error!("Failed to decode base64 for {}: {}", path, e); + } + } + } + Some("CONCAT") if parts.len() >= 3 => { + let vpath = parts[1]; + let source_paths: Vec = + parts[2..].iter().map(PathBuf::from).collect(); + let mut mapping = mapping_for_reader.lock().unwrap(); + mapping.register_concat(vpath, source_paths); + log::info!("Registered concat file: {} ({} sources)", vpath, parts.len() - 2); + } + Some("REMOVE") if parts.len() >= 2 => { + let path = parts[1]; + let mut mapping = mapping_for_reader.lock().unwrap(); + mapping.remove(path); + log::info!("Removed file: {}", path); + } + _ => { + log::warn!("Unknown command: {}", line); + } } } - } + log::info!("Stdin reader thread exiting"); + }); - let fs = EvalFs::new(file_mapping, read_fd, write_fd); + let fs = EvalFs::new(file_mapping); let options = vec![ MountOption::FSName("slosh-eval".to_string()), - MountOption::AllowOther, + MountOption::AutoUnmount, MountOption::DefaultPermissions, ]; - let mut session = Session::new(fs, mount_point.as_ref(), &options).expect("Failed to mount"); + let mut session = Session::new(fs, mount_point.as_ref(), &options) + .expect("Failed to create FUSE session"); - // Run until unmounted - session.run().expect("Failed to run session"); -} \ No newline at end of file + session.run().expect("FUSE session failed"); +} diff --git a/slosh-fuse/src/eval_fs.rs b/slosh-fuse/src/eval_fs.rs index f140c24f4a..1d24de839b 100644 --- a/slosh-fuse/src/eval_fs.rs +++ b/slosh-fuse/src/eval_fs.rs @@ -7,8 +7,6 @@ use std::collections::HashMap; use std::ffi::OsStr; use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; -use std::os::unix::io::{RawFd, FromRawFd, IntoRawFd}; -use std::io::{Read, Write}; use crate::file_mapping::FileMapping; @@ -19,19 +17,15 @@ pub struct EvalFs { inode_to_path: HashMap, path_to_inode: HashMap, next_inode: u64, - eval_pipe_read: RawFd, - eval_pipe_write: RawFd, } impl EvalFs { - pub fn new(file_mapping: Arc>, eval_pipe_read: RawFd, eval_pipe_write: RawFd) -> Self { + pub fn new(file_mapping: Arc>) -> Self { let mut fs = Self { file_mapping, inode_to_path: HashMap::new(), path_to_inode: HashMap::new(), next_inode: 2, // 1 is reserved for root - eval_pipe_read, - eval_pipe_write, }; // Pre-populate inodes for registered files @@ -59,56 +53,10 @@ impl EvalFs { inode } - fn get_inode_for_path(&self, path: &str) -> Option { - self.path_to_inode.get(path).copied() - } - fn get_path_for_inode(&self, inode: u64) -> Option<&str> { self.inode_to_path.get(&inode).map(|s| s.as_str()) } - fn evaluate_expression(&self, expression: &str) -> String { - // Send expression to parent process for evaluation - let msg = format!("{}\n", expression); - let mut pipe_write = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_write) }; - if let Err(e) = pipe_write.write_all(msg.as_bytes()) { - log::error!("Failed to send expression for evaluation: {}", e); - // Prevent closing the fd when File is dropped - let _ = pipe_write.into_raw_fd(); - return format!("ERROR: {}", e); - } - // Prevent closing the fd when File is dropped - let _ = pipe_write.into_raw_fd(); - - // Read response - let mut pipe_read = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_read) }; - let mut response = String::new(); - let mut buffer = [0u8; 4096]; - - loop { - match pipe_read.read(&mut buffer) { - Ok(0) => break, - Ok(n) => { - response.push_str(&String::from_utf8_lossy(&buffer[..n])); - if response.ends_with('\n') { - response.pop(); // Remove trailing newline - break; - } - } - Err(e) => { - log::error!("Failed to read evaluation result: {}", e); - // Prevent closing the fd when File is dropped - let _ = pipe_read.into_raw_fd(); - return format!("ERROR: {}", e); - } - } - } - // Prevent closing the fd when File is dropped - let _ = pipe_read.into_raw_fd(); - - response - } - fn file_attr(inode: u64, size: u64) -> FileAttr { let now = SystemTime::now(); FileAttr { @@ -161,14 +109,14 @@ impl Filesystem for EvalFs { let name_str = name.to_string_lossy(); - let found = { + let size = { let mapping = self.file_mapping.lock().unwrap(); - mapping.get(&name_str).is_some() + mapping.get(&name_str).map(|e| e.total_size()) }; - if found { + if let Some(size) = size { let inode = self.allocate_inode(&name_str); - reply.entry(&TTL, &Self::file_attr(inode, 0), 0); + reply.entry(&TTL, &Self::file_attr(inode, size), 0); } else { reply.error(ENOENT); } @@ -178,23 +126,13 @@ impl Filesystem for EvalFs { if ino == FUSE_ROOT_ID { reply.attr(&TTL, &Self::dir_attr(FUSE_ROOT_ID)); } else if let Some(path) = self.get_path_for_inode(ino) { - let (is_valid, expression) = { + let size = { let mapping = self.file_mapping.lock().unwrap(); - if let Some(entry) = mapping.get(path) { - if entry.is_cache_valid() { - let content = entry.cached_value.as_ref().unwrap().clone(); - reply.attr(&TTL, &Self::file_attr(ino, content.len() as u64)); - return; - } - (true, entry.expression.clone()) - } else { - (false, String::new()) - } + mapping.get(path).map(|e| e.total_size()) }; - if is_valid { - let content = self.evaluate_expression(&expression); - reply.attr(&TTL, &Self::file_attr(ino, content.len() as u64)); + if let Some(size) = size { + reply.attr(&TTL, &Self::file_attr(ino, size)); } else { reply.error(ENOENT); } @@ -215,34 +153,12 @@ impl Filesystem for EvalFs { reply: ReplyData, ) { if let Some(path) = self.get_path_for_inode(ino) { - let mut mapping = self.file_mapping.lock().unwrap(); - if let Some(entry) = mapping.get_mut(path) { - let content = if entry.is_cache_valid() { - entry.cached_value.as_ref().unwrap().clone() - } else { - let expr = entry.expression.clone(); - drop(mapping); - let result = self.evaluate_expression(&expr); - - // Update cache if caching is enabled - let mut mapping = self.file_mapping.lock().unwrap(); - if let Some(entry) = mapping.get_mut(path) { - if entry.cache_ttl_secs > 0 { - entry.update_cache(result.clone()); - } - } - result - }; - - let data = content.as_bytes(); - let start = offset as usize; - let end = (start + size as usize).min(data.len()); - - if start < data.len() { - reply.data(&data[start..end]); - } else { - reply.data(&[]); - } + let data = { + let mapping = self.file_mapping.lock().unwrap(); + mapping.get(path).map(|e| e.read_range(offset as u64, size)) + }; + if let Some(data) = data { + reply.data(&data); } else { reply.error(ENOENT); } @@ -293,4 +209,4 @@ impl Filesystem for EvalFs { reply.ok(); } -} \ No newline at end of file +} diff --git a/slosh-fuse/src/file_mapping.rs b/slosh-fuse/src/file_mapping.rs index 8ebbe040b8..9d70f53156 100644 --- a/slosh-fuse/src/file_mapping.rs +++ b/slosh-fuse/src/file_mapping.rs @@ -1,41 +1,114 @@ use std::collections::HashMap; -use std::time::SystemTime; +use std::fs; +use std::io::{Read, Seek, SeekFrom}; +use std::path::PathBuf; #[derive(Clone)] -pub struct FileEntry { - pub expression: String, - pub cached_value: Option, - pub cache_time: Option, - pub cache_ttl_secs: u64, +pub enum FileEntry { + Static { content: Vec }, + Concat { source_paths: Vec }, } impl FileEntry { - pub fn new(expression: String) -> Self { - Self { - expression, - cached_value: None, - cache_time: None, - cache_ttl_secs: 0, // No caching by default - } + pub fn new_static(content: Vec) -> Self { + FileEntry::Static { content } } - pub fn is_cache_valid(&self) -> bool { - if self.cache_ttl_secs == 0 { - return false; + pub fn new_concat(source_paths: Vec) -> Self { + FileEntry::Concat { source_paths } + } + + pub fn total_size(&self) -> u64 { + match self { + FileEntry::Static { content } => content.len() as u64, + FileEntry::Concat { source_paths } => { + source_paths + .iter() + .filter_map(|p| fs::metadata(p).ok()) + .map(|m| m.len()) + .sum() + } } + } - if let (Some(cached_time), Some(_)) = (self.cache_time, &self.cached_value) { - if let Ok(elapsed) = cached_time.elapsed() { - return elapsed.as_secs() < self.cache_ttl_secs; + pub fn read_range(&self, offset: u64, size: u32) -> Vec { + match self { + FileEntry::Static { content } => { + let start = offset as usize; + let end = (start + size as usize).min(content.len()); + if start < content.len() { + content[start..end].to_vec() + } else { + Vec::new() + } + } + FileEntry::Concat { source_paths } => { + read_concat_range(source_paths, offset, size) } } - false } +} + +fn read_concat_range(source_paths: &[PathBuf], offset: u64, size: u32) -> Vec { + let mut result = Vec::new(); + let mut remaining = size as u64; + let mut cursor = 0u64; // cumulative offset across all files + + for path in source_paths { + if remaining == 0 { + break; + } + + let file_len = match fs::metadata(path) { + Ok(m) => m.len(), + Err(e) => { + log::warn!("Concat: skipping missing/unreadable file {:?}: {}", path, e); + continue; + } + }; + + let file_end = cursor + file_len; + + if offset >= file_end { + // Requested range starts after this file + cursor = file_end; + continue; + } - pub fn update_cache(&mut self, value: String) { - self.cached_value = Some(value); - self.cache_time = Some(SystemTime::now()); + // How far into this file do we start reading? + let local_offset = if offset > cursor { offset - cursor } else { 0 }; + let bytes_available = file_len - local_offset; + let to_read = remaining.min(bytes_available); + + match fs::File::open(path) { + Ok(mut f) => { + if local_offset > 0 { + if let Err(e) = f.seek(SeekFrom::Start(local_offset)) { + log::warn!("Concat: seek failed on {:?}: {}", path, e); + cursor = file_end; + continue; + } + } + let mut buf = vec![0u8; to_read as usize]; + match f.read_exact(&mut buf) { + Ok(()) => { + result.extend_from_slice(&buf); + remaining -= to_read; + } + Err(e) => { + log::warn!("Concat: read failed on {:?}: {}", path, e); + } + } + } + Err(e) => { + log::warn!("Concat: failed to open {:?}: {}", path, e); + } + } + + cursor = file_end; } + + result } pub struct FileMapping { @@ -49,47 +122,156 @@ impl FileMapping { } } - pub fn register(&mut self, path: &str, expression: String) { - let normalized_path = if path.starts_with('/') { + fn normalize_path(path: &str) -> String { + if path.starts_with('/') { path[1..].to_string() } else { path.to_string() - }; + } + } - self.files.insert(normalized_path, FileEntry::new(expression)); + pub fn register(&mut self, path: &str, content: Vec) { + let normalized = Self::normalize_path(path); + self.files.insert(normalized, FileEntry::new_static(content)); + } + + pub fn register_concat(&mut self, path: &str, source_paths: Vec) { + let normalized = Self::normalize_path(path); + self.files.insert(normalized, FileEntry::new_concat(source_paths)); } pub fn get(&self, path: &str) -> Option<&FileEntry> { - let normalized_path = if path.starts_with('/') { + let normalized = if path.starts_with('/') { &path[1..] } else { path }; - - self.files.get(normalized_path) + self.files.get(normalized) } pub fn get_mut(&mut self, path: &str) -> Option<&mut FileEntry> { - let normalized_path = if path.starts_with('/') { - path[1..].to_string() - } else { - path.to_string() - }; - - self.files.get_mut(&normalized_path) + let normalized = Self::normalize_path(path); + self.files.get_mut(&normalized) } pub fn remove(&mut self, path: &str) -> Option { - let normalized_path = if path.starts_with('/') { - path[1..].to_string() - } else { - path.to_string() - }; - - self.files.remove(&normalized_path) + let normalized = Self::normalize_path(path); + self.files.remove(&normalized) } pub fn list_files(&self) -> Vec { self.files.keys().cloned().collect() } -} \ No newline at end of file +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn create_temp_file(content: &[u8]) -> (tempfile::NamedTempFile, PathBuf) { + let mut f = tempfile::NamedTempFile::new().unwrap(); + f.write_all(content).unwrap(); + f.flush().unwrap(); + let path = f.path().to_path_buf(); + (f, path) + } + + #[test] + fn static_entry_total_size() { + let entry = FileEntry::new_static(b"hello".to_vec()); + assert_eq!(entry.total_size(), 5); + } + + #[test] + fn static_entry_read_range() { + let entry = FileEntry::new_static(b"hello world".to_vec()); + assert_eq!(entry.read_range(0, 5), b"hello"); + assert_eq!(entry.read_range(6, 5), b"world"); + assert_eq!(entry.read_range(6, 100), b"world"); + assert_eq!(entry.read_range(100, 5), b""); + } + + #[test] + fn concat_total_size() { + let (_f1, p1) = create_temp_file(b"aaa"); + let (_f2, p2) = create_temp_file(b"bbbbb"); + let entry = FileEntry::new_concat(vec![p1, p2]); + assert_eq!(entry.total_size(), 8); + } + + #[test] + fn concat_read_within_one_file() { + let (_f1, p1) = create_temp_file(b"hello"); + let (_f2, p2) = create_temp_file(b"world"); + let entry = FileEntry::new_concat(vec![p1, p2]); + assert_eq!(entry.read_range(0, 5), b"hello"); + } + + #[test] + fn concat_read_spanning_two_files() { + let (_f1, p1) = create_temp_file(b"hello"); + let (_f2, p2) = create_temp_file(b"world"); + let entry = FileEntry::new_concat(vec![p1, p2]); + assert_eq!(entry.read_range(3, 4), b"lowo"); + } + + #[test] + fn concat_read_second_file_only() { + let (_f1, p1) = create_temp_file(b"hello"); + let (_f2, p2) = create_temp_file(b"world"); + let entry = FileEntry::new_concat(vec![p1, p2]); + assert_eq!(entry.read_range(5, 5), b"world"); + } + + #[test] + fn concat_read_past_end() { + let (_f1, p1) = create_temp_file(b"hello"); + let (_f2, p2) = create_temp_file(b"world"); + let entry = FileEntry::new_concat(vec![p1, p2]); + assert_eq!(entry.read_range(100, 5), b""); + } + + #[test] + fn concat_missing_file_skipped() { + let (_f1, p1) = create_temp_file(b"hello"); + let missing = PathBuf::from("/tmp/nonexistent-slosh-fuse-test-file"); + let (_f2, p2) = create_temp_file(b"world"); + let entry = FileEntry::new_concat(vec![p1, missing, p2]); + // total_size skips missing + assert_eq!(entry.total_size(), 10); + // read spans across present files, skipping missing + assert_eq!(entry.read_range(0, 10), b"helloworld"); + } + + #[test] + fn concat_reflects_file_changes() { + let (mut f1, p1) = create_temp_file(b"old"); + let (_f2, p2) = create_temp_file(b"data"); + let entry = FileEntry::new_concat(vec![p1, p2]); + assert_eq!(entry.total_size(), 7); + assert_eq!(entry.read_range(0, 7), b"olddata"); + + // Overwrite first file with longer content + f1.as_file_mut().set_len(0).unwrap(); + f1.seek(SeekFrom::Start(0)).unwrap(); + f1.write_all(b"newcontent").unwrap(); + f1.flush().unwrap(); + + assert_eq!(entry.total_size(), 13); // 10 + 3... wait: "newcontent" = 10, "data" = 4 + assert_eq!(entry.read_range(0, 14), b"newcontentdata"); + } + + #[test] + fn file_mapping_register_concat() { + let (_f1, p1) = create_temp_file(b"aaa"); + let (_f2, p2) = create_temp_file(b"bbb"); + + let mut mapping = FileMapping::new(); + mapping.register_concat("combined.txt", vec![p1, p2]); + + let entry = mapping.get("combined.txt").unwrap(); + assert_eq!(entry.total_size(), 6); + assert_eq!(entry.read_range(0, 6), b"aaabbb"); + } +} diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs index c23d6f7811..485ab3c58e 100644 --- a/slosh-fuse/src/lib.rs +++ b/slosh-fuse/src/lib.rs @@ -5,38 +5,80 @@ pub mod proc_subst; pub use eval_fs::EvalFs; pub use file_mapping::{FileMapping, FileEntry}; -use std::sync::{Arc, Mutex}; +use base64::Engine; +use std::io::Write; use std::path::PathBuf; +use std::process::ChildStdin; pub struct FuseMount { pub mount_point: PathBuf, - pub file_mapping: Arc>, pub process_handle: Option, - pub comm_write_fd: Option, + pub stdin_handle: Option, + pub registered_paths: Vec, } impl FuseMount { pub fn new(mount_point: PathBuf) -> Self { Self { mount_point, - file_mapping: Arc::new(Mutex::new(FileMapping::new())), process_handle: None, - comm_write_fd: None, + stdin_handle: None, + registered_paths: Vec::new(), } } - pub fn register_file(&self, path: &str, expression: String) { - if let Ok(mut mapping) = self.file_mapping.lock() { - mapping.register(path, expression); + fn send_command(&mut self, cmd: &str) -> Result<(), std::io::Error> { + if let Some(ref mut stdin) = self.stdin_handle { + stdin.write_all(cmd.as_bytes())?; + stdin.flush()?; + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::NotConnected, + "FUSE server stdin not connected", + )) } - // Silently ignore lock poisoning - the file just won't be registered + } + + pub fn register_file(&mut self, path: &str, content: &[u8]) -> Result<(), std::io::Error> { + let b64 = base64::engine::general_purpose::STANDARD.encode(content); + let cmd = format!("REGISTER\t{}\t{}\n", path, b64); + self.send_command(&cmd)?; + if !self.registered_paths.contains(&path.to_string()) { + self.registered_paths.push(path.to_string()); + } + Ok(()) + } + + pub fn register_concat_file(&mut self, vpath: &str, source_paths: &[&str]) -> Result<(), std::io::Error> { + let mut cmd = format!("CONCAT\t{}", vpath); + for p in source_paths { + cmd.push('\t'); + cmd.push_str(p); + } + cmd.push('\n'); + self.send_command(&cmd)?; + if !self.registered_paths.contains(&vpath.to_string()) { + self.registered_paths.push(vpath.to_string()); + } + Ok(()) + } + + pub fn remove_file(&mut self, path: &str) -> Result<(), std::io::Error> { + let cmd = format!("REMOVE\t{}\n", path); + self.send_command(&cmd)?; + self.registered_paths.retain(|p| p != path); + Ok(()) } pub fn unmount(&mut self) -> Result<(), std::io::Error> { + // Drop stdin to signal EOF to the server's reader thread + self.stdin_handle.take(); + if let Some(mut handle) = self.process_handle.take() { handle.kill()?; handle.wait()?; } Ok(()) } -} \ No newline at end of file +} diff --git a/slosh-fuse/src/proc_subst.rs b/slosh-fuse/src/proc_subst.rs index bb6bf6fc2b..27d1858661 100644 --- a/slosh-fuse/src/proc_subst.rs +++ b/slosh-fuse/src/proc_subst.rs @@ -1,9 +1,8 @@ use std::collections::HashMap; use std::fs; -use std::io::Write; use std::os::unix::fs::symlink; -use std::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd}; -use std::path::{Path, PathBuf}; +use std::os::unix::io::AsRawFd; +use std::path::PathBuf; use std::process::{Child, Command, Stdio}; use std::sync::{Arc, Mutex}; @@ -34,7 +33,7 @@ impl ProcSubst { pub fn create_file(&self, name: &str, command: &str) -> Result { // Create the symlink path let symlink_path = self.base_dir.join(name); - + // Ensure parent directory exists if let Some(parent) = symlink_path.parent() { fs::create_dir_all(parent)?; @@ -58,7 +57,7 @@ impl ProcSubst { // Store process info (leak the fd to keep it open) std::mem::forget(stdout); - + let info = ProcInfo { process: child, fd, @@ -74,21 +73,21 @@ impl ProcSubst { /// Remove a virtual file pub fn remove_file(&self, name: &str) -> Result<(), std::io::Error> { let mut processes = self.processes.lock().unwrap(); - + if let Some(mut info) = processes.remove(name) { // Remove the symlink let _ = fs::remove_file(&info.symlink_path); - + // Kill the process let _ = info.process.kill(); let _ = info.process.wait(); - + // Close the file descriptor unsafe { libc::close(info.fd); } } - + Ok(()) } @@ -120,7 +119,7 @@ impl Drop for ProcSubst { libc::close(info.fd); } } - + // Remove base directory let _ = fs::remove_dir_all(&self.base_dir); } @@ -135,18 +134,18 @@ mod tests { #[test] fn test_proc_subst() { let ps = ProcSubst::new().unwrap(); - + // Create a virtual file let path = ps.create_file("test.txt", "echo 'Hello, World!'").unwrap(); - + // Read from it let mut file = File::open(&path).unwrap(); let mut contents = String::new(); file.read_to_string(&mut contents).unwrap(); - + assert_eq!(contents.trim(), "Hello, World!"); - + // Clean up ps.remove_file("test.txt").unwrap(); } -} \ No newline at end of file +} diff --git a/slosh-fuse/test-fuse.sh b/slosh-fuse/test-fuse.sh new file mode 100755 index 0000000000..24acf476dc --- /dev/null +++ b/slosh-fuse/test-fuse.sh @@ -0,0 +1,191 @@ +#!/bin/bash +set -e + +MOUNT_POINT="/tmp/test-eval-fs" +SERVER_BIN="./target/debug/slosh-fuse-server" +PASSED=0 +FAILED=0 + +pass() { PASSED=$((PASSED + 1)); echo " PASS: $1"; } +fail() { FAILED=$((FAILED + 1)); echo " FAIL: $1"; } + +cleanup() { + exec 3>&- 2>/dev/null || true + if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$SERVER_PID" 2>/dev/null + wait "$SERVER_PID" 2>/dev/null || true + fi + # AutoUnmount should handle this, but just in case + fusermount3 -u "$MOUNT_POINT" 2>/dev/null || true + rm -rf "$MOUNT_POINT" + rm -f "$FIFO" + rm -rf /tmp/fuse-concat-test-* 2>/dev/null || true +} +trap cleanup EXIT + +echo "=== FUSE Server Test ===" + +# Ensure binary exists +if [ ! -f "$SERVER_BIN" ]; then + echo "Building slosh-fuse-server..." + cargo build -p slosh-fuse +fi + +# Create mount point +mkdir -p "$MOUNT_POINT" + +# Start the server with stdin piped from a fifo. +# The server opens the FIFO read end in background, then we open the +# write end. This order avoids a deadlock: opening a FIFO for writing +# blocks until a reader exists, and vice versa. +FIFO="/tmp/fuse-test-fifo-$$" +mkfifo "$FIFO" +"$SERVER_BIN" "$MOUNT_POINT" < "$FIFO" & +SERVER_PID=$! +exec 3>"$FIFO" +echo "Server PID: $SERVER_PID" + +# Give the FUSE mount time to come up +sleep 2 + +# Test 1: Mount exists +echo "" +echo "--- Test 1: Mount point is a FUSE mount ---" +if mountpoint -q "$MOUNT_POINT" 2>/dev/null || mount | grep -q "$MOUNT_POINT"; then + pass "Mount point is active" +else + fail "Mount point not detected" +fi + +# Test 2: Register a file and read it +echo "" +echo "--- Test 2: Register and read a file ---" +CONTENT="hello from fuse" +B64_CONTENT=$(echo -n "$CONTENT" | base64) +printf "REGISTER\ttest.txt\t%s\n" "$B64_CONTENT" >&3 + +sleep 1 + +if [ -f "$MOUNT_POINT/test.txt" ]; then + ACTUAL=$(cat "$MOUNT_POINT/test.txt") + if [ "$ACTUAL" = "$CONTENT" ]; then + pass "File content matches: '$ACTUAL'" + else + fail "Content mismatch: expected '$CONTENT', got '$ACTUAL'" + fi +else + fail "File test.txt not found in mount" +fi + +# Test 3: Register a second file +echo "" +echo "--- Test 3: Register a second file ---" +CONTENT2="HOST=myhost" +B64_CONTENT2=$(echo -n "$CONTENT2" | base64) +printf "REGISTER\tconfig.env\t%s\n" "$B64_CONTENT2" >&3 + +sleep 1 + +if [ -f "$MOUNT_POINT/config.env" ]; then + ACTUAL2=$(cat "$MOUNT_POINT/config.env") + if [ "$ACTUAL2" = "$CONTENT2" ]; then + pass "config.env content matches" + else + fail "config.env content mismatch: expected '$CONTENT2', got '$ACTUAL2'" + fi +else + fail "File config.env not found in mount" +fi + +# Test 4: Directory listing shows both files +echo "" +echo "--- Test 4: Directory listing ---" +LISTING=$(ls "$MOUNT_POINT") +if echo "$LISTING" | grep -q "test.txt" && echo "$LISTING" | grep -q "config.env"; then + pass "Both files appear in directory listing" +else + fail "Directory listing incomplete: $LISTING" +fi + +# Test 5: Remove a file +echo "" +echo "--- Test 5: Remove a file ---" +printf "REMOVE\ttest.txt\n" >&3 + +sleep 1 + +if [ ! -f "$MOUNT_POINT/test.txt" ]; then + pass "test.txt removed successfully" +else + fail "test.txt still exists after REMOVE" +fi + +# Test 6: CONCAT - create real files and concatenate them +echo "" +echo "--- Test 6: CONCAT basic concatenation ---" +CONCAT_DIR="/tmp/fuse-concat-test-$$" +mkdir -p "$CONCAT_DIR" +echo -n "AAA" > "$CONCAT_DIR/a.txt" +echo -n "BBB" > "$CONCAT_DIR/b.txt" +echo -n "CCC" > "$CONCAT_DIR/c.txt" + +printf "CONCAT\tcombined.txt\t%s/a.txt\t%s/b.txt\t%s/c.txt\n" "$CONCAT_DIR" "$CONCAT_DIR" "$CONCAT_DIR" >&3 + +sleep 1 + +if [ -f "$MOUNT_POINT/combined.txt" ]; then + ACTUAL=$(cat "$MOUNT_POINT/combined.txt") + if [ "$ACTUAL" = "AAABBBCCC" ]; then + pass "CONCAT content matches: '$ACTUAL'" + else + fail "CONCAT content mismatch: expected 'AAABBBCCC', got '$ACTUAL'" + fi +else + fail "combined.txt not found in mount" +fi + +# Test 7: CONCAT reflects changes to source files +echo "" +echo "--- Test 7: CONCAT reflects source file changes ---" +echo -n "XXX" > "$CONCAT_DIR/b.txt" + +ACTUAL=$(cat "$MOUNT_POINT/combined.txt") +if [ "$ACTUAL" = "AAAXXXCCC" ]; then + pass "CONCAT reflects file change: '$ACTUAL'" +else + fail "CONCAT did not reflect change: expected 'AAAXXXCCC', got '$ACTUAL'" +fi + +# Test 8: grep works across CONCAT virtual file +echo "" +echo "--- Test 8: grep across CONCAT virtual file ---" +printf "line1 apple\n" > "$CONCAT_DIR/a.txt" +printf "line2 banana\n" > "$CONCAT_DIR/b.txt" +printf "line3 cherry\n" > "$CONCAT_DIR/c.txt" + +sleep 1 + +if grep -q "banana" "$MOUNT_POINT/combined.txt"; then + pass "grep found 'banana' in concatenated file" +else + fail "grep did not find 'banana' in concatenated file" +fi + +if grep -q "cherry" "$MOUNT_POINT/combined.txt"; then + pass "grep found 'cherry' in concatenated file" +else + fail "grep did not find 'cherry' in concatenated file" +fi + +# Clean up concat test files +rm -rf "$CONCAT_DIR" + +# Close the fifo writer fd +exec 3>&- +rm -f "$FIFO" + +echo "" +echo "=== Results: $PASSED passed, $FAILED failed ===" +if [ "$FAILED" -gt 0 ]; then + exit 1 +fi diff --git a/slosh_lib/Cargo.toml b/slosh_lib/Cargo.toml index cf3843ab3a..ca20bf656e 100644 --- a/slosh_lib/Cargo.toml +++ b/slosh_lib/Cargo.toml @@ -5,7 +5,7 @@ edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -fuse = ["dep:slosh-fuse", "dep:nix", "dep:uuid"] +fuse = ["dep:slosh-fuse", "dep:uuid"] [dependencies] unicode_reader = { workspace = true } @@ -24,7 +24,6 @@ static_assertions = { workspace = true } # Optional dependencies slosh-fuse = { path = "../slosh-fuse", optional = true } -nix = { version = "0.27", features = ["signal", "process", "fs", "mount"], optional = true } uuid = { version = "1.0", features = ["v4"], optional = true } [dev-dependencies] diff --git a/slosh_lib/src/fuse_bindings.rs b/slosh_lib/src/fuse_bindings.rs index 7dd7831ae5..5f2ce28849 100644 --- a/slosh_lib/src/fuse_bindings.rs +++ b/slosh_lib/src/fuse_bindings.rs @@ -5,11 +5,9 @@ use std::collections::HashMap; use std::sync::Mutex; use std::path::PathBuf; use std::process::{Command, Stdio}; -use nix::unistd::pipe; use slosh_fuse::FuseMount; -// Registry for active mounts - only created when first mount is created use std::sync::OnceLock; static MOUNT_REGISTRY: OnceLock>> = OnceLock::new(); @@ -33,29 +31,22 @@ fn mount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { .map_err(|e| VMError::new_vm(format!("Failed to create mount point: {}", e)))?; } - // Create communication pipes - let (_comm_read, comm_write) = pipe() - .map_err(|e| VMError::new_vm(format!("Failed to create communication pipe: {}", e)))?; - let (eval_read, eval_write) = pipe() - .map_err(|e| VMError::new_vm(format!("Failed to create eval pipe: {}", e)))?; - let mount_id = format!("mount-{}", uuid::Uuid::new_v4()); let mut mount = FuseMount::new(mount_point.clone()); - // Build the server binary name let server_name = "slosh-fuse-server"; - // Try multiple possible locations for the server binary let possible_paths = vec![ - // Same directory as current executable std::env::current_exe() .ok() .and_then(|p| p.parent().map(|p| p.join(server_name))), - // In target/debug or target/release std::env::current_dir() .ok() - .map(|p| p.join("target").join(if cfg!(debug_assertions) { "debug" } else { "release" }).join(server_name)), - // System PATH + .map(|p| { + p.join("target") + .join(if cfg!(debug_assertions) { "debug" } else { "release" }) + .join(server_name) + }), Some(PathBuf::from(server_name)), ]; @@ -63,22 +54,30 @@ fn mount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { .into_iter() .flatten() .find(|p| p.exists()) - .ok_or_else(|| VMError::new_vm(format!("Could not find slosh-fuse-server binary. Make sure to build it with 'cargo build --bin slosh-fuse-server'")))?; - - let child = Command::new(&server_path) + .ok_or_else(|| { + VMError::new_vm( + "Could not find slosh-fuse-server binary. \ + Build it with 'cargo build --bin slosh-fuse-server'" + .to_string(), + ) + })?; + + let mut child = Command::new(&server_path) .arg(mount_point.to_str().unwrap()) - .arg(eval_read.to_string()) - .arg(eval_write.to_string()) .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::inherit()) // Let errors go to stderr for debugging + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) .spawn() - .map_err(|e| VMError::new_vm(format!("Failed to spawn FUSE server at {:?}: {}", server_path, e)))?; - + .map_err(|e| { + VMError::new_vm(format!( + "Failed to spawn FUSE server at {:?}: {}", + server_path, e + )) + })?; + + mount.stdin_handle = child.stdin.take(); mount.process_handle = Some(child); - mount.comm_write_fd = Some(comm_write); - // Store mount in registry get_registry() .lock() .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))? @@ -90,20 +89,22 @@ fn mount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { fn register_eval_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult { if registers.len() != 3 { return Err(VMError::new_vm( - "register-eval-file: requires three arguments (mount-id path expression)".to_string(), + "register-eval-file: requires three arguments (mount-id path content)".to_string(), )); } let mount_id = registers[0].get_string(vm)?; let path = registers[1].get_string(vm)?; - let expression = registers[2].get_string(vm)?; + let content = registers[2].get_string(vm)?; - let registry = get_registry() + let mut registry = get_registry() .lock() .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - - if let Some(mount) = registry.get(mount_id) { - mount.register_file(&path, expression.to_string()); + + if let Some(mount) = registry.get_mut(mount_id) { + mount + .register_file(&path, content.as_bytes()) + .map_err(|e| VMError::new_vm(format!("Failed to register file: {}", e)))?; Ok(Value::True) } else { Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) @@ -122,16 +123,42 @@ fn unmount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { let mut registry = get_registry() .lock() .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - + if let Some(mut mount) = registry.remove(mount_id) { - mount.unmount() + mount + .unmount() .map_err(|e| VMError::new_vm(format!("Failed to unmount: {}", e)))?; - - // Also unmount the FUSE filesystem - if let Err(e) = nix::mount::umount(&mount.mount_point) { - // It's okay if unmount fails - the process termination should handle it - eprintln!("Warning: umount failed: {}", e); - } + Ok(Value::True) + } else { + Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) + } +} + +fn concat_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() < 3 { + return Err(VMError::new_vm( + "concat-eval-files: requires at least three arguments (mount-id vpath source-path...)".to_string(), + )); + } + + let mount_id = registers[0].get_string(vm)?; + let vpath = registers[1].get_string(vm)?; + + let source_paths: Vec = registers[2..] + .iter() + .map(|v| v.get_string(vm).map(|s| s.to_string())) + .collect::, _>>()?; + + let source_refs: Vec<&str> = source_paths.iter().map(|s| s.as_str()).collect(); + + let mut registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(mount) = registry.get_mut(mount_id) { + mount + .register_concat_file(&vpath, &source_refs) + .map_err(|e| VMError::new_vm(format!("Failed to register concat file: {}", e)))?; Ok(Value::True) } else { Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) @@ -150,17 +177,14 @@ fn list_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { let registry = get_registry() .lock() .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - + if let Some(mount) = registry.get(mount_id) { - let mapping = mount.file_mapping - .lock() - .map_err(|e| VMError::new_vm(format!("File mapping lock poisoned: {}", e)))?; - - let files: Vec = mapping.list_files() - .into_iter() - .map(|f| vm.alloc_string(f)) + let files: Vec = mount + .registered_paths + .iter() + .map(|f| vm.alloc_string(f.clone())) .collect(); - + Ok(vm.alloc_vector(files)) } else { Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) @@ -171,23 +195,17 @@ fn list_mounts(vm: &mut SloshVm, _registers: &[Value]) -> VMResult { let registry = get_registry() .lock() .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - + let mut mounts = Vec::new(); for (mount_id, mount) in registry.iter() { - let file_count = mount.file_mapping - .lock() - .map_err(|e| VMError::new_vm(format!("File mapping lock poisoned: {}", e)))? - .list_files() - .len(); - let mount_info = vec![ vm.alloc_string("mount-id".to_string()), vm.alloc_string(mount_id.clone()), vm.alloc_string("mount-point".to_string()), vm.alloc_string(mount.mount_point.display().to_string()), vm.alloc_string("file-count".to_string()), - Value::from(file_count as i64), + Value::from(mount.registered_paths.len() as i64), ]; mounts.push(vm.alloc_vector(mount_info)); } @@ -215,14 +233,32 @@ Example: env, "register-eval-file", register_eval_file, - r#"Usage: (register-eval-file mount-id path expression) + r#"Usage: (register-eval-file mount-id path content) + +Register a file with static content in the FUSE filesystem. +Content is evaluated at registration time and sent to the FUSE server. -Register a file with a slosh expression that will be evaluated when the file is read. +Section: fuse + +Example: +(register-eval-file mount-id "config.txt" "HOST=myhost\nPORT=8080\n") +"#, + ); + + add_builtin( + env, + "concat-eval-files", + concat_eval_files, + r#"Usage: (concat-eval-files mount-id vpath source-path...) + +Present multiple real files as a single virtual file in the FUSE filesystem. +The virtual file contents are the concatenation of the source files, read lazily +on demand so changes to source files are reflected automatically. Section: fuse Example: -(register-eval-file mount-id "config.env" "(str \"HOST=\" (hostname))") +(concat-eval-files mount-id "combined.txt" "/path/a.txt" "/path/b.txt" "/path/c.txt") "#, ); @@ -272,4 +308,4 @@ Example: ; ["mount-id" "mount-point" "file-count" ] "#, ); -} \ No newline at end of file +} From 7e3d564adddfd4aa443ec08ecb8ee05d14914e27 Mon Sep 17 00:00:00 2001 From: gpwclark Date: Sat, 7 Mar 2026 22:52:33 -0500 Subject: [PATCH 07/12] new fuse daemon model --- Cargo.lock | 1 + shell/src/config.rs | 18 +- slosh-fuse/Cargo.toml | 5 +- slosh-fuse/README.md | 51 ++++-- slosh-fuse/src/auto_start.rs | 67 +++++++ slosh-fuse/src/bin/slosh-fuse-server.rs | 85 --------- slosh-fuse/src/client.rs | 177 +++++++++++++++++++ slosh-fuse/src/daemon.rs | 219 +++++++++++++++++++++++ slosh-fuse/src/lib.rs | 131 +++++++++----- slosh-fuse/src/mount_manager.rs | 139 +++++++++++++++ slosh-fuse/src/socket_server.rs | 211 ++++++++++++++++++++++ slosh-fuse/test-fuse.sh | 224 +++++++++++------------- slosh/src/config.rs | 16 +- slosh_lib/src/fuse_bindings.rs | 74 +++----- slosh_lib/src/lib.rs | 13 ++ 15 files changed, 1107 insertions(+), 324 deletions(-) create mode 100644 slosh-fuse/src/auto_start.rs delete mode 100644 slosh-fuse/src/bin/slosh-fuse-server.rs create mode 100644 slosh-fuse/src/client.rs create mode 100644 slosh-fuse/src/daemon.rs create mode 100644 slosh-fuse/src/mount_manager.rs create mode 100644 slosh-fuse/src/socket_server.rs diff --git a/Cargo.lock b/Cargo.lock index 0a4afd1a46..7564f9520a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2206,6 +2206,7 @@ dependencies = [ "fuser", "libc", "log", + "nix", "tempfile", ] diff --git a/shell/src/config.rs b/shell/src/config.rs index 2e330ed409..c6d0ae30ed 100644 --- a/shell/src/config.rs +++ b/shell/src/config.rs @@ -5,6 +5,8 @@ pub struct Config { pub command: Option, pub script: Option, pub args: Vec, + pub fuse_daemon: bool, + pub fuse_daemon_foreground: bool, } pub const VERSION_STRING: &str = env!("VERSION_STRING"); @@ -16,8 +18,10 @@ USAGE: sl-sh [FLAGS] [OPTIONS] [args] FLAGS: - -v, --version Print the version, platform and revision of sl-sh then exit. - -h, --help Print help (this) and exit. + -v, --version Print the version, platform and revision of sl-sh then exit. + -h, --help Print help (this) and exit. + --fuse-daemon Start as FUSE daemon (detaches from terminal). + --fuse-daemon-foreground Start as FUSE daemon in foreground (for debugging). OPTIONS: -c Command to run instead of entering the REPL. @@ -47,6 +51,8 @@ pub fn get_config() -> Option { let mut command: Option = None; let mut script: Option = None; let mut command_args: Vec = Vec::new(); + let mut fuse_daemon = false; + let mut fuse_daemon_foreground = false; let mut args: Vec = env::args_os().collect(); @@ -72,6 +78,12 @@ pub fn get_config() -> Option { help(&exe_name); return None; } + "--fuse-daemon" => { + fuse_daemon = true; + } + "--fuse-daemon-foreground" => { + fuse_daemon_foreground = true; + } _ => { if command.is_none() && script.is_none() { script = Some(arg); @@ -90,5 +102,7 @@ pub fn get_config() -> Option { command, script, args: command_args, + fuse_daemon, + fuse_daemon_foreground, }) } diff --git a/slosh-fuse/Cargo.toml b/slosh-fuse/Cargo.toml index ef179feff0..af0cabfd4b 100644 --- a/slosh-fuse/Cargo.toml +++ b/slosh-fuse/Cargo.toml @@ -3,10 +3,6 @@ name = "slosh-fuse" version.workspace = true edition = "2021" -[[bin]] -name = "slosh-fuse-server" -path = "src/bin/slosh-fuse-server.rs" - [dependencies] fuser = "0.14" libc = "0.2" @@ -14,3 +10,4 @@ log = "0.4" env_logger = "0.11" base64 = "0.21" tempfile = "3.0" +nix = { workspace = true, features = ["signal", "process", "fs"] } diff --git a/slosh-fuse/README.md b/slosh-fuse/README.md index a2dac25856..af6750d814 100644 --- a/slosh-fuse/README.md +++ b/slosh-fuse/README.md @@ -14,17 +14,29 @@ cat /tmp/dynamic/config.env ## Architecture -Content is **evaluated at registration time** in the parent slosh process, then sent as static bytes to a child FUSE server process via stdin (base64-encoded, tab-delimited protocol). This avoids the complexity of IPC-based expression evaluation since the Slosh VM is `thread_local!` and can't be accessed from child processes. +A single per-user FUSE daemon manages all mounts. The daemon: +- Is the slosh binary itself, started with `--fuse-daemon` +- Manages multiple mount points (one FUSE session thread per mount) +- Accepts connections from multiple REPL instances via Unix domain socket +- Auto-starts when a REPL first needs it +- Detaches from terminal (double-fork + setsid) +- Uses a PID file with advisory lock for single-instance enforcement + +Content is **evaluated at registration time** in the REPL, then sent as static bytes to the daemon via Unix socket (base64-encoded, tab-delimited protocol). Components: -1. **slosh-fuse crate** - FUSE filesystem serving static file content, plus the `FuseMount` parent-side handle -2. **slosh-fuse-server binary** - Child process that mounts FUSE and reads `REGISTER`/`REMOVE` commands from stdin -3. **Builtin functions** (in slosh_lib) - `mount-eval-fs`, `register-eval-file`, `unmount-eval-fs`, `list-eval-files`, `list-mounts` +1. **slosh-fuse crate** - FUSE filesystem, daemon core, socket server, and client library +2. **Builtin functions** (in slosh_lib) - `mount-eval-fs`, `register-eval-file`, `unmount-eval-fs`, `concat-eval-files`, `list-eval-files`, `list-mounts` + +Runtime files (under `$HOME/.local/share/slosh/`): +- `fuse-daemon.pid` - PID file with advisory lock +- `fuse.sock` - Unix domain socket +- `fuse-daemon.log` - Daemon log output ## Usage ```lisp -;; Mount a dynamic filesystem +;; Mount a dynamic filesystem (auto-starts daemon if needed) (def mount-id (mount-eval-fs "/tmp/dynamic")) ;; Register a file with content (evaluated now, served statically) @@ -33,9 +45,8 @@ Components: ;; The file /tmp/dynamic/config.env is now readable -;; To update content, re-register: -(register-eval-file mount-id "config.env" - (str "HOST=" (hostname) "\n" "TYPE=" *my-global-var* "\n")) +;; Concatenate multiple real files as a single virtual file +(concat-eval-files mount-id "combined.txt" "/path/a.txt" "/path/b.txt") ;; Unmount when done (unmount-eval-fs mount-id) @@ -47,8 +58,7 @@ Requires Linux with libfuse3 (FUSE doesn't work natively on macOS). ```bash # On Linux -cargo build -p slosh-fuse -cargo build -p slosh_lib --features fuse +cargo build -p slosh --features fuse # On macOS, use an Apple Container container build -t slosh-fuse -f Containerfile . @@ -58,17 +68,26 @@ container run --name slosh-fuse slosh-fuse bash /src/slosh-fuse/test-fuse.sh ## Testing ```bash -# Standalone FUSE server test (inside container or on Linux) +# Daemon test (inside container or on Linux, requires socat) bash slosh-fuse/test-fuse.sh + +# Manual daemon testing +slosh --fuse-daemon-foreground & +echo -e "PING" | socat - UNIX-CONNECT:$HOME/.local/share/slosh/fuse.sock ``` ## Protocol -The parent communicates with the FUSE server child via stdin using tab-delimited lines: +The REPL communicates with the daemon via Unix domain socket using tab-delimited request/response lines: ``` -REGISTER\t\t\n -REMOVE\t\n +Request: Response: +MOUNT\t\n -> OK\t\n +REGISTER\t\t\t\n -> OK\n +CONCAT\t\t\t\t...\n -> OK\n +REMOVE\t\t\n -> OK\n +UNMOUNT\t\n -> OK\n +LIST\t\n -> OK\t\t...\n +PING\n -> OK\tPONG\n +(any error) -> ERR\t\n ``` - -The server uses `AutoUnmount` so the filesystem is automatically unmounted when the server process exits. diff --git a/slosh-fuse/src/auto_start.rs b/slosh-fuse/src/auto_start.rs new file mode 100644 index 0000000000..ab712d0980 --- /dev/null +++ b/slosh-fuse/src/auto_start.rs @@ -0,0 +1,67 @@ +use crate::client::DaemonClient; +use crate::daemon::{DaemonConfig, PidFile}; +use std::io; +use std::path::Path; + +/// Ensure the FUSE daemon is running and return a connected client. +/// +/// Tries to connect + ping. If that fails, cleans stale files and spawns +/// the current executable with `--fuse-daemon`, then polls for the socket. +pub fn ensure_daemon_running(config: &DaemonConfig) -> io::Result { + // Try connecting to an existing daemon + if let Ok(mut client) = DaemonClient::connect(&config.socket_path) { + if client.ping().is_ok() { + return Ok(client); + } + } + + // No daemon running. Clean up stale files. + if PidFile::check_running(&config.pid_file).is_none() { + let _ = std::fs::remove_file(&config.socket_path); + let _ = std::fs::remove_file(&config.pid_file); + } else { + // PID file is locked but socket isn't responding. Wait a bit + // in case the daemon is still starting up. + return wait_for_socket(&config.socket_path, 5); + } + + // Spawn daemon using current executable + let exe = std::env::current_exe().map_err(|e| { + io::Error::new(io::ErrorKind::NotFound, format!("cannot find current exe: {}", e)) + })?; + + std::process::Command::new(&exe) + .arg("--fuse-daemon") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .map_err(|e| { + io::Error::new( + io::ErrorKind::Other, + format!("failed to spawn FUSE daemon: {}", e), + ) + })?; + + wait_for_socket(&config.socket_path, 5) +} + +/// Poll for the daemon socket to appear, then connect + ping. +fn wait_for_socket(socket_path: &Path, timeout_secs: u64) -> io::Result { + let deadline = std::time::Instant::now() + + std::time::Duration::from_secs(timeout_secs); + + while std::time::Instant::now() < deadline { + if let Ok(mut client) = DaemonClient::connect(socket_path) { + if client.ping().is_ok() { + return Ok(client); + } + } + std::thread::sleep(std::time::Duration::from_millis(100)); + } + + Err(io::Error::new( + io::ErrorKind::TimedOut, + "timed out waiting for FUSE daemon to start", + )) +} diff --git a/slosh-fuse/src/bin/slosh-fuse-server.rs b/slosh-fuse/src/bin/slosh-fuse-server.rs deleted file mode 100644 index 03d7b930ce..0000000000 --- a/slosh-fuse/src/bin/slosh-fuse-server.rs +++ /dev/null @@ -1,85 +0,0 @@ -use base64::Engine; -use fuser::{MountOption, Session}; -use slosh_fuse::{EvalFs, FileMapping}; -use std::io::BufRead; -use std::path::PathBuf; -use std::sync::{Arc, Mutex}; -use std::env; - -fn main() { - env_logger::init(); - - let args: Vec = env::args().collect(); - if args.len() < 2 { - eprintln!("Usage: {} ", args[0]); - std::process::exit(1); - } - - let mount_point = &args[1]; - let file_mapping = Arc::new(Mutex::new(FileMapping::new())); - - // Spawn a background thread to read commands from stdin - let mapping_for_reader = Arc::clone(&file_mapping); - std::thread::spawn(move || { - let stdin = std::io::stdin(); - let reader = stdin.lock(); - for line in reader.lines() { - let line = match line { - Ok(l) => l, - Err(e) => { - log::error!("Error reading stdin: {}", e); - break; - } - }; - - let parts: Vec<&str> = line.split('\t').collect(); - match parts.first().copied() { - Some("REGISTER") if parts.len() == 3 => { - let path = parts[1]; - let b64_content = parts[2]; - match base64::engine::general_purpose::STANDARD.decode(b64_content) { - Ok(content) => { - let mut mapping = mapping_for_reader.lock().unwrap(); - mapping.register(path, content); - log::info!("Registered file: {}", path); - } - Err(e) => { - log::error!("Failed to decode base64 for {}: {}", path, e); - } - } - } - Some("CONCAT") if parts.len() >= 3 => { - let vpath = parts[1]; - let source_paths: Vec = - parts[2..].iter().map(PathBuf::from).collect(); - let mut mapping = mapping_for_reader.lock().unwrap(); - mapping.register_concat(vpath, source_paths); - log::info!("Registered concat file: {} ({} sources)", vpath, parts.len() - 2); - } - Some("REMOVE") if parts.len() >= 2 => { - let path = parts[1]; - let mut mapping = mapping_for_reader.lock().unwrap(); - mapping.remove(path); - log::info!("Removed file: {}", path); - } - _ => { - log::warn!("Unknown command: {}", line); - } - } - } - log::info!("Stdin reader thread exiting"); - }); - - let fs = EvalFs::new(file_mapping); - - let options = vec![ - MountOption::FSName("slosh-eval".to_string()), - MountOption::AutoUnmount, - MountOption::DefaultPermissions, - ]; - - let mut session = Session::new(fs, mount_point.as_ref(), &options) - .expect("Failed to create FUSE session"); - - session.run().expect("FUSE session failed"); -} diff --git a/slosh-fuse/src/client.rs b/slosh-fuse/src/client.rs new file mode 100644 index 0000000000..21b7cc396d --- /dev/null +++ b/slosh-fuse/src/client.rs @@ -0,0 +1,177 @@ +use base64::Engine; +use std::io::{self, BufRead, BufReader, Write}; +use std::os::unix::net::UnixStream; +use std::path::Path; + +/// Client for communicating with the FUSE daemon over a Unix socket. +pub struct DaemonClient { + reader: BufReader, + writer: UnixStream, +} + +impl DaemonClient { + /// Connect to the daemon socket. + pub fn connect(socket_path: &Path) -> io::Result { + let stream = UnixStream::connect(socket_path)?; + let reader = BufReader::new(stream.try_clone()?); + Ok(Self { + reader, + writer: stream, + }) + } + + /// Send a raw command line and read the response. + /// Returns `Ok(response_fields)` on `OK\t...` or `Err` on `ERR\t...`. + fn send_command(&mut self, cmd: &str) -> io::Result> { + self.writer.write_all(cmd.as_bytes())?; + if !cmd.ends_with('\n') { + self.writer.write_all(b"\n")?; + } + self.writer.flush()?; + + let mut line = String::new(); + self.reader.read_line(&mut line)?; + let line = line.trim_end_matches('\n').trim_end_matches('\r'); + + let parts: Vec<&str> = line.split('\t').collect(); + match parts.first().copied() { + Some("OK") => Ok(parts[1..].iter().map(|s| s.to_string()).collect()), + Some("ERR") => { + let msg = parts[1..].join("\t"); + Err(io::Error::new(io::ErrorKind::Other, msg)) + } + _ => Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unexpected response: {}", line), + )), + } + } + + /// Ping the daemon to verify it's alive. + pub fn ping(&mut self) -> io::Result<()> { + let fields = self.send_command("PING")?; + if fields.first().map(|s| s.as_str()) == Some("PONG") { + Ok(()) + } else { + Err(io::Error::new( + io::ErrorKind::InvalidData, + "unexpected ping response", + )) + } + } + + /// Request a new FUSE mount. Returns the mount-id. + pub fn mount(&mut self, mount_point: &str) -> io::Result { + let fields = self.send_command(&format!("MOUNT\t{}", mount_point))?; + fields.into_iter().next().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "no mount-id in response") + }) + } + + /// Register a file with static content. + pub fn register_file( + &mut self, + mount_id: &str, + path: &str, + content: &[u8], + ) -> io::Result<()> { + let b64 = base64::engine::general_purpose::STANDARD.encode(content); + self.send_command(&format!("REGISTER\t{}\t{}\t{}", mount_id, path, b64))?; + Ok(()) + } + + /// Register a concat file (virtual file backed by multiple real files). + pub fn register_concat( + &mut self, + mount_id: &str, + vpath: &str, + source_paths: &[&str], + ) -> io::Result<()> { + let mut cmd = format!("CONCAT\t{}\t{}", mount_id, vpath); + for p in source_paths { + cmd.push('\t'); + cmd.push_str(p); + } + self.send_command(&cmd)?; + Ok(()) + } + + /// Remove a file from a mount. + pub fn remove_file(&mut self, mount_id: &str, path: &str) -> io::Result<()> { + self.send_command(&format!("REMOVE\t{}\t{}", mount_id, path))?; + Ok(()) + } + + /// Unmount a FUSE filesystem. + pub fn unmount(&mut self, mount_id: &str) -> io::Result<()> { + self.send_command(&format!("UNMOUNT\t{}", mount_id))?; + Ok(()) + } + + /// List files in a mount. + pub fn list_files(&mut self, mount_id: &str) -> io::Result> { + self.send_command(&format!("LIST\t{}", mount_id)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::net::UnixListener; + + fn mock_server(socket_path: &Path, response: &str) -> std::thread::JoinHandle<()> { + let listener = UnixListener::bind(socket_path).unwrap(); + let resp = response.to_string(); + std::thread::spawn(move || { + let (stream, _) = listener.accept().unwrap(); + let mut reader = BufReader::new(stream.try_clone().unwrap()); + let mut writer = stream; + let mut line = String::new(); + reader.read_line(&mut line).unwrap(); + writer.write_all(resp.as_bytes()).unwrap(); + writer.write_all(b"\n").unwrap(); + writer.flush().unwrap(); + }) + } + + #[test] + fn client_ping() { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("test.sock"); + + let handle = mock_server(&sock, "OK\tPONG"); + + let mut client = DaemonClient::connect(&sock).unwrap(); + client.ping().unwrap(); + + handle.join().unwrap(); + } + + #[test] + fn client_mount() { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("test.sock"); + + let handle = mock_server(&sock, "OK\tmount-1"); + + let mut client = DaemonClient::connect(&sock).unwrap(); + let id = client.mount("/tmp/test").unwrap(); + assert_eq!(id, "mount-1"); + + handle.join().unwrap(); + } + + #[test] + fn client_error_response() { + let dir = tempfile::TempDir::new().unwrap(); + let sock = dir.path().join("test.sock"); + + let handle = mock_server(&sock, "ERR\tsomething went wrong"); + + let mut client = DaemonClient::connect(&sock).unwrap(); + let result = client.ping(); + assert!(result.is_err()); + + handle.join().unwrap(); + } +} diff --git a/slosh-fuse/src/daemon.rs b/slosh-fuse/src/daemon.rs new file mode 100644 index 0000000000..455f7c6157 --- /dev/null +++ b/slosh-fuse/src/daemon.rs @@ -0,0 +1,219 @@ +use std::fs::{self, File}; +use std::io; +use std::os::unix::io::AsRawFd; +use std::path::{Path, PathBuf}; + +/// Paths used by the FUSE daemon. +pub struct DaemonConfig { + pub pid_file: PathBuf, + pub socket_path: PathBuf, + pub log_file: PathBuf, +} + +impl DaemonConfig { + /// Build paths under `$HOME/.local/share/slosh/`. + pub fn default_for_user() -> io::Result { + let home = std::env::var("HOME").map_err(|_| { + io::Error::new(io::ErrorKind::NotFound, "HOME not set") + })?; + let dir = PathBuf::from(home).join(".local/share/slosh"); + fs::create_dir_all(&dir)?; + Ok(Self { + pid_file: dir.join("fuse-daemon.pid"), + socket_path: dir.join("fuse.sock"), + log_file: dir.join("fuse-daemon.log"), + }) + } +} + +/// Advisory-lock-based PID file. +/// +/// The lock is held for the lifetime of this struct (via the open fd). +/// On drop the file is removed and the lock released (fd close). +pub struct PidFile { + path: PathBuf, + _file: File, +} + +impl PidFile { + /// Create (or overwrite) the PID file and acquire an exclusive advisory + /// lock via `libc::flock`. Returns `Err` if another process holds the lock. + pub fn acquire(path: &Path) -> io::Result { + let file = fs::OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path)?; + + let fd = file.as_raw_fd(); + let ret = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) }; + if ret != 0 { + return Err(io::Error::new( + io::ErrorKind::AddrInUse, + "another FUSE daemon is already running", + )); + } + + use std::io::Write; + // Write PID via a borrow of the File + (&file).write_all(format!("{}", std::process::id()).as_bytes())?; + (&file).flush()?; + + Ok(Self { + path: path.to_path_buf(), + _file: file, + }) + } + + /// Check whether a daemon process is already running by trying to + /// acquire the lock. If the lock is held, returns the PID from the file. + pub fn check_running(path: &Path) -> Option { + let file = match fs::OpenOptions::new().read(true).open(path) { + Ok(f) => f, + Err(_) => return None, + }; + + let fd = file.as_raw_fd(); + let ret = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) }; + if ret == 0 { + // We got the lock -> no daemon running. Release immediately. + unsafe { libc::flock(fd, libc::LOCK_UN) }; + None + } else { + // Lock is held -> read PID from file + use std::io::Read; + let mut buf = String::new(); + let mut f = &file; + let _ = f.read_to_string(&mut buf); + buf.trim().parse::().ok() + } + } +} + +impl Drop for PidFile { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +/// Double-fork + setsid to become a proper daemon. +/// +/// Returns `Ok(())` in the daemon (grandchild) process. +/// The parent and intermediate child processes call `std::process::exit(0)`. +pub fn daemonize() -> io::Result<()> { + use nix::unistd::{ForkResult, fork, setsid}; + + // First fork + match unsafe { fork() } { + Ok(ForkResult::Parent { .. }) => std::process::exit(0), + Ok(ForkResult::Child) => {} + Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), + } + + // New session leader + setsid().map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + + // Second fork (prevent re-acquiring a controlling terminal) + match unsafe { fork() } { + Ok(ForkResult::Parent { .. }) => std::process::exit(0), + Ok(ForkResult::Child) => {} + Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), + } + + // Redirect stdio to /dev/null + let devnull = fs::OpenOptions::new() + .read(true) + .write(true) + .open("/dev/null")?; + let fd = devnull.as_raw_fd(); + unsafe { + libc::dup2(fd, 0); + libc::dup2(fd, 1); + libc::dup2(fd, 2); + } + + Ok(()) +} + +/// Set up file-based logging for the daemon. +pub fn init_daemon_logging(log_path: &Path) -> io::Result<()> { + let file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path)?; + + env_logger::Builder::new() + .target(env_logger::Target::Pipe(Box::new(file))) + .filter_level(log::LevelFilter::Info) + .format(|buf, record| { + use std::io::Write; + writeln!( + buf, + "{} [{}] {}", + epoch_secs(), + record.level(), + record.args() + ) + }) + .init(); + + Ok(()) +} + +fn epoch_secs() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[test] + fn pid_file_acquire_and_drop() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("test.pid"); + + { + let _pid = PidFile::acquire(&path).unwrap(); + assert!(path.exists()); + + // Second acquire should fail (lock held) + let result = PidFile::acquire(&path); + assert!(result.is_err()); + } + + // After drop, file is removed + assert!(!path.exists()); + } + + #[test] + fn check_running_no_file() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("test.pid"); + assert!(PidFile::check_running(&path).is_none()); + } + + #[test] + fn check_running_with_locked_pid() { + let dir = TempDir::new().unwrap(); + let path = dir.path().join("test.pid"); + + let _pid = PidFile::acquire(&path).unwrap(); + let running = PidFile::check_running(&path); + assert!(running.is_some()); + assert_eq!(running.unwrap(), std::process::id()); + } + + #[test] + fn daemon_config_default() { + if std::env::var("HOME").is_ok() { + let config = DaemonConfig::default_for_user().unwrap(); + assert!(config.pid_file.to_str().unwrap().contains("fuse-daemon.pid")); + assert!(config.socket_path.to_str().unwrap().contains("fuse.sock")); + } + } +} diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs index 485ab3c58e..ca98289c98 100644 --- a/slosh-fuse/src/lib.rs +++ b/slosh-fuse/src/lib.rs @@ -2,62 +2,58 @@ pub mod eval_fs; pub mod file_mapping; pub mod proc_subst; +pub mod auto_start; +pub mod client; +pub mod daemon; +pub mod mount_manager; +pub mod socket_server; + pub use eval_fs::EvalFs; pub use file_mapping::{FileMapping, FileEntry}; -use base64::Engine; -use std::io::Write; use std::path::PathBuf; -use std::process::ChildStdin; +use crate::client::DaemonClient; + +/// A handle to a single FUSE mount managed by the daemon. pub struct FuseMount { pub mount_point: PathBuf, - pub process_handle: Option, - pub stdin_handle: Option, + pub mount_id: String, + client: DaemonClient, pub registered_paths: Vec, } impl FuseMount { - pub fn new(mount_point: PathBuf) -> Self { - Self { + /// Create a new mount via the daemon. Calls `client.mount()` to get a + /// mount-id from the running daemon. + pub fn new_daemon( + mount_point: PathBuf, + mut client: DaemonClient, + ) -> Result { + let mount_id = client.mount(mount_point.to_str().unwrap_or(""))?; + Ok(Self { mount_point, - process_handle: None, - stdin_handle: None, + mount_id, + client, registered_paths: Vec::new(), - } - } - - fn send_command(&mut self, cmd: &str) -> Result<(), std::io::Error> { - if let Some(ref mut stdin) = self.stdin_handle { - stdin.write_all(cmd.as_bytes())?; - stdin.flush()?; - Ok(()) - } else { - Err(std::io::Error::new( - std::io::ErrorKind::NotConnected, - "FUSE server stdin not connected", - )) - } + }) } pub fn register_file(&mut self, path: &str, content: &[u8]) -> Result<(), std::io::Error> { - let b64 = base64::engine::general_purpose::STANDARD.encode(content); - let cmd = format!("REGISTER\t{}\t{}\n", path, b64); - self.send_command(&cmd)?; + self.client.register_file(&self.mount_id, path, content)?; if !self.registered_paths.contains(&path.to_string()) { self.registered_paths.push(path.to_string()); } Ok(()) } - pub fn register_concat_file(&mut self, vpath: &str, source_paths: &[&str]) -> Result<(), std::io::Error> { - let mut cmd = format!("CONCAT\t{}", vpath); - for p in source_paths { - cmd.push('\t'); - cmd.push_str(p); - } - cmd.push('\n'); - self.send_command(&cmd)?; + pub fn register_concat_file( + &mut self, + vpath: &str, + source_paths: &[&str], + ) -> Result<(), std::io::Error> { + self.client + .register_concat(&self.mount_id, vpath, source_paths)?; if !self.registered_paths.contains(&vpath.to_string()) { self.registered_paths.push(vpath.to_string()); } @@ -65,20 +61,69 @@ impl FuseMount { } pub fn remove_file(&mut self, path: &str) -> Result<(), std::io::Error> { - let cmd = format!("REMOVE\t{}\n", path); - self.send_command(&cmd)?; + self.client.remove_file(&self.mount_id, path)?; self.registered_paths.retain(|p| p != path); Ok(()) } pub fn unmount(&mut self) -> Result<(), std::io::Error> { - // Drop stdin to signal EOF to the server's reader thread - self.stdin_handle.take(); + self.client.unmount(&self.mount_id) + } - if let Some(mut handle) = self.process_handle.take() { - handle.kill()?; - handle.wait()?; - } - Ok(()) + pub fn list_files(&mut self) -> Result, std::io::Error> { + self.client.list_files(&self.mount_id) + } +} + +/// Entry point for the daemon process. Called from `slosh --fuse-daemon`. +pub fn run_fuse_daemon(foreground: bool) -> Result<(), Box> { + use crate::daemon::{DaemonConfig, PidFile, daemonize, init_daemon_logging}; + use crate::socket_server::run_server; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let config = DaemonConfig::default_for_user()?; + + if !foreground { + daemonize()?; + } + + // Acquire PID file (single instance check) + let _pid_file = PidFile::acquire(&config.pid_file)?; + + // Set up logging to file + init_daemon_logging(&config.log_file)?; + log::info!("FUSE daemon starting (pid={})", std::process::id()); + + let shutdown = Arc::new(AtomicBool::new(false)); + + // Install signal handler for SIGTERM + let shut = Arc::clone(&shutdown); + unsafe { + libc::signal(libc::SIGTERM, signal_handler as libc::sighandler_t); + libc::signal(libc::SIGINT, signal_handler as libc::sighandler_t); + } + SHUTDOWN_FLAG.store( + Arc::into_raw(shut) as usize, + std::sync::atomic::Ordering::SeqCst, + ); + + run_server(&config.socket_path, shutdown)?; + + log::info!("FUSE daemon exiting"); + Ok(()) +} + +// Global pointer to the shutdown flag for the signal handler +static SHUTDOWN_FLAG: std::sync::atomic::AtomicUsize = + std::sync::atomic::AtomicUsize::new(0); + +extern "C" fn signal_handler(_sig: libc::c_int) { + let ptr = SHUTDOWN_FLAG.load(std::sync::atomic::Ordering::SeqCst); + if ptr != 0 { + let flag = unsafe { + &*(ptr as *const std::sync::atomic::AtomicBool) + }; + flag.store(true, std::sync::atomic::Ordering::SeqCst); } } diff --git a/slosh-fuse/src/mount_manager.rs b/slosh-fuse/src/mount_manager.rs new file mode 100644 index 0000000000..1675103c3a --- /dev/null +++ b/slosh-fuse/src/mount_manager.rs @@ -0,0 +1,139 @@ +use crate::eval_fs::EvalFs; +use crate::file_mapping::FileMapping; +use fuser::{MountOption, Session}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +struct MountState { + mount_point: PathBuf, + mapping: Arc>, + _thread: JoinHandle<()>, +} + +pub struct MountManager { + mounts: HashMap, + next_id: u64, +} + +impl MountManager { + pub fn new() -> Self { + Self { + mounts: HashMap::new(), + next_id: 1, + } + } + + /// Mount a new FUSE filesystem. Returns a mount-id string. + pub fn mount(&mut self, mount_point: &str) -> Result { + let path = PathBuf::from(mount_point); + + // Create mount point if needed + if !path.exists() { + std::fs::create_dir_all(&path) + .map_err(|e| format!("failed to create mount point: {}", e))?; + } + + let mapping = Arc::new(Mutex::new(FileMapping::new())); + let mapping_for_fuse = Arc::clone(&mapping); + let mount_path = path.clone(); + + let mount_id = format!("mount-{}", self.next_id); + self.next_id += 1; + + let thread = std::thread::Builder::new() + .name(format!("fuse-{}", mount_id)) + .spawn(move || { + let fs = EvalFs::new(mapping_for_fuse); + let options = vec![ + MountOption::FSName("slosh-eval".to_string()), + MountOption::AutoUnmount, + MountOption::DefaultPermissions, + ]; + + match Session::new(fs, mount_path.as_ref(), &options) { + Ok(mut session) => { + log::info!("FUSE session started for {:?}", mount_path); + if let Err(e) = session.run() { + log::error!("FUSE session error for {:?}: {}", mount_path, e); + } + log::info!("FUSE session ended for {:?}", mount_path); + } + Err(e) => { + log::error!("Failed to create FUSE session for {:?}: {}", mount_path, e); + } + } + }) + .map_err(|e| format!("failed to spawn FUSE thread: {}", e))?; + + self.mounts.insert( + mount_id.clone(), + MountState { + mount_point: path, + mapping, + _thread: thread, + }, + ); + + Ok(mount_id) + } + + /// Get the FileMapping for a given mount-id. + pub fn get_mapping(&self, mount_id: &str) -> Option>> { + self.mounts.get(mount_id).map(|s| Arc::clone(&s.mapping)) + } + + /// Unmount a FUSE filesystem using fusermount3. + pub fn unmount(&mut self, mount_id: &str) -> Result<(), String> { + let state = self.mounts.remove(mount_id) + .ok_or_else(|| format!("unknown mount-id: {}", mount_id))?; + + let mount_str = state.mount_point.to_string_lossy().to_string(); + Self::fusermount_unmount(&mount_str)?; + + // The FUSE thread will exit once the session is unmounted. + // We don't join it here because fusermount3 -u is async with + // respect to the FUSE thread finishing. + + Ok(()) + } + + /// Unmount all active mounts. Used during shutdown. + pub fn unmount_all(&mut self) { + let ids: Vec = self.mounts.keys().cloned().collect(); + for id in ids { + if let Err(e) = self.unmount(&id) { + log::error!("Failed to unmount {}: {}", id, e); + } + } + } + + /// List mount IDs. + pub fn list_mounts(&self) -> Vec<(String, String)> { + self.mounts + .iter() + .map(|(id, state)| { + (id.clone(), state.mount_point.to_string_lossy().to_string()) + }) + .collect() + } + + fn fusermount_unmount(mount_point: &str) -> Result<(), String> { + let status = std::process::Command::new("fusermount3") + .arg("-u") + .arg(mount_point) + .status() + .map_err(|e| format!("failed to run fusermount3: {}", e))?; + + if status.success() { + Ok(()) + } else { + Err(format!( + "fusermount3 -u {} exited with {}", + mount_point, + status.code().unwrap_or(-1) + )) + } + } +} diff --git a/slosh-fuse/src/socket_server.rs b/slosh-fuse/src/socket_server.rs new file mode 100644 index 0000000000..d9dee78bfc --- /dev/null +++ b/slosh-fuse/src/socket_server.rs @@ -0,0 +1,211 @@ +use crate::file_mapping::FileMapping; +use crate::mount_manager::MountManager; +use base64::Engine; +use std::io::{BufRead, BufReader, Write}; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +/// Run the daemon accept loop. Blocks until `shutdown` is set. +pub fn run_server( + socket_path: &Path, + shutdown: Arc, +) -> std::io::Result<()> { + // Remove stale socket if it exists + let _ = std::fs::remove_file(socket_path); + + let listener = UnixListener::bind(socket_path)?; + listener.set_nonblocking(true)?; + + log::info!("Listening on {:?}", socket_path); + + let manager = Arc::new(Mutex::new(MountManager::new())); + + while !shutdown.load(Ordering::Relaxed) { + match listener.accept() { + Ok((stream, _addr)) => { + log::info!("Client connected"); + let mgr = Arc::clone(&manager); + let shut = Arc::clone(&shutdown); + std::thread::Builder::new() + .name("fuse-client".to_string()) + .spawn(move || { + if let Err(e) = handle_client(stream, mgr, shut) { + log::error!("Client handler error: {}", e); + } + log::info!("Client disconnected"); + }) + .ok(); + } + Err(ref e) if e.kind() == std::io::ErrorKind::WouldBlock => { + // No pending connection, sleep briefly + std::thread::sleep(std::time::Duration::from_millis(100)); + } + Err(e) => { + log::error!("Accept error: {}", e); + std::thread::sleep(std::time::Duration::from_millis(100)); + } + } + } + + log::info!("Shutting down server"); + + // Clean unmount all + { + let mut mgr = manager.lock().unwrap(); + mgr.unmount_all(); + } + + // Remove socket + let _ = std::fs::remove_file(socket_path); + + Ok(()) +} + +fn handle_client( + stream: UnixStream, + manager: Arc>, + shutdown: Arc, +) -> std::io::Result<()> { + stream.set_nonblocking(false)?; + let reader = BufReader::new(stream.try_clone()?); + let mut writer = stream; + + for line in reader.lines() { + if shutdown.load(Ordering::Relaxed) { + break; + } + + let line = line?; + if line.is_empty() { + continue; + } + + let response = dispatch_command(&line, &manager); + writer.write_all(response.as_bytes())?; + writer.write_all(b"\n")?; + writer.flush()?; + } + + Ok(()) +} + +fn dispatch_command( + line: &str, + manager: &Arc>, +) -> String { + let parts: Vec<&str> = line.split('\t').collect(); + let cmd = parts.first().copied().unwrap_or(""); + + match cmd { + "PING" => "OK\tPONG".to_string(), + + "MOUNT" if parts.len() == 2 => { + let mount_point = parts[1]; + match manager.lock().unwrap().mount(mount_point) { + Ok(mount_id) => format!("OK\t{}", mount_id), + Err(e) => format!("ERR\t{}", e), + } + } + + "REGISTER" if parts.len() == 4 => { + let mount_id = parts[1]; + let path = parts[2]; + let b64 = parts[3]; + + let mgr = manager.lock().unwrap(); + if let Some(mapping) = mgr.get_mapping(mount_id) { + match base64::engine::general_purpose::STANDARD.decode(b64) { + Ok(content) => { + mapping.lock().unwrap().register(path, content); + "OK".to_string() + } + Err(e) => format!("ERR\tbase64 decode failed: {}", e), + } + } else { + format!("ERR\tunknown mount-id: {}", mount_id) + } + } + + "CONCAT" if parts.len() >= 4 => { + let mount_id = parts[1]; + let vpath = parts[2]; + let source_paths: Vec = + parts[3..].iter().map(|p| PathBuf::from(p)).collect(); + + let mgr = manager.lock().unwrap(); + if let Some(mapping) = mgr.get_mapping(mount_id) { + mapping.lock().unwrap().register_concat(vpath, source_paths); + "OK".to_string() + } else { + format!("ERR\tunknown mount-id: {}", mount_id) + } + } + + "REMOVE" if parts.len() == 3 => { + let mount_id = parts[1]; + let path = parts[2]; + + let mgr = manager.lock().unwrap(); + if let Some(mapping) = mgr.get_mapping(mount_id) { + mapping.lock().unwrap().remove(path); + "OK".to_string() + } else { + format!("ERR\tunknown mount-id: {}", mount_id) + } + } + + "UNMOUNT" if parts.len() == 2 => { + let mount_id = parts[1]; + match manager.lock().unwrap().unmount(mount_id) { + Ok(()) => "OK".to_string(), + Err(e) => format!("ERR\t{}", e), + } + } + + "LIST" if parts.len() == 2 => { + let mount_id = parts[1]; + let mgr = manager.lock().unwrap(); + if let Some(mapping) = mgr.get_mapping(mount_id) { + let files = mapping.lock().unwrap().list_files(); + let mut resp = "OK".to_string(); + for f in files { + resp.push('\t'); + resp.push_str(&f); + } + resp + } else { + format!("ERR\tunknown mount-id: {}", mount_id) + } + } + + _ => format!("ERR\tunknown command: {}", line), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn dispatch_ping() { + let mgr = Arc::new(Mutex::new(MountManager::new())); + let resp = dispatch_command("PING", &mgr); + assert_eq!(resp, "OK\tPONG"); + } + + #[test] + fn dispatch_unknown() { + let mgr = Arc::new(Mutex::new(MountManager::new())); + let resp = dispatch_command("BOGUS", &mgr); + assert!(resp.starts_with("ERR")); + } + + #[test] + fn dispatch_register_unknown_mount() { + let mgr = Arc::new(Mutex::new(MountManager::new())); + let resp = dispatch_command("REGISTER\tbad-id\tfile.txt\tYQ==", &mgr); + assert!(resp.starts_with("ERR")); + } +} diff --git a/slosh-fuse/test-fuse.sh b/slosh-fuse/test-fuse.sh index 24acf476dc..c9356d7372 100755 --- a/slosh-fuse/test-fuse.sh +++ b/slosh-fuse/test-fuse.sh @@ -2,7 +2,7 @@ set -e MOUNT_POINT="/tmp/test-eval-fs" -SERVER_BIN="./target/debug/slosh-fuse-server" +SLOSH_BIN="./target/debug/slosh" PASSED=0 FAILED=0 @@ -10,180 +10,168 @@ pass() { PASSED=$((PASSED + 1)); echo " PASS: $1"; } fail() { FAILED=$((FAILED + 1)); echo " FAIL: $1"; } cleanup() { - exec 3>&- 2>/dev/null || true - if [ -n "$SERVER_PID" ] && kill -0 "$SERVER_PID" 2>/dev/null; then - kill "$SERVER_PID" 2>/dev/null - wait "$SERVER_PID" 2>/dev/null || true + # Stop daemon if we started it + if [ -n "$DAEMON_PID" ] && kill -0 "$DAEMON_PID" 2>/dev/null; then + kill "$DAEMON_PID" 2>/dev/null + wait "$DAEMON_PID" 2>/dev/null || true fi - # AutoUnmount should handle this, but just in case fusermount3 -u "$MOUNT_POINT" 2>/dev/null || true rm -rf "$MOUNT_POINT" - rm -f "$FIFO" rm -rf /tmp/fuse-concat-test-* 2>/dev/null || true + # Clean up daemon socket/pid + rm -f "$HOME/.local/share/slosh/fuse.sock" 2>/dev/null || true + rm -f "$HOME/.local/share/slosh/fuse-daemon.pid" 2>/dev/null || true } trap cleanup EXIT -echo "=== FUSE Server Test ===" +echo "=== FUSE Daemon Test ===" # Ensure binary exists -if [ ! -f "$SERVER_BIN" ]; then - echo "Building slosh-fuse-server..." - cargo build -p slosh-fuse +if [ ! -f "$SLOSH_BIN" ]; then + echo "Building slosh with fuse feature..." + cargo build -p slosh --features fuse +fi + +# Start daemon in foreground mode (background it ourselves) +"$SLOSH_BIN" --fuse-daemon-foreground & +DAEMON_PID=$! +echo "Daemon PID: $DAEMON_PID" + +# Wait for socket to appear +SOCK="$HOME/.local/share/slosh/fuse.sock" +for i in $(seq 1 30); do + if [ -S "$SOCK" ]; then + break + fi + sleep 0.2 +done + +if [ ! -S "$SOCK" ]; then + fail "Daemon socket did not appear" + exit 1 fi # Create mount point mkdir -p "$MOUNT_POINT" -# Start the server with stdin piped from a fifo. -# The server opens the FIFO read end in background, then we open the -# write end. This order avoids a deadlock: opening a FIFO for writing -# blocks until a reader exists, and vice versa. -FIFO="/tmp/fuse-test-fifo-$$" -mkfifo "$FIFO" -"$SERVER_BIN" "$MOUNT_POINT" < "$FIFO" & -SERVER_PID=$! -exec 3>"$FIFO" -echo "Server PID: $SERVER_PID" - -# Give the FUSE mount time to come up -sleep 2 - -# Test 1: Mount exists +# Test 1: PING echo "" -echo "--- Test 1: Mount point is a FUSE mount ---" -if mountpoint -q "$MOUNT_POINT" 2>/dev/null || mount | grep -q "$MOUNT_POINT"; then - pass "Mount point is active" +echo "--- Test 1: PING ---" +REPLY=$(echo -e "PING" | socat - UNIX-CONNECT:"$SOCK") +if [ "$REPLY" = "OK PONG" ]; then + pass "PING/PONG works" else - fail "Mount point not detected" + fail "PING response: '$REPLY'" fi -# Test 2: Register a file and read it +# Test 2: MOUNT echo "" -echo "--- Test 2: Register and read a file ---" -CONTENT="hello from fuse" -B64_CONTENT=$(echo -n "$CONTENT" | base64) -printf "REGISTER\ttest.txt\t%s\n" "$B64_CONTENT" >&3 +echo "--- Test 2: MOUNT ---" +MOUNT_ID=$(echo -e "MOUNT\t$MOUNT_POINT" | socat - UNIX-CONNECT:"$SOCK" | cut -f2) +if [ -n "$MOUNT_ID" ] && [ "$MOUNT_ID" != "" ]; then + pass "Got mount ID: $MOUNT_ID" +else + fail "No mount ID returned" +fi -sleep 1 +# Give FUSE mount time to come up +sleep 2 -if [ -f "$MOUNT_POINT/test.txt" ]; then - ACTUAL=$(cat "$MOUNT_POINT/test.txt") - if [ "$ACTUAL" = "$CONTENT" ]; then - pass "File content matches: '$ACTUAL'" - else - fail "Content mismatch: expected '$CONTENT', got '$ACTUAL'" - fi +# Test 3: Mount point is active +echo "" +echo "--- Test 3: Mount point is active ---" +if mountpoint -q "$MOUNT_POINT" 2>/dev/null || mount | grep -q "$MOUNT_POINT"; then + pass "Mount point is active" else - fail "File test.txt not found in mount" + fail "Mount point not detected" fi -# Test 3: Register a second file +# Test 4: Register a file and read it echo "" -echo "--- Test 3: Register a second file ---" -CONTENT2="HOST=myhost" -B64_CONTENT2=$(echo -n "$CONTENT2" | base64) -printf "REGISTER\tconfig.env\t%s\n" "$B64_CONTENT2" >&3 - -sleep 1 - -if [ -f "$MOUNT_POINT/config.env" ]; then - ACTUAL2=$(cat "$MOUNT_POINT/config.env") - if [ "$ACTUAL2" = "$CONTENT2" ]; then - pass "config.env content matches" +echo "--- Test 4: Register and read a file ---" +CONTENT="hello from fuse daemon" +B64_CONTENT=$(echo -n "$CONTENT" | base64) +REPLY=$(echo -e "REGISTER\t$MOUNT_ID\ttest.txt\t$B64_CONTENT" | socat - UNIX-CONNECT:"$SOCK") +if [ "$REPLY" = "OK" ]; then + sleep 1 + if [ -f "$MOUNT_POINT/test.txt" ]; then + ACTUAL=$(cat "$MOUNT_POINT/test.txt") + if [ "$ACTUAL" = "$CONTENT" ]; then + pass "File content matches: '$ACTUAL'" + else + fail "Content mismatch: expected '$CONTENT', got '$ACTUAL'" + fi else - fail "config.env content mismatch: expected '$CONTENT2', got '$ACTUAL2'" + fail "File test.txt not found in mount" fi else - fail "File config.env not found in mount" + fail "REGISTER failed: $REPLY" fi -# Test 4: Directory listing shows both files +# Test 5: LIST files echo "" -echo "--- Test 4: Directory listing ---" -LISTING=$(ls "$MOUNT_POINT") -if echo "$LISTING" | grep -q "test.txt" && echo "$LISTING" | grep -q "config.env"; then - pass "Both files appear in directory listing" +echo "--- Test 5: LIST files ---" +REPLY=$(echo -e "LIST\t$MOUNT_ID" | socat - UNIX-CONNECT:"$SOCK") +if echo "$REPLY" | grep -q "test.txt"; then + pass "LIST includes test.txt" else - fail "Directory listing incomplete: $LISTING" + fail "LIST response: '$REPLY'" fi -# Test 5: Remove a file +# Test 6: REMOVE a file echo "" -echo "--- Test 5: Remove a file ---" -printf "REMOVE\ttest.txt\n" >&3 - -sleep 1 - -if [ ! -f "$MOUNT_POINT/test.txt" ]; then - pass "test.txt removed successfully" +echo "--- Test 6: REMOVE a file ---" +REPLY=$(echo -e "REMOVE\t$MOUNT_ID\ttest.txt" | socat - UNIX-CONNECT:"$SOCK") +if [ "$REPLY" = "OK" ]; then + sleep 1 + if [ ! -f "$MOUNT_POINT/test.txt" ]; then + pass "test.txt removed successfully" + else + fail "test.txt still exists after REMOVE" + fi else - fail "test.txt still exists after REMOVE" + fail "REMOVE failed: $REPLY" fi -# Test 6: CONCAT - create real files and concatenate them +# Test 7: CONCAT echo "" -echo "--- Test 6: CONCAT basic concatenation ---" +echo "--- Test 7: CONCAT basic concatenation ---" CONCAT_DIR="/tmp/fuse-concat-test-$$" mkdir -p "$CONCAT_DIR" echo -n "AAA" > "$CONCAT_DIR/a.txt" echo -n "BBB" > "$CONCAT_DIR/b.txt" echo -n "CCC" > "$CONCAT_DIR/c.txt" -printf "CONCAT\tcombined.txt\t%s/a.txt\t%s/b.txt\t%s/c.txt\n" "$CONCAT_DIR" "$CONCAT_DIR" "$CONCAT_DIR" >&3 - -sleep 1 - -if [ -f "$MOUNT_POINT/combined.txt" ]; then - ACTUAL=$(cat "$MOUNT_POINT/combined.txt") - if [ "$ACTUAL" = "AAABBBCCC" ]; then - pass "CONCAT content matches: '$ACTUAL'" +REPLY=$(echo -e "CONCAT\t$MOUNT_ID\tcombined.txt\t$CONCAT_DIR/a.txt\t$CONCAT_DIR/b.txt\t$CONCAT_DIR/c.txt" | socat - UNIX-CONNECT:"$SOCK") +if [ "$REPLY" = "OK" ]; then + sleep 1 + if [ -f "$MOUNT_POINT/combined.txt" ]; then + ACTUAL=$(cat "$MOUNT_POINT/combined.txt") + if [ "$ACTUAL" = "AAABBBCCC" ]; then + pass "CONCAT content matches: '$ACTUAL'" + else + fail "CONCAT content mismatch: expected 'AAABBBCCC', got '$ACTUAL'" + fi else - fail "CONCAT content mismatch: expected 'AAABBBCCC', got '$ACTUAL'" + fail "combined.txt not found in mount" fi else - fail "combined.txt not found in mount" -fi - -# Test 7: CONCAT reflects changes to source files -echo "" -echo "--- Test 7: CONCAT reflects source file changes ---" -echo -n "XXX" > "$CONCAT_DIR/b.txt" - -ACTUAL=$(cat "$MOUNT_POINT/combined.txt") -if [ "$ACTUAL" = "AAAXXXCCC" ]; then - pass "CONCAT reflects file change: '$ACTUAL'" -else - fail "CONCAT did not reflect change: expected 'AAAXXXCCC', got '$ACTUAL'" + fail "CONCAT failed: $REPLY" fi -# Test 8: grep works across CONCAT virtual file +# Test 8: UNMOUNT echo "" -echo "--- Test 8: grep across CONCAT virtual file ---" -printf "line1 apple\n" > "$CONCAT_DIR/a.txt" -printf "line2 banana\n" > "$CONCAT_DIR/b.txt" -printf "line3 cherry\n" > "$CONCAT_DIR/c.txt" - -sleep 1 - -if grep -q "banana" "$MOUNT_POINT/combined.txt"; then - pass "grep found 'banana' in concatenated file" +echo "--- Test 8: UNMOUNT ---" +REPLY=$(echo -e "UNMOUNT\t$MOUNT_ID" | socat - UNIX-CONNECT:"$SOCK") +if [ "$REPLY" = "OK" ]; then + pass "UNMOUNT succeeded" else - fail "grep did not find 'banana' in concatenated file" -fi - -if grep -q "cherry" "$MOUNT_POINT/combined.txt"; then - pass "grep found 'cherry' in concatenated file" -else - fail "grep did not find 'cherry' in concatenated file" + fail "UNMOUNT failed: $REPLY" fi # Clean up concat test files rm -rf "$CONCAT_DIR" -# Close the fifo writer fd -exec 3>&- -rm -f "$FIFO" - echo "" echo "=== Results: $PASSED passed, $FAILED failed ===" if [ "$FAILED" -gt 0 ]; then diff --git a/slosh/src/config.rs b/slosh/src/config.rs index a1695d91ee..4cfca41683 100644 --- a/slosh/src/config.rs +++ b/slosh/src/config.rs @@ -10,8 +10,10 @@ USAGE: slosh [FLAGS] [OPTIONS] [args] FLAGS: - -v, --version Print the version, platform and revision of sl-sh then exit. - -h, --help Print help (this) and exit. + -v, --version Print the version, platform and revision of sl-sh then exit. + -h, --help Print help (this) and exit. + --fuse-daemon Start as FUSE daemon (detaches from terminal). + --fuse-daemon-foreground Start as FUSE daemon in foreground (for debugging). OPTIONS: -c Command to run instead of entering the REPL. @@ -41,6 +43,8 @@ pub fn get_config() -> Option { let mut command: Option = None; let mut script: Option = None; let mut command_args: Vec = Vec::new(); + let mut fuse_daemon = false; + let mut fuse_daemon_foreground = false; let mut args: Vec = env::args_os().collect(); @@ -66,6 +70,12 @@ pub fn get_config() -> Option { help(&exe_name); return None; } + "--fuse-daemon" => { + fuse_daemon = true; + } + "--fuse-daemon-foreground" => { + fuse_daemon_foreground = true; + } _ => { if command.is_none() && script.is_none() { script = Some(arg); @@ -84,5 +94,7 @@ pub fn get_config() -> Option { command, script, args: command_args, + fuse_daemon, + fuse_daemon_foreground, }) } diff --git a/slosh_lib/src/fuse_bindings.rs b/slosh_lib/src/fuse_bindings.rs index 5f2ce28849..bcf5683390 100644 --- a/slosh_lib/src/fuse_bindings.rs +++ b/slosh_lib/src/fuse_bindings.rs @@ -4,9 +4,10 @@ use slvm::{VMError, VMResult, Value}; use std::collections::HashMap; use std::sync::Mutex; use std::path::PathBuf; -use std::process::{Command, Stdio}; use slosh_fuse::FuseMount; +use slosh_fuse::auto_start::ensure_daemon_running; +use slosh_fuse::daemon::DaemonConfig; use std::sync::OnceLock; static MOUNT_REGISTRY: OnceLock>> = OnceLock::new(); @@ -31,52 +32,16 @@ fn mount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { .map_err(|e| VMError::new_vm(format!("Failed to create mount point: {}", e)))?; } - let mount_id = format!("mount-{}", uuid::Uuid::new_v4()); - let mut mount = FuseMount::new(mount_point.clone()); - - let server_name = "slosh-fuse-server"; - - let possible_paths = vec![ - std::env::current_exe() - .ok() - .and_then(|p| p.parent().map(|p| p.join(server_name))), - std::env::current_dir() - .ok() - .map(|p| { - p.join("target") - .join(if cfg!(debug_assertions) { "debug" } else { "release" }) - .join(server_name) - }), - Some(PathBuf::from(server_name)), - ]; - - let server_path = possible_paths - .into_iter() - .flatten() - .find(|p| p.exists()) - .ok_or_else(|| { - VMError::new_vm( - "Could not find slosh-fuse-server binary. \ - Build it with 'cargo build --bin slosh-fuse-server'" - .to_string(), - ) - })?; - - let mut child = Command::new(&server_path) - .arg(mount_point.to_str().unwrap()) - .stdin(Stdio::piped()) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn() - .map_err(|e| { - VMError::new_vm(format!( - "Failed to spawn FUSE server at {:?}: {}", - server_path, e - )) - })?; - - mount.stdin_handle = child.stdin.take(); - mount.process_handle = Some(child); + let config = DaemonConfig::default_for_user() + .map_err(|e| VMError::new_vm(format!("Failed to get daemon config: {}", e)))?; + + let client = ensure_daemon_running(&config) + .map_err(|e| VMError::new_vm(format!("Failed to connect to FUSE daemon: {}", e)))?; + + let mount = FuseMount::new_daemon(mount_point, client) + .map_err(|e| VMError::new_vm(format!("Failed to create mount: {}", e)))?; + + let mount_id = mount.mount_id.clone(); get_registry() .lock() @@ -174,18 +139,18 @@ fn list_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { let mount_id = registers[0].get_string(vm)?; - let registry = get_registry() + let mut registry = get_registry() .lock() .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - if let Some(mount) = registry.get(mount_id) { - let files: Vec = mount - .registered_paths + if let Some(mount) = registry.get_mut(mount_id) { + let files = mount.list_files() + .map_err(|e| VMError::new_vm(format!("Failed to list files: {}", e)))?; + let values: Vec = files .iter() .map(|f| vm.alloc_string(f.clone())) .collect(); - - Ok(vm.alloc_vector(files)) + Ok(vm.alloc_vector(values)) } else { Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) } @@ -221,6 +186,7 @@ pub fn add_fuse_builtins(env: &mut SloshVm) { r#"Usage: (mount-eval-fs path) Mount an evaluation filesystem at the specified path. Returns a mount ID. +Uses a shared per-user FUSE daemon (auto-started if needed). Section: fuse @@ -236,7 +202,7 @@ Example: r#"Usage: (register-eval-file mount-id path content) Register a file with static content in the FUSE filesystem. -Content is evaluated at registration time and sent to the FUSE server. +Content is evaluated at registration time and sent to the FUSE daemon. Section: fuse diff --git a/slosh_lib/src/lib.rs b/slosh_lib/src/lib.rs index d916193d03..638e493530 100644 --- a/slosh_lib/src/lib.rs +++ b/slosh_lib/src/lib.rs @@ -648,6 +648,19 @@ pub fn run(modify_vm: impl FnOnce(&mut SloshVm)) -> i32 { pub fn run_slosh(modify_vm: impl FnOnce(&mut SloshVm)) -> i32 { let mut status = 0; if let Some(config) = get_config() { + // Handle --fuse-daemon before VM initialization (daemon doesn't need VM) + #[cfg(feature = "fuse")] + if config.fuse_daemon || config.fuse_daemon_foreground { + let foreground = config.fuse_daemon_foreground; + match slosh_fuse::run_fuse_daemon(foreground) { + Ok(()) => return 0, + Err(e) => { + eprintln!("FUSE daemon error: {}", e); + return 1; + } + } + } + ENV.with(|renv| { let mut env = renv.borrow_mut(); env.pause_gc(); From e336f68466f698c1004c3c2941d2bdd541f171b3 Mon Sep 17 00:00:00 2001 From: gpwclark Date: Sun, 8 Mar 2026 11:40:26 -0400 Subject: [PATCH 08/12] edits to daemon mode --- Cargo.lock | 21 +++++++++++++++++++++ slosh-fuse/Cargo.toml | 1 + slosh-fuse/src/daemon.rs | 16 +++++++++++----- slosh-fuse/src/lib.rs | 29 +++-------------------------- slosh-fuse/src/socket_server.rs | 5 ++--- 5 files changed, 38 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7564f9520a..660601a97a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2115,6 +2115,26 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + [[package]] name = "siphasher" version = "0.3.11" @@ -2207,6 +2227,7 @@ dependencies = [ "libc", "log", "nix", + "signal-hook", "tempfile", ] diff --git a/slosh-fuse/Cargo.toml b/slosh-fuse/Cargo.toml index af0cabfd4b..c52b5ab18c 100644 --- a/slosh-fuse/Cargo.toml +++ b/slosh-fuse/Cargo.toml @@ -11,3 +11,4 @@ env_logger = "0.11" base64 = "0.21" tempfile = "3.0" nix = { workspace = true, features = ["signal", "process", "fs"] } +signal-hook = "0.3" diff --git a/slosh-fuse/src/daemon.rs b/slosh-fuse/src/daemon.rs index 455f7c6157..f54ce08353 100644 --- a/slosh-fuse/src/daemon.rs +++ b/slosh-fuse/src/daemon.rs @@ -2,6 +2,8 @@ use std::fs::{self, File}; use std::io; use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; +use nix::sys::stat::umask; + /// Paths used by the FUSE daemon. pub struct DaemonConfig { @@ -120,17 +122,21 @@ pub fn daemonize() -> io::Result<()> { Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), } + // Clear the file mode creation mask + umask(Mode::empty()); + + // Change working directory so we don't hold a mountpoint + std::env::set_current_dir("/")?; + // Redirect stdio to /dev/null let devnull = fs::OpenOptions::new() .read(true) .write(true) .open("/dev/null")?; let fd = devnull.as_raw_fd(); - unsafe { - libc::dup2(fd, 0); - libc::dup2(fd, 1); - libc::dup2(fd, 2); - } + unistd::dup2(fd, 0); + unistd::dup2(fd, 1); + unistd::dup2(fd, 2); Ok(()) } diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs index ca98289c98..c7ec5d5f03 100644 --- a/slosh-fuse/src/lib.rs +++ b/slosh-fuse/src/lib.rs @@ -79,8 +79,8 @@ impl FuseMount { pub fn run_fuse_daemon(foreground: bool) -> Result<(), Box> { use crate::daemon::{DaemonConfig, PidFile, daemonize, init_daemon_logging}; use crate::socket_server::run_server; + use std::sync::atomic::AtomicBool; use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; let config = DaemonConfig::default_for_user()?; @@ -96,34 +96,11 @@ pub fn run_fuse_daemon(foreground: bool) -> Result<(), Box>, - shutdown: Arc, + shutdown: &AtomicBool, ) -> std::io::Result<()> { stream.set_nonblocking(false)?; let reader = BufReader::new(stream.try_clone()?); From d61b4d688672b4c219ceaf6fa19fe43a286a0672 Mon Sep 17 00:00:00 2001 From: gpwclark Date: Sun, 8 Mar 2026 15:11:34 -0400 Subject: [PATCH 09/12] remove unused examples code, update to allow concatenating files that are also fuse hosted --- slosh-fuse/Containerfile | 56 ++ slosh-fuse/Containerfile_static | 56 ++ slosh-fuse/README.md | 46 +- slosh-fuse/examples/slosh-fuse/Cargo.toml | 19 - slosh-fuse/examples/slosh-fuse/README.md | 63 --- .../slosh-fuse/src/bin/slosh-fuse-server.rs | 44 -- slosh-fuse/examples/slosh-fuse/src/eval_fs.rs | 282 ---------- .../examples/slosh-fuse/src/file_mapping.rs | 95 ---- slosh-fuse/examples/slosh-fuse/src/lib.rs | 37 -- slosh-fuse/src/daemon.rs | 12 +- slosh-fuse/src/eval_fs.rs | 29 +- slosh-fuse/src/lib.rs | 2 + slosh-fuse/src/mount_manager.rs | 12 +- slosh-fuse/src/resolve.rs | 493 ++++++++++++++++++ 14 files changed, 685 insertions(+), 561 deletions(-) create mode 100644 slosh-fuse/Containerfile create mode 100644 slosh-fuse/Containerfile_static delete mode 100644 slosh-fuse/examples/slosh-fuse/Cargo.toml delete mode 100644 slosh-fuse/examples/slosh-fuse/README.md delete mode 100644 slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs delete mode 100644 slosh-fuse/examples/slosh-fuse/src/eval_fs.rs delete mode 100644 slosh-fuse/examples/slosh-fuse/src/file_mapping.rs delete mode 100644 slosh-fuse/examples/slosh-fuse/src/lib.rs create mode 100644 slosh-fuse/src/resolve.rs diff --git a/slosh-fuse/Containerfile b/slosh-fuse/Containerfile new file mode 100644 index 0000000000..e7c5593a95 --- /dev/null +++ b/slosh-fuse/Containerfile @@ -0,0 +1,56 @@ +# Containerfile — development toolchain for slosh-fuse (volume-mounted source) +# +# This is the primary development container. Source code is mounted at runtime +# (not copied), so edits are reflected instantly and incremental builds are fast. +# Includes socat for debugging the daemon's Unix domain socket. +# Runs as non-root user "slosh" with XDG dirs pre-created. +# +# Build the image (from repo root, once or after changing this file): +# docker build -t dyn-slosh-fuse -f slosh-fuse/Containerfile . +# +# Run unit tests (no FUSE device needed): +# docker run --rm -v "$PWD:/home/slosh/src" dyn-slosh-fuse cargo test -p slosh-fuse +# +# Run the full integration test (needs FUSE device access): +# docker run --rm -v "$PWD:/home/slosh/src" --cap-add SYS_ADMIN --device /dev/fuse \ +# dyn-slosh-fuse bash slosh-fuse/test-fuse.sh +# +# Build slosh-fuse: +# docker run --rm -v "$PWD:/home/slosh/src" dyn-slosh-fuse cargo build -p slosh-fuse +# +# Interactive shell for debugging: +# docker run --rm -it -v "$PWD:/home/slosh/src" --cap-add SYS_ADMIN --device /dev/fuse \ +# dyn-slosh-fuse bash + +FROM ubuntu:24.04 + +RUN apt-get update && apt-get install -y \ + fuse3 \ + libfuse3-dev \ + build-essential \ + curl \ + pkg-config \ + git \ + socat \ + && rm -rf /var/lib/apt/lists/* + +# Allow non-root users to use allow_other mount option +RUN echo 'user_allow_other' >> /etc/fuse.conf + +# Create non-root user with XDG directories +RUN useradd -m -s /bin/bash slosh \ + && mkdir -p /home/slosh/.config /home/slosh/.local/share \ + && chown -R slosh:slosh /home/slosh + +# Install Rust for the slosh user +USER slosh +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/home/slosh/.cargo/bin:${PATH}" + +# Trust volume-mounted source (host UID differs from container user) +RUN git config --global --add safe.directory /home/slosh/src + +# Pre-fetch crate index so first build is faster +RUN cargo init /tmp/warmup && cd /tmp/warmup && cargo fetch 2>/dev/null; rm -rf /tmp/warmup + +WORKDIR /home/slosh/src \ No newline at end of file diff --git a/slosh-fuse/Containerfile_static b/slosh-fuse/Containerfile_static new file mode 100644 index 0000000000..163ce8082e --- /dev/null +++ b/slosh-fuse/Containerfile_static @@ -0,0 +1,56 @@ +# Containerfile_static — self-contained build (source copied into image) +# +# This container copies the source tree into the image at build time and +# compiles everything during `docker build`. The resulting image is standalone +# and doesn't need the host source tree at runtime. Slower to rebuild after +# edits (re-sends build context + full recompile), but useful for testing a +# stable version more robustly or for CI. +# Runs as non-root user "slosh" with XDG dirs pre-created. +# +# Make sure .dockerignore excludes target/ and .git/ to keep context small. +# +# Build the image (from repo root, copies source and compiles): +# docker build -t slosh-fuse-static -f slosh-fuse/Containerfile_static . +# +# Run unit tests: +# docker run --rm slosh-fuse-static cargo test -p slosh-fuse +# +# Run the full integration test: +# docker run --rm --cap-add SYS_ADMIN --device /dev/fuse \ +# slosh-fuse-static bash slosh-fuse/test-fuse.sh + +FROM ubuntu:24.04 + +RUN apt-get update && apt-get install -y \ + fuse3 \ + libfuse3-dev \ + build-essential \ + curl \ + pkg-config \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Allow non-root users to use allow_other mount option +RUN echo 'user_allow_other' >> /etc/fuse.conf + +# Create non-root user with XDG directories +RUN useradd -m -s /bin/bash slosh \ + && mkdir -p /home/slosh/.config /home/slosh/.local/share \ + && chown -R slosh:slosh /home/slosh + +# Install Rust for the slosh user +USER slosh +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y +ENV PATH="/home/slosh/.cargo/bin:${PATH}" + +# Pre-fetch crate index so first build is faster +RUN cargo init /tmp/warmup && cd /tmp/warmup && cargo fetch 2>/dev/null; rm -rf /tmp/warmup + +COPY --chown=slosh:slosh . /home/slosh/src +WORKDIR /home/slosh/src + +# Build slosh-fuse crate (binary + lib) +RUN cargo build -p slosh-fuse + +# Build slosh with fuse feature +RUN cargo build -p slosh_lib --features fuse \ No newline at end of file diff --git a/slosh-fuse/README.md b/slosh-fuse/README.md index af6750d814..c1738f45be 100644 --- a/slosh-fuse/README.md +++ b/slosh-fuse/README.md @@ -59,10 +59,50 @@ Requires Linux with libfuse3 (FUSE doesn't work natively on macOS). ```bash # On Linux cargo build -p slosh --features fuse +``` + +### Containers (for macOS or CI) + +Two Containerfiles live in `slosh-fuse/`. Both are built from the **repo root** and +run as a non-root `slosh` user with `~/.config` and `~/.local/share` pre-created. + +#### `Containerfile` — active development + +Source code is **volume-mounted** at runtime, not copied into the image. Edits are +reflected instantly and incremental builds are fast. Includes `socat` for socket debugging. + +```bash +# Build the toolchain image (once, or after changing the Containerfile) +docker build -t dyn-slosh-fuse -f slosh-fuse/Containerfile . + +# Run unit tests (no FUSE device needed) +docker run --rm -v "$PWD:/home/slosh/src" dyn-slosh-fuse cargo test -p slosh-fuse + +# Run the full integration test +docker run --rm -v "$PWD:/home/slosh/src" --cap-add SYS_ADMIN --device /dev/fuse \ + dyn-slosh-fuse bash slosh-fuse/test-fuse.sh + +# Interactive shell for debugging +docker run --rm -it -v "$PWD:/home/slosh/src" --cap-add SYS_ADMIN --device /dev/fuse \ + dyn-slosh-fuse bash +``` + +#### `Containerfile_static` — stable/CI testing + +Source is **copied into the image** and compiled during `docker build`. The resulting +image is self-contained. Slower to rebuild after edits, but useful for testing a stable +version more robustly or in CI where you want a reproducible artifact. + +```bash +# Build (copies source, compiles everything) +docker build -t slosh-fuse-static -f slosh-fuse/Containerfile_static . + +# Run unit tests +docker run --rm slosh-fuse-static cargo test -p slosh-fuse -# On macOS, use an Apple Container -container build -t slosh-fuse -f Containerfile . -container run --name slosh-fuse slosh-fuse bash /src/slosh-fuse/test-fuse.sh +# Run the full integration test +docker run --rm --cap-add SYS_ADMIN --device /dev/fuse \ + slosh-fuse-static bash slosh-fuse/test-fuse.sh ``` ## Testing diff --git a/slosh-fuse/examples/slosh-fuse/Cargo.toml b/slosh-fuse/examples/slosh-fuse/Cargo.toml deleted file mode 100644 index 086bcaabe4..0000000000 --- a/slosh-fuse/examples/slosh-fuse/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "slosh-fuse" -version.workspace = true -edition = "2024" - -[dependencies] -fuser = "0.14" -libc = "0.2" -log = "0.4" -env_logger = "0.11" - -# Local slosh dependencies -vm = { path = "../vm" } -compile_state = { path = "../compile_state" } -slosh_lib = { path = "../slosh_lib" } - -[dependencies.nix] -version = "0.27" -features = ["signal", "process", "fs"] \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/README.md b/slosh-fuse/examples/slosh-fuse/README.md deleted file mode 100644 index 99a26fd5c6..0000000000 --- a/slosh-fuse/examples/slosh-fuse/README.md +++ /dev/null @@ -1,63 +0,0 @@ -# Slosh FUSE Integration - -This module provides FUSE (Filesystem in Userspace) integration for slosh, allowing you to create dynamic files whose contents are generated by evaluating slosh expressions. - -## Use Case - -The primary use case is creating dynamic configuration files for services like systemd, where environment files need to contain values from slosh variables, environment variables, or computed values. - -For example, a systemd service file at `~/.config/systemd/user/my-service.env` could dynamically generate content like: - -```bash -HOST=meow -CUSTOM_TYPE=value-of-global -``` - -Where `meow` is the hostname and `value-of-global` comes from a slosh global variable. - -## Architecture - -The implementation consists of: - -1. **slosh-fuse crate**: Implements the FUSE filesystem that serves dynamic files -2. **Builtin functions**: Slosh functions to mount/unmount filesystems and register files -3. **Expression evaluation**: Files contain slosh expressions that are evaluated on read - -## Usage - -```lisp -;; Mount a dynamic filesystem -(def mount-id (mount-eval-fs "/tmp/dynamic")) - -;; Register a file with dynamic content -(register-eval-file mount-id "config.env" - "(str \"HOST=\" (hostname) \"\\n\" - \"TYPE=\" *my-global-var* \"\\n\")") - -;; The file /tmp/dynamic/config.env will now return evaluated content when read - -;; Unmount when done -(unmount-eval-fs mount-id) -``` - -## Implementation Status - -The current implementation provides: -- Basic FUSE filesystem structure -- File registration and mapping system -- Builtin functions for mounting and file management -- Example scripts demonstrating the concept - -Full FUSE integration with expression evaluation requires: -- Process forking and IPC for evaluation -- FUSE mount/unmount handling -- Security considerations for file access - -## Alternative Approach - -For simpler use cases, the `dynamic-env-generator.sl` example shows how to achieve similar functionality without FUSE by: -- Registering file paths with generator functions -- Periodically regenerating files when configuration changes -- Using standard file I/O operations - -This approach is simpler to implement and doesn't require FUSE support. \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs b/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs deleted file mode 100644 index e546353baa..0000000000 --- a/slosh-fuse/examples/slosh-fuse/src/bin/slosh-fuse-server.rs +++ /dev/null @@ -1,44 +0,0 @@ -use fuser::{MountOption, Session}; -use slosh_fuse::{EvalFs, FileMapping}; -use std::sync::{Arc, Mutex}; -use std::env; -use std::os::unix::io::{RawFd, FromRawFd}; - -fn main() { - env_logger::init(); - - let args: Vec = env::args().collect(); - if args.len() < 4 { - eprintln!("Usage: {} ", args[0]); - std::process::exit(1); - } - - let mount_point = &args[1]; - let read_fd: RawFd = args[2].parse().expect("Invalid read fd"); - let write_fd: RawFd = args[3].parse().expect("Invalid write fd"); - - let file_mapping = Arc::new(Mutex::new(FileMapping::new())); - - // Pre-populate with test data if provided - if args.len() > 4 { - let mut mapping = file_mapping.lock().unwrap(); - for i in (4..args.len()).step_by(2) { - if i + 1 < args.len() { - mapping.register(&args[i], args[i + 1].clone()); - } - } - } - - let fs = EvalFs::new(file_mapping, read_fd, write_fd); - - let options = vec![ - MountOption::FSName("slosh-eval".to_string()), - MountOption::AllowOther, - MountOption::DefaultPermissions, - ]; - - let session = Session::new(fs, &mount_point, &options).expect("Failed to mount"); - - // Run until unmounted - session.run().expect("Failed to run session"); -} \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs b/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs deleted file mode 100644 index e06147ec80..0000000000 --- a/slosh-fuse/examples/slosh-fuse/src/eval_fs.rs +++ /dev/null @@ -1,282 +0,0 @@ -use fuser::{ - FileAttr, FileType, Filesystem, ReplyAttr, ReplyData, ReplyDirectory, ReplyEntry, - Request, FUSE_ROOT_ID, -}; -use libc::{ENOENT, ENOTDIR}; -use std::collections::HashMap; -use std::ffi::OsStr; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use std::os::unix::io::{RawFd, FromRawFd, IntoRawFd}; -use std::io::{Read, Write}; - -use crate::file_mapping::FileMapping; - -const TTL: Duration = Duration::from_secs(1); - -pub struct EvalFs { - file_mapping: Arc>, - inode_to_path: HashMap, - path_to_inode: HashMap, - next_inode: u64, - eval_pipe_read: RawFd, - eval_pipe_write: RawFd, -} - -impl EvalFs { - pub fn new(file_mapping: Arc>, eval_pipe_read: RawFd, eval_pipe_write: RawFd) -> Self { - let mut fs = Self { - file_mapping, - inode_to_path: HashMap::new(), - path_to_inode: HashMap::new(), - next_inode: 2, // 1 is reserved for root - eval_pipe_read, - eval_pipe_write, - }; - - // Pre-populate inodes for registered files - let mapping = fs.file_mapping.lock().unwrap(); - for path in mapping.list_files() { - fs.allocate_inode(&path); - } - drop(mapping); - - fs - } - - fn allocate_inode(&mut self, path: &str) -> u64 { - if let Some(&inode) = self.path_to_inode.get(path) { - return inode; - } - - let inode = self.next_inode; - self.next_inode += 1; - self.inode_to_path.insert(inode, path.to_string()); - self.path_to_inode.insert(path.to_string(), inode); - inode - } - - fn get_inode_for_path(&self, path: &str) -> Option { - self.path_to_inode.get(path).copied() - } - - fn get_path_for_inode(&self, inode: u64) -> Option<&str> { - self.inode_to_path.get(&inode).map(|s| s.as_str()) - } - - fn evaluate_expression(&self, expression: &str) -> String { - // Send expression to parent process for evaluation - let msg = format!("{}\n", expression); - let mut pipe_write = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_write) }; - if let Err(e) = pipe_write.write_all(msg.as_bytes()) { - log::error!("Failed to send expression for evaluation: {}", e); - // Prevent closing the fd when File is dropped - let _ = pipe_write.into_raw_fd(); - return format!("ERROR: {}", e); - } - // Prevent closing the fd when File is dropped - let _ = pipe_write.into_raw_fd(); - - // Read response - let mut pipe_read = unsafe { std::fs::File::from_raw_fd(self.eval_pipe_read) }; - let mut response = String::new(); - let mut buffer = [0u8; 4096]; - - loop { - match pipe_read.read(&mut buffer) { - Ok(0) => break, - Ok(n) => { - response.push_str(&String::from_utf8_lossy(&buffer[..n])); - if response.ends_with('\n') { - response.pop(); // Remove trailing newline - break; - } - } - Err(e) => { - log::error!("Failed to read evaluation result: {}", e); - // Prevent closing the fd when File is dropped - let _ = pipe_read.into_raw_fd(); - return format!("ERROR: {}", e); - } - } - } - // Prevent closing the fd when File is dropped - let _ = pipe_read.into_raw_fd(); - - response - } - - fn file_attr(inode: u64, size: u64) -> FileAttr { - let now = SystemTime::now(); - FileAttr { - ino: inode, - size, - blocks: 1, - atime: now, - mtime: now, - ctime: now, - crtime: now, - kind: FileType::RegularFile, - perm: 0o644, - nlink: 1, - uid: unsafe { libc::getuid() }, - gid: unsafe { libc::getgid() }, - rdev: 0, - blksize: 512, - flags: 0, - } - } - - fn dir_attr(inode: u64) -> FileAttr { - let now = SystemTime::now(); - FileAttr { - ino: inode, - size: 0, - blocks: 0, - atime: now, - mtime: now, - ctime: now, - crtime: now, - kind: FileType::Directory, - perm: 0o755, - nlink: 2, - uid: unsafe { libc::getuid() }, - gid: unsafe { libc::getgid() }, - rdev: 0, - blksize: 512, - flags: 0, - } - } -} - -impl Filesystem for EvalFs { - fn lookup(&mut self, _req: &Request, parent: u64, name: &OsStr, reply: ReplyEntry) { - if parent != FUSE_ROOT_ID { - reply.error(ENOENT); - return; - } - - let name_str = name.to_string_lossy(); - let mapping = self.file_mapping.lock().unwrap(); - - if mapping.get(&name_str).is_some() { - let inode = self.allocate_inode(&name_str); - reply.entry(&TTL, &Self::file_attr(inode, 0), 0); - } else { - reply.error(ENOENT); - } - } - - fn getattr(&mut self, _req: &Request, ino: u64, reply: ReplyAttr) { - if ino == FUSE_ROOT_ID { - reply.attr(&TTL, &Self::dir_attr(FUSE_ROOT_ID)); - } else if let Some(path) = self.get_path_for_inode(ino) { - let mapping = self.file_mapping.lock().unwrap(); - if let Some(entry) = mapping.get(path) { - // Get content to determine size - let content = if entry.is_cache_valid() { - entry.cached_value.as_ref().unwrap().clone() - } else { - drop(mapping); - self.evaluate_expression(&entry.expression) - }; - reply.attr(&TTL, &Self::file_attr(ino, content.len() as u64)); - } else { - reply.error(ENOENT); - } - } else { - reply.error(ENOENT); - } - } - - fn read( - &mut self, - _req: &Request, - ino: u64, - _fh: u64, - offset: i64, - size: u32, - _flags: i32, - _lock: Option, - reply: ReplyData, - ) { - if let Some(path) = self.get_path_for_inode(ino) { - let mut mapping = self.file_mapping.lock().unwrap(); - if let Some(entry) = mapping.get_mut(path) { - let content = if entry.is_cache_valid() { - entry.cached_value.as_ref().unwrap().clone() - } else { - let expr = entry.expression.clone(); - drop(mapping); - let result = self.evaluate_expression(&expr); - - // Update cache if caching is enabled - let mut mapping = self.file_mapping.lock().unwrap(); - if let Some(entry) = mapping.get_mut(path) { - if entry.cache_ttl_secs > 0 { - entry.update_cache(result.clone()); - } - } - result - }; - - let data = content.as_bytes(); - let start = offset as usize; - let end = (start + size as usize).min(data.len()); - - if start < data.len() { - reply.data(&data[start..end]); - } else { - reply.data(&[]); - } - } else { - reply.error(ENOENT); - } - } else { - reply.error(ENOENT); - } - } - - fn readdir( - &mut self, - _req: &Request, - ino: u64, - _fh: u64, - offset: i64, - mut reply: ReplyDirectory, - ) { - if ino != FUSE_ROOT_ID { - reply.error(ENOTDIR); - return; - } - - let entries = vec![ - (FUSE_ROOT_ID, FileType::Directory, "."), - (FUSE_ROOT_ID, FileType::Directory, ".."), - ]; - - for (i, (inode, file_type, name)) in entries.iter().enumerate() { - if offset <= i as i64 { - if reply.add(*inode, (i + 1) as i64, *file_type, name) { - break; - } - } - } - - let mapping = self.file_mapping.lock().unwrap(); - let files = mapping.list_files(); - drop(mapping); - - for (i, path) in files.iter().enumerate() { - let index = i + entries.len(); - if offset <= index as i64 { - let inode = self.allocate_inode(path); - if reply.add(inode, (index + 1) as i64, FileType::RegularFile, path) { - break; - } - } - } - - reply.ok(); - } -} \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs b/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs deleted file mode 100644 index 8ebbe040b8..0000000000 --- a/slosh-fuse/examples/slosh-fuse/src/file_mapping.rs +++ /dev/null @@ -1,95 +0,0 @@ -use std::collections::HashMap; -use std::time::SystemTime; - -#[derive(Clone)] -pub struct FileEntry { - pub expression: String, - pub cached_value: Option, - pub cache_time: Option, - pub cache_ttl_secs: u64, -} - -impl FileEntry { - pub fn new(expression: String) -> Self { - Self { - expression, - cached_value: None, - cache_time: None, - cache_ttl_secs: 0, // No caching by default - } - } - - pub fn is_cache_valid(&self) -> bool { - if self.cache_ttl_secs == 0 { - return false; - } - - if let (Some(cached_time), Some(_)) = (self.cache_time, &self.cached_value) { - if let Ok(elapsed) = cached_time.elapsed() { - return elapsed.as_secs() < self.cache_ttl_secs; - } - } - false - } - - pub fn update_cache(&mut self, value: String) { - self.cached_value = Some(value); - self.cache_time = Some(SystemTime::now()); - } -} - -pub struct FileMapping { - files: HashMap, -} - -impl FileMapping { - pub fn new() -> Self { - Self { - files: HashMap::new(), - } - } - - pub fn register(&mut self, path: &str, expression: String) { - let normalized_path = if path.starts_with('/') { - path[1..].to_string() - } else { - path.to_string() - }; - - self.files.insert(normalized_path, FileEntry::new(expression)); - } - - pub fn get(&self, path: &str) -> Option<&FileEntry> { - let normalized_path = if path.starts_with('/') { - &path[1..] - } else { - path - }; - - self.files.get(normalized_path) - } - - pub fn get_mut(&mut self, path: &str) -> Option<&mut FileEntry> { - let normalized_path = if path.starts_with('/') { - path[1..].to_string() - } else { - path.to_string() - }; - - self.files.get_mut(&normalized_path) - } - - pub fn remove(&mut self, path: &str) -> Option { - let normalized_path = if path.starts_with('/') { - path[1..].to_string() - } else { - path.to_string() - }; - - self.files.remove(&normalized_path) - } - - pub fn list_files(&self) -> Vec { - self.files.keys().cloned().collect() - } -} \ No newline at end of file diff --git a/slosh-fuse/examples/slosh-fuse/src/lib.rs b/slosh-fuse/examples/slosh-fuse/src/lib.rs deleted file mode 100644 index 53a0d1322c..0000000000 --- a/slosh-fuse/examples/slosh-fuse/src/lib.rs +++ /dev/null @@ -1,37 +0,0 @@ -pub mod eval_fs; -pub mod file_mapping; - -pub use eval_fs::EvalFs; -pub use file_mapping::{FileMapping, FileEntry}; - -use std::sync::{Arc, Mutex}; -use std::path::PathBuf; - -pub struct FuseMount { - pub mount_point: PathBuf, - pub file_mapping: Arc>, - pub process_handle: Option, -} - -impl FuseMount { - pub fn new(mount_point: PathBuf) -> Self { - Self { - mount_point, - file_mapping: Arc::new(Mutex::new(FileMapping::new())), - process_handle: None, - } - } - - pub fn register_file(&self, path: &str, expression: String) { - let mut mapping = self.file_mapping.lock().unwrap(); - mapping.register(path, expression); - } - - pub fn unmount(&mut self) -> Result<(), std::io::Error> { - if let Some(mut handle) = self.process_handle.take() { - handle.kill()?; - handle.wait()?; - } - Ok(()) - } -} \ No newline at end of file diff --git a/slosh-fuse/src/daemon.rs b/slosh-fuse/src/daemon.rs index f54ce08353..ebd47bfd5a 100644 --- a/slosh-fuse/src/daemon.rs +++ b/slosh-fuse/src/daemon.rs @@ -2,8 +2,7 @@ use std::fs::{self, File}; use std::io; use std::os::unix::io::AsRawFd; use std::path::{Path, PathBuf}; -use nix::sys::stat::umask; - +use nix::sys::stat::{umask, Mode}; /// Paths used by the FUSE daemon. pub struct DaemonConfig { @@ -103,7 +102,7 @@ impl Drop for PidFile { /// Returns `Ok(())` in the daemon (grandchild) process. /// The parent and intermediate child processes call `std::process::exit(0)`. pub fn daemonize() -> io::Result<()> { - use nix::unistd::{ForkResult, fork, setsid}; + use nix::unistd::{self, ForkResult, fork, setsid}; // First fork match unsafe { fork() } { @@ -134,9 +133,9 @@ pub fn daemonize() -> io::Result<()> { .write(true) .open("/dev/null")?; let fd = devnull.as_raw_fd(); - unistd::dup2(fd, 0); - unistd::dup2(fd, 1); - unistd::dup2(fd, 2); + unistd::dup2(fd, 0)?; + unistd::dup2(fd, 1)?; + unistd::dup2(fd, 2)?; Ok(()) } @@ -151,6 +150,7 @@ pub fn init_daemon_logging(log_path: &Path) -> io::Result<()> { env_logger::Builder::new() .target(env_logger::Target::Pipe(Box::new(file))) .filter_level(log::LevelFilter::Info) + .parse_default_env() .format(|buf, record| { use std::io::Write; writeln!( diff --git a/slosh-fuse/src/eval_fs.rs b/slosh-fuse/src/eval_fs.rs index 1d24de839b..56b8a2d2e7 100644 --- a/slosh-fuse/src/eval_fs.rs +++ b/slosh-fuse/src/eval_fs.rs @@ -9,20 +9,23 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; use crate::file_mapping::FileMapping; +use crate::resolve::{FileResolver, MountRegistry}; const TTL: Duration = Duration::from_secs(1); pub struct EvalFs { file_mapping: Arc>, + registry: MountRegistry, inode_to_path: HashMap, path_to_inode: HashMap, next_inode: u64, } impl EvalFs { - pub fn new(file_mapping: Arc>) -> Self { + pub fn new(file_mapping: Arc>, registry: MountRegistry) -> Self { let mut fs = Self { file_mapping, + registry, inode_to_path: HashMap::new(), path_to_inode: HashMap::new(), next_inode: 2, // 1 is reserved for root @@ -109,12 +112,14 @@ impl Filesystem for EvalFs { let name_str = name.to_string_lossy(); - let size = { + let entry = { let mapping = self.file_mapping.lock().unwrap(); - mapping.get(&name_str).map(|e| e.total_size()) + mapping.get(&name_str).cloned() }; - if let Some(size) = size { + if let Some(entry) = entry { + let mut resolver = FileResolver::new(&self.registry); + let size = resolver.total_size(&entry); let inode = self.allocate_inode(&name_str); reply.entry(&TTL, &Self::file_attr(inode, size), 0); } else { @@ -126,12 +131,14 @@ impl Filesystem for EvalFs { if ino == FUSE_ROOT_ID { reply.attr(&TTL, &Self::dir_attr(FUSE_ROOT_ID)); } else if let Some(path) = self.get_path_for_inode(ino) { - let size = { + let entry = { let mapping = self.file_mapping.lock().unwrap(); - mapping.get(path).map(|e| e.total_size()) + mapping.get(path).cloned() }; - if let Some(size) = size { + if let Some(entry) = entry { + let mut resolver = FileResolver::new(&self.registry); + let size = resolver.total_size(&entry); reply.attr(&TTL, &Self::file_attr(ino, size)); } else { reply.error(ENOENT); @@ -153,11 +160,13 @@ impl Filesystem for EvalFs { reply: ReplyData, ) { if let Some(path) = self.get_path_for_inode(ino) { - let data = { + let entry = { let mapping = self.file_mapping.lock().unwrap(); - mapping.get(path).map(|e| e.read_range(offset as u64, size)) + mapping.get(path).cloned() }; - if let Some(data) = data { + if let Some(entry) = entry { + let mut resolver = FileResolver::new(&self.registry); + let data = resolver.read_range(&entry, offset as u64, size); reply.data(&data); } else { reply.error(ENOENT); diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs index c7ec5d5f03..9158db487b 100644 --- a/slosh-fuse/src/lib.rs +++ b/slosh-fuse/src/lib.rs @@ -1,6 +1,7 @@ pub mod eval_fs; pub mod file_mapping; pub mod proc_subst; +pub mod resolve; pub mod auto_start; pub mod client; @@ -10,6 +11,7 @@ pub mod socket_server; pub use eval_fs::EvalFs; pub use file_mapping::{FileMapping, FileEntry}; +pub use resolve::{MountRegistry, FileResolver}; use std::path::PathBuf; diff --git a/slosh-fuse/src/mount_manager.rs b/slosh-fuse/src/mount_manager.rs index 1675103c3a..0f00110f9d 100644 --- a/slosh-fuse/src/mount_manager.rs +++ b/slosh-fuse/src/mount_manager.rs @@ -1,5 +1,6 @@ use crate::eval_fs::EvalFs; use crate::file_mapping::FileMapping; +use crate::resolve::MountRegistry; use fuser::{MountOption, Session}; use std::collections::HashMap; use std::path::PathBuf; @@ -15,6 +16,7 @@ struct MountState { pub struct MountManager { mounts: HashMap, next_id: u64, + registry: MountRegistry, } impl MountManager { @@ -22,6 +24,7 @@ impl MountManager { Self { mounts: HashMap::new(), next_id: 1, + registry: MountRegistry::new(), } } @@ -42,14 +45,17 @@ impl MountManager { let mount_id = format!("mount-{}", self.next_id); self.next_id += 1; + self.registry.add(&path, Arc::clone(&mapping)); + let registry_for_fuse = self.registry.clone(); + let thread = std::thread::Builder::new() .name(format!("fuse-{}", mount_id)) .spawn(move || { - let fs = EvalFs::new(mapping_for_fuse); + let fs = EvalFs::new(mapping_for_fuse, registry_for_fuse); let options = vec![ MountOption::FSName("slosh-eval".to_string()), MountOption::AutoUnmount, - MountOption::DefaultPermissions, + MountOption::AllowOther, ]; match Session::new(fs, mount_path.as_ref(), &options) { @@ -89,6 +95,8 @@ impl MountManager { let state = self.mounts.remove(mount_id) .ok_or_else(|| format!("unknown mount-id: {}", mount_id))?; + self.registry.remove(&state.mount_point); + let mount_str = state.mount_point.to_string_lossy().to_string(); Self::fusermount_unmount(&mount_str)?; diff --git a/slosh-fuse/src/resolve.rs b/slosh-fuse/src/resolve.rs new file mode 100644 index 0000000000..1904973737 --- /dev/null +++ b/slosh-fuse/src/resolve.rs @@ -0,0 +1,493 @@ +use std::collections::HashSet; +use std::fs; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; + +use crate::file_mapping::{FileEntry, FileMapping}; + +struct MountInfo { + original_mount: PathBuf, + canonical_mount: PathBuf, + mapping: Arc>, +} + +/// Shared registry of active FUSE mount points. Clone is cheap (Arc). +#[derive(Clone)] +pub struct MountRegistry { + mounts: Arc>>, +} + +impl MountRegistry { + pub fn new() -> Self { + Self { + mounts: Arc::new(Mutex::new(Vec::new())), + } + } + + /// Register a mount point and its file mapping. + pub fn add(&self, path: &Path, mapping: Arc>) { + let canonical = fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let mut mounts = self.mounts.lock().unwrap(); + mounts.push(MountInfo { + original_mount: path.to_path_buf(), + canonical_mount: canonical, + mapping, + }); + } + + /// Unregister a mount point. + pub fn remove(&self, path: &Path) { + let mut mounts = self.mounts.lock().unwrap(); + mounts.retain(|m| m.original_mount != path); + } + + /// Check if `path` is under any registered mount. Returns the virtual + /// path (relative to mount root) and the mapping if found. + /// Checks both original and canonical mount paths. + /// Lock is held briefly; the Arc> is cloned out. + fn resolve_by_prefix(&self, path: &Path) -> Option<(String, Arc>)> { + let mounts = self.mounts.lock().unwrap(); + for info in mounts.iter() { + if let Ok(rel) = path.strip_prefix(&info.original_mount) { + let vpath = rel.to_string_lossy().to_string(); + return Some((vpath, Arc::clone(&info.mapping))); + } + if let Ok(rel) = path.strip_prefix(&info.canonical_mount) { + let vpath = rel.to_string_lossy().to_string(); + return Some((vpath, Arc::clone(&info.mapping))); + } + } + None + } +} + +/// Per-request file resolver with cycle detection. +/// Created fresh for each FUSE read/lookup to track visited paths. +pub struct FileResolver<'a> { + registry: &'a MountRegistry, + visited: HashSet, +} + +impl<'a> FileResolver<'a> { + pub fn new(registry: &'a MountRegistry) -> Self { + Self { + registry, + visited: HashSet::new(), + } + } + + /// Compute total size for a FileEntry, resolving internal paths. + pub fn total_size(&mut self, entry: &FileEntry) -> u64 { + match entry { + FileEntry::Static { content } => content.len() as u64, + FileEntry::Concat { source_paths } => { + source_paths.iter().map(|p| self.file_size(p)).sum() + } + } + } + + /// Read a range from a FileEntry, resolving internal paths. + pub fn read_range(&mut self, entry: &FileEntry, offset: u64, size: u32) -> Vec { + match entry { + FileEntry::Static { content } => { + let start = offset as usize; + let end = (start + size as usize).min(content.len()); + if start < content.len() { + content[start..end].to_vec() + } else { + Vec::new() + } + } + FileEntry::Concat { source_paths } => { + self.read_concat(source_paths, offset, size) + } + } + } + + /// Get the size of a file at `path`, resolving internal paths and detecting cycles. + fn file_size(&mut self, path: &Path) -> u64 { + if self.visited.contains(path) { + log::warn!("Cycle detected resolving size for {:?}", path); + return 0; + } + + if let Some((vpath, mapping)) = self.try_resolve_internal(path) { + self.visited.insert(path.to_path_buf()); + let entry = { + let m = mapping.lock().unwrap(); + m.get(&vpath).cloned() + }; + let result = match entry { + Some(entry) => self.total_size(&entry), + None => 0, + }; + self.visited.remove(path); + return result; + } + + // External file — use filesystem metadata + fs::metadata(path).map(|m| m.len()).unwrap_or(0) + } + + /// Read a range spanning multiple source files (concat logic). + fn read_concat(&mut self, source_paths: &[PathBuf], offset: u64, size: u32) -> Vec { + let mut result = Vec::new(); + let mut remaining = size as u64; + let mut cursor = 0u64; + + for path in source_paths { + if remaining == 0 { + break; + } + + let file_len = self.file_size(path); + if file_len == 0 { + continue; + } + + let file_end = cursor + file_len; + + if offset >= file_end { + cursor = file_end; + continue; + } + + let local_offset = if offset > cursor { offset - cursor } else { 0 }; + let bytes_available = file_len - local_offset; + let to_read = remaining.min(bytes_available); + + let data = self.read_file(path, local_offset, to_read as u32); + remaining -= data.len() as u64; + result.extend_from_slice(&data); + + cursor = file_end; + } + + result + } + + /// Read bytes from a file, resolving internal paths and detecting cycles. + fn read_file(&mut self, path: &Path, offset: u64, size: u32) -> Vec { + if self.visited.contains(path) { + log::warn!("Cycle detected reading {:?}", path); + return Vec::new(); + } + + if let Some((vpath, mapping)) = self.try_resolve_internal(path) { + self.visited.insert(path.to_path_buf()); + let entry = { + let m = mapping.lock().unwrap(); + m.get(&vpath).cloned() + }; + let result = match entry { + Some(entry) => self.read_range(&entry, offset, size), + None => Vec::new(), + }; + self.visited.remove(path); + return result; + } + + // External file + read_external_file(path, offset, size) + } + + /// Try to resolve a path as internal to a FUSE mount. + /// First tries raw prefix match (no syscalls, safe for mount-internal paths). + /// Then tries fs::canonicalize on the path — only safe because we only + /// reach here if the raw prefix match failed, meaning the path is NOT + /// directly under a mount (could be a symlink pointing into one). + fn try_resolve_internal(&self, path: &Path) -> Option<(String, Arc>)> { + // Raw prefix match first — no syscalls, no deadlock risk + if let Some(result) = self.registry.resolve_by_prefix(path) { + return Some(result); + } + + // Canonicalize fallback — safe because path is NOT under a mount + // (raw prefix match would have caught it). This handles symlinks + // pointing into a mount from outside. + if let Ok(canonical) = fs::canonicalize(path) { + if canonical != path { + return self.registry.resolve_by_prefix(&canonical); + } + } + + None + } +} + +/// Read bytes from an external (non-FUSE) file on disk. +pub fn read_external_file(path: &Path, offset: u64, size: u32) -> Vec { + let mut file = match fs::File::open(path) { + Ok(f) => f, + Err(e) => { + log::warn!("Failed to open external file {:?}: {}", path, e); + return Vec::new(); + } + }; + + if offset > 0 { + if let Err(e) = file.seek(SeekFrom::Start(offset)) { + log::warn!("Seek failed on {:?}: {}", path, e); + return Vec::new(); + } + } + + let mut buf = vec![0u8; size as usize]; + match file.read(&mut buf) { + Ok(n) => { + buf.truncate(n); + buf + } + Err(e) => { + log::warn!("Read failed on {:?}: {}", path, e); + Vec::new() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn create_temp_file(content: &[u8]) -> (tempfile::NamedTempFile, PathBuf) { + let mut f = tempfile::NamedTempFile::new().unwrap(); + f.write_all(content).unwrap(); + f.flush().unwrap(); + let path = f.path().to_path_buf(); + (f, path) + } + + fn make_registry() -> MountRegistry { + MountRegistry::new() + } + + fn make_mapping() -> Arc> { + Arc::new(Mutex::new(FileMapping::new())) + } + + // Test 1: Static entry passthrough + #[test] + fn static_entry_passthrough() { + let registry = make_registry(); + let mut resolver = FileResolver::new(®istry); + let entry = FileEntry::new_static(b"hello world".to_vec()); + + assert_eq!(resolver.total_size(&entry), 11); + assert_eq!(resolver.read_range(&entry, 0, 5), b"hello"); + assert_eq!(resolver.read_range(&entry, 6, 5), b"world"); + assert_eq!(resolver.read_range(&entry, 100, 5), b""); + } + + // Test 2: Concat of external files + #[test] + fn concat_external_files() { + let registry = make_registry(); + let (_f1, p1) = create_temp_file(b"aaa"); + let (_f2, p2) = create_temp_file(b"bbb"); + let entry = FileEntry::new_concat(vec![p1, p2]); + + let mut resolver = FileResolver::new(®istry); + assert_eq!(resolver.total_size(&entry), 6); + assert_eq!(resolver.read_range(&entry, 0, 6), b"aaabbb"); + assert_eq!(resolver.read_range(&entry, 2, 3), b"ab"); + } + + // Test 3: Concat with internal static file + #[test] + fn concat_with_internal_static() { + let mount_dir = tempfile::tempdir().unwrap(); + let mount_path = mount_dir.path(); + + let mapping = make_mapping(); + { + let mut m = mapping.lock().unwrap(); + m.register("inner.txt", b"INNER".to_vec()); + } + + let registry = make_registry(); + registry.add(mount_path, Arc::clone(&mapping)); + + let (_f_ext, p_ext) = create_temp_file(b"EXT"); + let internal_path = mount_path.join("inner.txt"); + + let entry = FileEntry::new_concat(vec![p_ext, internal_path]); + let mut resolver = FileResolver::new(®istry); + + assert_eq!(resolver.total_size(&entry), 8); // 3 + 5 + assert_eq!(resolver.read_range(&entry, 0, 8), b"EXTINNER"); + } + + // Test 4: Concat with internal concat (recursive) + #[test] + fn concat_recursive_internal() { + let mount_dir = tempfile::tempdir().unwrap(); + let mount_path = mount_dir.path(); + + let (_f_ext, p_ext) = create_temp_file(b"EXT"); + + let mapping = make_mapping(); + { + let mut m = mapping.lock().unwrap(); + // inner_concat.txt is itself a concat of an external file + static + m.register("leaf.txt", b"LEAF".to_vec()); + m.register_concat( + "inner_concat.txt", + vec![p_ext.clone(), mount_path.join("leaf.txt")], + ); + } + + let registry = make_registry(); + registry.add(mount_path, Arc::clone(&mapping)); + + // Top-level concat references inner_concat.txt + let entry = FileEntry::new_concat(vec![mount_path.join("inner_concat.txt")]); + let mut resolver = FileResolver::new(®istry); + + assert_eq!(resolver.total_size(&entry), 7); // EXT(3) + LEAF(4) + assert_eq!(resolver.read_range(&entry, 0, 7), b"EXTLEAF"); + } + + // Test 5: Direct cycle (A -> A) returns empty + #[test] + fn direct_cycle_returns_empty() { + let mount_dir = tempfile::tempdir().unwrap(); + let mount_path = mount_dir.path(); + + let mapping = make_mapping(); + { + let mut m = mapping.lock().unwrap(); + // a.txt sources from itself + m.register_concat("a.txt", vec![mount_path.join("a.txt")]); + } + + let registry = make_registry(); + registry.add(mount_path, Arc::clone(&mapping)); + + let entry = FileEntry::new_concat(vec![mount_path.join("a.txt")]); + let mut resolver = FileResolver::new(®istry); + + assert_eq!(resolver.total_size(&entry), 0); + assert_eq!(resolver.read_range(&entry, 0, 100), b""); + } + + // Test 6: Indirect cycle (A -> B -> A) returns empty + #[test] + fn indirect_cycle_returns_empty() { + let mount_dir = tempfile::tempdir().unwrap(); + let mount_path = mount_dir.path(); + + let mapping = make_mapping(); + { + let mut m = mapping.lock().unwrap(); + m.register_concat("a.txt", vec![mount_path.join("b.txt")]); + m.register_concat("b.txt", vec![mount_path.join("a.txt")]); + } + + let registry = make_registry(); + registry.add(mount_path, Arc::clone(&mapping)); + + let entry = FileEntry::new_concat(vec![mount_path.join("a.txt")]); + let mut resolver = FileResolver::new(®istry); + + assert_eq!(resolver.total_size(&entry), 0); + assert_eq!(resolver.read_range(&entry, 0, 100), b""); + } + + // Test 7: Mixed internal/external total_size + #[test] + fn mixed_internal_external_size() { + let mount_dir = tempfile::tempdir().unwrap(); + let mount_path = mount_dir.path(); + + let (_f_ext, p_ext) = create_temp_file(b"12345"); + + let mapping = make_mapping(); + { + let mut m = mapping.lock().unwrap(); + m.register("internal.txt", b"abc".to_vec()); + } + + let registry = make_registry(); + registry.add(mount_path, Arc::clone(&mapping)); + + let entry = FileEntry::new_concat(vec![ + p_ext, + mount_path.join("internal.txt"), + ]); + let mut resolver = FileResolver::new(®istry); + + assert_eq!(resolver.total_size(&entry), 8); // 5 + 3 + } + + // Test 8: Symlink into mount detected via canonicalize + #[test] + fn symlink_into_mount_resolved_internally() { + use std::os::unix::fs::symlink; + + let mount_dir = tempfile::tempdir().unwrap(); + let mount_path = mount_dir.path(); + + // We need a real file at the mount path for canonicalize to resolve + // the symlink target. Write a placeholder. + let real_file = mount_path.join("inner.txt"); + fs::write(&real_file, b"PLACEHOLDER").unwrap(); + + let mapping = make_mapping(); + { + let mut m = mapping.lock().unwrap(); + m.register("inner.txt", b"INNER".to_vec()); + } + + let registry = make_registry(); + registry.add(mount_path, Arc::clone(&mapping)); + + // Create symlink from outside the mount pointing into it + let link_dir = tempfile::tempdir().unwrap(); + let link_path = link_dir.path().join("link.txt"); + symlink(&real_file, &link_path).unwrap(); + + let entry = FileEntry::new_concat(vec![link_path]); + let mut resolver = FileResolver::new(®istry); + + // Should resolve via canonicalize -> internal mapping, not read PLACEHOLDER + assert_eq!(resolver.total_size(&entry), 5); + assert_eq!(resolver.read_range(&entry, 0, 10), b"INNER"); + } + + // Test 9: Symlink cycle + #[test] + fn symlink_cycle_returns_empty() { + use std::os::unix::fs::symlink; + + let mount_dir = tempfile::tempdir().unwrap(); + let mount_path = mount_dir.path(); + + // Create placeholder file at mount path so canonicalize works + let real_file = mount_path.join("a.txt"); + fs::write(&real_file, b"PLACEHOLDER").unwrap(); + + // Create symlink elsewhere -> mount/a.txt + let link_dir = tempfile::tempdir().unwrap(); + let link_path = link_dir.path().join("link.txt"); + symlink(&real_file, &link_path).unwrap(); + + let mapping = make_mapping(); + { + let mut m = mapping.lock().unwrap(); + // a.txt is a concat that sources from the symlink + m.register_concat("a.txt", vec![link_path.clone()]); + } + + let registry = make_registry(); + registry.add(mount_path, Arc::clone(&mapping)); + + // Reading a.txt -> link.txt -> (canonicalize) -> mount/a.txt -> cycle! + let entry = FileEntry::new_concat(vec![mount_path.join("a.txt")]); + let mut resolver = FileResolver::new(®istry); + + assert_eq!(resolver.total_size(&entry), 0); + assert_eq!(resolver.read_range(&entry, 0, 100), b""); + } +} From e1746d4b7c001d1091405e18692674a0ec737a65 Mon Sep 17 00:00:00 2001 From: gpwclark Date: Sun, 8 Mar 2026 16:43:06 -0400 Subject: [PATCH 10/12] update fuse tests --- slosh-fuse/README.md | 2 +- slosh-fuse/examples/dynamic-env-file.sl | 38 ------ slosh-fuse/examples/dynamic-env-generator.sl | 109 ----------------- slosh-fuse/examples/dynamic-file-demo.sl | 32 ----- .../examples/test-concat-external.slosh | 72 ++++++++++++ .../examples/test-concat-self-cycle.slosh | 30 +++++ .../examples/test-concat-with-fuse-file.slosh | 75 ++++++++++++ .../examples/test-dynamic-file-demo.slosh | 82 +++++++++++++ slosh-fuse/examples/test-fuse-systemd.sl | 64 ---------- slosh-fuse/examples/test-fuse-systemd.slosh | 110 ++++++++++++++++++ .../examples/test-list-and-remove.slosh | 44 +++++++ slosh-fuse/examples/test-static-file.slosh | 66 +++++++++++ slosh_lib/src/fuse_bindings.rs | 39 +++++++ 13 files changed, 519 insertions(+), 244 deletions(-) delete mode 100644 slosh-fuse/examples/dynamic-env-file.sl delete mode 100644 slosh-fuse/examples/dynamic-env-generator.sl delete mode 100644 slosh-fuse/examples/dynamic-file-demo.sl create mode 100644 slosh-fuse/examples/test-concat-external.slosh create mode 100644 slosh-fuse/examples/test-concat-self-cycle.slosh create mode 100644 slosh-fuse/examples/test-concat-with-fuse-file.slosh create mode 100644 slosh-fuse/examples/test-dynamic-file-demo.slosh delete mode 100644 slosh-fuse/examples/test-fuse-systemd.sl create mode 100644 slosh-fuse/examples/test-fuse-systemd.slosh create mode 100644 slosh-fuse/examples/test-list-and-remove.slosh create mode 100644 slosh-fuse/examples/test-static-file.slosh diff --git a/slosh-fuse/README.md b/slosh-fuse/README.md index c1738f45be..40d6975c32 100644 --- a/slosh-fuse/README.md +++ b/slosh-fuse/README.md @@ -8,7 +8,7 @@ Dynamic configuration files for services like systemd, where environment files n ```bash cat /tmp/dynamic/config.env -# => HOST=meow +# => HOST=my_hostname # => CUSTOM_TYPE=production ``` diff --git a/slosh-fuse/examples/dynamic-env-file.sl b/slosh-fuse/examples/dynamic-env-file.sl deleted file mode 100644 index fd09f59061..0000000000 --- a/slosh-fuse/examples/dynamic-env-file.sl +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env slosh - -;; Dynamic Environment File Generator -;; This demonstrates generating dynamic env files for systemd - -(def *custom-type* "production") -(def *app-version* "1.0.0") - -(defn generate-env-file [path] - "Generate a dynamic environment file at the given path" - (let [content (str "# Generated by slosh at " (current-time-millis) "\n" - "HOST=" (or (env "HOSTNAME") "localhost") "\n" - "CUSTOM_TYPE=" *custom-type* "\n" - "APP_VERSION=" *app-version* "\n" - "USER=" (env "USER") "\n" - "PATH=" (env "PATH") "\n")] - (fs-write path content) - (println "Generated" path "with content:") - (println content))) - -;; Example usage -(def env-file-path "/tmp/my-service.env") -(generate-env-file env-file-path) - -;; Function to create a background process that updates the file periodically -(defn start-dynamic-env-updater [path interval-ms] - "Start a background process that updates the env file every interval-ms milliseconds" - (fork - (loop [] - (generate-env-file path) - (sleep interval-ms) - (recur)))) - -;; Start updating the file every 5 seconds -(def updater-proc (start-dynamic-env-updater env-file-path 5000)) -(println "Started background updater process:" updater-proc) -(println "File will be updated every 5 seconds") -(println "Kill the process with: (kill" (pid updater-proc) "9)") \ No newline at end of file diff --git a/slosh-fuse/examples/dynamic-env-generator.sl b/slosh-fuse/examples/dynamic-env-generator.sl deleted file mode 100644 index ee32e6af0e..0000000000 --- a/slosh-fuse/examples/dynamic-env-generator.sl +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env slosh - -;; Dynamic Environment File Generator -;; A practical implementation that generates env files based on slosh expressions - -(def *env-expressions* (hash)) - -(defn register-env-file [path expr-fn] - "Register an environment file path with a function that generates its content" - (hash-set! *env-expressions* path expr-fn)) - -(defn generate-env-file [path] - "Generate the environment file by evaluating its registered expression" - (if-let [expr-fn (hash-get *env-expressions* path)] - (let [content (expr-fn) - file (open path :create :truncate)] - (write-string file content) - (close file) - (println "Generated" path)) - (println "No expression registered for" path))) - -(defn generate-all-env-files [] - "Generate all registered environment files" - (doseq [[path _] *env-expressions*] - (generate-env-file path))) - -;; Example: Register systemd environment files - -;; Define some global configuration -(def *app-config* - {:environment "production" - :version "1.0.0" - :debug false}) - -;; Register my-service.env -(register-env-file "/tmp/my-service.env" - (fn [] - (str "# Generated by slosh at " (current-time-millis) "\n" - "HOST=" (or (env "HOSTNAME") "meow") "\n" - "CUSTOM_TYPE=" (:environment *app-config*) "\n" - "APP_VERSION=" (:version *app-config*) "\n" - "DEBUG=" (if (:debug *app-config*) "true" "false") "\n" - "USER=" (env "USER") "\n" - "PATH=" (env "PATH") "\n"))) - -;; Register database.env -(register-env-file "/tmp/database.env" - (fn [] - (str "# Database configuration\n" - "DB_HOST=" (if (= (:environment *app-config*) "production") - "prod-db.example.com" - "localhost") "\n" - "DB_PORT=5432\n" - "DB_NAME=" (:environment *app-config*) "_db\n" - "DB_USER=" (env "USER") "_app\n" - "DB_POOL_SIZE=" (if (= (:environment *app-config*) "production") "20" "5") "\n"))) - -;; Generate the files -(println "Generating environment files with current configuration:") -(println "*app-config*:" *app-config*) -(generate-all-env-files) - -;; Show the generated content -(println "\n--- Generated my-service.env ---") -(let [file (open "/tmp/my-service.env" :read)] - (println (read-string file 10000)) - (close file)) - -(println "--- Generated database.env ---") -(let [file (open "/tmp/database.env" :read)] - (println (read-string file 10000)) - (close file)) - -;; Demonstrate dynamic updates -(println "\n--- Updating configuration to development ---") -(set! *app-config* (assoc *app-config* - :environment "development" - :debug true)) - -(generate-all-env-files) - -(println "\n--- Updated my-service.env ---") -(let [file (open "/tmp/my-service.env" :read)] - (println (read-string file 10000)) - (close file)) - -(println "--- Updated database.env ---") -(let [file (open "/tmp/database.env" :read)] - (println (read-string file 10000)) - (close file)) - -;; Create a watcher function that regenerates files when config changes -(defn watch-and-regenerate [check-interval-ms] - "Watch for config changes and regenerate files" - (let [last-config (atom *app-config*)] - (fork - (loop [] - (when (not= @last-config *app-config*) - (println "Configuration changed, regenerating files...") - (generate-all-env-files) - (reset! last-config *app-config*)) - (sleep check-interval-ms) - (recur))))) - -(println "\n--- Starting config watcher ---") -(def watcher (watch-and-regenerate 1000)) -(println "Watcher started, will check for config changes every second") -(println "Try changing *app-config* and the files will auto-regenerate!") -(println "Stop watcher with: (kill" (pid watcher) "9)") \ No newline at end of file diff --git a/slosh-fuse/examples/dynamic-file-demo.sl b/slosh-fuse/examples/dynamic-file-demo.sl deleted file mode 100644 index ed0f53cb8e..0000000000 --- a/slosh-fuse/examples/dynamic-file-demo.sl +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env slosh - -;; Dynamic File Demo -;; Demonstrates FUSE integration with static content evaluated at registration time - -;; Mount a dynamic filesystem -(def mount-id (mount-eval-fs "/tmp/slosh-dynamic")) - -;; Register a systemd-style environment file -;; Content is evaluated NOW and sent to the FUSE server -(register-eval-file mount-id "my-service.env" - (str "HOST=" (or (env "HOSTNAME") "localhost") "\n" - "CUSTOM_TYPE=" (or *custom-type* "default") "\n")) - -;; Register another file showing system info -(register-eval-file mount-id "system-info.txt" - (str "User: " (env "USER") "\n" - "Home: " (env "HOME") "\n" - "Shell: " (env "SHELL") "\n")) - -;; List registered files -(println "Registered files:") -(doseq [file (list-eval-files mount-id)] - (println " " file)) - -;; Files are now readable at the mount point: -;; cat /tmp/slosh-dynamic/my-service.env -;; cat /tmp/slosh-dynamic/system-info.txt - -;; Clean up -(unmount-eval-fs mount-id) -(println "Filesystem unmounted.") diff --git a/slosh-fuse/examples/test-concat-external.slosh b/slosh-fuse/examples/test-concat-external.slosh new file mode 100644 index 0000000000..f0085c793b --- /dev/null +++ b/slosh-fuse/examples/test-concat-external.slosh @@ -0,0 +1,72 @@ +#!/usr/bin/env slosh +;; Test: Concat external files, read back, modify source and re-read (lazy). + +(load "test.slosh") + +(defn wait-for-file (path) + (loop (tries) (0) + (if (fs-exists? path) + #t + (if (>= tries 10) + (do (eprn (str "Timed out waiting for " path)) (exit 1)) + (do (sleep 100) (recur (+ tries 1))))))) + +(defn read-file (path) + (wait-for-file path) + (let (f (fopen path :read), result "") + (defer (fclose f)) + (loop (line) ((read-line f)) + (if line + (do (set! result (str result line)) + (recur (read-line f))) + result)) + result)) + +(defn assert-file-content (path expected label) + (prn (str " Checking " label ": " path " == '" expected "'")) + (let (actual (read-file path)) + (when (and (not (= expected "")) (= actual "")) + (eprn (str "FAIL [" label "]: read returned empty string (file unreadable or missing?)")) + (exit 1)) + (if (= actual expected) + (prn (str " OK: " label)) + (do + (eprn (str "FAIL [" label "]: expected '" expected "' but got '" actual "'")) + (exit 1))))) + +(defn write-file (path content) + (let (f (fopen path :create :truncate :write)) + (fpr f content) + (fclose f))) + +;; create source files +(prn "Creating external source files...") +$(mkdir -p /tmp/concat-src) +(write-file "/tmp/concat-src/a.txt" "AAA") +(write-file "/tmp/concat-src/b.txt" "BBB") +(write-file "/tmp/concat-src/c.txt" "CCC") + +;; mount +(prn "Mounting filesystem at /tmp/test-concat-ext...") +(def mount-id (mount-eval-fs "/tmp/test-concat-ext")) +(prn (str " Got mount-id: " mount-id)) + +;; register concat +(prn "Registering concat file combined.txt from a.txt + b.txt + c.txt...") +(concat-eval-files mount-id "combined.txt" + "/tmp/concat-src/a.txt" "/tmp/concat-src/b.txt" "/tmp/concat-src/c.txt") + +;; read back +(prn "Reading back combined.txt via FUSE...") +(assert-file-content "/tmp/test-concat-ext/combined.txt" "AAABBBCCC" "initial concat") + +;; modify source B and re-read (lazy — no re-registration needed) +(prn "Modifying b.txt to 'XXX' and re-reading...") +(write-file "/tmp/concat-src/b.txt" "XXX") +(assert-file-content "/tmp/test-concat-ext/combined.txt" "AAAXXXCCC" "lazy re-read after modify") + +;; cleanup +(prn "Unmounting...") +(unmount-eval-fs mount-id) +$(rm -rf /tmp/concat-src) +(prn "PASS: test-concat-external") diff --git a/slosh-fuse/examples/test-concat-self-cycle.slosh b/slosh-fuse/examples/test-concat-self-cycle.slosh new file mode 100644 index 0000000000..1973828f0e --- /dev/null +++ b/slosh-fuse/examples/test-concat-self-cycle.slosh @@ -0,0 +1,30 @@ +#!/usr/bin/env slosh +;; Test: Concat that references its own mount path (self-cycle). Should return empty, not hang. + +(load "test.slosh") + +;; mount +(prn "Mounting filesystem at /tmp/test-cycle...") +(def mount-id (mount-eval-fs "/tmp/test-cycle")) +(prn (str " Got mount-id: " mount-id)) + +;; register a concat file that sources from itself (cycle) +(prn "Registering loop.txt that sources from its own mount path...") +(concat-eval-files mount-id "loop.txt" "/tmp/test-cycle/loop.txt") + +;; read with a timeout — if cycle detection fails, this would hang forever +;; We use the shell timeout command to cap it at 5 seconds. +(prn "Reading loop.txt via FUSE (5s timeout, expecting empty due to cycle detection)...") +(def result ($sh "timeout" "5" "cat" "/tmp/test-cycle/loop.txt")) + +(prn (str " Got result: '" result "'")) +(if (= result "") + (prn " OK: cycle detected, empty content returned") + (do + (eprn (str "FAIL [self-cycle]: expected empty string but got '" result "'")) + (exit 1))) + +;; cleanup +(prn "Unmounting...") +(unmount-eval-fs mount-id) +(prn "PASS: test-concat-self-cycle") diff --git a/slosh-fuse/examples/test-concat-with-fuse-file.slosh b/slosh-fuse/examples/test-concat-with-fuse-file.slosh new file mode 100644 index 0000000000..618c68fa65 --- /dev/null +++ b/slosh-fuse/examples/test-concat-with-fuse-file.slosh @@ -0,0 +1,75 @@ +#!/usr/bin/env slosh +;; Test: Concat that includes a FUSE-hosted file (internal path resolution via resolve.rs). + +(load "test.slosh") + +(defn wait-for-file (path) + (loop (tries) (0) + (if (fs-exists? path) + #t + (if (>= tries 10) + (do (eprn (str "Timed out waiting for " path)) (exit 1)) + (do (sleep 100) (recur (+ tries 1))))))) + +(defn read-file (path) + (wait-for-file path) + (let (f (fopen path :read), result "") + (defer (fclose f)) + (loop (line) ((read-line f)) + (if line + (do (set! result (str result line)) + (recur (read-line f))) + result)) + result)) + +(defn assert-file-content (path expected label) + (prn (str " Checking " label ": " path " == '" expected "'")) + (let (actual (read-file path)) + (when (and (not (= expected "")) (= actual "")) + (eprn (str "FAIL [" label "]: read returned empty string (file unreadable or missing?)")) + (exit 1)) + (if (= actual expected) + (prn (str " OK: " label)) + (do + (eprn (str "FAIL [" label "]: expected '" expected "' but got '" actual "'")) + (exit 1))))) + +(defn write-file (path content) + (let (f (fopen path :create :truncate :write)) + (fpr f content) + (fclose f))) + +;; create an external source file +(prn "Creating external source file...") +$(mkdir -p /tmp/concat-mixed-src) +(write-file "/tmp/concat-mixed-src/ext.txt" "EXTERNAL") + +;; mount +(prn "Mounting filesystem at /tmp/test-concat-mixed...") +(def mount-id (mount-eval-fs "/tmp/test-concat-mixed")) +(prn (str " Got mount-id: " mount-id)) + +;; register a static file inside the FUSE mount +(prn "Registering inner.txt with content 'HELLO'...") +(register-eval-file mount-id "inner.txt" "HELLO") + +;; verify inner.txt is readable +(prn "Reading back inner.txt via FUSE...") +(assert-file-content "/tmp/test-concat-mixed/inner.txt" "HELLO" "inner static read") + +;; register a concat that references both external and FUSE-internal paths +;; The resolve.rs module should detect /tmp/test-concat-mixed/inner.txt as internal +;; and read it directly from the file mapping, avoiding FUSE deadlock. +(prn "Registering concat combo.txt from ext.txt + inner.txt...") +(concat-eval-files mount-id "combo.txt" + "/tmp/concat-mixed-src/ext.txt" "/tmp/test-concat-mixed/inner.txt") + +;; read back +(prn "Reading back combo.txt via FUSE...") +(assert-file-content "/tmp/test-concat-mixed/combo.txt" "EXTERNALHELLO" "mixed concat") + +;; cleanup +(prn "Unmounting...") +(unmount-eval-fs mount-id) +$(rm -rf /tmp/concat-mixed-src) +(prn "PASS: test-concat-with-fuse-file") diff --git a/slosh-fuse/examples/test-dynamic-file-demo.slosh b/slosh-fuse/examples/test-dynamic-file-demo.slosh new file mode 100644 index 0000000000..d8ea2e48c1 --- /dev/null +++ b/slosh-fuse/examples/test-dynamic-file-demo.slosh @@ -0,0 +1,82 @@ +#!/usr/bin/env slosh +;; Test: Dynamic file demo — register files with evaluated content, read back, assert. + +(load "test.slosh") + +(defn wait-for-file (path) + (loop (tries) (0) + (if (fs-exists? path) + #t + (if (>= tries 10) + (do (eprn (str "Timed out waiting for " path)) (exit 1)) + (do (sleep 100) (recur (+ tries 1))))))) + +(defn read-file (path) + (wait-for-file path) + (let (f (fopen path :read), result "") + (defer (fclose f)) + (loop (line) ((read-line f)) + (if line + (do (set! result (str result line)) + (recur (read-line f))) + result)) + result)) + +(defn assert-file-content (path expected label) + (prn (str " Checking " label ": " path)) + (let (actual (read-file path)) + (when (and (not (= expected "")) (= actual "")) + (eprn (str "FAIL [" label "]: read returned empty string (file unreadable or missing?)")) + (exit 1)) + (if (= actual expected) + (prn (str " OK: " label)) + (do + (eprn (str "FAIL [" label "]: expected '" expected "' but got '" actual "'")) + (exit 1))))) + +;; Set known values for env vars we'll use +(def test-hostname (or (env "HOSTNAME") "localhost")) +(def test-user (or (env "USER") "slosh")) +(def test-home (or (env "HOME") "/tmp")) + +;; Mount +(prn "Mounting dynamic filesystem at /tmp/test-dynamic-demo...") +(def mount-id (mount-eval-fs "/tmp/test-dynamic-demo")) +(prn (str " Got mount-id: " mount-id)) + +;; Register a service env file with evaluated content +(def service-content + (str "HOST=" test-hostname "\n" + "CUSTOM_TYPE=default\n")) + +(prn "Registering my-service.env...") +(register-eval-file mount-id "my-service.env" service-content) + +;; Register a system info file +(def sysinfo-content + (str "User: " test-user "\n" + "Home: " test-home "\n")) + +(prn "Registering system-info.txt...") +(register-eval-file mount-id "system-info.txt" sysinfo-content) + +;; Verify file list +(prn "Listing registered files...") +(def files (list-eval-files mount-id)) +(prn (str " Got: " files)) +(test::assert-equal 2 (len files) " should have exactly 2 files") +(test::assert-true (str-contains (str files) "my-service.env") " should contain my-service.env") +(test::assert-true (str-contains (str files) "system-info.txt") " should contain system-info.txt") +(prn " OK: file list correct") + +;; Read back and assert content matches +(prn "Reading back my-service.env via FUSE...") +(assert-file-content "/tmp/test-dynamic-demo/my-service.env" service-content "service env") + +(prn "Reading back system-info.txt via FUSE...") +(assert-file-content "/tmp/test-dynamic-demo/system-info.txt" sysinfo-content "system info") + +;; Clean up +(prn "Unmounting...") +(unmount-eval-fs mount-id) +(prn "PASS: test-dynamic-file-demo") diff --git a/slosh-fuse/examples/test-fuse-systemd.sl b/slosh-fuse/examples/test-fuse-systemd.sl deleted file mode 100644 index e4748bfaf8..0000000000 --- a/slosh-fuse/examples/test-fuse-systemd.sl +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env slosh - -;; Test FUSE dynamic files for systemd environment -;; Content is evaluated at registration time and served statically by the FUSE server - -;; Set some global variables that will be used in the env file -(def *custom-type* "production") -(def *app-version* "1.2.3") - -;; Mount the dynamic filesystem -(println "Mounting dynamic filesystem...") -(def mount-id (mount-eval-fs "/tmp/slosh-systemd-test")) - -;; Register a systemd-style environment file with content evaluated now -(println "Registering my-service.env...") -(register-eval-file mount-id "my-service.env" - (str "# Generated by slosh\n" - "HOST=" (or (env "HOSTNAME") "meow") "\n" - "CUSTOM_TYPE=" *custom-type* "\n" - "APP_VERSION=" *app-version* "\n" - "USER=" (env "USER") "\n" - "HOME=" (env "HOME") "\n")) - -;; Register another file for database config -(register-eval-file mount-id "database.env" - (str "DB_HOST=localhost\n" - "DB_PORT=5432\n" - "DB_NAME=" *custom-type* "_db\n" - "DB_USER=" (env "USER") "_app\n")) - -;; List all registered files -(println "\nRegistered files:") -(doseq [file (list-eval-files mount-id)] - (println " -" file)) - -;; Files are now readable: -(println "\nFiles are now accessible at:") -(println " /tmp/slosh-systemd-test/my-service.env") -(println " /tmp/slosh-systemd-test/database.env") -(println "And systemd could read them directly!") - -;; To update content after changing a global, re-register: -(println "\n--- Changing *custom-type* to 'development' and re-registering ---") -(set! *custom-type* "development") - -(register-eval-file mount-id "my-service.env" - (str "# Generated by slosh\n" - "HOST=" (or (env "HOSTNAME") "meow") "\n" - "CUSTOM_TYPE=" *custom-type* "\n" - "APP_VERSION=" *app-version* "\n" - "USER=" (env "USER") "\n" - "HOME=" (env "HOME") "\n")) - -(register-eval-file mount-id "database.env" - (str "DB_HOST=localhost\n" - "DB_PORT=5432\n" - "DB_NAME=" *custom-type* "_db\n" - "DB_USER=" (env "USER") "_app\n")) - -(println "Files re-registered with new content.") - -;; Clean up -(unmount-eval-fs mount-id) -(println "\nFilesystem unmounted.") diff --git a/slosh-fuse/examples/test-fuse-systemd.slosh b/slosh-fuse/examples/test-fuse-systemd.slosh new file mode 100644 index 0000000000..0ee1648ec1 --- /dev/null +++ b/slosh-fuse/examples/test-fuse-systemd.slosh @@ -0,0 +1,110 @@ +#!/usr/bin/env slosh +;; Test: Systemd-style env files — register, read, re-register with changed values, read again. + +(load "test.slosh") + +(defn wait-for-file (path) + (loop (tries) (0) + (if (fs-exists? path) + #t + (if (>= tries 10) + (do (eprn (str "Timed out waiting for " path)) (exit 1)) + (do (sleep 100) (recur (+ tries 1))))))) + +(defn read-file (path) + (wait-for-file path) + (let (f (fopen path :read), result "") + (defer (fclose f)) + (loop (line) ((read-line f)) + (if line + (do (set! result (str result line)) + (recur (read-line f))) + result)) + result)) + +(defn assert-file-content (path expected label) + (prn (str " Checking " label ": " path)) + (let (actual (read-file path)) + (when (and (not (= expected "")) (= actual "")) + (eprn (str "FAIL [" label "]: read returned empty string (file unreadable or missing?)")) + (exit 1)) + (if (= actual expected) + (prn (str " OK: " label)) + (do + (eprn (str "FAIL [" label "]: expected '" expected "' but got '" actual "'")) + (exit 1))))) + +;; Known values +(def *custom-type* "production") +(def *app-version* "1.2.3") +(def test-hostname (or (env "HOSTNAME") "testhost")) +(def test-user (or (env "USER") "slosh")) +(def test-home (or (env "HOME") "/tmp")) + +;; Mount +(prn "Mounting dynamic filesystem at /tmp/test-systemd...") +(def mount-id (mount-eval-fs "/tmp/test-systemd")) +(prn (str " Got mount-id: " mount-id)) + +;; Build and register service env +(defn make-service-content () + (str "# Generated by slosh\n" + "HOST=" test-hostname "\n" + "CUSTOM_TYPE=" *custom-type* "\n" + "APP_VERSION=" *app-version* "\n" + "USER=" test-user "\n" + "HOME=" test-home "\n")) + +(defn make-db-content () + (str "DB_HOST=localhost\n" + "DB_PORT=5432\n" + "DB_NAME=" *custom-type* "_db\n" + "DB_USER=" test-user "_app\n")) + +(prn "Registering my-service.env (production)...") +(def svc1 (make-service-content)) +(register-eval-file mount-id "my-service.env" svc1) + +(prn "Registering database.env (production)...") +(def db1 (make-db-content)) +(register-eval-file mount-id "database.env" db1) + +;; Verify file list +(prn "Listing registered files...") +(def files (list-eval-files mount-id)) +(prn (str " Got: " files)) +(test::assert-equal 2 (len files) " should have 2 files") + +;; Read back and assert production content +(prn "Reading back my-service.env (production)...") +(assert-file-content "/tmp/test-systemd/my-service.env" svc1 "service env production") + +(prn "Reading back database.env (production)...") +(assert-file-content "/tmp/test-systemd/database.env" db1 "db env production") + +;; Change config and re-register +(prn "Changing *custom-type* to 'development' and re-registering...") +(set! *custom-type* "development") + +(def svc2 (make-service-content)) +(register-eval-file mount-id "my-service.env" svc2) + +(def db2 (make-db-content)) +(register-eval-file mount-id "database.env" db2) + +;; Read back and assert development content +(prn "Reading back my-service.env (development)...") +(assert-file-content "/tmp/test-systemd/my-service.env" svc2 "service env development") + +(prn "Reading back database.env (development)...") +(assert-file-content "/tmp/test-systemd/database.env" db2 "db env development") + +;; Verify the content actually changed +(test::assert-true (str-contains svc2 "CUSTOM_TYPE=development") " should have development type") +(test::assert-true (str-contains db2 "DB_NAME=development_db") " should have development db name") +(prn " OK: content correctly reflects changed config") + +;; Clean up +(prn "Unmounting...") +(unmount-eval-fs mount-id) +(prn "PASS: test-fuse-systemd") diff --git a/slosh-fuse/examples/test-list-and-remove.slosh b/slosh-fuse/examples/test-list-and-remove.slosh new file mode 100644 index 0000000000..aa9e3893c1 --- /dev/null +++ b/slosh-fuse/examples/test-list-and-remove.slosh @@ -0,0 +1,44 @@ +#!/usr/bin/env slosh +;; Test: Register files, list them, remove one, list again, verify. + +(load "test.slosh") + +(defn vec-contains? (v item) + (str-contains (str v) item)) + +;; mount +(prn "Mounting filesystem at /tmp/test-list...") +(def mount-id (mount-eval-fs "/tmp/test-list")) +(prn (str " Got mount-id: " mount-id)) + +;; register two files +(prn "Registering alpha.txt and beta.txt...") +(register-eval-file mount-id "alpha.txt" "aaa") +(register-eval-file mount-id "beta.txt" "bbb") + +;; list and verify both present +(prn "Listing files (expecting alpha.txt and beta.txt)...") +(def files1 (list-eval-files mount-id)) +(prn (str " Got: " files1)) +(test::assert-true (vec-contains? files1 "alpha.txt") " alpha.txt should be in list") +(test::assert-true (vec-contains? files1 "beta.txt") " beta.txt should be in list") +(test::assert-equal 2 (len files1) " should have exactly 2 files") +(prn " OK: both files present") + +;; remove one +(prn "Removing alpha.txt...") +(remove-eval-file mount-id "alpha.txt") + +;; list and verify only beta remains +(prn "Listing files after removal (expecting only beta.txt)...") +(def files2 (list-eval-files mount-id)) +(prn (str " Got: " files2)) +(test::assert-true (vec-contains? files2 "beta.txt") " beta.txt should still be in list") +(test::assert-false (vec-contains? files2 "alpha.txt") " alpha.txt should be gone") +(test::assert-equal 1 (len files2) " should have exactly 1 file") +(prn " OK: only beta.txt remains") + +;; cleanup +(prn "Unmounting...") +(unmount-eval-fs mount-id) +(prn "PASS: test-list-and-remove") diff --git a/slosh-fuse/examples/test-static-file.slosh b/slosh-fuse/examples/test-static-file.slosh new file mode 100644 index 0000000000..711522b6ca --- /dev/null +++ b/slosh-fuse/examples/test-static-file.slosh @@ -0,0 +1,66 @@ +#!/usr/bin/env slosh +;; Test: Mount, register a static file, read it back via FUSE cat, assert content matches. + +(load "test.slosh") + +(defn wait-for-file (path) + (loop (tries) (0) + (if (fs-exists? path) + #t + (if (>= tries 10) + (do (eprn (str "Timed out waiting for " path)) (exit 1)) + (do (sleep 100) (recur (+ tries 1))))))) + +(defn read-file (path) + (wait-for-file path) + (let (f (fopen path :read), result "") + (defer (fclose f)) + (loop (line) ((read-line f)) + (if line + (do (set! result (str result line)) + (recur (read-line f))) + result)) + result)) + +(defn assert-file-content (path expected label) + (prn (str " Checking " label ": " path " == '" expected "'")) + (let (actual (read-file path)) + (when (and (not (= expected "")) (= actual "")) + (eprn (str "FAIL [" label "]: read returned empty string (file unreadable or missing?)")) + (exit 1)) + (if (= actual expected) + (prn (str " OK: " label)) + (do + (eprn (str "FAIL [" label "]: expected '" expected "' but got '" actual "'")) + (exit 1))))) + +;; setup +(prn "Mounting filesystem at /tmp/test-static...") +(def mount-id (mount-eval-fs "/tmp/test-static")) +(prn (str " Got mount-id: " mount-id)) + +;; register and read a simple file +(prn "Registering file hello.txt with content 'hello world'...") +(register-eval-file mount-id "hello.txt" "hello world") + +(prn "Reading back hello.txt via FUSE...") +(assert-file-content "/tmp/test-static/hello.txt" "hello world" "static read") + +;; register a second file to make sure multiple files work +(prn "Registering file num.txt with content '12345'...") +(register-eval-file mount-id "num.txt" "12345") + +(prn "Reading back num.txt via FUSE...") +(assert-file-content "/tmp/test-static/num.txt" "12345" "second static read") + +;; overwrite the first file and verify +(prn "Re-registering hello.txt with content 'updated'...") +(register-eval-file mount-id "hello.txt" "updated") + +(prn "Reading back hello.txt after update...") +(assert-file-content "/tmp/test-static/hello.txt" "updated" "overwrite read") + +;; cleanup +(prn "Unmounting...") +(unmount-eval-fs mount-id) +(prn "PASS: test-static-file") diff --git a/slosh_lib/src/fuse_bindings.rs b/slosh_lib/src/fuse_bindings.rs index bcf5683390..96214ddc66 100644 --- a/slosh_lib/src/fuse_bindings.rs +++ b/slosh_lib/src/fuse_bindings.rs @@ -130,6 +130,30 @@ fn concat_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { } } +fn remove_eval_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 2 { + return Err(VMError::new_vm( + "remove-eval-file: requires two arguments (mount-id path)".to_string(), + )); + } + + let mount_id = registers[0].get_string(vm)?; + let path = registers[1].get_string(vm)?; + + let mut registry = get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; + + if let Some(mount) = registry.get_mut(mount_id) { + mount + .remove_file(&path) + .map_err(|e| VMError::new_vm(format!("Failed to remove file: {}", e)))?; + Ok(Value::True) + } else { + Err(VMError::new_vm(format!("Invalid mount id: {}", mount_id))) + } +} + fn list_eval_files(vm: &mut SloshVm, registers: &[Value]) -> VMResult { if registers.len() != 1 { return Err(VMError::new_vm( @@ -228,6 +252,21 @@ Example: "#, ); + add_builtin( + env, + "remove-eval-file", + remove_eval_file, + r#"Usage: (remove-eval-file mount-id path) + +Remove a file from the FUSE filesystem. + +Section: fuse + +Example: +(remove-eval-file mount-id "config.txt") +"#, + ); + add_builtin( env, "unmount-eval-fs", From 4300458af6372cacdeb5c6b236d614341302a96d Mon Sep 17 00:00:00 2001 From: gpwclark Date: Sun, 8 Mar 2026 17:25:35 -0400 Subject: [PATCH 11/12] fuse overlay --- doc/mdbook-slosh-eval/Cargo.lock | 13 -- slosh-fuse/Cargo.toml | 2 +- slosh-fuse/examples/test-overlay.slosh | 90 ++++++++++++ slosh-fuse/src/backing_dir.rs | 186 +++++++++++++++++++++++++ slosh-fuse/src/client.rs | 8 ++ slosh-fuse/src/eval_fs.rs | 51 ++++++- slosh-fuse/src/lib.rs | 17 +++ slosh-fuse/src/mount_manager.rs | 24 +++- slosh-fuse/src/socket_server.rs | 8 ++ slosh_lib/src/fuse_bindings.rs | 54 +++++++ 10 files changed, 433 insertions(+), 20 deletions(-) create mode 100644 slosh-fuse/examples/test-overlay.slosh create mode 100644 slosh-fuse/src/backing_dir.rs diff --git a/doc/mdbook-slosh-eval/Cargo.lock b/doc/mdbook-slosh-eval/Cargo.lock index 6a341f1043..164e633cad 100644 --- a/doc/mdbook-slosh-eval/Cargo.lock +++ b/doc/mdbook-slosh-eval/Cargo.lock @@ -202,7 +202,6 @@ dependencies = [ "bridge_types", "compile_state", "glob", - "lazy_static", "rand", "same-file", "shell", @@ -211,7 +210,6 @@ dependencies = [ "static_assertions", "unicode-segmentation", "unicode_reader", - "uuid", "walkdir", ] @@ -2383,17 +2381,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" -[[package]] -name = "uuid" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33196643e165781c20a5ead5582283a7dacbb87855d867fbc2df3f81eddc1be" -dependencies = [ - "getrandom 0.3.1", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "version_check" version = "0.9.5" diff --git a/slosh-fuse/Cargo.toml b/slosh-fuse/Cargo.toml index c52b5ab18c..e5ed47ef8e 100644 --- a/slosh-fuse/Cargo.toml +++ b/slosh-fuse/Cargo.toml @@ -10,5 +10,5 @@ log = "0.4" env_logger = "0.11" base64 = "0.21" tempfile = "3.0" -nix = { workspace = true, features = ["signal", "process", "fs"] } +nix = { workspace = true, features = ["signal", "process", "fs", "dir"] } signal-hook = "0.3" diff --git a/slosh-fuse/examples/test-overlay.slosh b/slosh-fuse/examples/test-overlay.slosh new file mode 100644 index 0000000000..b5cccf89f0 --- /dev/null +++ b/slosh-fuse/examples/test-overlay.slosh @@ -0,0 +1,90 @@ +#!/usr/bin/env slosh +;; Test: Overlay mount on existing directory — real files visible, +;; virtual files shadow real ones, readdir merges both sets. + +(load "test.slosh") + +(defn wait-for-file (path) + (loop (tries) (0) + (if (fs-exists? path) + #t + (if (>= tries 10) + (do (eprn (str "Timed out waiting for " path)) (exit 1)) + (do (sleep 100) (recur (+ tries 1))))))) + +(defn read-file (path) + (wait-for-file path) + (let (f (fopen path :read), result "") + (defer (fclose f)) + (loop (line) ((read-line f)) + (if line + (do (set! result (str result line)) + (recur (read-line f))) + result)) + result)) + +(defn assert-file-content (path expected label) + (prn (str " Checking " label ": " path)) + (let (actual (read-file path)) + (when (and (not (= expected "")) (= actual "")) + (eprn (str "FAIL [" label "]: read returned empty string")) + (exit 1)) + (if (= actual expected) + (prn (str " OK: " label)) + (do + (eprn (str "FAIL [" label "]: expected '" expected "' but got '" actual "'")) + (exit 1))))) + +(defn write-file (path content) + (let (f (fopen path :create :truncate :write)) + (fpr f content) + (fclose f))) + +;; --- Setup: create a real directory with real files --- +(def overlay-dir "/tmp/test-overlay-dir") +$(mkdir -p /tmp/test-overlay-dir) +(write-file "/tmp/test-overlay-dir/real-a.txt" "real-a-content") +(write-file "/tmp/test-overlay-dir/real-b.txt" "real-b-content") + +;; --- Mount overlay --- +(prn "Mounting overlay on /tmp/test-overlay-dir...") +(def mount-id (mount-eval-fs-overlay overlay-dir)) +(prn (str " Got mount-id: " mount-id)) + +;; --- Test 1: Real files are visible through FUSE --- +(prn "Test 1: Real files visible through overlay...") +(assert-file-content (str overlay-dir "/real-a.txt") "real-a-content" "real file A") +(assert-file-content (str overlay-dir "/real-b.txt") "real-b-content" "real file B") + +;; --- Test 2: Register a virtual file, verify it's visible --- +(prn "Test 2: Virtual file visible...") +(register-eval-file mount-id "virtual.txt" "virtual-content") +(assert-file-content (str overlay-dir "/virtual.txt") "virtual-content" "virtual file") + +;; --- Test 3: Virtual file shadows a real file --- +(prn "Test 3: Virtual file shadows real file...") +(register-eval-file mount-id "real-a.txt" "shadowed-content") +(assert-file-content (str overlay-dir "/real-a.txt") "shadowed-content" "shadow overrides real") + +;; --- Test 4: Unshadowed real file still readable --- +(prn "Test 4: Unshadowed real file still readable...") +(assert-file-content (str overlay-dir "/real-b.txt") "real-b-content" "unshadowed real file") + +;; --- Test 5: Remove shadow, real file reappears --- +(prn "Test 5: Remove shadow, real file reappears...") +(remove-eval-file mount-id "real-a.txt") +(assert-file-content (str overlay-dir "/real-a.txt") "real-a-content" "real file after unshadow") + +;; --- Cleanup --- +(prn "Unmounting...") +(unmount-eval-fs mount-id) + +;; Verify real files survived unmount +(prn "Verifying real files survived unmount...") +(assert-file-content (str overlay-dir "/real-a.txt") "real-a-content" "real file post-unmount A") +(assert-file-content (str overlay-dir "/real-b.txt") "real-b-content" "real file post-unmount B") + +;; Clean up temp dir +$(rm -rf /tmp/test-overlay-dir) + +(prn "PASS: test-overlay") diff --git a/slosh-fuse/src/backing_dir.rs b/slosh-fuse/src/backing_dir.rs new file mode 100644 index 0000000000..ceeb730304 --- /dev/null +++ b/slosh-fuse/src/backing_dir.rs @@ -0,0 +1,186 @@ +use nix::dir::Dir; +use nix::fcntl::{AtFlags, OFlag}; +use nix::sys::stat::{self, Mode}; +use std::io::{Read, Seek, SeekFrom}; +use std::os::unix::io::{AsRawFd, FromRawFd}; + +/// Holds a pre-opened directory file descriptor for passthrough reads. +/// +/// Opened BEFORE the FUSE mount covers the directory, so all operations +/// via `openat`/`fstatat` relative to this fd bypass the FUSE layer entirely. +pub struct BackingDir { + dir: Dir, +} + +impl BackingDir { + /// Open a directory fd BEFORE the FUSE mount covers it. + pub fn open(path: &str) -> Result { + let dir = Dir::open( + path, + OFlag::O_RDONLY | OFlag::O_DIRECTORY, + Mode::empty(), + ) + .map_err(|e| format!("failed to open backing dir {:?}: {}", path, e))?; + + Ok(Self { dir }) + } + + /// Stat a file in the backing dir. Returns None if not found. + /// Returns (size, is_regular_file). + pub fn stat(&self, name: &str) -> Option<(u64, bool)> { + let fd = self.dir.as_raw_fd(); + match stat::fstatat(Some(fd), name, AtFlags::AT_SYMLINK_NOFOLLOW) { + Ok(st) => { + let is_regular = (st.st_mode & libc::S_IFMT) == libc::S_IFREG; + Some((st.st_size as u64, is_regular)) + } + Err(_) => None, + } + } + + /// Read bytes from a file in the backing dir. + pub fn read_file(&self, name: &str, offset: u64, size: u32) -> Vec { + let fd = self.dir.as_raw_fd(); + let file_fd = match nix::fcntl::openat( + Some(fd), + name, + OFlag::O_RDONLY, + Mode::empty(), + ) { + Ok(fd) => fd, + Err(e) => { + log::warn!("backing_dir: failed to open {:?}: {}", name, e); + return Vec::new(); + } + }; + + // Safety: file_fd is a valid fd we just opened + let mut file = unsafe { std::fs::File::from_raw_fd(file_fd) }; + + if offset > 0 { + if let Err(e) = file.seek(SeekFrom::Start(offset)) { + log::warn!("backing_dir: seek failed on {:?}: {}", name, e); + return Vec::new(); + } + } + + let mut buf = vec![0u8; size as usize]; + match file.read(&mut buf) { + Ok(n) => { + buf.truncate(n); + buf + } + Err(e) => { + log::warn!("backing_dir: read failed on {:?}: {}", name, e); + Vec::new() + } + } + } + + /// List all regular files in the backing dir. + pub fn list_files(&self) -> Vec { + let fd = self.dir.as_raw_fd(); + // Open a fresh Dir from the same fd so we can iterate without + // consuming self. We dup the fd first. + let dup_fd = match nix::unistd::dup(fd) { + Ok(d) => d, + Err(e) => { + log::warn!("backing_dir: dup failed: {}", e); + return Vec::new(); + } + }; + + let mut dir = match Dir::from_fd(dup_fd) { + Ok(d) => d, + Err(e) => { + log::warn!("backing_dir: Dir::from_fd failed: {}", e); + let _ = nix::unistd::close(dup_fd); + return Vec::new(); + } + }; + + let mut files = Vec::new(); + for entry in dir.iter() { + match entry { + Ok(e) => { + let name = e.file_name().to_string_lossy().to_string(); + if name == "." || name == ".." { + continue; + } + // Only include regular files + if let Some((_, true)) = self.stat(&name) { + files.push(name); + } + } + Err(e) => { + log::warn!("backing_dir: readdir error: {}", e); + break; + } + } + } + + files + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn open_and_stat() { + let dir = tempfile::tempdir().unwrap(); + let file_path = dir.path().join("hello.txt"); + std::fs::write(&file_path, b"hello").unwrap(); + + let backing = BackingDir::open(dir.path().to_str().unwrap()).unwrap(); + let (size, is_regular) = backing.stat("hello.txt").unwrap(); + assert_eq!(size, 5); + assert!(is_regular); + + assert!(backing.stat("nonexistent.txt").is_none()); + } + + #[test] + fn read_file_full() { + let dir = tempfile::tempdir().unwrap(); + let file_path = dir.path().join("data.txt"); + std::fs::write(&file_path, b"hello world").unwrap(); + + let backing = BackingDir::open(dir.path().to_str().unwrap()).unwrap(); + let data = backing.read_file("data.txt", 0, 100); + assert_eq!(data, b"hello world"); + } + + #[test] + fn read_file_with_offset() { + let dir = tempfile::tempdir().unwrap(); + let file_path = dir.path().join("data.txt"); + std::fs::write(&file_path, b"hello world").unwrap(); + + let backing = BackingDir::open(dir.path().to_str().unwrap()).unwrap(); + let data = backing.read_file("data.txt", 6, 5); + assert_eq!(data, b"world"); + } + + #[test] + fn list_files_returns_regular_only() { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("a.txt"), b"a").unwrap(); + std::fs::write(dir.path().join("b.txt"), b"b").unwrap(); + std::fs::create_dir(dir.path().join("subdir")).unwrap(); + + let backing = BackingDir::open(dir.path().to_str().unwrap()).unwrap(); + let mut files = backing.list_files(); + files.sort(); + assert_eq!(files, vec!["a.txt", "b.txt"]); + } + + #[test] + fn read_nonexistent_returns_empty() { + let dir = tempfile::tempdir().unwrap(); + let backing = BackingDir::open(dir.path().to_str().unwrap()).unwrap(); + let data = backing.read_file("nope.txt", 0, 100); + assert!(data.is_empty()); + } +} diff --git a/slosh-fuse/src/client.rs b/slosh-fuse/src/client.rs index 21b7cc396d..7ce53b1b00 100644 --- a/slosh-fuse/src/client.rs +++ b/slosh-fuse/src/client.rs @@ -68,6 +68,14 @@ impl DaemonClient { }) } + /// Request an overlay FUSE mount on an existing directory. Returns the mount-id. + pub fn mount_overlay(&mut self, mount_point: &str) -> io::Result { + let fields = self.send_command(&format!("MOUNT_OVERLAY\t{}", mount_point))?; + fields.into_iter().next().ok_or_else(|| { + io::Error::new(io::ErrorKind::InvalidData, "no mount-id in response") + }) + } + /// Register a file with static content. pub fn register_file( &mut self, diff --git a/slosh-fuse/src/eval_fs.rs b/slosh-fuse/src/eval_fs.rs index 56b8a2d2e7..ca619ed9fe 100644 --- a/slosh-fuse/src/eval_fs.rs +++ b/slosh-fuse/src/eval_fs.rs @@ -3,11 +3,12 @@ use fuser::{ Request, FUSE_ROOT_ID, }; use libc::{ENOENT, ENOTDIR}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::ffi::OsStr; use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; +use crate::backing_dir::BackingDir; use crate::file_mapping::FileMapping; use crate::resolve::{FileResolver, MountRegistry}; @@ -19,16 +20,22 @@ pub struct EvalFs { inode_to_path: HashMap, path_to_inode: HashMap, next_inode: u64, + backing: Option, } impl EvalFs { - pub fn new(file_mapping: Arc>, registry: MountRegistry) -> Self { + pub fn new( + file_mapping: Arc>, + registry: MountRegistry, + backing: Option, + ) -> Self { let mut fs = Self { file_mapping, registry, inode_to_path: HashMap::new(), path_to_inode: HashMap::new(), next_inode: 2, // 1 is reserved for root + backing, }; // Pre-populate inodes for registered files @@ -122,6 +129,9 @@ impl Filesystem for EvalFs { let size = resolver.total_size(&entry); let inode = self.allocate_inode(&name_str); reply.entry(&TTL, &Self::file_attr(inode, size), 0); + } else if let Some((size, true)) = self.backing.as_ref().and_then(|b| b.stat(&name_str)) { + let inode = self.allocate_inode(&name_str); + reply.entry(&TTL, &Self::file_attr(inode, size), 0); } else { reply.error(ENOENT); } @@ -131,15 +141,20 @@ impl Filesystem for EvalFs { if ino == FUSE_ROOT_ID { reply.attr(&TTL, &Self::dir_attr(FUSE_ROOT_ID)); } else if let Some(path) = self.get_path_for_inode(ino) { + let path = path.to_string(); let entry = { let mapping = self.file_mapping.lock().unwrap(); - mapping.get(path).cloned() + mapping.get(&path).cloned() }; if let Some(entry) = entry { let mut resolver = FileResolver::new(&self.registry); let size = resolver.total_size(&entry); reply.attr(&TTL, &Self::file_attr(ino, size)); + } else if let Some((size, true)) = + self.backing.as_ref().and_then(|b| b.stat(&path)) + { + reply.attr(&TTL, &Self::file_attr(ino, size)); } else { reply.error(ENOENT); } @@ -160,14 +175,18 @@ impl Filesystem for EvalFs { reply: ReplyData, ) { if let Some(path) = self.get_path_for_inode(ino) { + let path = path.to_string(); let entry = { let mapping = self.file_mapping.lock().unwrap(); - mapping.get(path).cloned() + mapping.get(&path).cloned() }; if let Some(entry) = entry { let mut resolver = FileResolver::new(&self.registry); let data = resolver.read_range(&entry, offset as u64, size); reply.data(&data); + } else if let Some(ref backing) = self.backing { + let data = backing.read_file(&path, offset as u64, size); + reply.data(&data); } else { reply.error(ENOENT); } @@ -206,12 +225,34 @@ impl Filesystem for EvalFs { let files = mapping.list_files(); drop(mapping); + let mut seen = HashSet::new(); + for (i, path) in files.iter().enumerate() { let index = i + entries.len(); + seen.insert(path.clone()); if offset <= index as i64 { let inode = self.allocate_inode(path); if reply.add(inode, (index + 1) as i64, FileType::RegularFile, path) { - break; + reply.ok(); + return; + } + } + } + + // Merge backing dir entries, skipping virtual file names + if let Some(ref backing) = self.backing { + let backing_files = backing.list_files(); + let base = entries.len() + files.len(); + for (i, name) in backing_files.iter().enumerate() { + if seen.contains(name) { + continue; + } + let index = base + i; + if offset <= index as i64 { + let inode = self.allocate_inode(name); + if reply.add(inode, (index + 1) as i64, FileType::RegularFile, name) { + break; + } } } } diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs index 9158db487b..7f3be84d5e 100644 --- a/slosh-fuse/src/lib.rs +++ b/slosh-fuse/src/lib.rs @@ -1,3 +1,4 @@ +pub mod backing_dir; pub mod eval_fs; pub mod file_mapping; pub mod proc_subst; @@ -9,6 +10,7 @@ pub mod daemon; pub mod mount_manager; pub mod socket_server; +pub use backing_dir::BackingDir; pub use eval_fs::EvalFs; pub use file_mapping::{FileMapping, FileEntry}; pub use resolve::{MountRegistry, FileResolver}; @@ -41,6 +43,21 @@ impl FuseMount { }) } + /// Create an overlay mount via the daemon. Real files in the directory + /// remain visible; virtual files shadow them. + pub fn new_overlay_daemon( + mount_point: PathBuf, + mut client: DaemonClient, + ) -> Result { + let mount_id = client.mount_overlay(mount_point.to_str().unwrap_or(""))?; + Ok(Self { + mount_point, + mount_id, + client, + registered_paths: Vec::new(), + }) + } + pub fn register_file(&mut self, path: &str, content: &[u8]) -> Result<(), std::io::Error> { self.client.register_file(&self.mount_id, path, content)?; if !self.registered_paths.contains(&path.to_string()) { diff --git a/slosh-fuse/src/mount_manager.rs b/slosh-fuse/src/mount_manager.rs index 0f00110f9d..ff3f4d1c17 100644 --- a/slosh-fuse/src/mount_manager.rs +++ b/slosh-fuse/src/mount_manager.rs @@ -1,3 +1,4 @@ +use crate::backing_dir::BackingDir; use crate::eval_fs::EvalFs; use crate::file_mapping::FileMapping; use crate::resolve::MountRegistry; @@ -30,6 +31,27 @@ impl MountManager { /// Mount a new FUSE filesystem. Returns a mount-id string. pub fn mount(&mut self, mount_point: &str) -> Result { + self.mount_inner(mount_point, None) + } + + /// Mount an overlay FUSE filesystem on an existing directory. + /// Real files in the directory remain visible; virtual files shadow them. + pub fn mount_overlay(&mut self, mount_point: &str) -> Result { + let path = PathBuf::from(mount_point); + if !path.is_dir() { + return Err(format!("{:?} is not an existing directory", mount_point)); + } + + // Open the backing dir fd BEFORE the FUSE mount covers it + let backing = BackingDir::open(mount_point)?; + self.mount_inner(mount_point, Some(backing)) + } + + fn mount_inner( + &mut self, + mount_point: &str, + backing: Option, + ) -> Result { let path = PathBuf::from(mount_point); // Create mount point if needed @@ -51,7 +73,7 @@ impl MountManager { let thread = std::thread::Builder::new() .name(format!("fuse-{}", mount_id)) .spawn(move || { - let fs = EvalFs::new(mapping_for_fuse, registry_for_fuse); + let fs = EvalFs::new(mapping_for_fuse, registry_for_fuse, backing); let options = vec![ MountOption::FSName("slosh-eval".to_string()), MountOption::AutoUnmount, diff --git a/slosh-fuse/src/socket_server.rs b/slosh-fuse/src/socket_server.rs index ea875a1276..c33a574d66 100644 --- a/slosh-fuse/src/socket_server.rs +++ b/slosh-fuse/src/socket_server.rs @@ -108,6 +108,14 @@ fn dispatch_command( } } + "MOUNT_OVERLAY" if parts.len() == 2 => { + let mount_point = parts[1]; + match manager.lock().unwrap().mount_overlay(mount_point) { + Ok(mount_id) => format!("OK\t{}", mount_id), + Err(e) => format!("ERR\t{}", e), + } + } + "REGISTER" if parts.len() == 4 => { let mount_id = parts[1]; let path = parts[2]; diff --git a/slosh_lib/src/fuse_bindings.rs b/slosh_lib/src/fuse_bindings.rs index 96214ddc66..f27b78ec6a 100644 --- a/slosh_lib/src/fuse_bindings.rs +++ b/slosh_lib/src/fuse_bindings.rs @@ -76,6 +76,42 @@ fn register_eval_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult } } +fn mount_eval_fs_overlay(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.len() != 1 { + return Err(VMError::new_vm( + "mount-eval-fs-overlay: requires one argument (mount-path)".to_string(), + )); + } + + let mount_path = registers[0].get_string(vm)?; + let mount_point = PathBuf::from(mount_path); + + if !mount_point.is_dir() { + return Err(VMError::new_vm(format!( + "mount-eval-fs-overlay: {:?} is not an existing directory", + mount_path + ))); + } + + let config = DaemonConfig::default_for_user() + .map_err(|e| VMError::new_vm(format!("Failed to get daemon config: {}", e)))?; + + let client = ensure_daemon_running(&config) + .map_err(|e| VMError::new_vm(format!("Failed to connect to FUSE daemon: {}", e)))?; + + let mount = FuseMount::new_overlay_daemon(mount_point, client) + .map_err(|e| VMError::new_vm(format!("Failed to create overlay mount: {}", e)))?; + + let mount_id = mount.mount_id.clone(); + + get_registry() + .lock() + .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))? + .insert(mount_id.clone(), mount); + + Ok(vm.alloc_string(mount_id)) +} + fn unmount_eval_fs(vm: &mut SloshVm, registers: &[Value]) -> VMResult { if registers.len() != 1 { return Err(VMError::new_vm( @@ -219,6 +255,24 @@ Example: "#, ); + add_builtin( + env, + "mount-eval-fs-overlay", + mount_eval_fs_overlay, + r#"Usage: (mount-eval-fs-overlay path) + +Mount an overlay evaluation filesystem on an existing directory. Real files +in the directory remain visible through the FUSE mount. Virtual files +registered via register-eval-file shadow real files of the same name. +Returns a mount ID. + +Section: fuse + +Example: +(def mount-id (mount-eval-fs-overlay "/existing/dir")) +"#, + ); + add_builtin( env, "register-eval-file", From c7132680e62d104b17ae2f7cddedabea8757cfe1 Mon Sep 17 00:00:00 2001 From: gpwclark Date: Sat, 14 Mar 2026 08:39:13 -0400 Subject: [PATCH 12/12] do an overlay inside an existing directory, can even shadow files --- Cargo.lock | 1 + slosh-fuse/examples/bug-when-recur.slosh | 15 + slosh-fuse/examples/docs-mount.slosh | 47 + .../examples/test-overlay-concat-shadow.slosh | 88 + slosh-fuse/src/eval_fs.rs | 52 +- slosh-fuse/src/file_mapping.rs | 7 +- slosh-fuse/src/lib.rs | 1 - slosh-fuse/src/mount_manager.rs | 4 +- slosh-fuse/src/proc_subst.rs | 151 -- slosh-fuse/src/resolve.rs | 153 +- slosh_lib/Cargo.toml | 1 + slosh_lib/src/docs.rs | 1525 +++++++++++++++++ slosh_lib/src/lib.rs | 20 +- slosh_lib/src/proc_subst_bindings.rs | 288 ---- slosh_test_lib/src/docs.rs | 1499 +--------------- slosh_test_lib/src/docs/legacy.rs | 8 +- 16 files changed, 1885 insertions(+), 1975 deletions(-) create mode 100644 slosh-fuse/examples/bug-when-recur.slosh create mode 100644 slosh-fuse/examples/docs-mount.slosh create mode 100644 slosh-fuse/examples/test-overlay-concat-shadow.slosh delete mode 100644 slosh-fuse/src/proc_subst.rs create mode 100644 slosh_lib/src/docs.rs delete mode 100644 slosh_lib/src/proc_subst_bindings.rs diff --git a/Cargo.lock b/Cargo.lock index 660601a97a..96e4758ca3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2245,6 +2245,7 @@ dependencies = [ "criterion", "glob", "iai-callgrind", + "regex", "shell", "sl-compiler", "sl-liner", diff --git a/slosh-fuse/examples/bug-when-recur.slosh b/slosh-fuse/examples/bug-when-recur.slosh new file mode 100644 index 0000000000..8dbe45c09f --- /dev/null +++ b/slosh-fuse/examples/bug-when-recur.slosh @@ -0,0 +1,15 @@ +#!/usr/bin/env slosh +;; Bug reproduction: `when` + `recur` only executes the first iteration. +;; Expected: loop runs 3 times, result is "abc" +;; Actual: loop runs once, result is "a" + +(load "test.slosh") + +;; This FAILS — only the first iteration runs +(let (result "") + (loop (items) ('("a" "b" "c")) + (when (not (= items '())) + (set! result (str result (car items))) + (recur (cdr items)))) + (prn (str "when+recur result: '" result "' (expected 'abc')")) + (test::assert-equal "abc" result " when+recur should iterate all items")) diff --git a/slosh-fuse/examples/docs-mount.slosh b/slosh-fuse/examples/docs-mount.slosh new file mode 100644 index 0000000000..ad1ee7ffd5 --- /dev/null +++ b/slosh-fuse/examples/docs-mount.slosh @@ -0,0 +1,47 @@ +#!/usr/bin/env slosh +;; Mount/unmount slosh documentation as markdown files via FUSE. +;; +;; Usage: +;; slosh docs-mount.slosh mount Mount docs at +;; slosh docs-mount.slosh unmount Unmount docs at +;; +;; Each section becomes a .md file under the mount point. + +(defn do-mount (path) + (prn (str "Mounting docs at " path "...")) + (let (mid (mount-eval-fs path) + sections (docs-by-section)) + (seq-for key in (hash-keys sections) + (let (filename (str key ".md") + content (get sections key)) + (register-eval-file mid filename content) + (prn (str " registered " filename)))) + (prn (str "Done. mount-id=" mid)) + (prn "Press enter to unmount, or Ctrl-C to leave mounted.") + (read-line) + (prn "Unmounting...") + (unmount-eval-fs mid) + (prn "Unmounted."))) + +(defn do-unmount (path) + ;; fusermount -u is the standard way to unmount a FUSE fs externally + ($sh "fusermount" "-u" path) + (prn (str "Unmounted " path))) + +(defn print-usage () + (prn "Usage: slosh docs-mount.slosh ") + (prn " mount Mount docs as .md files at ") + (prn " unmount Unmount docs at ")) + +(let (args *args*) + (if (< (len args) 3) + (do (print-usage) (exit 1)) + (let (cmd (get args 1) + path (get args 2)) + (if (= cmd "mount") + (do-mount path) + (if (= cmd "unmount") + (do-unmount path) + (do (eprn (str "Unknown command: " cmd)) + (print-usage) + (exit 1))))))) diff --git a/slosh-fuse/examples/test-overlay-concat-shadow.slosh b/slosh-fuse/examples/test-overlay-concat-shadow.slosh new file mode 100644 index 0000000000..8865605ae9 --- /dev/null +++ b/slosh-fuse/examples/test-overlay-concat-shadow.slosh @@ -0,0 +1,88 @@ +#!/usr/bin/env slosh +;; Test: Overlay concat that shadows a real file and appends to it. +;; The concat references the same filename it shadows — the resolver +;; should read the original from the backing dir (not cycle). + +(load "test.slosh") + +(defn wait-for-file (path) + (loop (tries) (0) + (if (fs-exists? path) + #t + (if (>= tries 10) + (do (eprn (str "Timed out waiting for " path)) (exit 1)) + (do (sleep 100) (recur (+ tries 1))))))) + +(defn read-file (path) + (wait-for-file path) + (let (f (fopen path :read), result "") + (defer (fclose f)) + (loop (line) ((read-line f)) + (if line + (do (set! result (str result line)) + (recur (read-line f))) + result)) + result)) + +(defn assert-file-content (path expected label) + (prn (str " Checking " label ": " path)) + (let (actual (read-file path)) + (when (and (not (= expected "")) (= actual "")) + (eprn (str "FAIL [" label "]: read returned empty string")) + (exit 1)) + (if (= actual expected) + (prn (str " OK: " label)) + (do + (eprn (str "FAIL [" label "]: expected '" expected "' but got '" actual "'")) + (exit 1))))) + +(defn write-file (path content) + (let (f (fopen path :create :truncate :write)) + (fpr f content) + (fclose f))) + +;; --- Setup: create directory with a real config file --- +(def overlay-dir "/tmp/test-overlay-concat") +$(mkdir -p /tmp/test-overlay-concat) +(write-file "/tmp/test-overlay-concat/config.txt" "line1=original") + +;; Also create an extra-lines file outside the overlay +(write-file "/tmp/test-overlay-extra.txt" "\nline2=appended") + +;; --- Mount overlay --- +(prn "Mounting overlay...") +(def mount-id (mount-eval-fs-overlay overlay-dir)) +(prn (str " Got mount-id: " mount-id)) + +;; --- Test 1: Real file visible before any concat --- +(prn "Test 1: Real file visible before concat...") +(assert-file-content + (str overlay-dir "/config.txt") + "line1=original" + "real config before shadow") + +;; --- Test 2: Shadow config.txt with concat of itself + extra --- +(prn "Test 2: Concat shadow (original + appended line)...") +(concat-eval-files mount-id "config.txt" + (str overlay-dir "/config.txt") + "/tmp/test-overlay-extra.txt") + +(assert-file-content + (str overlay-dir "/config.txt") + "line1=original\nline2=appended" + "concat shadow with append") + +;; --- Test 3: Remove shadow, original reappears --- +(prn "Test 3: Remove shadow, original reappears...") +(remove-eval-file mount-id "config.txt") +(assert-file-content + (str overlay-dir "/config.txt") + "line1=original" + "original after unshadow") + +;; --- Cleanup --- +(prn "Unmounting...") +(unmount-eval-fs mount-id) +$(rm -rf /tmp/test-overlay-concat /tmp/test-overlay-extra.txt) + +(prn "PASS: test-overlay-concat-shadow") diff --git a/slosh-fuse/src/eval_fs.rs b/slosh-fuse/src/eval_fs.rs index ca619ed9fe..e77aabaf35 100644 --- a/slosh-fuse/src/eval_fs.rs +++ b/slosh-fuse/src/eval_fs.rs @@ -5,6 +5,7 @@ use fuser::{ use libc::{ENOENT, ENOTDIR}; use std::collections::{HashMap, HashSet}; use std::ffi::OsStr; +use std::path::PathBuf; use std::sync::{Arc, Mutex}; use std::time::{Duration, SystemTime}; @@ -20,14 +21,15 @@ pub struct EvalFs { inode_to_path: HashMap, path_to_inode: HashMap, next_inode: u64, - backing: Option, + /// For overlay mounts: (mount_point, backing_dir). + backing: Option<(PathBuf, BackingDir)>, } impl EvalFs { pub fn new( file_mapping: Arc>, registry: MountRegistry, - backing: Option, + backing: Option<(PathBuf, BackingDir)>, ) -> Self { let mut fs = Self { file_mapping, @@ -67,6 +69,10 @@ impl EvalFs { self.inode_to_path.get(&inode).map(|s| s.as_str()) } + fn backing_ref(&self) -> Option<(&PathBuf, &BackingDir)> { + self.backing.as_ref().map(|(p, b)| (p, b)) + } + fn file_attr(inode: u64, size: u64) -> FileAttr { let now = SystemTime::now(); FileAttr { @@ -125,13 +131,21 @@ impl Filesystem for EvalFs { }; if let Some(entry) = entry { - let mut resolver = FileResolver::new(&self.registry); + let backing = self.backing_ref(); + let mut resolver = FileResolver::new(&self.registry, backing); + if let Some((ref mp, _)) = self.backing { + resolver.mark_visited(mp.join(name_str.as_ref())); + } let size = resolver.total_size(&entry); let inode = self.allocate_inode(&name_str); reply.entry(&TTL, &Self::file_attr(inode, size), 0); - } else if let Some((size, true)) = self.backing.as_ref().and_then(|b| b.stat(&name_str)) { - let inode = self.allocate_inode(&name_str); - reply.entry(&TTL, &Self::file_attr(inode, size), 0); + } else if let Some((_, ref b)) = self.backing { + if let Some((size, true)) = b.stat(&name_str) { + let inode = self.allocate_inode(&name_str); + reply.entry(&TTL, &Self::file_attr(inode, size), 0); + } else { + reply.error(ENOENT); + } } else { reply.error(ENOENT); } @@ -148,13 +162,19 @@ impl Filesystem for EvalFs { }; if let Some(entry) = entry { - let mut resolver = FileResolver::new(&self.registry); + let backing = self.backing_ref(); + let mut resolver = FileResolver::new(&self.registry, backing); + if let Some((ref mp, _)) = self.backing { + resolver.mark_visited(mp.join(&path)); + } let size = resolver.total_size(&entry); reply.attr(&TTL, &Self::file_attr(ino, size)); - } else if let Some((size, true)) = - self.backing.as_ref().and_then(|b| b.stat(&path)) - { - reply.attr(&TTL, &Self::file_attr(ino, size)); + } else if let Some((_, ref b)) = self.backing { + if let Some((size, true)) = b.stat(&path) { + reply.attr(&TTL, &Self::file_attr(ino, size)); + } else { + reply.error(ENOENT); + } } else { reply.error(ENOENT); } @@ -181,10 +201,14 @@ impl Filesystem for EvalFs { mapping.get(&path).cloned() }; if let Some(entry) = entry { - let mut resolver = FileResolver::new(&self.registry); + let backing = self.backing_ref(); + let mut resolver = FileResolver::new(&self.registry, backing); + if let Some((ref mp, _)) = self.backing { + resolver.mark_visited(mp.join(&path)); + } let data = resolver.read_range(&entry, offset as u64, size); reply.data(&data); - } else if let Some(ref backing) = self.backing { + } else if let Some((_, ref backing)) = self.backing { let data = backing.read_file(&path, offset as u64, size); reply.data(&data); } else { @@ -240,7 +264,7 @@ impl Filesystem for EvalFs { } // Merge backing dir entries, skipping virtual file names - if let Some(ref backing) = self.backing { + if let Some((_, ref backing)) = self.backing { let backing_files = backing.list_files(); let base = entries.len() + files.len(); for (i, name) in backing_files.iter().enumerate() { diff --git a/slosh-fuse/src/file_mapping.rs b/slosh-fuse/src/file_mapping.rs index 9d70f53156..5e55903e28 100644 --- a/slosh-fuse/src/file_mapping.rs +++ b/slosh-fuse/src/file_mapping.rs @@ -2,16 +2,17 @@ use std::collections::HashMap; use std::fs; use std::io::{Read, Seek, SeekFrom}; use std::path::PathBuf; +use std::sync::Arc; #[derive(Clone)] pub enum FileEntry { - Static { content: Vec }, + Static { content: Arc> }, Concat { source_paths: Vec }, } impl FileEntry { pub fn new_static(content: Vec) -> Self { - FileEntry::Static { content } + FileEntry::Static { content: Arc::new(content) } } pub fn new_concat(source_paths: Vec) -> Self { @@ -258,7 +259,7 @@ mod tests { f1.write_all(b"newcontent").unwrap(); f1.flush().unwrap(); - assert_eq!(entry.total_size(), 13); // 10 + 3... wait: "newcontent" = 10, "data" = 4 + assert_eq!(entry.total_size(), 14); // "newcontent" = 10, "data" = 4 assert_eq!(entry.read_range(0, 14), b"newcontentdata"); } diff --git a/slosh-fuse/src/lib.rs b/slosh-fuse/src/lib.rs index 7f3be84d5e..65f886cdd5 100644 --- a/slosh-fuse/src/lib.rs +++ b/slosh-fuse/src/lib.rs @@ -1,7 +1,6 @@ pub mod backing_dir; pub mod eval_fs; pub mod file_mapping; -pub mod proc_subst; pub mod resolve; pub mod auto_start; diff --git a/slosh-fuse/src/mount_manager.rs b/slosh-fuse/src/mount_manager.rs index ff3f4d1c17..b0340752d1 100644 --- a/slosh-fuse/src/mount_manager.rs +++ b/slosh-fuse/src/mount_manager.rs @@ -70,10 +70,12 @@ impl MountManager { self.registry.add(&path, Arc::clone(&mapping)); let registry_for_fuse = self.registry.clone(); + let backing_pair = backing.map(|b| (path.clone(), b)); + let thread = std::thread::Builder::new() .name(format!("fuse-{}", mount_id)) .spawn(move || { - let fs = EvalFs::new(mapping_for_fuse, registry_for_fuse, backing); + let fs = EvalFs::new(mapping_for_fuse, registry_for_fuse, backing_pair); let options = vec![ MountOption::FSName("slosh-eval".to_string()), MountOption::AutoUnmount, diff --git a/slosh-fuse/src/proc_subst.rs b/slosh-fuse/src/proc_subst.rs deleted file mode 100644 index 27d1858661..0000000000 --- a/slosh-fuse/src/proc_subst.rs +++ /dev/null @@ -1,151 +0,0 @@ -use std::collections::HashMap; -use std::fs; -use std::os::unix::fs::symlink; -use std::os::unix::io::AsRawFd; -use std::path::PathBuf; -use std::process::{Child, Command, Stdio}; -use std::sync::{Arc, Mutex}; - -/// Process substitution implementation that creates /dev/fd/* symlinks -pub struct ProcSubst { - /// Map of virtual paths to their backing processes - processes: Arc>>, - /// Base directory for symlinks (e.g., /tmp/slosh-proc-subst-XXXXX) - base_dir: PathBuf, -} - -struct ProcInfo { - process: Child, - fd: i32, - symlink_path: PathBuf, -} - -impl ProcSubst { - pub fn new() -> Result { - let base_dir = tempfile::tempdir()?.into_path(); - Ok(Self { - processes: Arc::new(Mutex::new(HashMap::new())), - base_dir, - }) - } - - /// Create a virtual file backed by a process - pub fn create_file(&self, name: &str, command: &str) -> Result { - // Create the symlink path - let symlink_path = self.base_dir.join(name); - - // Ensure parent directory exists - if let Some(parent) = symlink_path.parent() { - fs::create_dir_all(parent)?; - } - - // Spawn the process - let mut child = Command::new("sh") - .arg("-c") - .arg(command) - .stdout(Stdio::piped()) - .spawn()?; - - // Get the file descriptor - let stdout = child.stdout.take() - .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "No stdout"))?; - let fd = stdout.as_raw_fd(); - - // Create symlink to /dev/fd/N - let fd_path = format!("/dev/fd/{}", fd); - symlink(&fd_path, &symlink_path)?; - - // Store process info (leak the fd to keep it open) - std::mem::forget(stdout); - - let info = ProcInfo { - process: child, - fd, - symlink_path: symlink_path.clone(), - }; - - self.processes.lock().unwrap() - .insert(name.to_string(), info); - - Ok(symlink_path) - } - - /// Remove a virtual file - pub fn remove_file(&self, name: &str) -> Result<(), std::io::Error> { - let mut processes = self.processes.lock().unwrap(); - - if let Some(mut info) = processes.remove(name) { - // Remove the symlink - let _ = fs::remove_file(&info.symlink_path); - - // Kill the process - let _ = info.process.kill(); - let _ = info.process.wait(); - - // Close the file descriptor - unsafe { - libc::close(info.fd); - } - } - - Ok(()) - } - - /// List all virtual files - pub fn list_files(&self) -> Vec { - self.processes.lock().unwrap() - .keys() - .cloned() - .collect() - } - - /// Get the path to a virtual file - pub fn get_file_path(&self, name: &str) -> Option { - self.processes.lock().unwrap() - .get(name) - .map(|info| info.symlink_path.clone()) - } -} - -impl Drop for ProcSubst { - fn drop(&mut self) { - // Clean up all processes - let mut processes = self.processes.lock().unwrap(); - for (_, mut info) in processes.drain() { - let _ = fs::remove_file(&info.symlink_path); - let _ = info.process.kill(); - let _ = info.process.wait(); - unsafe { - libc::close(info.fd); - } - } - - // Remove base directory - let _ = fs::remove_dir_all(&self.base_dir); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs::File; - use std::io::Read; - - #[test] - fn test_proc_subst() { - let ps = ProcSubst::new().unwrap(); - - // Create a virtual file - let path = ps.create_file("test.txt", "echo 'Hello, World!'").unwrap(); - - // Read from it - let mut file = File::open(&path).unwrap(); - let mut contents = String::new(); - file.read_to_string(&mut contents).unwrap(); - - assert_eq!(contents.trim(), "Hello, World!"); - - // Clean up - ps.remove_file("test.txt").unwrap(); - } -} diff --git a/slosh-fuse/src/resolve.rs b/slosh-fuse/src/resolve.rs index 1904973737..9b01d87f46 100644 --- a/slosh-fuse/src/resolve.rs +++ b/slosh-fuse/src/resolve.rs @@ -4,6 +4,7 @@ use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex}; +use crate::backing_dir::BackingDir; use crate::file_mapping::{FileEntry, FileMapping}; struct MountInfo { @@ -67,16 +68,31 @@ impl MountRegistry { pub struct FileResolver<'a> { registry: &'a MountRegistry, visited: HashSet, + /// Optional backing dir for overlay mounts: (mount_point, backing_dir). + /// When a concat source would cycle back to a file under this mount, + /// the resolver reads from the backing dir instead. + backing: Option<(&'a PathBuf, &'a BackingDir)>, } impl<'a> FileResolver<'a> { - pub fn new(registry: &'a MountRegistry) -> Self { + pub fn new( + registry: &'a MountRegistry, + backing: Option<(&'a PathBuf, &'a BackingDir)>, + ) -> Self { Self { registry, visited: HashSet::new(), + backing, } } + /// Mark a path as already visited. Use this to pre-seed the visited set + /// with the file currently being resolved, so that self-referencing + /// concat sources fall through to the backing dir immediately. + pub fn mark_visited(&mut self, path: PathBuf) { + self.visited.insert(path); + } + /// Compute total size for a FileEntry, resolving internal paths. pub fn total_size(&mut self, entry: &FileEntry) -> u64 { match entry { @@ -108,6 +124,10 @@ impl<'a> FileResolver<'a> { /// Get the size of a file at `path`, resolving internal paths and detecting cycles. fn file_size(&mut self, path: &Path) -> u64 { if self.visited.contains(path) { + // Cycle — try backing dir fallback + if let Some(size) = self.backing_dir_size(path) { + return size; + } log::warn!("Cycle detected resolving size for {:?}", path); return 0; } @@ -120,7 +140,10 @@ impl<'a> FileResolver<'a> { }; let result = match entry { Some(entry) => self.total_size(&entry), - None => 0, + None => { + // Not in mapping — try backing dir + self.backing_dir_size(path).unwrap_or(0) + } }; self.visited.remove(path); return result; @@ -170,6 +193,10 @@ impl<'a> FileResolver<'a> { /// Read bytes from a file, resolving internal paths and detecting cycles. fn read_file(&mut self, path: &Path, offset: u64, size: u32) -> Vec { if self.visited.contains(path) { + // Cycle — try backing dir fallback + if let Some(data) = self.backing_dir_read(path, offset, size) { + return data; + } log::warn!("Cycle detected reading {:?}", path); return Vec::new(); } @@ -182,7 +209,11 @@ impl<'a> FileResolver<'a> { }; let result = match entry { Some(entry) => self.read_range(&entry, offset, size), - None => Vec::new(), + None => { + // Not in mapping — try backing dir + self.backing_dir_read(path, offset, size) + .unwrap_or_default() + } }; self.visited.remove(path); return result; @@ -192,6 +223,25 @@ impl<'a> FileResolver<'a> { read_external_file(path, offset, size) } + /// Try to get the size of a file via the backing dir. + /// Only succeeds if path is under the backing dir's mount point. + fn backing_dir_size(&self, path: &Path) -> Option { + let (mount_point, backing) = self.backing?; + let rel = path.strip_prefix(mount_point).ok()?; + let name = rel.to_string_lossy(); + let (size, is_regular) = backing.stat(&name)?; + if is_regular { Some(size) } else { None } + } + + /// Try to read from a file via the backing dir. + /// Only succeeds if path is under the backing dir's mount point. + fn backing_dir_read(&self, path: &Path, offset: u64, size: u32) -> Option> { + let (mount_point, backing) = self.backing?; + let rel = path.strip_prefix(mount_point).ok()?; + let name = rel.to_string_lossy(); + Some(backing.read_file(&name, offset, size)) + } + /// Try to resolve a path as internal to a FUSE mount. /// First tries raw prefix match (no syscalls, safe for mount-internal paths). /// Then tries fs::canonicalize on the path — only safe because we only @@ -271,7 +321,7 @@ mod tests { #[test] fn static_entry_passthrough() { let registry = make_registry(); - let mut resolver = FileResolver::new(®istry); + let mut resolver = FileResolver::new(®istry, None); let entry = FileEntry::new_static(b"hello world".to_vec()); assert_eq!(resolver.total_size(&entry), 11); @@ -288,10 +338,10 @@ mod tests { let (_f2, p2) = create_temp_file(b"bbb"); let entry = FileEntry::new_concat(vec![p1, p2]); - let mut resolver = FileResolver::new(®istry); + let mut resolver = FileResolver::new(®istry, None); assert_eq!(resolver.total_size(&entry), 6); assert_eq!(resolver.read_range(&entry, 0, 6), b"aaabbb"); - assert_eq!(resolver.read_range(&entry, 2, 3), b"ab"); + assert_eq!(resolver.read_range(&entry, 2, 3), b"abb"); } // Test 3: Concat with internal static file @@ -313,7 +363,7 @@ mod tests { let internal_path = mount_path.join("inner.txt"); let entry = FileEntry::new_concat(vec![p_ext, internal_path]); - let mut resolver = FileResolver::new(®istry); + let mut resolver = FileResolver::new(®istry, None); assert_eq!(resolver.total_size(&entry), 8); // 3 + 5 assert_eq!(resolver.read_range(&entry, 0, 8), b"EXTINNER"); @@ -343,7 +393,7 @@ mod tests { // Top-level concat references inner_concat.txt let entry = FileEntry::new_concat(vec![mount_path.join("inner_concat.txt")]); - let mut resolver = FileResolver::new(®istry); + let mut resolver = FileResolver::new(®istry, None); assert_eq!(resolver.total_size(&entry), 7); // EXT(3) + LEAF(4) assert_eq!(resolver.read_range(&entry, 0, 7), b"EXTLEAF"); @@ -366,7 +416,7 @@ mod tests { registry.add(mount_path, Arc::clone(&mapping)); let entry = FileEntry::new_concat(vec![mount_path.join("a.txt")]); - let mut resolver = FileResolver::new(®istry); + let mut resolver = FileResolver::new(®istry, None); assert_eq!(resolver.total_size(&entry), 0); assert_eq!(resolver.read_range(&entry, 0, 100), b""); @@ -389,7 +439,7 @@ mod tests { registry.add(mount_path, Arc::clone(&mapping)); let entry = FileEntry::new_concat(vec![mount_path.join("a.txt")]); - let mut resolver = FileResolver::new(®istry); + let mut resolver = FileResolver::new(®istry, None); assert_eq!(resolver.total_size(&entry), 0); assert_eq!(resolver.read_range(&entry, 0, 100), b""); @@ -416,7 +466,7 @@ mod tests { p_ext, mount_path.join("internal.txt"), ]); - let mut resolver = FileResolver::new(®istry); + let mut resolver = FileResolver::new(®istry, None); assert_eq!(resolver.total_size(&entry), 8); // 5 + 3 } @@ -449,7 +499,7 @@ mod tests { symlink(&real_file, &link_path).unwrap(); let entry = FileEntry::new_concat(vec![link_path]); - let mut resolver = FileResolver::new(®istry); + let mut resolver = FileResolver::new(®istry, None); // Should resolve via canonicalize -> internal mapping, not read PLACEHOLDER assert_eq!(resolver.total_size(&entry), 5); @@ -485,9 +535,86 @@ mod tests { // Reading a.txt -> link.txt -> (canonicalize) -> mount/a.txt -> cycle! let entry = FileEntry::new_concat(vec![mount_path.join("a.txt")]); - let mut resolver = FileResolver::new(®istry); + let mut resolver = FileResolver::new(®istry, None); assert_eq!(resolver.total_size(&entry), 0); assert_eq!(resolver.read_range(&entry, 0, 100), b""); } + + // Test 10: Concat shadows backing dir file — reads original via backing dir + #[test] + fn concat_shadow_reads_backing_dir() { + let mount_dir = tempfile::tempdir().unwrap(); + let mount_path = mount_dir.path(); + + // Create a real file in the directory (before it would be overlaid) + fs::write(mount_path.join("config.txt"), b"ORIGINAL").unwrap(); + + let backing = BackingDir::open(mount_path.to_str().unwrap()).unwrap(); + + // Create a concat that shadows config.txt: original + extra line + let (_f_extra, p_extra) = create_temp_file(b"\nEXTRA"); + + let mapping = make_mapping(); + { + let mut m = mapping.lock().unwrap(); + // config.txt is a concat of itself (backing) + extra + m.register_concat( + "config.txt", + vec![mount_path.join("config.txt"), p_extra], + ); + } + + let registry = make_registry(); + registry.add(mount_path, Arc::clone(&mapping)); + + let entry = mapping.lock().unwrap().get("config.txt").unwrap().clone(); + let mp = mount_path.to_path_buf(); + let mut resolver = FileResolver::new(®istry, Some((&mp, &backing))); + // Pre-seed visited with the file being resolved (as eval_fs.rs does), + // so self-referencing sources fall through to backing dir immediately. + resolver.mark_visited(mount_path.join("config.txt")); + + // Without backing dir fallback this would be 0 (cycle). + // With it, we get ORIGINAL (8) + \nEXTRA (6) = 14. + assert_eq!(resolver.total_size(&entry), 14); + + let mut resolver2 = FileResolver::new(®istry, Some((&mp, &backing))); + resolver2.mark_visited(mount_path.join("config.txt")); + assert_eq!(resolver2.read_range(&entry, 0, 20), b"ORIGINAL\nEXTRA"); + } + + // Test 11: Backing dir fallback for file not in mapping + #[test] + fn backing_dir_fallback_not_in_mapping() { + let mount_dir = tempfile::tempdir().unwrap(); + let mount_path = mount_dir.path(); + + // Create a real file that won't be in the mapping + fs::write(mount_path.join("real.txt"), b"REALDATA").unwrap(); + + let backing = BackingDir::open(mount_path.to_str().unwrap()).unwrap(); + + // Create a concat that references real.txt (not in mapping) + let mapping = make_mapping(); + { + let mut m = mapping.lock().unwrap(); + m.register_concat( + "combined.txt", + vec![mount_path.join("real.txt")], + ); + } + + let registry = make_registry(); + registry.add(mount_path, Arc::clone(&mapping)); + + let entry = mapping.lock().unwrap().get("combined.txt").unwrap().clone(); + let mp = mount_path.to_path_buf(); + let mut resolver = FileResolver::new(®istry, Some((&mp, &backing))); + + // real.txt is under the mount but not in the mapping. + // Without backing dir, this would return 0. With it, reads the real file. + assert_eq!(resolver.total_size(&entry), 8); + assert_eq!(resolver.read_range(&entry, 0, 20), b"REALDATA"); + } } diff --git a/slosh_lib/Cargo.toml b/slosh_lib/Cargo.toml index ca20bf656e..cad1669bef 100644 --- a/slosh_lib/Cargo.toml +++ b/slosh_lib/Cargo.toml @@ -21,6 +21,7 @@ shell = { path = "../shell" } unicode-width = { workspace = true } glob = { workspace = true } static_assertions = { workspace = true } +regex = { workspace = true } # Optional dependencies slosh-fuse = { path = "../slosh-fuse", optional = true } diff --git a/slosh_lib/src/docs.rs b/slosh_lib/src/docs.rs new file mode 100644 index 0000000000..3bc7d8e3fe --- /dev/null +++ b/slosh_lib/src/docs.rs @@ -0,0 +1,1525 @@ +use bridge_adapters::add_builtin; +use bridge_adapters::lisp_adapters::SlFrom; +use compile_state::state::{SloshVm, SloshVmTrait}; +use regex::{Regex, RegexBuilder}; +use slvm::vm_hashmap::VMHashMap; +use slvm::{Interned, SLOSH_NIL, VMError, VMErrorObj, VMResult, Value}; +use std::borrow::Cow; +use std::cmp::Ordering; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::error::Error; +use std::fmt::{Debug, Display, Formatter}; +use std::hash::{Hash, Hasher}; +use std::string::ToString; +use std::sync::LazyLock; + +pub const USER_FORMS: &str = "User Forms"; +pub const GLOBAL_NAMESPACE: &str = "root"; +const USAGE: &str = "usage"; +const DESCRIPTION: &str = "description"; +const SECTION: &str = "section"; +const EXAMPLE: &str = "example"; + +pub static DOC_REGEX: LazyLock = LazyLock::new(|| { + RegexBuilder::new(r#"(\s*?Usage:(.+?)$\n\n|\s*?)(\S{1}.*)\n\n\s*Section:(.+?)$(\n\n\s*Example:\n(.*)|\s*)"#) + .multi_line(true) + .dot_matches_new_line(true) + .crlf(true) + .build() + .unwrap() +}); + +// TODO #229 save off list of EXEMPTIONS triggered and write to a md file for docs. +pub static EXEMPTIONS: LazyLock> = LazyLock::new(|| { + let mut exemption_set = HashSet::new(); + exemption_set.insert("version"); + exemption_set.insert("this-fn"); + exemption_set.insert("identical?"); + exemption_set.insert("type"); + exemption_set.insert("call/cc"); + exemption_set.insert("defer"); + exemption_set.insert("while"); + exemption_set.insert("doc-string"); + exemption_set.insert("get"); + exemption_set.insert("return"); + exemption_set.insert("*int-min*"); + exemption_set.insert("*int-max*"); + exemption_set.insert("prn"); + exemption_set.insert("pr"); + exemption_set.insert("fprn"); + exemption_set.insert("fpr"); + exemption_set.insert("eprn"); + exemption_set.insert("epr"); + exemption_set.insert("dump-regs"); + exemption_set.insert("dasm"); + exemption_set.insert("*int-bits*"); + exemption_set.insert("*stdout*"); + exemption_set.insert("*prn*"); + + // slosh specific colors + exemption_set.insert("get-rgb-seq"); + exemption_set.insert("tok-slsh-form-color"); + exemption_set.insert("tok-slsh-fcn-color"); + exemption_set.insert("tok-default-color"); + exemption_set.insert("tok-sys-command-color"); + exemption_set.insert("tok-sys-alias-color"); + exemption_set.insert("tok-string-color"); + exemption_set.insert("tok-invalid-color"); + + exemption_set.insert("*fg-default*"); + exemption_set.insert("*fg-black*"); + exemption_set.insert("*fg-red*"); + exemption_set.insert("*fg-green*"); + exemption_set.insert("*fg-yellow*"); + exemption_set.insert("*fg-blue*"); + exemption_set.insert("*fg-magenta*"); + exemption_set.insert("*fg-cyan*"); + exemption_set.insert("*fg-white*"); + + exemption_set.insert("*bg-default*"); + exemption_set.insert("*bg-black*"); + exemption_set.insert("*bg-red*"); + exemption_set.insert("*bg-green*"); + exemption_set.insert("*bg-yellow*"); + exemption_set.insert("*bg-blue*"); + exemption_set.insert("*bg-magenta*"); + exemption_set.insert("*bg-cyan*"); + exemption_set.insert("*bg-white*"); + + // default init.slosh + exemption_set.insert("*ns*"); + exemption_set.insert("__prompt"); + exemption_set.insert("__line_handler"); + exemption_set.insert("get-pwd"); + exemption_set.insert("set-prompt-tail"); + exemption_set.insert("parse-git-branch"); + + // in runtime + exemption_set.insert("#"); + + exemption_set +}); + +#[derive(Debug, Clone, Eq, Hash, PartialEq)] +pub enum Namespace { + Global, + Other(Interned), +} + +impl Namespace { + pub fn display(&self, vm: &mut SloshVm) -> String { + match self { + Namespace::Global => GLOBAL_NAMESPACE.to_string(), + Namespace::Other(s) => { + let s = vm.get_interned(*s); + s.to_string() + } + } + } + + pub fn get_doc( + &self, + interned: &Interned, + docs: &mut Vec, + vm: &mut SloshVm, + require_proper_format: bool, + ) -> DocResult<()> { + let slosh_doc = SloshDoc::new(*interned, vm, self.clone()); + match slosh_doc { + Ok(slosh_doc) => { + docs.push(slosh_doc); + } + Err(e) => match e { + _ if !require_proper_format => { + let incomplete_doc = SloshDoc::new_incomplete(*interned, vm, self.clone())?; + docs.push(incomplete_doc); + } + _ => { + return Err(e); + } + }, + } + Ok(()) + } + + pub fn add_docs( + &self, + docs: &mut Vec, + vm: &mut SloshVm, + require_proper_format: bool, + ) -> DocResult<()> { + match self { + Namespace::Global => { + for g in vm.globals().clone().keys() { + self.get_doc(g, docs, vm, require_proper_format)?; + } + } + Namespace::Other(i) => { + let value = builtins::retrieve_in_namespace(vm, i); + for v in value { + if let Value::Symbol(sym) = v { + self.get_doc(&sym, docs, vm, require_proper_format)?; + } + } + } + } + docs.sort(); + docs.dedup(); + Ok(()) + } +} + +#[derive(Debug, Clone, Eq, Hash, PartialEq)] +pub struct DocStringSection { + pub usage: Option, + pub description: String, + pub section: String, + pub example: Option, +} + +impl Display for DocStringSection { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let usage = self + .usage + .clone() + .map(|usage| format!("Usage:{}\n\n", usage)) + .unwrap_or_default(); + let example = self + .example + .clone() + .map(|example| format!("Example:\n{}", example)) + .unwrap_or_default(); + write!( + f, + "{usage}\n{description}\n\nSection: {section}\n\n{example}", + usage = usage, + description = self.description, + section = self.section, + example = example, + ) + } +} + +impl DocStringSection { + pub fn from_symbol(slot: u32, sym: Value, vm: &mut SloshVm) -> DocResult { + let sym_str = sym.display_value(vm); + let raw_doc_string = Self::raw_docstring(slot, vm); + let backup_usage = crate::usage(vm, slot, &sym); + Self::parse_doc_string(Cow::Owned(sym_str), raw_doc_string, backup_usage) + } + + fn raw_docstring(slot: u32, vm: &mut SloshVm) -> String { + let docstring_key = vm.intern_static("doc-string"); + vm.get_global_property(slot, docstring_key) + .and_then(|x| match x { + Value::String(h) => Some(vm.get_string(h).to_string()), + Value::StringConst(i) => Some(vm.get_interned(i).to_string()), + _ => None, + }) + // return default empty string and have parse_doc_string handle error if no doc provided. + .unwrap_or_default() + } + + /// Given the rules for parsing slosh docstrings, parse one! See [`static@DOC_REGEX`] + /// for the specification. + pub fn parse_doc_string( + symbol: Cow<'_, str>, + raw_doc_string: String, + backup_usage: String, + ) -> DocResult { + let cap = + DOC_REGEX + .captures(raw_doc_string.as_str()) + .ok_or_else(|| DocError::NoDocString { + symbol: symbol.to_string(), + }); + if EXEMPTIONS.contains(symbol.as_ref()) && cap.is_err() { + let usage = Some("unknown".to_string()); + let description = "unknown".to_string(); + let section = "undocumented".to_string(); + let example = None; + return Ok(DocStringSection { + usage, + description, + section, + example, + }); + } + let cap = cap?; + let mut usage = cap.get(2).map(|x| x.as_str().trim().to_string()); + if usage.is_none() && !backup_usage.trim().is_empty() { + usage = Some(backup_usage); + } + let description = cap + .get(3) + .ok_or_else(|| DocError::DocStringMissingSection { + symbol: symbol.to_string(), + section: "Description".to_string(), + }) + .map(|x| x.as_str().to_string())?; + let section = cap + .get(4) + .ok_or_else(|| DocError::DocStringMissingSection { + symbol: symbol.to_string(), + section: "Section".to_string(), + }) + .map(|x| x.as_str().trim().to_string())?; + let example = cap.get(6).map(|x| x.as_str().trim().to_string()); + if EXEMPTIONS.contains(symbol.as_ref()) { + Err(DocError::RemoveExemption { + symbol: symbol.to_string(), + }) + } else { + Ok(DocStringSection { + usage, + description, + section, + example, + }) + } + } + + /// Just write everything that was in the doc section to the description section. + /// TODO #229 track incomplete documentation in a page. + pub fn new_incomplete(slot: u32, sym: &Value, vm: &mut SloshVm) -> Self { + let description = Self::raw_docstring(slot, vm); + let usage = Some(crate::usage(vm, slot, sym)); + DocStringSection { + usage, + description, + section: SLOSH_NIL.to_string(), + example: None, + } + } + + fn into_styled_output(mut self, query: &DocSearchQuery, style: &StyleOptions) -> Self { + (self.usage, self.description, self.section, self.example) = { + // use destructing to force compiler error if structure changes. + let Self { + usage, + description, + section, + example, + } = &self; + ( + usage.as_ref().map(|x| highlight_matches(x, query, style)), + highlight_matches(description, query, style), + highlight_matches(section, query, style), + example.as_ref().map(|x| highlight_matches(x, query, style)), + ) + }; + self + } +} + +pub trait AsMd { + fn as_md(&self) -> String; +} + +impl AsMd for SloshDoc { + fn as_md(&self) -> String { + let mut content = format!(" ### {}\n", self.symbol); + if let Some(usage) = &self.doc_string.usage { + content += &format!("**Usage:** {}\n\n", usage); + } + content = content + &format!("**Namespace:** {}\n\n", self.namespace); + content = content + &format!("{}\n", self.doc_string.description); + if let Some(example) = &self.doc_string.example { + content += "\n\nExample:\n```\n"; + content += example; + content += "\n``` \n"; + } else { + content += "\n\nNo Examples\n"; + } + content + } +} + +#[derive(Eq, Debug, Clone)] +pub struct SloshDoc { + pub symbol: String, + pub symbol_type: String, + pub namespace: String, + pub doc_string: DocStringSection, +} + +impl Display for SloshDoc { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{symbol}\nType: {symbol_type}\nNamespace: {namespace}\n\n{doc_string}", + symbol = self.symbol, + symbol_type = self.symbol_type, + namespace = self.namespace, + doc_string = self.doc_string + ) + } +} + +impl PartialEq for SloshDoc { + fn eq(&self, other: &Self) -> bool { + self.symbol == other.symbol && self.namespace == other.namespace + } +} + +impl Hash for SloshDoc { + fn hash(&self, state: &mut H) { + state.write(self.symbol.as_bytes()); + state.write(self.namespace.as_bytes()); + } +} + +impl PartialOrd for SloshDoc { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for SloshDoc { + fn cmp(&self, other: &Self) -> Ordering { + self.symbol + .cmp(&other.symbol) + .then_with(|| self.namespace.cmp(&other.namespace)) + } +} + +impl SloshDoc { + pub fn new(g: Interned, vm: &mut SloshVm, namespace: Namespace) -> DocResult { + let sym = Value::Symbol(g); + let slot = vm.global_intern_slot(g); + if let Some(slot) = slot { + let doc_string = DocStringSection::from_symbol(slot, sym, vm)?; + let symbol = sym.display_value(vm); + let mut full_name: Vec<_> = symbol.split("::").collect(); + let symbol = full_name + .pop() + .expect("Symbol should never be an empty.") + .to_string(); + let symbol_type = sym.display_type(vm).to_string(); + let namespace = namespace.display(vm); + Ok(SloshDoc { + symbol, + symbol_type, + namespace, + doc_string, + }) + } else { + Err(DocError::NoSymbol { + symbol: sym.display_value(vm).to_string(), + }) + } + } + + pub fn new_incomplete(g: Interned, vm: &mut SloshVm, namespace: Namespace) -> DocResult { + let sym = Value::Symbol(g); + let slot = vm.global_intern_slot(g); + if let Some(slot) = slot { + let doc_string = DocStringSection::new_incomplete(slot, &sym, vm); + let symbol = sym.display_value(vm); + let symbol_type = sym.display_type(vm).to_string(); + let namespace = namespace.display(vm); + Ok(SloshDoc { + symbol, + symbol_type, + namespace, + doc_string, + }) + } else { + Err(DocError::NoSymbol { + symbol: sym.display_value(vm).to_string(), + }) + } + } + + /// Provide the fully qualified name + pub fn fully_qualified_name(&self) -> String { + self.namespace.to_string() + "::" + self.symbol.as_ref() + } + + fn into_styled_output(mut self, query: &DocSearchQuery, style: &StyleOptions) -> Self { + ( + self.symbol, + self.namespace, + self.symbol_type, + self.doc_string, + ) = { + // use destructing to force compiler error if structure changes. + let Self { + symbol, + namespace, + symbol_type, + doc_string, + } = &self; + ( + highlight_matches(symbol, query, style), + highlight_matches(namespace, query, style), + highlight_matches(symbol_type, query, style), + doc_string.clone().into_styled_output(query, style), + ) + }; + self + } +} + +pub enum DocError { + NoSymbol { symbol: String }, + NoDocString { symbol: String }, + RemoveExemption { symbol: String }, + DocStringMissingSection { symbol: String, section: String }, +} + +impl Debug for DocError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(self, f) + } +} + +impl Display for DocError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let str = match self { + DocError::NoSymbol { symbol } => { + format!( + "No documentation string provided for symbol {symbol}." + ) + } + DocError::NoDocString { symbol } => { + format!( + "Either documentation provided does not conform to conventional layout or no documentation string provided for symbol {symbol} all slosh functions with documentation must have a string that conforms to the conventional layout." + ) + } + DocError::DocStringMissingSection { symbol, section } => { + format!("Invalid documentation string for symbol {symbol}, missing required section {section:?}") + } + DocError::RemoveExemption { symbol} => { + format!("Documentation has been added for {symbol}, remove it from EXEMPTIONS list in slosh_test::docs::EXEMPTIONS.") + } + } + .to_string(); + write!(f, "{}", str) + } +} + +impl Error for DocError {} + +impl From for VMError { + fn from(value: DocError) -> Self { + VMError { + key: "doc", + obj: VMErrorObj::Message(value.to_string()), + } + } +} + +pub type DocResult = Result; + +#[derive(Debug, Clone)] +struct StyleOptions { + highlight_color: String, + default_color: String, + use_background: bool, +} + +impl Default for StyleOptions { + fn default() -> Self { + StyleOptions { + highlight_color: "\x1b[43m\x1b[30m".to_string(), // yellow background with black text + default_color: "\x1b[0m".to_string(), // reset + use_background: true, + } + } +} + +impl StyleOptions { + fn from_handle(vm: &mut SloshVm, handle: slvm::Handle) -> Self { + let mut options = Self::default(); + + // Intern all keys upfront + let highlight_color_key = Value::Keyword(vm.intern("highlight-color")); + let default_color_key = Value::Keyword(vm.intern("default-color")); + let use_background_key = Value::Keyword(vm.intern("use-background")); + + let map = vm.get_map(handle); + + // Extract colors from map + if let Some(val) = map.get(vm, highlight_color_key) { + match val { + Value::String(s) => options.highlight_color = vm.get_string(s).to_string(), + Value::StringConst(i) => options.highlight_color = vm.get_interned(i).to_string(), + _ => {} + } + } + + if let Some(val) = map.get(vm, default_color_key) { + match val { + Value::String(s) => options.default_color = vm.get_string(s).to_string(), + Value::StringConst(i) => options.default_color = vm.get_interned(i).to_string(), + _ => {} + } + } + + if let Some(Value::False) = map.get(vm, use_background_key) { + options.use_background = false; + } + + options + } +} + +fn highlight_matches(text: &str, query: &DocSearchQuery, style: &StyleOptions) -> String { + if query.use_regex { + highlight_regex_matches(text, &query.query, style) + } else if query.basic { + highlight_basic_matches(text, &query.query, style) + } else { + highlight_substring_matches(text, &query.query, style) + } +} + +fn highlight_substring_matches(text: &str, query: &str, style: &StyleOptions) -> String { + let text_lower = text.to_lowercase(); + let query_lower = query.to_lowercase(); + let mut result = String::new(); + let mut last_end = 0; + + for (start, _) in text_lower.match_indices(&query_lower) { + // Add text before match + result.push_str(&text[last_end..start]); + // Add highlighted match + result.push_str(&style.highlight_color); + result.push_str(&text[start..start + query.len()]); + result.push_str(&style.default_color); + last_end = start + query.len(); + } + + // Add remaining text + result.push_str(&text[last_end..]); + result +} + +fn highlight_regex_matches(text: &str, pattern: &str, style: &StyleOptions) -> String { + if let Ok(re) = Regex::new(pattern) { + let mut result = String::new(); + let mut last_end = 0; + + for mat in re.find_iter(text) { + // Add text before match + result.push_str(&text[last_end..mat.start()]); + // Add highlighted match + result.push_str(&style.highlight_color); + result.push_str(mat.as_str()); + result.push_str(&style.default_color); + last_end = mat.end(); + } + + // Add remaining text + result.push_str(&text[last_end..]); + result + } else { + text.to_string() + } +} + +fn highlight_basic_matches(text: &str, pattern: &str, style: &StyleOptions) -> String { + let pattern_lower = pattern.to_lowercase(); + let mut result = String::new(); + let mut pattern_chars = pattern_lower.chars().peekable(); + let mut in_match = false; + + for ch in text.chars() { + if let Some(&pattern_ch) = pattern_chars.peek() { + if ch.to_lowercase().next() == Some(pattern_ch) { + if !in_match { + result.push_str(&style.highlight_color); + in_match = true; + } + result.push(ch); + pattern_chars.next(); + } else { + if in_match { + result.push_str(&style.default_color); + in_match = false; + } + result.push(ch); + } + } else { + if in_match { + result.push_str(&style.default_color); + in_match = false; + } + result.push(ch); + } + } + + if in_match { + result.push_str(&style.default_color); + } + + result +} + +fn generate_styled_output( + _vm: &mut SloshVm, + results: &BTreeSet, + query: &DocSearchQuery, + style: &StyleOptions, +) -> String { + let mut output = String::new(); + let len = results.len(); + + for (i, doc) in results.iter().enumerate() { + if i > 0 { + output.push_str("\n────────────────────────────────────────\n\n"); + } + + // Result header with highlighted symbol + output.push_str(&format!("Result {}: ", i + 1)); + let mut doc = doc.clone(); + doc = doc.into_styled_output(query, style); + output.push_str(&format!("{doc}")); + } + + output.push_str(&format!( + "\nFound {} result{}.", + len, + if len == 1 { "" } else { "s" } + )); + + output +} + +fn insert_section(map: &mut VMHashMap, key: &'static str, value: String, vm: &mut SloshVm) { + let key_const = Value::Keyword(vm.intern_static(key)); + let value_text = vm.alloc_string(value); + map.insert(vm, key_const, value_text); +} + +fn insert_nil_section(map: &mut VMHashMap, key: &'static str, vm: &mut SloshVm) { + let key_const = Value::Keyword(vm.intern_static(key)); + map.insert(vm, key_const, Value::Nil); +} + +impl SlFrom for VMHashMap { + fn sl_from(value: SloshDoc, vm: &mut SloshVm) -> VMResult { + let mut map = Self::with_capacity(4); + match (value.doc_string.usage, value.doc_string.example) { + (Some(usage), Some(example)) => { + insert_section(&mut map, USAGE, usage, vm); + insert_section(&mut map, EXAMPLE, example, vm); + } + (Some(usage), None) => { + insert_section(&mut map, USAGE, usage, vm); + insert_nil_section(&mut map, EXAMPLE, vm); + } + (None, Some(example)) => { + insert_section(&mut map, EXAMPLE, example, vm); + insert_nil_section(&mut map, USAGE, vm); + } + (None, None) => { + insert_nil_section(&mut map, EXAMPLE, vm); + insert_nil_section(&mut map, USAGE, vm); + } + } + insert_section(&mut map, SECTION, value.doc_string.section, vm); + insert_section(&mut map, DESCRIPTION, value.doc_string.description, vm); + Ok(map) + } +} + +impl SlFrom for Value { + fn sl_from(value: SloshDoc, vm: &mut SloshVm) -> VMResult { + let map = VMHashMap::sl_from(value, vm)?; + Ok(vm.alloc_map(map)) + } +} + +fn doc_map(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + let mut i = registers.iter(); + match (i.next(), i.next()) { + (Some(Value::Symbol(g)), None) => { + // Pause GC so that we don't wind up collecting any strings used to build the doc map + // before they get rooted via the map. + vm.pause_gc(); + + let res = match SloshDoc::new(*g, vm, Namespace::Global) { + Ok(slosh_doc) => Value::sl_from(slosh_doc, vm), + Err(e) => Err(VMError::from(e)), + }; + // Unpause GC, this MUST happen so no early returns (looking at you ?). + vm.unpause_gc(); + res + } + _ => Err(VMError::new_vm("takes one argument (symbol)".to_string())), + } +} + +/// Each doc has a tag in its `Section:` definition by convention that logically groups functions. +/// Using a HashMap store the section tags as keys and add all slosh docs from to a vector as a value +/// corresponding to its section. +pub fn get_docs_by_section( + vm: &mut SloshVm, + require_proper_format: bool, +) -> HashMap> { + let mut docs_by_section: HashMap> = HashMap::new(); + let mut docs: Vec = vec![]; + Namespace::Global + .add_docs(&mut docs, vm, require_proper_format) + .unwrap(); + let namespaces = builtins::get_namespaces_interned(vm); + for i in namespaces { + let namespace = Namespace::Other(i); + namespace + .add_docs(&mut docs, vm, require_proper_format) + .unwrap(); + } + for d in docs { + let d = d.clone(); + let section = d.doc_string.section.clone(); + docs_by_section.entry(section).or_default().insert(d); + } + docs_by_section +} + +fn get_exemptions(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if !registers.is_empty() { + return Err(VMError::new_vm( + "get-exemptions: takes no arguments".to_string(), + )); + } + let mut exemptions = EXEMPTIONS.iter().copied().collect::>(); + exemptions.sort(); + let exemptions = exemptions + .iter() + .map(|x| Value::Symbol(vm.intern(x))) + .collect::>(); + Ok(vm.alloc_vector(exemptions)) +} + +fn get_globals_sorted(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if !registers.is_empty() { + return Err(VMError::new_vm( + "get-globals-sorted: takes no arguments".to_string(), + )); + } + let mut result = BTreeMap::new(); + for g in vm.globals().keys() { + let sym = Value::Symbol(*g); + let val: String = sym.display_value(vm); + result.insert(val, sym); + } + let v = result.values().cloned().collect(); + Ok(vm.alloc_vector(v)) +} + +#[derive(Debug, Clone)] +struct DocSearchQuery { + query: String, + use_regex: bool, + basic: bool, + fields: HashSet, + namespace_filter: Option, + section_filter: Option, +} + +impl DocSearchQuery { + fn new(query: String) -> Self { + DocSearchQuery { + query, + use_regex: false, + basic: false, + fields: HashSet::new(), + namespace_filter: None, + section_filter: None, + } + } + + fn with_regex(mut self) -> Self { + self.use_regex = true; + self.basic = false; + self + } + + fn with_basic(mut self) -> Self { + self.basic = true; + self.use_regex = false; + self + } + + fn with_fields(mut self, fields: Vec) -> Self { + self.fields = fields.into_iter().collect(); + self + } + + fn with_namespace(mut self, namespace: String) -> Self { + self.namespace_filter = Some(namespace); + self + } + + fn with_section(mut self, section: String) -> Self { + self.section_filter = Some(section); + self + } + + fn should_search_field(&self, field: &str) -> bool { + self.fields.is_empty() || self.fields.contains(field) + } + + fn matches_text(&self, text: &str) -> bool { + if self.use_regex { + if let Ok(re) = Regex::new(&self.query) { + re.is_match(text) + } else { + false + } + } else if self.basic { + basic_match(text, &self.query) + } else { + text.to_lowercase().contains(&self.query.to_lowercase()) + } + } +} + +fn basic_match(text: &str, pattern: &str) -> bool { + let text_lower = text.to_lowercase(); + let pattern_lower = pattern.to_lowercase(); + let mut pattern_chars = pattern_lower.chars(); + let mut current_char = pattern_chars.next(); + + for text_char in text_lower.chars() { + if let Some(p_char) = current_char { + if text_char == p_char { + current_char = pattern_chars.next(); + } + } else { + return true; + } + } + + current_char.is_none() +} + +fn search_docs( + vm: &mut SloshVm, + query: &DocSearchQuery, + require_proper_format: bool, +) -> BTreeSet { + let docs_by_section = get_docs_by_section(vm, require_proper_format); + let mut results = BTreeSet::new(); + + for (section, docs) in docs_by_section { + // Check section filter + if let Some(ref section_filter) = query.section_filter { + if !query.matches_text(§ion) && section != *section_filter { + continue; + } + } + + for doc in docs { + // Check namespace filter + if let Some(ref ns_filter) = query.namespace_filter { + if !doc.namespace.contains(ns_filter) && doc.namespace != *ns_filter { + continue; + } + } + + let mut matched = false; + + // Search in symbol name + if query.should_search_field("name") + || query.should_search_field("symbol") && query.matches_text(&doc.symbol) + { + matched = true; + } + + // Search in usage + if !matched + && query.should_search_field(USAGE) + && let Some(ref usage) = doc.doc_string.usage + && query.matches_text(usage) + { + matched = true; + } + + // Search in description + if !matched + && query.should_search_field(DESCRIPTION) + && query.matches_text(&doc.doc_string.description) + { + matched = true; + } + + // Search in section + if !matched + && query.should_search_field(SECTION) + && query.matches_text(&doc.doc_string.section) + { + matched = true; + } + + // Search in example + if !matched + && query.should_search_field(EXAMPLE) + && let Some(ref example) = doc.doc_string.example + && query.matches_text(example) + { + matched = true; + } + + // If no specific fields specified, search all fields + if !matched && query.fields.is_empty() { + matched = query.matches_text(&doc.symbol) + || doc + .doc_string + .usage + .as_ref() + .map(|u| query.matches_text(u)) + .unwrap_or(false) + || query.matches_text(&doc.doc_string.description) + || query.matches_text(&doc.doc_string.section) + || doc + .doc_string + .example + .as_ref() + .map(|e| query.matches_text(e)) + .unwrap_or(false); + } + + if matched { + results.insert(doc); + } + } + } + + results +} + +fn doc_search(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if registers.is_empty() { + return Err(VMError::new_vm( + "doc-search requires at least a query string".to_string(), + )); + } + + let query_str = match ®isters[0] { + Value::String(s) => vm.get_string(*s).to_string(), + Value::StringConst(i) => vm.get_interned(*i).to_string(), + _ => { + return Err(VMError::new_vm( + "First argument must be a query string".to_string(), + )); + } + }; + + let mut query = DocSearchQuery::new(query_str); + + // Parse options if provided as a hash map + let mut use_styled = true; + let mut style_options = StyleOptions::default(); + let mut style_map_handle_opt = None; + if registers.len() > 1 { + match ®isters[1] { + Value::Map(handle) => { + // Intern all keys first to avoid borrowing issues + let regex_key = Value::Keyword(vm.intern_static("regex")); + let basic_search_key = Value::Keyword(vm.intern_static("basic")); + let fields_key = Value::Keyword(vm.intern_static("fields")); + let ns_key = Value::Keyword(vm.intern_static("namespace")); + let section_key = Value::Keyword(vm.intern_static("section")); + let styled_key = Value::Keyword(vm.intern_static("styled")); + let style_key = Value::Keyword(vm.intern_static("style")); + + let map = vm.get_map(*handle); + + // Check for regex option + if let Some(Value::True) = map.get(vm, regex_key) { + query = query.with_regex(); + } + + // Check for basic option + if let Some(Value::True) = map.get(vm, basic_search_key) { + query = query.with_basic(); + } + + // Check for fields option + if let Some(Value::Vector(v_handle)) = map.get(vm, fields_key) { + let vec = vm.get_vector(v_handle); + let mut fields = Vec::new(); + for val in vec.iter() { + match val { + Value::String(s) => fields.push(vm.get_string(*s).to_string()), + Value::StringConst(i) => fields.push(vm.get_interned(*i).to_string()), + Value::Keyword(i) => fields.push(vm.get_interned(*i).to_string()), + _ => {} + } + } + query = query.with_fields(fields); + } + + // Check for namespace filter + if let Some(ns_val) = map.get(vm, ns_key) { + match ns_val { + Value::String(s) => { + query = query.with_namespace(vm.get_string(s).to_string()) + } + Value::StringConst(i) => { + query = query.with_namespace(vm.get_interned(i).to_string()) + } + _ => {} + } + } + + // Check for section filter + if let Some(section_val) = map.get(vm, section_key) { + match section_val { + Value::String(s) => { + query = query.with_section(vm.get_string(s).to_string()) + } + Value::StringConst(i) => { + query = query.with_section(vm.get_interned(i).to_string()) + } + _ => {} + } + } + + // Check for styled option + if let Some(Value::False) = map.get(vm, styled_key) { + use_styled = false; + } + + // Check for style customization + if let Some(Value::Map(style_handle)) = map.get(vm, style_key) { + use_styled = true; // Enable styling if style options are provided + style_map_handle_opt = Some(style_handle); + } + } + _ => { + return Err(VMError::new_vm( + "Second argument must be a map of options".to_string(), + )); + } + } + } + + // Process style options after we're done with the initial map + if let Some(style_handle) = style_map_handle_opt { + style_options = StyleOptions::from_handle(vm, style_handle); + } + + // Check if markdown output is requested + let mut use_markdown = false; + if registers.len() > 1 { + if let Value::Map(handle) = ®isters[1] { + let markdown_key = Value::Keyword(vm.intern_static("markdown")); + let map = vm.get_map(*handle); + if let Some(Value::True) = map.get(vm, markdown_key) { + use_markdown = true; + } + } + } + + let results = search_docs(vm, &query, false); + + let result = if use_markdown { + // Generate markdown output + let mut markdown_output = String::new(); + let len = results.len(); + for (i, doc) in results.iter().enumerate() { + if i > 0 { + markdown_output.push_str("\n---\n\n"); + } + markdown_output.push_str(&format!( + "**Result {}: `{}::{}`**\n\n", + i + 1, + doc.namespace, + doc.symbol + )); + markdown_output.push_str(&doc.as_md()); + } + markdown_output.push_str(&format!( + "\nFound {len} result{}.", + if len == 1 { "" } else { "s" } + )); + vm.alloc_string(markdown_output) + } else if use_styled { + // Generate styled string output + let styled_output = generate_styled_output(vm, &results, &query, &style_options); + vm.alloc_string(styled_output) + } else { + // Generate vector of doc maps + let mut result_values = Vec::new(); + for doc in results { + match Value::sl_from(doc, vm) { + Ok(val) => result_values.push(val), + Err(e) => { + vm.unpause_gc(); + return Err(e); + } + } + } + vm.alloc_vector(result_values) + }; + + Ok(result) +} + +fn docs_by_section_builtin(vm: &mut SloshVm, registers: &[Value]) -> VMResult { + if !registers.is_empty() { + return Err(VMError::new_vm( + "docs-by-section: takes no arguments".to_string(), + )); + } + vm.pause_gc(); + let sections = get_docs_by_section(vm, false); + let mut map = VMHashMap::with_capacity(sections.len()); + for (section, docs) in §ions { + let key = vm.alloc_string(section.clone()); + let md: String = docs.iter().map(|d| d.as_md()).collect::>().join("\n---\n\n"); + let val = vm.alloc_string(md); + map.insert(vm, key, val); + } + let result = vm.alloc_map(map); + vm.unpause_gc(); + Ok(result) +} + +pub fn add_doc_search_builtins(env: &mut SloshVm) { + add_builtin( + env, + "doc-map", + doc_map, + "Usage: (doc-map symbol) + +Returns documentation for given symbol as map. Keyword is a documentation fragment +(usage, section, description, example) and value is text describing given fragment. + +Section: doc + +Example: +#t +", + ); + + add_builtin( + env, + "doc-search", + doc_search, + "Usage: (doc-search query-string [options-map]) + +Search through all documentation for functions matching the query. +The query string is searched across all documentation fields by default. + +Options map can contain: +- :regex #t - Use regular expressions for matching +- :basic #t - Use basic matching (characters must appear in order) +- :fields [\"usage\" \"description\"] - Limit search to specific fields +- :namespace \"namespace-name\" - Filter by namespace +- :section \"section-name\" - Filter by section +- :markdown #t - Return results as a markdown-formatted string instead of vector +- :styled #t - Return results with highlighted search matches +- :style {:highlight-color \"\\x1b[43m\\x1b[30m\" ...} - Customize highlight style + +Style options (when :style map is provided): +- :highlight-color - Color for highlighting matches (default: yellow background, black text) +- :default-color - Color to reset after highlight (default: \\x1b[0m) +- :use-background #f - Use foreground color instead of background + +When :styled is true, matching text in results is highlighted to show why +each result matched the search query. + +Returns a vector of documentation maps, markdown string, or styled string. + +Section: doc + +Example: +(doc-search \"file\") +(doc-search \"^str-\" {:regex #t}) +(doc-search \"core\" {:fields [\"section\"]}) +(doc-search \"map\" {:basic #t :section \"core\"}) +(doc-search \"string\" {:markdown #t}) +(doc-search \"string\" {:styled #t}) +(doc-search \"map\" {:style {:highlight-color *fg-red* :use-background #f}}) +", + ); + + add_builtin( + env, + "get-globals-sorted", + get_globals_sorted, + "Usage: (get-globals-sorted) + +Return a vector containing all the symbols currently defined globally in sorted order (alphanumerically). + +Section: doc +", + ); + + add_builtin( + env, + "get-exemptions", + get_exemptions, + "Usage: (get-exemptions) + +Return a vector containing all the symbols currently exempted from docs +(so the build passes), Ideally this will be 0. + +Section: doc +", + ); + + add_builtin( + env, + "docs-by-section", + docs_by_section_builtin, + "Usage: (docs-by-section) + +Returns a map where keys are section names and values are markdown-formatted +documentation strings for all symbols in that section. Useful for generating +documentation files or mounting docs via FUSE. + +Section: doc + +Example: +(docs-by-section) +", + ); +} + +#[cfg(test)] +mod test { + use super::*; + use std::borrow::Cow; + + static REGEX_TEST_CASES: LazyLock> = + LazyLock::new(|| { + let mut set = BTreeMap::new(); + + set.insert( + (true, "no whitespace around content"), + "Usage: (defmacro name doc_string? argument_list body) + +Create a macro and bind it to a symbol in the current scope. + +Section: core + +Example: +(def test-mac-x 2)", + ); + + set.insert( + (true, "newlines at beginning and end"), + " +Usage: (defmacro name doc_string? argument_list body) + +Create a macro and bind it to a symbol in the current scope. + +Section: core + +Example: +(def test-mac-x 2) +", + ); + + set.insert( + (true, "mixed whitespace at beginning and end"), + " + + Usage: (defmacro name doc_string? argument_list body) + +Create a macro and bind it to a symbol in the current scope. + +Section: core + +Example: +(def test-mac-x 2) + + + ", + ); + + set.insert( + (true, "no usage, no whitespace around content"), + "Create a macro and bind it to a symbol in the current scope. + +Section: core + +Example: +(def test-mac-x 2)", + ); + + set.insert( + (true, "no usage, newlines at beginning and end"), + " +Create a macro and bind it to a symbol in the current scope. + +Section: core + +Example: +(def test-mac-x 2) +", + ); + + set.insert( + (true, "no usage, mixed whitespace at beginning and end"), + " + + Create a macro and bind it to a symbol in the current scope. + +Section: core + +Example: +(def test-mac-x 2) + + + ", + ); + + set.insert( + (true, "no whitespace around content"), + "Usage: (defmacro name doc_string? argument_list body) + + Create a macro and bind it to a symbol in the current scope. + + Section: core + + Example: + (def test-mac-x 2)", + ); + + set.insert( + (true, "newlines at beginning and end"), + " + Usage: (defmacro name doc_string? argument_list body) + +Create a macro and bind it to a symbol in the current scope. + +Section: +core + +Example: +(def test-mac-x 2) +", + ); + + set.insert( + (true, "mixed whitespace at beginning and end"), + " + + Usage: (defmacro name doc_string? argument_list body) + +Create a macro and bind it to a symbol in the current scope. + +Section: core + + + Example: +(def test-mac-x 2) + + + ", + ); + + set.insert( + (true, "no usage, extra whitespace around content"), + "Create a macro and bind it to a symbol in the current scope. + + Section: core + + Example: + (def test-mac-x 2)", + ); + + set.insert( + ( + false, + "no description, no usage, no example, newlines at beginning and end", + ), + " + +Section: +core +", + ); + + set.insert( + (true, "no usage, no example, newlines at beginning and end, extra whitespace around content."), + " + Create a macro and bind it to a symbol in the current scope. + + Section: + core + +", + ); + + set.insert( + (true, "no usage, mixed whitespace at beginning and end, extra whitespace around content."), + " + + Create a macro and bind it to a symbol in the current scope. + +Section: core + + Example: +(def test-mac-x 2) + + + ", + ); + + set.insert( + ( + false, + "no usage, no section, mixed whitespace at beginning and end, extra whitespace around content.", + ), + " + + Create a macro and bind it to a symbol in the current scope. + +core + + Example: +(def test-mac-x 2) + + + ", + ); + + set.insert( + (true, "str-sub test"), + " Usage: (str-sub string start [length]) -> string\n\n Return a substring from a string given start (0 based) and optional length.\n If length is 0 or not provided produces the rest of the string from start to\n string end.\n\n Section: string\n\n Example:\n (test::assert-equal \"string\" (str-sub \"stringxxxyyyxxxsome\" 0 6))\n (test::assert-equal \"some\" (str-sub \"stringxxxyyyxxxsome\" 15 4))\n (test::assert-equal \"yyy\" (str-sub \"stringxxxyyyxxxsome\" 9 3))\n (test::assert-equal \"some\" (str-sub \"stringxxxyyyxxxsome\" 15))\n", + ); + + set + }); + + #[test] + fn test_doc_string_regex() { + for ((result, label), test_case) in REGEX_TEST_CASES.iter() { + let fake_symbol = Cow::Owned("fake-symbol".to_string()); + match DocStringSection::parse_doc_string( + fake_symbol, + test_case.to_string(), + "".to_string(), + ) { + ok @ Ok(_) => { + assert_eq!( + ok.is_ok(), + *result, + "Case: {}, Regex succeeded but should not have with test case: {}!", + label, + test_case + ); + } + ref err @ Err(_) => { + assert_ne!( + err.is_err(), + *result, + "Case: {}, Regex failed but should not have, got err ({:?}) with tests case: {}!", + label, + err, + test_case + ); + } + } + } + } +} diff --git a/slosh_lib/src/lib.rs b/slosh_lib/src/lib.rs index 638e493530..0253d90961 100644 --- a/slosh_lib/src/lib.rs +++ b/slosh_lib/src/lib.rs @@ -33,6 +33,7 @@ use builtins::{add_global_value, add_misc_builtins}; use sl_liner::vi::AlphanumericAndVariableKeywordRule; use sl_liner::{ColorClosure, Context, Prompt, keymap}; +pub mod docs; mod completions; pub mod debug; mod liner_rules; @@ -441,19 +442,16 @@ pub fn set_builtins(env: &mut SloshVm) { add_rand_builtins(env); add_math_builtins(env); add_doc_builtins(env); + docs::add_doc_search_builtins(env); } #[cfg(feature = "fuse")] mod fuse_bindings; -#[cfg(feature = "fuse")] -mod proc_subst_bindings; - /// Add FUSE builtins if the feature is enabled #[cfg(feature = "fuse")] pub fn add_fuse_builtins(env: &mut SloshVm) { fuse_bindings::add_fuse_builtins(env); - proc_subst_bindings::add_proc_subst_builtins(env); } /// Loads the user's sloshrc file, has side-effects, and sets some important @@ -528,13 +526,19 @@ Section: shell"# env.set_global_property(si, key, s); } -pub fn set_builtins_and_shell_builtins(env: &mut SloshVm) { +pub fn set_builtins_and_shell_builtins_with_vm(env: &mut SloshVm, modify_vm: impl FnOnce(&mut SloshVm)) { set_builtins(env); + modify_vm(env); set_shell_builtins(env); #[cfg(feature = "fuse")] add_fuse_builtins(env); } +pub fn set_builtins_and_shell_builtins(env: &mut SloshVm) { + let noop = |_| {}; + set_builtins_and_shell_builtins_with_vm(env, noop); +} + pub fn set_shell_builtins(env: &mut SloshVm) { set_environment(env); add_shell_builtins(env); @@ -664,11 +668,7 @@ pub fn run_slosh(modify_vm: impl FnOnce(&mut SloshVm)) -> i32 { ENV.with(|renv| { let mut env = renv.borrow_mut(); env.pause_gc(); - set_builtins(&mut env); - modify_vm(&mut env); - set_shell_builtins(&mut env); - #[cfg(feature = "fuse")] - add_fuse_builtins(&mut env); + set_builtins_and_shell_builtins_with_vm(&mut env, modify_vm); env.unpause_gc(); }); if config.command.is_none() && config.script.is_none() { diff --git a/slosh_lib/src/proc_subst_bindings.rs b/slosh_lib/src/proc_subst_bindings.rs deleted file mode 100644 index b36c690018..0000000000 --- a/slosh_lib/src/proc_subst_bindings.rs +++ /dev/null @@ -1,288 +0,0 @@ -use bridge_adapters::add_builtin; -use compile_state::state::SloshVm; -use slvm::{VMError, VMResult, Value}; -use std::collections::HashMap; -use std::sync::Mutex; - -use slosh_fuse::proc_subst::ProcSubst; - -// Registry for process substitution instances -use std::sync::OnceLock; -static PROC_SUBST_REGISTRY: OnceLock>> = OnceLock::new(); - -fn get_registry() -> &'static Mutex> { - PROC_SUBST_REGISTRY.get_or_init(|| Mutex::new(HashMap::new())) -} - -fn create_proc_subst(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 0 { - return Err(VMError::new_vm( - "create-proc-subst: requires no arguments".to_string(), - )); - } - - let ps = ProcSubst::new() - .map_err(|e| VMError::new_vm(format!("Failed to create proc subst: {}", e)))?; - - let ps_id = format!("proc-subst-{}", uuid::Uuid::new_v4()); - - get_registry() - .lock() - .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))? - .insert(ps_id.clone(), ps); - - Ok(vm.alloc_string(ps_id)) -} - -fn proc_subst_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 3 { - return Err(VMError::new_vm( - "proc-subst-file: requires three arguments (ps-id name command)".to_string(), - )); - } - - let ps_id = registers[0].get_string(vm)?; - let name = registers[1].get_string(vm)?; - let command = registers[2].get_string(vm)?; - - let registry = get_registry() - .lock() - .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - - if let Some(ps) = registry.get(ps_id) { - let path = ps.create_file(&name, &command) - .map_err(|e| VMError::new_vm(format!("Failed to create file: {}", e)))?; - - Ok(vm.alloc_string(path.display().to_string())) - } else { - Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) - } -} - -fn proc_subst_eval_file(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 3 { - return Err(VMError::new_vm( - "proc-subst-eval-file: requires three arguments (ps-id name expression)".to_string(), - )); - } - - let ps_id = registers[0].get_string(vm)?; - let name = registers[1].get_string(vm)?; - let expression = registers[2].get_string(vm)?; - - // Build slosh command to evaluate the expression - let command = format!("slosh -c 'pr {}'", expression); - - let registry = get_registry() - .lock() - .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - - if let Some(ps) = registry.get(ps_id) { - let path = ps.create_file(&name, &command) - .map_err(|e| VMError::new_vm(format!("Failed to create file: {}", e)))?; - - Ok(vm.alloc_string(path.display().to_string())) - } else { - Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) - } -} - -fn proc_subst_remove(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 2 { - return Err(VMError::new_vm( - "proc-subst-remove: requires two arguments (ps-id name)".to_string(), - )); - } - - let ps_id = registers[0].get_string(vm)?; - let name = registers[1].get_string(vm)?; - - let registry = get_registry() - .lock() - .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - - if let Some(ps) = registry.get(ps_id) { - ps.remove_file(&name) - .map_err(|e| VMError::new_vm(format!("Failed to remove file: {}", e)))?; - Ok(Value::True) - } else { - Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) - } -} - -fn proc_subst_list(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 1 { - return Err(VMError::new_vm( - "proc-subst-list: requires one argument (ps-id)".to_string(), - )); - } - - let ps_id = registers[0].get_string(vm)?; - - let registry = get_registry() - .lock() - .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - - if let Some(ps) = registry.get(ps_id) { - let files: Vec = ps.list_files() - .into_iter() - .map(|f| vm.alloc_string(f)) - .collect(); - - Ok(vm.alloc_vector(files)) - } else { - Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) - } -} - -fn proc_subst_path(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 2 { - return Err(VMError::new_vm( - "proc-subst-path: requires two arguments (ps-id name)".to_string(), - )); - } - - let ps_id = registers[0].get_string(vm)?; - let name = registers[1].get_string(vm)?; - - let registry = get_registry() - .lock() - .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - - if let Some(ps) = registry.get(ps_id) { - if let Some(path) = ps.get_file_path(&name) { - Ok(vm.alloc_string(path.display().to_string())) - } else { - Err(VMError::new_vm(format!("File not found: {}", name))) - } - } else { - Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) - } -} - -fn destroy_proc_subst(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.len() != 1 { - return Err(VMError::new_vm( - "destroy-proc-subst: requires one argument (ps-id)".to_string(), - )); - } - - let ps_id = registers[0].get_string(vm)?; - - let mut registry = get_registry() - .lock() - .map_err(|e| VMError::new_vm(format!("Registry lock poisoned: {}", e)))?; - - if registry.remove(ps_id).is_some() { - Ok(Value::True) - } else { - Err(VMError::new_vm(format!("Invalid proc-subst id: {}", ps_id))) - } -} - -pub fn add_proc_subst_builtins(env: &mut SloshVm) { - add_builtin( - env, - "create-proc-subst", - create_proc_subst, - r#"Usage: (create-proc-subst) - -Create a new process substitution context. Returns an ID. - -Section: proc-subst - -Example: -(def ps-id (create-proc-subst)) -"#, - ); - - add_builtin( - env, - "proc-subst-file", - proc_subst_file, - r#"Usage: (proc-subst-file ps-id name command) - -Create a virtual file backed by a command. Returns the file path. - -Section: proc-subst - -Example: -(proc-subst-file ps-id "date.txt" "date +%Y-%m-%d") -"#, - ); - - add_builtin( - env, - "proc-subst-eval-file", - proc_subst_eval_file, - r#"Usage: (proc-subst-eval-file ps-id name expression) - -Create a virtual file backed by a slosh expression. Returns the file path. - -Section: proc-subst - -Example: -(proc-subst-eval-file ps-id "config.env" "(str \"HOST=\" (hostname))") -"#, - ); - - add_builtin( - env, - "proc-subst-remove", - proc_subst_remove, - r#"Usage: (proc-subst-remove ps-id name) - -Remove a virtual file. - -Section: proc-subst - -Example: -(proc-subst-remove ps-id "date.txt") -"#, - ); - - add_builtin( - env, - "proc-subst-list", - proc_subst_list, - r#"Usage: (proc-subst-list ps-id) - -List all virtual files in a proc-subst context. - -Section: proc-subst - -Example: -(proc-subst-list ps-id) -"#, - ); - - add_builtin( - env, - "proc-subst-path", - proc_subst_path, - r#"Usage: (proc-subst-path ps-id name) - -Get the path to a virtual file. - -Section: proc-subst - -Example: -(proc-subst-path ps-id "date.txt") -"#, - ); - - add_builtin( - env, - "destroy-proc-subst", - destroy_proc_subst, - r#"Usage: (destroy-proc-subst ps-id) - -Destroy a process substitution context and clean up all files. - -Section: proc-subst - -Example: -(destroy-proc-subst ps-id) -"#, - ); -} \ No newline at end of file diff --git a/slosh_test_lib/src/docs.rs b/slosh_test_lib/src/docs.rs index ac98f46a1c..6a7f45d6a6 100644 --- a/slosh_test_lib/src/docs.rs +++ b/slosh_test_lib/src/docs.rs @@ -1,794 +1,21 @@ -use crate::docs::legacy as legacy_docs; use bridge_adapters::add_builtin; -use bridge_adapters::lisp_adapters::SlFrom; -use compile_state::state::{SloshVm, SloshVmTrait}; -use lazy_static::lazy_static; +use compile_state::state::SloshVm; use mdbook::book::{Book, Chapter}; use mdbook::{BookItem, MDBook}; -use regex::{Regex, RegexBuilder}; -use slvm::vm_hashmap::VMHashMap; -use slvm::{Interned, SLOSH_NIL, VMError, VMErrorObj, VMResult, Value}; -use std::borrow::Cow; -use std::cmp::Ordering; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::error::Error; -use std::fmt::{Debug, Display, Formatter}; +use slvm::{VMError, VMResult, Value}; +use std::collections::{BTreeMap, BTreeSet}; use std::fs::{self, File}; -use std::hash::{Hash, Hasher}; use std::io::{self, Write}; use std::path::Path; use std::path::PathBuf; -use std::string::ToString; pub mod legacy; -const USER_FORMS: &str = "User Forms"; -const GLOBAL_NAMESPACE: &str = "root"; -const USAGE: &str = "usage"; -const DESCRIPTION: &str = "description"; -const SECTION: &str = "section"; -const EXAMPLE: &str = "example"; - -lazy_static! { - static ref DOC_REGEX: Regex = - RegexBuilder::new(r#"(\s*?Usage:(.+?)$\n\n|\s*?)(\S{1}.*)\n\n\s*Section:(.+?)$(\n\n\s*Example:\n(.*)|\s*)"#) - .multi_line(true) - .dot_matches_new_line(true) - .crlf(true) - .build() - .unwrap(); - // TODO #229 save off list of EXEMPTIONS triggered and write to a md file for docs. - pub static ref EXEMPTIONS: HashSet<&'static str> = { - let mut exemption_set = HashSet::new(); - exemption_set.insert("version"); - exemption_set.insert("this-fn"); - exemption_set.insert("identical?"); - exemption_set.insert("type"); - exemption_set.insert("call/cc"); - exemption_set.insert("defer"); - exemption_set.insert("while"); - exemption_set.insert("doc-string"); - exemption_set.insert("get"); - exemption_set.insert("return"); - exemption_set.insert("*int-min*"); - exemption_set.insert("*int-max*"); - exemption_set.insert("prn"); - exemption_set.insert("pr"); - exemption_set.insert("fprn"); - exemption_set.insert("fpr"); - exemption_set.insert("eprn"); - exemption_set.insert("epr"); - exemption_set.insert("dump-regs"); - exemption_set.insert("dasm"); - exemption_set.insert("*int-bits*"); - exemption_set.insert("*stdout*"); - exemption_set.insert("*prn*"); - - // slosh specific colors - exemption_set.insert("get-rgb-seq"); - exemption_set.insert("tok-slsh-form-color"); - exemption_set.insert("tok-slsh-fcn-color"); - exemption_set.insert("tok-default-color"); - exemption_set.insert("tok-sys-command-color"); - exemption_set.insert("tok-sys-alias-color"); - exemption_set.insert("tok-string-color"); - exemption_set.insert("tok-invalid-color"); - - exemption_set.insert("*fg-default*"); - exemption_set.insert("*fg-black*"); - exemption_set.insert("*fg-red*"); - exemption_set.insert("*fg-green*"); - exemption_set.insert("*fg-yellow*"); - exemption_set.insert("*fg-blue*"); - exemption_set.insert("*fg-magenta*"); - exemption_set.insert("*fg-cyan*"); - exemption_set.insert("*fg-white*"); - - exemption_set.insert("*bg-default*"); - exemption_set.insert("*bg-black*"); - exemption_set.insert("*bg-red*"); - exemption_set.insert("*bg-green*"); - exemption_set.insert("*bg-yellow*"); - exemption_set.insert("*bg-blue*"); - exemption_set.insert("*bg-magenta*"); - exemption_set.insert("*bg-cyan*"); - exemption_set.insert("*bg-white*"); - - // default init.slosh - exemption_set.insert("*ns*"); - exemption_set.insert("__prompt"); - exemption_set.insert("__line_handler"); - exemption_set.insert("get-pwd"); - exemption_set.insert("set-prompt-tail"); - exemption_set.insert("parse-git-branch"); - - // in runtime - exemption_set.insert("#"); - - exemption_set - }; -} - -#[derive(Debug, Clone, Eq, Hash, PartialEq)] -enum Namespace { - Global, - Other(Interned), -} - -impl Namespace { - fn display(&self, vm: &mut SloshVm) -> String { - match self { - Namespace::Global => GLOBAL_NAMESPACE.to_string(), - Namespace::Other(s) => { - let s = vm.get_interned(*s); - s.to_string() - } - } - } - - fn get_doc( - &self, - interned: &Interned, - docs: &mut Vec, - vm: &mut SloshVm, - require_proper_format: bool, - ) -> DocResult<()> { - let slosh_doc = SloshDoc::new(*interned, vm, self.clone()); - match slosh_doc { - Ok(slosh_doc) => { - docs.push(slosh_doc); - } - Err(e) => match e { - _ if !require_proper_format => { - let incomplete_doc = SloshDoc::new_incomplete(*interned, vm, self.clone())?; - docs.push(incomplete_doc); - } - _ => { - return Err(e); - } - }, - } - Ok(()) - } - - fn add_docs( - &self, - docs: &mut Vec, - vm: &mut SloshVm, - require_proper_format: bool, - ) -> DocResult<()> { - match self { - Namespace::Global => { - for g in vm.globals().clone().keys() { - self.get_doc(g, docs, vm, require_proper_format)?; - } - } - Namespace::Other(i) => { - let value = builtins::retrieve_in_namespace(vm, i); - for v in value { - if let Value::Symbol(sym) = v { - self.get_doc(&sym, docs, vm, require_proper_format)?; - } - } - } - } - docs.sort(); - docs.dedup(); - Ok(()) - } -} - -#[derive(Debug, Clone, Eq, Hash, PartialEq)] -struct DocStringSection { - usage: Option, - description: String, - section: String, - example: Option, -} - -impl Display for DocStringSection { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let usage = self - .usage - .clone() - .map(|usage| format!("Usage:{}\n\n", usage)) - .unwrap_or_default(); - let example = self - .example - .clone() - .map(|example| format!("Example:\n{}", example)) - .unwrap_or_default(); - write!( - f, - "{usage}\n{description}\n\nSection: {section}\n\n{example}", - usage = usage, - description = self.description, - section = self.section, - example = example, - ) - } -} - -impl DocStringSection { - pub fn from_symbol(slot: u32, sym: Value, vm: &mut SloshVm) -> DocResult { - let sym_str = sym.display_value(vm); - let raw_doc_string = Self::raw_docstring(slot, vm); - let backup_usage = slosh_lib::usage(vm, slot, &sym); - Self::parse_doc_string(Cow::Owned(sym_str), raw_doc_string, backup_usage) - } - - fn raw_docstring(slot: u32, vm: &mut SloshVm) -> String { - let docstring_key = vm.intern_static("doc-string"); - vm.get_global_property(slot, docstring_key) - .and_then(|x| match x { - Value::String(h) => Some(vm.get_string(h).to_string()), - Value::StringConst(i) => Some(vm.get_interned(i).to_string()), - _ => None, - }) - // return default empty string and have parse_doc_string handle error if no doc provided. - .unwrap_or_default() - } - - /// Given the rules for parsing slosh docstrings, parse one! See [`static@DOC_REGEX`] - /// for the specification. - pub fn parse_doc_string( - symbol: Cow<'_, str>, - raw_doc_string: String, - backup_usage: String, - ) -> DocResult { - let cap = - DOC_REGEX - .captures(raw_doc_string.as_str()) - .ok_or_else(|| DocError::NoDocString { - symbol: symbol.to_string(), - }); - if EXEMPTIONS.contains(symbol.as_ref()) && cap.is_err() { - let usage = Some("unknown".to_string()); - let description = "unknown".to_string(); - let section = "undocumented".to_string(); - let example = None; - return Ok(DocStringSection { - usage, - description, - section, - example, - }); - } - let cap = cap?; - let mut usage = cap.get(2).map(|x| x.as_str().trim().to_string()); - if usage.is_none() && !backup_usage.trim().is_empty() { - usage = Some(backup_usage); - } - let description = cap - .get(3) - .ok_or_else(|| DocError::DocStringMissingSection { - symbol: symbol.to_string(), - section: "Description".to_string(), - }) - .map(|x| x.as_str().to_string())?; - let section = cap - .get(4) - .ok_or_else(|| DocError::DocStringMissingSection { - symbol: symbol.to_string(), - section: "Section".to_string(), - }) - .map(|x| x.as_str().trim().to_string())?; - let example = cap.get(6).map(|x| x.as_str().trim().to_string()); - if EXEMPTIONS.contains(symbol.as_ref()) { - Err(DocError::RemoveExemption { - symbol: symbol.to_string(), - }) - } else { - Ok(DocStringSection { - usage, - description, - section, - example, - }) - } - } - - /// Just write everything that was in the doc section to the description section. - /// TODO #229 track incomplete documenntation in a page. - pub fn new_incomplete(slot: u32, sym: &Value, vm: &mut SloshVm) -> Self { - let description = Self::raw_docstring(slot, vm); - let usage = Some(slosh_lib::usage(vm, slot, sym)); - DocStringSection { - usage, - description, - section: SLOSH_NIL.to_string(), - example: None, - } - } - - fn into_styled_output(mut self, query: &DocSearchQuery, style: &StyleOptions) -> Self { - (self.usage, self.description, self.section, self.example) = { - // use destructing to force compiler error if structure changes. - let Self { - usage, - description, - section, - example, - } = &self; - ( - usage.as_ref().map(|x| highlight_matches(x, query, style)), - highlight_matches(description, query, style), - highlight_matches(section, query, style), - example.as_ref().map(|x| highlight_matches(x, query, style)), - ) - }; - self - } -} - -pub trait AsMd { - fn as_md(&self) -> String; -} - -impl AsMd for SloshDoc { - fn as_md(&self) -> String { - let mut content = format!(" ### {}\n", self.symbol); - //content = content + &format!("- type: {}\n", docs.symbol_type); - //content = content + &format!("- namespace: {}\n", docs.namespace); - if let Some(usage) = &self.doc_string.usage { - content += &format!("**Usage:** {}\n\n", usage); - } - content = content + &format!("**Namespace:** {}\n\n", self.namespace); - content = content + &format!("{}\n", self.doc_string.description); - if let Some(example) = &self.doc_string.example { - content += "\n\nExample:\n```\n"; - content += example; - content += "\n``` \n"; - } else { - content += "\n\nNo Examples\n"; - } - content - } -} - -#[derive(Eq, Debug, Clone)] -pub struct SloshDoc { - symbol: String, - symbol_type: String, - namespace: String, - doc_string: DocStringSection, -} - -impl Display for SloshDoc { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - write!( - f, - "{symbol}\nType: {symbol_type}\nNamespace: {namespace}\n\n{doc_string}", - symbol = self.symbol, - symbol_type = self.symbol_type, - namespace = self.namespace, - doc_string = self.doc_string - ) - } -} - -impl PartialEq for SloshDoc { - fn eq(&self, other: &Self) -> bool { - self.symbol == other.symbol && self.namespace == other.namespace - } -} - -impl Hash for SloshDoc { - fn hash(&self, state: &mut H) { - state.write(self.symbol.as_bytes()); - state.write(self.namespace.as_bytes()); - } -} - -impl PartialOrd for SloshDoc { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for SloshDoc { - fn cmp(&self, other: &Self) -> Ordering { - self.symbol - .cmp(&other.symbol) - .then_with(|| self.namespace.cmp(&other.namespace)) - } -} - -impl SloshDoc { - fn new(g: Interned, vm: &mut SloshVm, namespace: Namespace) -> DocResult { - let sym = Value::Symbol(g); - let slot = vm.global_intern_slot(g); - if let Some(slot) = slot { - let doc_string = DocStringSection::from_symbol(slot, sym, vm)?; - let symbol = sym.display_value(vm); - let mut full_name: Vec<_> = symbol.split("::").collect(); - let symbol = full_name - .pop() - .expect("Symbol should never be an empty.") - .to_string(); - let symbol_type = sym.display_type(vm).to_string(); - let namespace = namespace.display(vm); - Ok(SloshDoc { - symbol, - symbol_type, - namespace, - doc_string, - }) - } else { - Err(DocError::NoSymbol { - symbol: sym.display_value(vm).to_string(), - }) - } - } - - fn new_incomplete(g: Interned, vm: &mut SloshVm, namespace: Namespace) -> DocResult { - let sym = Value::Symbol(g); - let slot = vm.global_intern_slot(g); - if let Some(slot) = slot { - let doc_string = DocStringSection::new_incomplete(slot, &sym, vm); - let symbol = sym.display_value(vm); - let symbol_type = sym.display_type(vm).to_string(); - let namespace = namespace.display(vm); - Ok(SloshDoc { - symbol, - symbol_type, - namespace, - doc_string, - }) - } else { - Err(DocError::NoSymbol { - symbol: sym.display_value(vm).to_string(), - }) - } - } - - /// Provide the fully - pub fn fully_qualified_name(&self) -> String { - self.namespace.to_string() + "::" + self.symbol.as_ref() - } - - fn into_styled_output(mut self, query: &DocSearchQuery, style: &StyleOptions) -> Self { - ( - self.symbol, - self.namespace, - self.symbol_type, - self.doc_string, - ) = { - // use destructing to force compiler error if structure changes. - let Self { - symbol, - namespace, - symbol_type, - doc_string, - } = &self; - ( - highlight_matches(symbol, query, style), - highlight_matches(namespace, query, style), - highlight_matches(symbol_type, query, style), - doc_string.clone().into_styled_output(query, style), - ) - }; - self - } -} - -enum DocError { - NoSymbol { symbol: String }, - NoDocString { symbol: String }, - RemoveExemption { symbol: String }, - DocStringMissingSection { symbol: String, section: String }, -} - -impl Debug for DocError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - Display::fmt(self, f) - } -} - -impl Display for DocError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let str = match self { - DocError::NoSymbol { symbol } => { - format!( - "No documentation string provided for symbol {symbol}." - ) - } - DocError::NoDocString { symbol } => { - format!( - "Either documentation provided does not conform to conventional layout or no documentation string provided for symbol {symbol} all slosh functions with documentation must have a string that conforms to the conventional layout." - ) - } - DocError::DocStringMissingSection { symbol, section } => { - format!("Invalid documentation string for symbol {symbol}, missing required section {section:?}") - } - DocError::RemoveExemption { symbol} => { - format!("Documentation has been added for {symbol}, remove it from EXEMPTIONS list in slosh_test::docs::EXEMPTIONS.") - } - } - .to_string(); - write!(f, "{}", str) - } -} - -impl Error for DocError {} - -impl From for VMError { - fn from(value: DocError) -> Self { - VMError { - key: "doc", - obj: VMErrorObj::Message(value.to_string()), - } - } -} - -type DocResult = Result; - -#[derive(Debug, Clone)] -struct StyleOptions { - highlight_color: String, - default_color: String, - use_background: bool, -} - -impl Default for StyleOptions { - fn default() -> Self { - StyleOptions { - highlight_color: "\x1b[43m\x1b[30m".to_string(), // yellow background with black text - default_color: "\x1b[0m".to_string(), // reset - use_background: true, - } - } -} - -impl StyleOptions { - fn from_handle(vm: &mut SloshVm, handle: slvm::Handle) -> Self { - let mut options = Self::default(); - - // Intern all keys upfront - let highlight_color_key = Value::Keyword(vm.intern("highlight-color")); - let default_color_key = Value::Keyword(vm.intern("default-color")); - let use_background_key = Value::Keyword(vm.intern("use-background")); - - let map = vm.get_map(handle); - - // Extract colors from map - if let Some(val) = map.get(vm, highlight_color_key) { - match val { - Value::String(s) => options.highlight_color = vm.get_string(s).to_string(), - Value::StringConst(i) => options.highlight_color = vm.get_interned(i).to_string(), - _ => {} - } - } - - if let Some(val) = map.get(vm, default_color_key) { - match val { - Value::String(s) => options.default_color = vm.get_string(s).to_string(), - Value::StringConst(i) => options.default_color = vm.get_interned(i).to_string(), - _ => {} - } - } - - if let Some(Value::False) = map.get(vm, use_background_key) { - options.use_background = false; - } - - options - } -} - -fn highlight_matches(text: &str, query: &DocSearchQuery, style: &StyleOptions) -> String { - if query.use_regex { - highlight_regex_matches(text, &query.query, style) - } else if query.basic { - highlight_basic_matches(text, &query.query, style) - } else { - highlight_substring_matches(text, &query.query, style) - } -} - -fn highlight_substring_matches(text: &str, query: &str, style: &StyleOptions) -> String { - let text_lower = text.to_lowercase(); - let query_lower = query.to_lowercase(); - let mut result = String::new(); - let mut last_end = 0; - - for (start, _) in text_lower.match_indices(&query_lower) { - // Add text before match - result.push_str(&text[last_end..start]); - // Add highlighted match - result.push_str(&style.highlight_color); - result.push_str(&text[start..start + query.len()]); - result.push_str(&style.default_color); - last_end = start + query.len(); - } - - // Add remaining text - result.push_str(&text[last_end..]); - result -} - -fn highlight_regex_matches(text: &str, pattern: &str, style: &StyleOptions) -> String { - if let Ok(re) = Regex::new(pattern) { - let mut result = String::new(); - let mut last_end = 0; - - for mat in re.find_iter(text) { - // Add text before match - result.push_str(&text[last_end..mat.start()]); - // Add highlighted match - result.push_str(&style.highlight_color); - result.push_str(mat.as_str()); - result.push_str(&style.default_color); - last_end = mat.end(); - } - - // Add remaining text - result.push_str(&text[last_end..]); - result - } else { - text.to_string() - } -} - -fn highlight_basic_matches(text: &str, pattern: &str, style: &StyleOptions) -> String { - let pattern_lower = pattern.to_lowercase(); - let mut result = String::new(); - let mut pattern_chars = pattern_lower.chars().peekable(); - let mut in_match = false; - - for ch in text.chars() { - if let Some(&pattern_ch) = pattern_chars.peek() { - if ch.to_lowercase().next() == Some(pattern_ch) { - if !in_match { - result.push_str(&style.highlight_color); - in_match = true; - } - result.push(ch); - pattern_chars.next(); - } else { - if in_match { - result.push_str(&style.default_color); - in_match = false; - } - result.push(ch); - } - } else { - if in_match { - result.push_str(&style.default_color); - in_match = false; - } - result.push(ch); - } - } - - if in_match { - result.push_str(&style.default_color); - } - - result -} - -fn generate_styled_output( - _vm: &mut SloshVm, - results: &BTreeSet, - query: &DocSearchQuery, - style: &StyleOptions, -) -> String { - let mut output = String::new(); - let len = results.len(); - - for (i, doc) in results.iter().enumerate() { - if i > 0 { - output.push_str("\n────────────────────────────────────────\n\n"); - } - - // Result header with highlighted symbol - output.push_str(&format!("Result {}: ", i + 1)); - let mut doc = doc.clone(); - doc = doc.into_styled_output(query, style); - output.push_str(&format!("{doc}")); - } - - output.push_str(&format!( - "\nFound {} result{}.", - len, - if len == 1 { "" } else { "s" } - )); - - output -} - -fn insert_section(map: &mut VMHashMap, key: &'static str, value: String, vm: &mut SloshVm) { - let key_const = Value::Keyword(vm.intern_static(key)); - let value_text = vm.alloc_string(value); - map.insert(vm, key_const, value_text); -} - -fn insert_nil_section(map: &mut VMHashMap, key: &'static str, vm: &mut SloshVm) { - let key_const = Value::Keyword(vm.intern_static(key)); - map.insert(vm, key_const, Value::Nil); -} - -impl SlFrom for VMHashMap { - fn sl_from(value: SloshDoc, vm: &mut SloshVm) -> VMResult { - let mut map = Self::with_capacity(4); - match (value.doc_string.usage, value.doc_string.example) { - (Some(usage), Some(example)) => { - insert_section(&mut map, USAGE, usage, vm); - insert_section(&mut map, EXAMPLE, example, vm); - } - (Some(usage), None) => { - insert_section(&mut map, USAGE, usage, vm); - insert_nil_section(&mut map, EXAMPLE, vm); - } - (None, Some(example)) => { - insert_section(&mut map, EXAMPLE, example, vm); - insert_nil_section(&mut map, USAGE, vm); - } - (None, None) => { - insert_nil_section(&mut map, EXAMPLE, vm); - insert_nil_section(&mut map, USAGE, vm); - } - } - insert_section(&mut map, SECTION, value.doc_string.section, vm); - insert_section(&mut map, DESCRIPTION, value.doc_string.description, vm); - Ok(map) - } -} - -impl SlFrom for Value { - fn sl_from(value: SloshDoc, vm: &mut SloshVm) -> VMResult { - let map = VMHashMap::sl_from(value, vm)?; - Ok(vm.alloc_map(map)) - } -} - -fn doc_map(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - let mut i = registers.iter(); - match (i.next(), i.next()) { - (Some(Value::Symbol(g)), None) => { - // Pause GC so that we don't wind up collecting any strings used to build the doc map - // before they get rooted via the map. - vm.pause_gc(); - - let res = match SloshDoc::new(*g, vm, Namespace::Global) { - Ok(slosh_doc) => Value::sl_from(slosh_doc, vm), - Err(e) => Err(VMError::from(e)), - }; - // Unpause GC, this MUST happen so no early returns (looking at you ?). - vm.unpause_gc(); - res - } - _ => Err(VMError::new_vm("takes one argument (symbol)".to_string())), - } -} - -/// Each doc has a tag in its `Section:` definition by convention that logically groups functions. -/// Using a HashMap store the section tags as keys and add all slosh docs from to a vector as a value -/// corresponding to its section. -fn get_docs_by_section( - vm: &mut SloshVm, - require_proper_format: bool, -) -> HashMap> { - let mut docs_by_section: HashMap> = HashMap::new(); - let mut docs: Vec = vec![]; - Namespace::Global - .add_docs(&mut docs, vm, require_proper_format) - .unwrap(); - let namespaces = builtins::get_namespaces_interned(vm); - for i in namespaces { - let namespace = Namespace::Other(i); - namespace - .add_docs(&mut docs, vm, require_proper_format) - .unwrap(); - } - for d in docs { - let d = d.clone(); - let section = d.doc_string.section.clone(); - docs_by_section.entry(section).or_default().insert(d); - } - docs_by_section -} +// Re-export core doc types from slosh_lib for consumers +pub use slosh_lib::docs::{ + AsMd, DocError, DocResult, DocStringSection, EXEMPTIONS, GLOBAL_NAMESPACE, Namespace, + SloshDoc, USER_FORMS, get_docs_by_section, +}; fn build_symbols_list( docs_by_section: &BTreeMap>, @@ -895,8 +122,6 @@ fn build_each_docs_section_chapter( // If there is a section file header include it for preprocessing. if let Ok(file_str) = fs::read_to_string(&file) { content = content + &file_str; - // could never get it to work right w/ the #include directive... - //content = content + &format!("{{{{#include section-docs/{}.md}}}}\n\n\n", section); } else if fs::metadata(&file).is_ok() { eprintln!("Error processing file: {}", file); } @@ -1087,390 +312,6 @@ fn capitalize_first(s: &str) -> String { } } -fn get_exemptions(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if !registers.is_empty() { - return Err(VMError::new_vm( - "get-exemptions: takes no arguments".to_string(), - )); - } - let mut exemptions = EXEMPTIONS.iter().copied().collect::>(); - exemptions.sort(); - let exemptions = exemptions - .iter() - .map(|x| Value::Symbol(vm.intern(x))) - .collect::>(); - Ok(vm.alloc_vector(exemptions)) -} - -fn get_globals_sorted(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if !registers.is_empty() { - return Err(VMError::new_vm( - "get-globals-sorted: takes no arguments".to_string(), - )); - } - let mut result = BTreeMap::new(); - for g in vm.globals().keys() { - let sym = Value::Symbol(*g); - let val: String = sym.display_value(vm); - result.insert(val, sym); - } - let v = result.values().cloned().collect(); - Ok(vm.alloc_vector(v)) -} - -#[derive(Debug, Clone)] -struct DocSearchQuery { - query: String, - use_regex: bool, - basic: bool, - fields: HashSet, - namespace_filter: Option, - section_filter: Option, -} - -impl DocSearchQuery { - fn new(query: String) -> Self { - DocSearchQuery { - query, - use_regex: false, - basic: false, - fields: HashSet::new(), - namespace_filter: None, - section_filter: None, - } - } - - fn with_regex(mut self) -> Self { - self.use_regex = true; - self.basic = false; - self - } - - fn with_basic(mut self) -> Self { - self.basic = true; - self.use_regex = false; - self - } - - fn with_fields(mut self, fields: Vec) -> Self { - self.fields = fields.into_iter().collect(); - self - } - - fn with_namespace(mut self, namespace: String) -> Self { - self.namespace_filter = Some(namespace); - self - } - - fn with_section(mut self, section: String) -> Self { - self.section_filter = Some(section); - self - } - - fn should_search_field(&self, field: &str) -> bool { - self.fields.is_empty() || self.fields.contains(field) - } - - fn matches_text(&self, text: &str) -> bool { - if self.use_regex { - if let Ok(re) = Regex::new(&self.query) { - re.is_match(text) - } else { - false - } - } else if self.basic { - basic_match(text, &self.query) - } else { - text.to_lowercase().contains(&self.query.to_lowercase()) - } - } -} - -fn basic_match(text: &str, pattern: &str) -> bool { - let text_lower = text.to_lowercase(); - let pattern_lower = pattern.to_lowercase(); - let mut pattern_chars = pattern_lower.chars(); - let mut current_char = pattern_chars.next(); - - for text_char in text_lower.chars() { - if let Some(p_char) = current_char { - if text_char == p_char { - current_char = pattern_chars.next(); - } - } else { - return true; - } - } - - current_char.is_none() -} - -fn search_docs( - vm: &mut SloshVm, - query: &DocSearchQuery, - require_proper_format: bool, -) -> BTreeSet { - let docs_by_section = get_docs_by_section(vm, require_proper_format); - let mut results = BTreeSet::new(); - - for (section, docs) in docs_by_section { - // Check section filter - if let Some(ref section_filter) = query.section_filter { - if !query.matches_text(§ion) && section != *section_filter { - continue; - } - } - - for doc in docs { - // Check namespace filter - if let Some(ref ns_filter) = query.namespace_filter { - if !doc.namespace.contains(ns_filter) && doc.namespace != *ns_filter { - continue; - } - } - - let mut matched = false; - - // Search in symbol name - if query.should_search_field("name") - || query.should_search_field("symbol") && query.matches_text(&doc.symbol) - { - matched = true; - } - - // Search in usage - if !matched - && query.should_search_field(USAGE) - && let Some(ref usage) = doc.doc_string.usage - && query.matches_text(usage) - { - matched = true; - } - - // Search in description - if !matched - && query.should_search_field(DESCRIPTION) - && query.matches_text(&doc.doc_string.description) - { - matched = true; - } - - // Search in section - if !matched - && query.should_search_field(SECTION) - && query.matches_text(&doc.doc_string.section) - { - matched = true; - } - - // Search in example - if !matched - && query.should_search_field(EXAMPLE) - && let Some(ref example) = doc.doc_string.example - && query.matches_text(example) - { - matched = true; - } - - // If no specific fields specified, search all fields - if !matched && query.fields.is_empty() { - matched = query.matches_text(&doc.symbol) - || doc - .doc_string - .usage - .as_ref() - .map(|u| query.matches_text(u)) - .unwrap_or(false) - || query.matches_text(&doc.doc_string.description) - || query.matches_text(&doc.doc_string.section) - || doc - .doc_string - .example - .as_ref() - .map(|e| query.matches_text(e)) - .unwrap_or(false); - } - - if matched { - results.insert(doc); - } - } - } - - results -} - -fn doc_search(vm: &mut SloshVm, registers: &[Value]) -> VMResult { - if registers.is_empty() { - return Err(VMError::new_vm( - "doc-search requires at least a query string".to_string(), - )); - } - - let query_str = match ®isters[0] { - Value::String(s) => vm.get_string(*s).to_string(), - Value::StringConst(i) => vm.get_interned(*i).to_string(), - _ => { - return Err(VMError::new_vm( - "First argument must be a query string".to_string(), - )); - } - }; - - let mut query = DocSearchQuery::new(query_str); - - // Parse options if provided as a hash map - let mut use_styled = true; - let mut style_options = StyleOptions::default(); - let mut style_map_handle_opt = None; - if registers.len() > 1 { - match ®isters[1] { - Value::Map(handle) => { - // Intern all keys first to avoid borrowing issues - let regex_key = Value::Keyword(vm.intern_static("regex")); - let basic_search_key = Value::Keyword(vm.intern_static("basic")); - let fields_key = Value::Keyword(vm.intern_static("fields")); - let ns_key = Value::Keyword(vm.intern_static("namespace")); - let section_key = Value::Keyword(vm.intern_static("section")); - let styled_key = Value::Keyword(vm.intern_static("styled")); - let style_key = Value::Keyword(vm.intern_static("style")); - - let map = vm.get_map(*handle); - - // Check for regex option - if let Some(Value::True) = map.get(vm, regex_key) { - query = query.with_regex(); - } - - // Check for basic option - if let Some(Value::True) = map.get(vm, basic_search_key) { - query = query.with_basic(); - } - - // Check for fields option - if let Some(Value::Vector(v_handle)) = map.get(vm, fields_key) { - let vec = vm.get_vector(v_handle); - let mut fields = Vec::new(); - for val in vec.iter() { - match val { - Value::String(s) => fields.push(vm.get_string(*s).to_string()), - Value::StringConst(i) => fields.push(vm.get_interned(*i).to_string()), - Value::Keyword(i) => fields.push(vm.get_interned(*i).to_string()), - _ => {} - } - } - query = query.with_fields(fields); - } - - // Check for namespace filter - if let Some(ns_val) = map.get(vm, ns_key) { - match ns_val { - Value::String(s) => { - query = query.with_namespace(vm.get_string(s).to_string()) - } - Value::StringConst(i) => { - query = query.with_namespace(vm.get_interned(i).to_string()) - } - _ => {} - } - } - - // Check for section filter - if let Some(section_val) = map.get(vm, section_key) { - match section_val { - Value::String(s) => { - query = query.with_section(vm.get_string(s).to_string()) - } - Value::StringConst(i) => { - query = query.with_section(vm.get_interned(i).to_string()) - } - _ => {} - } - } - - // Check for styled option - if let Some(Value::False) = map.get(vm, styled_key) { - use_styled = false; - } - - // Check for style customization - if let Some(Value::Map(style_handle)) = map.get(vm, style_key) { - use_styled = true; // Enable styling if style options are provided - style_map_handle_opt = Some(style_handle); - } - } - _ => { - return Err(VMError::new_vm( - "Second argument must be a map of options".to_string(), - )); - } - } - } - - // Process style options after we're done with the initial map - if let Some(style_handle) = style_map_handle_opt { - style_options = StyleOptions::from_handle(vm, style_handle); - } - - // Check if markdown output is requested - let mut use_markdown = false; - if registers.len() > 1 { - if let Value::Map(handle) = ®isters[1] { - let markdown_key = Value::Keyword(vm.intern_static("markdown")); - let map = vm.get_map(*handle); - if let Some(Value::True) = map.get(vm, markdown_key) { - use_markdown = true; - } - } - } - - let results = search_docs(vm, &query, false); - - let result = if use_markdown { - // Generate markdown output - let mut markdown_output = String::new(); - let len = results.len(); - for (i, doc) in results.iter().enumerate() { - // Don't print to stdout when generating markdown - the numbered list is included in the markdown output instead - if i > 0 { - markdown_output.push_str("\n---\n\n"); - } - // Add a header showing which result this is - markdown_output.push_str(&format!( - "**Result {}: `{}::{}`**\n\n", - i + 1, - doc.namespace, - doc.symbol - )); - markdown_output.push_str(&doc.as_md()); - } - markdown_output.push_str(&format!( - "\nFound {len} result{}.", - if len == 1 { "" } else { "s" } - )); - vm.alloc_string(markdown_output) - } else if use_styled { - // Generate styled string output - let styled_output = generate_styled_output(vm, &results, &query, &style_options); - vm.alloc_string(styled_output) - } else { - // Generate vector of doc maps - let mut result_values = Vec::new(); - for doc in results { - match Value::sl_from(doc, vm) { - Ok(val) => result_values.push(val), - Err(e) => { - vm.unpause_gc(); - return Err(e); - } - } - } - vm.alloc_vector(result_values) - }; - - Ok(result) -} - fn legacy_report(vm: &mut SloshVm, _registers: &[Value]) -> VMResult { let report = legacy::build_report(vm)?; Ok(vm.alloc_string(report)) @@ -1489,70 +330,13 @@ Section: doc Example: #t -", - ); - add_builtin( - env, - "doc-map", - doc_map, - "Usage: (doc-map symbol) - -Returns documentation for given symbol as map. Keyword is a documentation fragment -(usage, section, description, example) and value is text describing given fragment. - -Section: doc - -Example: -#t -", - ); - - add_builtin( - env, - "doc-search", - doc_search, - "Usage: (doc-search query-string [options-map]) - -Search through all documentation for functions matching the query. -The query string is searched across all documentation fields by default. - -Options map can contain: -- :regex #t - Use regular expressions for matching -- :basic #t - Use basic matching (characters must appear in order) -- :fields [\"usage\" \"description\"] - Limit search to specific fields -- :namespace \"namespace-name\" - Filter by namespace -- :section \"section-name\" - Filter by section -- :markdown #t - Return results as a markdown-formatted string instead of vector -- :styled #t - Return results with highlighted search matches -- :style {:highlight-color \"\\x1b[43m\\x1b[30m\" ...} - Customize highlight style - -Style options (when :style map is provided): -- :highlight-color - Color for highlighting matches (default: yellow background, black text) -- :default-color - Color to reset after highlight (default: \\x1b[0m) -- :use-background #f - Use foreground color instead of background - -When :styled is true, matching text in results is highlighted to show why -each result matched the search query. - -Returns a vector of documentation maps, markdown string, or styled string. - -Section: doc - -Example: -(doc-search \"file\") -(doc-search \"^str-\" {:regex #t}) -(doc-search \"core\" {:fields [\"section\"]}) -(doc-search \"map\" {:basic #t :section \"core\"}) -(doc-search \"string\" {:markdown #t}) -(doc-search \"string\" {:styled #t}) -(doc-search \"map\" {:style {:highlight-color *fg-red* :use-background #f}}) ", ); add_builtin( env, "legacy_forms", - legacy_docs::get_legacy_sl_sh_form_syms, + legacy::get_legacy_sl_sh_form_syms, "Usage: (legacy_forms) Gets list of all forms that were used in the previous version of sl_sh. @@ -1564,31 +348,6 @@ Example: ", ); - add_builtin( - env, - "get-globals-sorted", - get_globals_sorted, - "Usage: (get-globals-sorted) - -Return a vector containing all the symbols currently defined globally in sorted order (alphanumerically). - -Section: doc -", - ); - - add_builtin( - env, - "get-exemptions", - get_exemptions, - "Usage: (get-exemptions) - -Return a vector containing all the symbols currently exempted from docs -(so the build passes), Ideally this will be 0. - -Section: doc -", - ); - add_builtin( env, "build-doc", @@ -1609,12 +368,12 @@ Example: mod test { use super::*; use compile_state::state::new_slosh_vm; + use compile_state::state::SloshVmTrait; use sl_compiler::Reader; use slosh_lib::{ ENV, load_builtins_lisp_less_sloshrc, run_reader, set_builtins_and_shell_builtins, set_initial_load_path, }; - use std::collections::BTreeMap; use std::ops::DerefMut; use tempfile::TempDir; @@ -1688,242 +447,4 @@ mod test { ); } } - - lazy_static! { - static ref REGEX_TEST_CASES: BTreeMap<(bool, &'static str), &'static str> = { - let mut set = BTreeMap::new(); - - set.insert( - (true, "no whitespace around content"), - "Usage: (defmacro name doc_string? argument_list body) - -Create a macro and bind it to a symbol in the current scope. - -Section: core - -Example: -(def test-mac-x 2)", - ); - - set.insert( - (true, "newlines at beginning and end"), - " -Usage: (defmacro name doc_string? argument_list body) - -Create a macro and bind it to a symbol in the current scope. - -Section: core - -Example: -(def test-mac-x 2) -", - ); - - set.insert( - (true, "mixed whitespace at beginning and end"), - " - - Usage: (defmacro name doc_string? argument_list body) - -Create a macro and bind it to a symbol in the current scope. - -Section: core - -Example: -(def test-mac-x 2) - - - ", - ); - - set.insert( - (true, "no usage, no whitespace around content"), - "Create a macro and bind it to a symbol in the current scope. - -Section: core - -Example: -(def test-mac-x 2)", - ); - - set.insert( - (true, "no usage, newlines at beginning and end"), - " -Create a macro and bind it to a symbol in the current scope. - -Section: core - -Example: -(def test-mac-x 2) -", - ); - - set.insert( - (true, "no usage, mixed whitespace at beginning and end"), - " - - Create a macro and bind it to a symbol in the current scope. - -Section: core - -Example: -(def test-mac-x 2) - - - ", - ); - - set.insert( - (true, "no whitespace around content"), - "Usage: (defmacro name doc_string? argument_list body) - - Create a macro and bind it to a symbol in the current scope. - - Section: core - - Example: - (def test-mac-x 2)", - ); - - set.insert( - (true, "newlines at beginning and end"), - " - Usage: (defmacro name doc_string? argument_list body) - -Create a macro and bind it to a symbol in the current scope. - -Section: -core - -Example: -(def test-mac-x 2) -", - ); - - set.insert( - (true, "mixed whitespace at beginning and end"), - " - - Usage: (defmacro name doc_string? argument_list body) - -Create a macro and bind it to a symbol in the current scope. - -Section: core - - - Example: -(def test-mac-x 2) - - - ", - ); - - set.insert( - (true, "no usage, extra whitespace around content"), - "Create a macro and bind it to a symbol in the current scope. - - Section: core - - Example: - (def test-mac-x 2)", - ); - - set.insert( - ( - false, - "no description, no usage, no example, newlines at beginning and end", - ), - " - -Section: -core -", - ); - - set.insert( - (true, "no usage, no example, newlines at beginning and end, extra whitespace around content."), - " - Create a macro and bind it to a symbol in the current scope. - - Section: - core - -", - ); - - set.insert( - (true, "no usage, mixed whitespace at beginning and end, extra whitespace around content."), - " - - Create a macro and bind it to a symbol in the current scope. - -Section: core - - Example: -(def test-mac-x 2) - - - ", - ); - - set.insert( - ( - false, - "no usage, no section, mixed whitespace at beginning and end, extra whitespace around content.", - ), - " - - Create a macro and bind it to a symbol in the current scope. - -core - - Example: -(def test-mac-x 2) - - - ", - ); - - set.insert( - (true, "str-sub test"), - " Usage: (str-sub string start [length]) -> string\n\n Return a substring from a string given start (0 based) and optional length.\n If length is 0 or not provided produces the r -est of the string from start to\n string end.\n\n Section: string\n\n Example:\n (test::assert-equal \"string\" (str-sub \"stringxxxyyyxxxsome\" 0 6))\n (test::assert-equal \"some\" (str-sub - \"stringxxxyyyxxxsome\" 15 4))\n (test::assert-equal \"yyy\" (str-sub \"stringxxxyyyxxxsome\" 9 3))\n (test::assert-equal \"some\" (str-sub \"stringxxxyyyxxxsome\" 15))\n", - ); - - set - }; - } - - #[test] - fn test_doc_string_regex() { - for ((result, label), test_case) in REGEX_TEST_CASES.iter() { - let fake_symbol = Cow::Owned("fake-symbol".to_string()); - match DocStringSection::parse_doc_string( - fake_symbol, - test_case.to_string(), - "".to_string(), - ) { - ok @ Ok(_) => { - assert_eq!( - ok.is_ok(), - *result, - "Case: {}, Regex succeeded but should not have with test case: {}!", - label, - test_case - ); - } - ref err @ Err(_) => { - assert_ne!( - err.is_err(), - *result, - "Case: {}, Regex failed but should not have, got err ({:?}) with tests case: {}!", - label, - err, - test_case - ); - } - } - } - } } diff --git a/slosh_test_lib/src/docs/legacy.rs b/slosh_test_lib/src/docs/legacy.rs index 0a2849e74f..8634353fa0 100644 --- a/slosh_test_lib/src/docs/legacy.rs +++ b/slosh_test_lib/src/docs/legacy.rs @@ -1,5 +1,3 @@ -use crate::docs::{legacy, legacy_docs}; -use bridge_macros::sl_sh_fn; use compile_state::state::{SloshVm, SloshVmTrait}; use slvm::{VMResult, Value}; use std::collections::HashSet; @@ -141,7 +139,7 @@ pub(crate) fn unimplemented_report(vm: &mut SloshVm) -> VMResult { } } - let metadata = legacy_docs::full_legacy_sl_sh_forms_metadata(); + let metadata = full_legacy_sl_sh_forms_metadata(); let mut completed = 0; let mut yet_to_be_implemented = 0; let mut status_entries = Vec::new(); @@ -224,9 +222,9 @@ pub(crate) fn unimplemented_report(vm: &mut SloshVm) -> VMResult { /// /// Example: /// #t -#[sl_sh_fn(fn_name = "legacy-report", takes_env = true)] +#[bridge_macros::sl_sh_fn(fn_name = "legacy-report", takes_env = true)] pub fn legacy_report(environment: &mut SloshVm) -> VMResult { - legacy::build_report(environment) + build_report(environment) } pub fn get_legacy_sl_sh_form_syms(vm: &mut SloshVm, _registers: &[Value]) -> VMResult {