diff --git a/README.md b/README.md index 8cf70b6..ac4d82e 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,30 @@ This creates: - `web-darwin-amd64` - macOS Intel - `web-linux-amd64` - Linux x86_64 +### Nix / NixOS + +If you use Nix, you can build and run directly from the flake: + +```bash +# Run directly from GitHub without installing +nix run github:chrismccord/web -- https://example.com + +# Run from local checkout +nix run . -- https://example.com + +# Build the package +nix build + +# Enter development shell (includes Go, Firefox, geckodriver) +nix develop +``` + +The flake provides: +- **`packages.default`** - The `web` binary with Firefox and geckodriver bundled in PATH +- **`devShells.default`** - Development environment with all dependencies + +This eliminates the need for automatic Firefox/geckodriver downloads since Nix provides them. + ## Usage Examples ```bash diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000..304d1ac --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1769789167, + "narHash": "sha256-kKB3bqYJU5nzYeIROI82Ef9VtTbu4uA3YydSk/Bioa8=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "62c8382960464ceb98ea593cb8321a2cf8f9e3e5", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000..bcde094 --- /dev/null +++ b/flake.nix @@ -0,0 +1,67 @@ +{ + description = "web - portable web scraper for LLMs"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + isDarwin = pkgs.stdenv.isDarwin; + + # On macOS, Firefox is an app bundle; create a wrapper script + firefoxWrapper = if isDarwin then + pkgs.writeShellScriptBin "firefox" '' + exec "${pkgs.firefox}/Applications/Firefox.app/Contents/MacOS/firefox" "$@" + '' + else + pkgs.firefox; + in + { + packages.default = pkgs.buildGoModule { + pname = "web"; + version = "0.1.0"; + src = ./.; + + vendorHash = "sha256-Pc/ZMqesG0bNPAqBnbIkRbJfAmz6NW0JvcQ2kwWpG6M="; + + nativeBuildInputs = [ pkgs.makeWrapper ]; + + # Skip tests during build - they require network access and browser + # Run tests manually in devShell: go test -v -timeout=300s + doCheck = false; + + postInstall = '' + wrapProgram $out/bin/web \ + --prefix PATH : ${pkgs.lib.makeBinPath [ firefoxWrapper pkgs.geckodriver ]} + ''; + + meta = with pkgs.lib; { + description = "Portable web scraper for LLMs with Firefox automation"; + license = licenses.mit; + mainProgram = "web"; + }; + }; + + devShells.default = pkgs.mkShell { + buildInputs = [ + pkgs.go + firefoxWrapper + pkgs.geckodriver + ]; + + shellHook = '' + echo "web development shell" + echo "Firefox: $(which firefox)" + echo "Geckodriver: $(which geckodriver)" + echo "" + echo "Build: go build -o web ." + echo "Run tests: go test -v -timeout=300s" + ''; + }; + } + ); +} diff --git a/main.go b/main.go index ce5e043..8a84690 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "os" + "os/exec" "path/filepath" "runtime" "strconv" @@ -19,6 +20,36 @@ import ( const DEFAULT_TRUNCATE_AFTER = 100000 +// getFirefoxPath returns the path to Firefox, checking PATH first (for Nix/system installs), +// then falling back to the downloaded location in ~/.web-firefox/ +func getFirefoxPath() string { + // Check PATH first (works with Nix, system Firefox, etc.) + if path, err := exec.LookPath("firefox"); err == nil { + return path + } + // Fall back to downloaded location + homeDir, _ := os.UserHomeDir() + switch runtime.GOOS { + case "darwin": + return filepath.Join(homeDir, ".web-firefox", "firefox", "Nightly.app", "Contents", "MacOS", "firefox") + case "linux": + return filepath.Join(homeDir, ".web-firefox", "firefox", "firefox") + } + return "" +} + +// getGeckodriverPath returns the path to geckodriver, checking PATH first (for Nix/system installs), +// then falling back to the downloaded location in ~/.web-firefox/ +func getGeckodriverPath() string { + // Check PATH first (works with Nix, system geckodriver, etc.) + if path, err := exec.LookPath("geckodriver"); err == nil { + return path + } + // Fall back to downloaded location + homeDir, _ := os.UserHomeDir() + return filepath.Join(homeDir, ".web-firefox", "geckodriver", "geckodriver") +} + type FormInput struct { Name string Value string @@ -68,6 +99,13 @@ func main() { } func ensureFirefox() error { + // Check if Firefox is already available (via PATH - Nix, system install, etc.) + if path, err := exec.LookPath("firefox"); err == nil { + // Firefox found in PATH, no download needed + _ = path + return nil + } + // Get home directory for our isolated Firefox installation homeDir, err := os.UserHomeDir() if err != nil { @@ -100,7 +138,7 @@ func ensureFirefox() error { return fmt.Errorf("unsupported platform: %s", runtime.GOOS) } - // Check if Firefox executable exists + // Check if Firefox executable exists in downloaded location if _, err := os.Stat(firefoxExec); err == nil { return nil } @@ -122,6 +160,13 @@ func ensureFirefox() error { } func ensureGeckodriver() error { + // Check if geckodriver is already available (via PATH - Nix, system install, etc.) + if path, err := exec.LookPath("geckodriver"); err == nil { + // Geckodriver found in PATH, no download needed + _ = path + return nil + } + homeDir, err := os.UserHomeDir() if err != nil { return fmt.Errorf("could not get home directory: %v", err) @@ -147,7 +192,7 @@ func ensureGeckodriver() error { return fmt.Errorf("unsupported platform: %s", runtime.GOOS) } - // Check if geckodriver exists + // Check if geckodriver exists in downloaded location if _, err := os.Stat(geckoExec); err == nil { return nil } @@ -357,23 +402,16 @@ func extractZip(src, dest string) error { func processRequest(config Config) (string, error) { baseURL := ensureProtocol(config.URL) - // Get Firefox and geckodriver paths + // Get Firefox and geckodriver paths (checks PATH first, then falls back to ~/.web-firefox/) + firefoxExec := getFirefoxPath() + geckoDriverPath := getGeckodriverPath() + + // Get home directory for profile storage homeDir, err := os.UserHomeDir() if err != nil { return "", fmt.Errorf("could not get home directory: %v", err) } - firefoxDir := filepath.Join(homeDir, ".web-firefox") - geckoDriverPath := filepath.Join(firefoxDir, "geckodriver", "geckodriver") - - var firefoxExec string - switch runtime.GOOS { - case "darwin": - firefoxExec = filepath.Join(firefoxDir, "firefox", "Nightly.app", "Contents", "MacOS", "firefox") - case "linux": - firefoxExec = filepath.Join(firefoxDir, "firefox", "firefox") - } - // Start geckodriver service service, err := selenium.NewGeckoDriverService(geckoDriverPath, 4444) if err != nil { @@ -381,7 +419,7 @@ func processRequest(config Config) (string, error) { } defer service.Stop() - // Configure Firefox with profile + // Configure Firefox with profile (profiles always stored in ~/.web-firefox/profiles/) profileDir := filepath.Join(homeDir, ".web-firefox", "profiles", config.Profile) os.MkdirAll(profileDir, 0755)