From 6abb1ee497968da87f9aba0ed5b9ebc620ac19b8 Mon Sep 17 00:00:00 2001 From: mja00 Date: Mon, 13 Jul 2026 19:42:08 -0400 Subject: [PATCH] feat(memsync): key memory sync by project fingerprint, not local path Mirror directories are now keyed by a machine-independent fingerprint (normalized git remote URL, then root-commit SHA, then the path fallback for non-git dirs) instead of Claude's path-derived directory name, so the same repo checked out at different paths on different machines syncs and merges. Real project paths are recovered from the cwd in Claude's session transcripts, and a per-PC .state/index.json maps localHash to key. Legacy path-keyed layouts auto-migrate on upgrade, union-merging collisions. The foreground daemon now logs startup and sync activity so it no longer looks hung. --- README.md | 3 +- cmd/claude-memsync/init.go | 20 ++- docs/claude-memsync.md | 65 +++++++-- internal/project/fingerprint.go | 133 ++++++++++++++++++ internal/project/fingerprint_test.go | 152 ++++++++++++++++++++ internal/project/index.go | 144 +++++++++++++++++++ internal/project/index_test.go | 117 ++++++++++++++++ internal/project/resolve.go | 104 ++++++++++++++ internal/project/resolve_test.go | 76 ++++++++++ internal/sync/keyed_test.go | 151 ++++++++++++++++++++ internal/sync/loop.go | 94 ++++++++++--- internal/sync/mirror.go | 202 ++++++++++++++++++++------- internal/sync/reconcile_test.go | 10 +- 13 files changed, 1181 insertions(+), 90 deletions(-) create mode 100644 internal/project/fingerprint.go create mode 100644 internal/project/fingerprint_test.go create mode 100644 internal/project/index.go create mode 100644 internal/project/index_test.go create mode 100644 internal/project/resolve.go create mode 100644 internal/project/resolve_test.go create mode 100644 internal/sync/keyed_test.go diff --git a/README.md b/README.md index b582809..2e7335e 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,8 @@ Full instructions: |---|---| | `cmd/claude-memsync` | Daemon binary: lifecycle + watcher + sync loop | | `cmd/claude-memmerge` | Git custom merge driver for `MEMORY.md` | -| `internal/sync` | Mirror, reconcile, watcher loop, manifest | +| `internal/sync` | Mirror, reconcile, watcher loop, manifest, layout migration | +| `internal/project` | Project fingerprinting (path resolution, key derivation, per-PC index) | | `internal/merge` | Section-block parser + 3-way semantic merge | | `internal/distill` | Distilled-memory catalog index + reconcile | | `internal/gitwt` | Wrapper around the `git` CLI scoped to the sync work-tree | diff --git a/cmd/claude-memsync/init.go b/cmd/claude-memsync/init.go index a761488..0d3c6d9 100644 --- a/cmd/claude-memsync/init.go +++ b/cmd/claude-memsync/init.go @@ -11,6 +11,7 @@ import ( "github.com/MarimerLLC/claude-utils/internal/config" "github.com/MarimerLLC/claude-utils/internal/gitwt" + "github.com/MarimerLLC/claude-utils/internal/project" syncpkg "github.com/MarimerLLC/claude-utils/internal/sync" ) @@ -130,14 +131,31 @@ func bootstrap(cfg config.Config, force bool) error { return err } roots := syncpkg.Roots{Claude: cfg.ClaudeProjectsDir, Mirror: mirrorProjects} + + // Build the fingerprint index and migrate any legacy path-keyed dirs the + // remote already carries (from a pre-fingerprint version or another machine) + // to their fingerprint keys before reconciling local memories in. + idx, err := project.Build(cfg.ClaudeProjectsDir) + if err != nil { + return fmt.Errorf("build project index: %w", err) + } + if migRep, err := syncpkg.MigrateLayout(roots, idx); err != nil { + return fmt.Errorf("migrate layout: %w", err) + } else { + printReport(migRep) + } + // First-time init: no manifest yet. Pass nil so Reconcile doesn't try // to propagate deletes (with no prior state, "missing in Claude" can't // be distinguished from "this PC never had it"). - report, err := syncpkg.Reconcile(roots, nil) + report, err := syncpkg.Reconcile(roots, idx, nil) if err != nil { return fmt.Errorf("reconcile: %w", err) } printReport(report) + if err := idx.Save(cfg.SyncDir); err != nil { + fmt.Fprintln(os.Stderr, "save project index:", err) + } // Step 6: stage + commit + push. if err := repo.AddAll(); err != nil { diff --git a/docs/claude-memsync.md b/docs/claude-memsync.md index 22b5523..baffd03 100644 --- a/docs/claude-memsync.md +++ b/docs/claude-memsync.md @@ -27,9 +27,12 @@ of its own — just a git remote you can push to. inherits your normal git credentials (Git Credential Manager on Windows, SSH agent or `~/.gitconfig` credential helpers on Linux/macOS). Confirm `git push` works against the remote before installing. -- The same project paths on each PC. Claude's project-hash directory names are - derived from absolute paths, so if your repos live at the same drive letter / - mount point on every PC, you're set. (See [Limitations](#limitations--known-issues).) +- Repos checked out at **different paths on different PCs still merge**: the + daemon fingerprints each project (by its git remote URL, falling back to the + root-commit SHA) and keys the sync repo by that fingerprint rather than the + local path. Non-git directories keep the old path-based key and only merge + across PCs that use identical paths. (See [How it works](#how-it-works) and + [Limitations](#limitations--known-issues).) ## Setup @@ -133,22 +136,45 @@ and SSH keys. None require root. ## How it works ``` -~/.claude/projects//memory/ (Claude reads + writes here — authoritative) - │ +~/.claude/projects//memory/ (Claude reads + writes here — authoritative) + │ localHash = path with / → -, per machine │ fsnotify watcher, 3s debounce + │ fingerprint the project → stable key ▼ -~/.claudesync/projects//memory/ (mirror — git work-tree) +~/.claudesync/projects//memory/ (mirror — git work-tree, keyed by fingerprint) │ │ git add -A, commit, pull --rebase, push ▼ - (private GitHub repo) + (private GitHub repo) │ │ ls-remote check on 1h timer (or after local push); │ full pull only when remote SHA actually moved ▼ - propagate pull-driven changes back to ~/.claude/projects/... + propagate pull-driven changes back to ~/.claude/projects//... ``` +**Project fingerprinting.** Claude names each project directory by dash-encoding +its absolute path (`/Users/me/code/foo` → `-Users-me-code-foo`), which differs +per machine. Keying the sync repo by that name would split one repo into several. +Instead the daemon resolves each project's real path (from the `cwd` Claude +records in its session transcripts) and derives a machine-independent **key**: + +- `g-` — from the normalized `origin` remote URL (all of + `git@github.com:Acme/app.git`, `https://github.com/acme/app`, and + `…/app.git` collapse to the same key). +- `r-` — from the git root-commit SHA, when the repo has no remote. +- the dash-encoded path itself — fallback for non-git directories (unchanged + behavior; only merges across PCs with identical paths). + +A per-PC index at `~/.claudesync/.state/index.json` (gitignored) records the +`localHash → key` mapping for observability and to route inbound changes back to +the right local directory. + +**Lazy materialization.** A project synced from another PC but never opened on +this one stays pending in the mirror — there's no local path to write it to yet. +The first time you open that project here, its directory appears, fingerprints to +the pending key, and the next reconcile copies its memories in. + Idle ticks cost a single `git ls-remote` round-trip — no pull, no push, no merge driver invocation. The full pull/push cycle runs only when origin's branch SHA differs from the local `refs/remotes/origin/` or when there are unpushed @@ -176,6 +202,7 @@ commits. `~/.claude/history.jsonl`, `~/.claude/settings.json`, etc. - `~/.claudesync/config.json` — per-PC (paths embed your machine layout) - `~/.claudesync/.state/manifest.json` — per-PC (delete-detection state) +- `~/.claudesync/.state/index.json` — per-PC (localHash → fingerprint key map) - `~/.claudesync/daemon.pid` — runtime state - `~/.claudesync/distilled/DISTILLED.md` — derived index, regenerated per-PC (the distilled `.md` entry files themselves **are** synced) @@ -206,7 +233,7 @@ This means: ## On-disk layout ``` -~/.claude/projects//memory/ # what Claude reads + writes +~/.claude/projects//memory/ # what Claude reads + writes ├── MEMORY.md └── .md ... @@ -217,10 +244,11 @@ This means: ├── .gitattributes # MEMORY.md merge=claude-memory-index ├── .gitignore # excludes config.json, .state/, etc. ├── .state/manifest.json # per-PC delete-detection state + ├── .state/index.json # per-PC localHash → fingerprint key map ├── distilled/ # shared distilled-memory catalog │ ├── DISTILLED.md # derived index (gitignored) │ └── .md ... # distilled entries (synced) - └── projects//memory/... # git work-tree mirror + └── projects//memory/... # git work-tree mirror (keyed by fingerprint) ``` ## Auth notes @@ -246,10 +274,19 @@ If that works without prompting, the daemon will too. ## Limitations / known issues -- **Path consistency required**: Claude derives the per-project memory directory - name by escaping the project's absolute path. PCs that open the same repo at - different paths (e.g. `C:\src\foo` vs. `D:\dev\foo`) will see them as different - projects. +- **Git projects merge across paths; non-git ones don't**: git repos are keyed + by their remote URL (or root-commit SHA), so opening the same repo at different + paths on different PCs still merges. A non-git directory falls back to a + path-based key, so it only merges across PCs that use the identical absolute + path. +- **Upgrading from a path-keyed layout**: the first daemon run after upgrading + migrates existing mirror directories to fingerprint keys, merging any that + collapse to the same key. Both PCs must be upgraded for a project's previously + divergent histories to converge on the shared key. +- **Fingerprint edge cases**: a shallow clone (`git clone --depth 1`) has no true + root commit, so a remote-less shallow repo falls back to the path key. A repo + whose `origin` URL is later changed (renamed org, new host) re-keys and will + re-merge under the new key. - **No live conflict UI**: when the merge driver emits actual conflict markers (rare; only on overlapping line edits within the same `MEMORY.md` section body), the file is committed and pushed with markers. The next time you edit on diff --git a/internal/project/fingerprint.go b/internal/project/fingerprint.go new file mode 100644 index 0000000..9966a24 --- /dev/null +++ b/internal/project/fingerprint.go @@ -0,0 +1,133 @@ +package project + +import ( + "crypto/sha256" + "encoding/hex" + "sort" + "strings" + + "github.com/MarimerLLC/claude-utils/internal/gitwt" +) + +// Method records how a project's key was derived, for diagnostics in the index. +type Method string + +const ( + MethodRemote Method = "remote" // normalized origin remote URL (preferred) + MethodRoot Method = "root" // git root-commit SHA (git repo, no remote) + MethodPath Method = "path" // fallback: Claude's dash-encoded dir name +) + +// Key prefixes distinguish fingerprint keys from legacy path keys (which start +// with "-"), so migration can tell what to move and the two never collide. +const ( + remotePrefix = "g-" + rootPrefix = "r-" +) + +// Fingerprint computes the machine-independent mirror key for the project at +// realPath. localHash is Claude's directory name, used verbatim as the path +// fallback so non-git projects keep today's per-machine behavior (they simply +// won't merge across machines, but never merge with an unrelated project). +// +// Preference order: normalized origin remote URL, then git root-commit SHA when +// there is no remote, then the path fallback. +func Fingerprint(realPath, localHash string) (key string, method Method) { + if realPath != "" { + repo := gitwt.New(realPath) + if repo.IsRepo() { + if url, ok := originURL(repo); ok { + return remotePrefix + shortHash(NormalizeRemoteURL(url)), MethodRemote + } + if root, ok := rootCommit(repo); ok { + return rootPrefix + shortHash(root), MethodRoot + } + } + } + return localHash, MethodPath +} + +func originURL(repo *gitwt.Repo) (string, bool) { + out, err := repo.Run("config", "--get", "remote.origin.url") + if err != nil { + return "", false + } + url := strings.TrimSpace(out) + return url, url != "" +} + +// rootCommit returns a stable identifier built from the repository's root +// commit(s). A shallow clone's history is truncated, so its "root" is a +// per-clone boundary commit rather than the true first commit — useless as an +// identity — so we bail and let the caller fall back to the path key. Repos with +// multiple root commits (merged orphan histories) are handled by sorting and +// joining so the result is order-independent. +func rootCommit(repo *gitwt.Repo) (string, bool) { + if out, err := repo.Run("rev-parse", "--is-shallow-repository"); err == nil { + if strings.TrimSpace(out) == "true" { + return "", false + } + } + out, err := repo.Run("rev-list", "--max-parents=0", "HEAD") + if err != nil { + return "", false + } + roots := strings.Fields(out) + if len(roots) == 0 { + return "", false + } + sort.Strings(roots) + return strings.Join(roots, ","), true +} + +// NormalizeRemoteURL reduces the many equivalent spellings of a git remote to a +// single canonical "host/path" form so the same repository yields one key across +// machines and protocols. All of git@github.com:Acme/app.git, +// https://github.com/acme/app, and ssh://git@github.com:22/acme/app.git collapse +// to github.com/acme/app. +func NormalizeRemoteURL(raw string) string { + s := strings.TrimSpace(raw) + + if i := strings.Index(s, "://"); i >= 0 { + // scheme://[user@]host[:port]/path + s = s[i+3:] + if at := strings.Index(s, "@"); at >= 0 { + if slash := strings.Index(s, "/"); slash < 0 || at < slash { + s = s[at+1:] + } + } + } else if colon := strings.Index(s, ":"); colon >= 0 { + // scp-like: [user@]host:path + host := s[:colon] + path := s[colon+1:] + if at := strings.LastIndex(host, "@"); at >= 0 { + host = host[at+1:] + } + s = host + "/" + path + } + + // Strip a :port from the host segment (everything before the first slash). + if slash := strings.Index(s, "/"); slash > 0 { + host, rest := s[:slash], s[slash:] + if c := strings.Index(host, ":"); c >= 0 { + host = host[:c] + } + s = host + rest + } + + s = strings.TrimSuffix(strings.TrimSuffix(strings.TrimSuffix(s, "/"), ".git"), "/") + return strings.ToLower(s) +} + +// IsFingerprintKey reports whether name is a fingerprint key (g-/r- prefixed) +// rather than a legacy path key. A fingerprint key can't be reversed to a local +// path, so a mirror dir with one and no matching local project must wait to +// materialize until that project is opened here. +func IsFingerprintKey(name string) bool { + return strings.HasPrefix(name, remotePrefix) || strings.HasPrefix(name, rootPrefix) +} + +func shortHash(s string) string { + sum := sha256.Sum256([]byte(s)) + return hex.EncodeToString(sum[:])[:12] +} diff --git a/internal/project/fingerprint_test.go b/internal/project/fingerprint_test.go new file mode 100644 index 0000000..3b606d3 --- /dev/null +++ b/internal/project/fingerprint_test.go @@ -0,0 +1,152 @@ +package project + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestNormalizeRemoteURL(t *testing.T) { + cases := []struct { + in, want string + }{ + {"git@github.com:Acme/app.git", "github.com/acme/app"}, + {"https://github.com/acme/app", "github.com/acme/app"}, + {"https://github.com/acme/app.git", "github.com/acme/app"}, + {"https://github.com/acme/app/", "github.com/acme/app"}, + {"ssh://git@github.com/acme/app.git", "github.com/acme/app"}, + {"ssh://git@github.com:22/acme/app.git", "github.com/acme/app"}, + {"https://user:token@github.com/acme/app.git", "github.com/acme/app"}, + {"git@gitlab.example.com:group/sub/repo.git", "gitlab.example.com/group/sub/repo"}, + } + for _, c := range cases { + if got := NormalizeRemoteURL(c.in); got != c.want { + t.Errorf("NormalizeRemoteURL(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestNormalizeRemoteURL_EquivalentFormsCollapse(t *testing.T) { + forms := []string{ + "git@github.com:Acme/app.git", + "https://github.com/acme/app", + "https://github.com/acme/app.git", + "ssh://git@github.com/Acme/app.git", + } + want := NormalizeRemoteURL(forms[0]) + for _, f := range forms[1:] { + if got := NormalizeRemoteURL(f); got != want { + t.Errorf("form %q normalized to %q, want %q", f, got, want) + } + } +} + +func TestIsFingerprintKey(t *testing.T) { + yes := []string{"g-abc123", "r-deadbeef"} + no := []string{"-Users-me-code-foo", "proj1", "-home-matta-repos-foo"} + for _, s := range yes { + if !IsFingerprintKey(s) { + t.Errorf("IsFingerprintKey(%q) = false, want true", s) + } + } + for _, s := range no { + if IsFingerprintKey(s) { + t.Errorf("IsFingerprintKey(%q) = true, want false", s) + } + } +} + +// gitCmd runs git in dir, failing the test on error. +func gitCmd(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } +} + +func initRepoWithCommit(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + gitCmd(t, dir, "init", "-q", "-b", "main") + if err := os.WriteFile(filepath.Join(dir, "f.txt"), []byte("hi"), 0600); err != nil { + t.Fatal(err) + } + gitCmd(t, dir, "add", "-A") + gitCmd(t, dir, "commit", "-q", "-m", "init") +} + +func TestFingerprint_RemoteURL(t *testing.T) { + dir := filepath.Join(t.TempDir(), "repo") + initRepoWithCommit(t, dir) + gitCmd(t, dir, "remote", "add", "origin", "git@github.com:Acme/app.git") + + key, method := Fingerprint(dir, "-fallback") + if method != MethodRemote { + t.Fatalf("method = %q, want remote", method) + } + if !strings.HasPrefix(key, "g-") { + t.Errorf("key = %q, want g- prefix", key) + } + + // A different URL spelling for the same repo yields the same key. + dir2 := filepath.Join(t.TempDir(), "repo2") + initRepoWithCommit(t, dir2) + gitCmd(t, dir2, "remote", "add", "origin", "https://github.com/acme/app") + key2, _ := Fingerprint(dir2, "-other") + if key != key2 { + t.Errorf("equivalent remotes gave different keys: %q vs %q", key, key2) + } +} + +func TestFingerprint_RootCommitFallback(t *testing.T) { + dir := filepath.Join(t.TempDir(), "repo") + initRepoWithCommit(t, dir) + // No remote configured → root-commit fallback. + + key, method := Fingerprint(dir, "-fallback") + if method != MethodRoot { + t.Fatalf("method = %q, want root", method) + } + if !strings.HasPrefix(key, "r-") { + t.Errorf("key = %q, want r- prefix", key) + } + + // A clone shares the root commit, so it fingerprints identically. (Cloning + // auto-adds an origin pointing at the local source; drop it so the + // root-commit path is exercised on both sides.) + clone := filepath.Join(t.TempDir(), "clone") + gitCmd(t, t.TempDir(), "clone", "-q", dir, clone) + gitCmd(t, clone, "remote", "remove", "origin") + key2, method2 := Fingerprint(clone, "-clone") + if method2 != MethodRoot || key2 != key { + t.Errorf("clone key/method = %q/%q, want %q/root", key2, method2, key) + } +} + +func TestFingerprint_PathFallbackForNonGit(t *testing.T) { + dir := t.TempDir() // not a git repo + key, method := Fingerprint(dir, "-Users-me-code-foo") + if method != MethodPath { + t.Fatalf("method = %q, want path", method) + } + if key != "-Users-me-code-foo" { + t.Errorf("key = %q, want the localHash fallback", key) + } +} + +func TestFingerprint_EmptyPathIsPathFallback(t *testing.T) { + key, method := Fingerprint("", "-Users-me-code-foo") + if method != MethodPath || key != "-Users-me-code-foo" { + t.Errorf("empty path: key/method = %q/%q, want fallback/path", key, method) + } +} diff --git a/internal/project/index.go b/internal/project/index.go new file mode 100644 index 0000000..926dfd3 --- /dev/null +++ b/internal/project/index.go @@ -0,0 +1,144 @@ +package project + +import ( + "encoding/json" + "errors" + "io/fs" + "os" + "path/filepath" + "sort" +) + +// Index maps each local Claude project directory name (localHash) to the +// machine-independent key its memories sync under. It is per-PC state: localHash +// values encode this machine's paths, so the index never propagates across +// workstations (it lives under .state/, which is gitignored). +type Index struct { + byLocal map[string]Entry +} + +// Entry is one project's resolved identity. +type Entry struct { + Key string `json:"key"` + Method Method `json:"method"` + Path string `json:"path,omitempty"` // resolved real path, for diagnostics +} + +type indexJSON struct { + Version int `json:"version"` + Entries map[string]Entry `json:"entries"` +} + +const ( + indexStateDir = ".state" + indexFileName = "index.json" + indexVersion = 1 +) + +// NewIndex returns an empty index. +func NewIndex() *Index { return &Index{byLocal: map[string]Entry{}} } + +// IndexPath returns the absolute path of the index file inside syncDir. +func IndexPath(syncDir string) string { + return filepath.Join(syncDir, indexStateDir, indexFileName) +} + +// Build scans claudeProjectsDir and resolves every project directory to its key. +// Directories whose real path can't be resolved, or that aren't git repos, fall +// back to the path key (their own name). +func Build(claudeProjectsDir string) (*Index, error) { + idx := NewIndex() + entries, err := os.ReadDir(claudeProjectsDir) + if errors.Is(err, fs.ErrNotExist) { + return idx, nil + } + if err != nil { + return nil, err + } + for _, e := range entries { + if !e.IsDir() { + continue + } + localHash := e.Name() + realPath, _ := ResolvePath(filepath.Join(claudeProjectsDir, localHash)) + key, method := Fingerprint(realPath, localHash) + idx.byLocal[localHash] = Entry{Key: key, Method: method, Path: realPath} + } + return idx, nil +} + +// KeyFor returns the mirror key for a local project directory, computing and +// caching it on a miss (so the daemon's incremental path doesn't need a full +// rebuild when a new project appears mid-session). +func (idx *Index) KeyFor(claudeProjectsDir, localHash string) string { + if e, ok := idx.byLocal[localHash]; ok { + return e.Key + } + realPath, _ := ResolvePath(filepath.Join(claudeProjectsDir, localHash)) + key, method := Fingerprint(realPath, localHash) + idx.byLocal[localHash] = Entry{Key: key, Method: method, Path: realPath} + return key +} + +// LocalsFor returns the local project directories that map to key. Usually one, +// but a repo checked out at two paths on the same machine yields several — all of +// them should receive inbound memories for that project. +func (idx *Index) LocalsFor(key string) []string { + var out []string + for lh, e := range idx.byLocal { + if e.Key == key { + out = append(out, lh) + } + } + sort.Strings(out) + return out +} + +// Get returns the entry for a local project directory, if known. +func (idx *Index) Get(localHash string) (Entry, bool) { + e, ok := idx.byLocal[localHash] + return e, ok +} + +// All returns a copy of the localHash→Entry mapping. +func (idx *Index) All() map[string]Entry { + out := make(map[string]Entry, len(idx.byLocal)) + for k, v := range idx.byLocal { + out[k] = v + } + return out +} + +// Load reads the index from syncDir. A missing file yields an empty index and no +// error, so first-run callers need no special-casing. +func Load(syncDir string) (*Index, error) { + b, err := os.ReadFile(IndexPath(syncDir)) + if errors.Is(err, fs.ErrNotExist) { + return NewIndex(), nil + } + if err != nil { + return nil, err + } + var raw indexJSON + if err := json.Unmarshal(b, &raw); err != nil { + return nil, err + } + idx := NewIndex() + for lh, e := range raw.Entries { + idx.byLocal[lh] = e + } + return idx, nil +} + +// Save writes the index to syncDir as pretty-printed JSON. +func (idx *Index) Save(syncDir string) error { + b, err := json.MarshalIndent(indexJSON{Version: indexVersion, Entries: idx.byLocal}, "", " ") + if err != nil { + return err + } + b = append(b, '\n') + if err := os.MkdirAll(filepath.Dir(IndexPath(syncDir)), 0700); err != nil { + return err + } + return os.WriteFile(IndexPath(syncDir), b, 0600) +} diff --git a/internal/project/index_test.go b/internal/project/index_test.go new file mode 100644 index 0000000..58c6efc --- /dev/null +++ b/internal/project/index_test.go @@ -0,0 +1,117 @@ +package project + +import ( + "os" + "path/filepath" + "testing" +) + +func writeTranscript(t *testing.T, projectDir, cwd string) { + t.Helper() + if err := os.MkdirAll(projectDir, 0700); err != nil { + t.Fatal(err) + } + // json-encode the cwd so backslashes/quotes are handled. + b, err := os.OpenFile(filepath.Join(projectDir, "s.jsonl"), os.O_CREATE|os.O_WRONLY, 0600) + if err != nil { + t.Fatal(err) + } + defer b.Close() + if _, err := b.WriteString(`{"type":"user","cwd":` + quoteJSON(cwd) + "}\n"); err != nil { + t.Fatal(err) + } +} + +func quoteJSON(s string) string { + out := `"` + for _, r := range s { + switch r { + case '\\': + out += `\\` + case '"': + out += `\"` + default: + out += string(r) + } + } + return out + `"` +} + +func TestBuild_MixedProjects(t *testing.T) { + repo := filepath.Join(t.TempDir(), "app") + initRepoWithCommit(t, repo) + gitCmd(t, repo, "remote", "add", "origin", "git@github.com:acme/app.git") + + projects := t.TempDir() + gitProj := filepath.Join(projects, "-fake-localhash") + writeTranscript(t, gitProj, repo) + + plain := filepath.Join(projects, "-plain-dir") + if err := os.MkdirAll(plain, 0700); err != nil { // no transcript → path fallback + t.Fatal(err) + } + + idx, err := Build(projects) + if err != nil { + t.Fatal(err) + } + + gitEntry, ok := idx.Get("-fake-localhash") + if !ok || gitEntry.Method != MethodRemote { + t.Fatalf("git project entry = %+v, ok=%v; want remote", gitEntry, ok) + } + if locals := idx.LocalsFor(gitEntry.Key); len(locals) != 1 || locals[0] != "-fake-localhash" { + t.Errorf("LocalsFor(%q) = %v; want [-fake-localhash]", gitEntry.Key, locals) + } + + plainEntry, ok := idx.Get("-plain-dir") + if !ok || plainEntry.Method != MethodPath || plainEntry.Key != "-plain-dir" { + t.Errorf("plain entry = %+v; want path fallback keyed by its own name", plainEntry) + } +} + +func TestIndex_SaveLoadRoundTrip(t *testing.T) { + syncDir := t.TempDir() + idx := NewIndex() + idx.byLocal["-a"] = Entry{Key: "g-abc", Method: MethodRemote, Path: "/x/a"} + idx.byLocal["-b"] = Entry{Key: "-b", Method: MethodPath} + if err := idx.Save(syncDir); err != nil { + t.Fatal(err) + } + loaded, err := Load(syncDir) + if err != nil { + t.Fatal(err) + } + if e, ok := loaded.Get("-a"); !ok || e.Key != "g-abc" || e.Method != MethodRemote { + t.Errorf("round-trip lost -a: %+v ok=%v", e, ok) + } + if e, ok := loaded.Get("-b"); !ok || e.Key != "-b" { + t.Errorf("round-trip lost -b: %+v ok=%v", e, ok) + } +} + +func TestLoad_MissingReturnsEmpty(t *testing.T) { + idx, err := Load(t.TempDir()) + if err != nil { + t.Fatalf("missing index must not error; got %v", err) + } + if len(idx.All()) != 0 { + t.Errorf("expected empty index") + } +} + +func TestKeyFor_CachesOnMiss(t *testing.T) { + projects := t.TempDir() + plain := filepath.Join(projects, "-plain") + if err := os.MkdirAll(plain, 0700); err != nil { + t.Fatal(err) + } + idx := NewIndex() + key := idx.KeyFor(projects, "-plain") + if key != "-plain" { + t.Fatalf("KeyFor = %q; want path fallback -plain", key) + } + if _, ok := idx.Get("-plain"); !ok { + t.Errorf("KeyFor should cache the computed entry") + } +} diff --git a/internal/project/resolve.go b/internal/project/resolve.go new file mode 100644 index 0000000..9e9838d --- /dev/null +++ b/internal/project/resolve.go @@ -0,0 +1,104 @@ +// Package project derives a machine-independent key ("fingerprint") for each +// Claude Code project so the same repository syncs to one location regardless of +// where it is checked out on each machine. +// +// Claude names its per-project directory by dash-encoding the absolute project +// path (e.g. /Users/me/code/foo -> -Users-me-code-foo). That name differs across +// machines that check the repo out at different paths, so keying the sync repo by +// it splits one project into several. This package instead resolves the project's +// real path from Claude's session transcripts and fingerprints the git repo +// living there, falling back to the dash-encoded name for non-git directories. +package project + +import ( + "bufio" + "bytes" + "encoding/json" + "os" + "path/filepath" + "sort" + "strings" +) + +// maxJSONLLine bounds the scanner buffer. Transcript lines embed tool output and +// can be large, so we allow well beyond bufio's 64K default. +const maxJSONLLine = 16 * 1024 * 1024 + +// ResolvePath recovers the real absolute project path for a Claude project +// directory (~/.claude/projects/) by reading the "cwd" field Claude +// records in its session transcripts. The directory name itself is a lossy +// dash-encoding of the path (a dash inside a path segment is indistinguishable +// from a separator), so the transcript is the authoritative source. +// +// It scans transcripts newest-first and returns the first cwd found. ok is false +// when the directory has no transcript carrying a cwd (e.g. a stale or synthetic +// project dir), in which case callers fall back to the path key. +func ResolvePath(dir string) (path string, ok bool) { + for _, f := range newestJSONLFirst(dir) { + if cwd, found := scanCwd(f); found { + return cwd, true + } + } + return "", false +} + +// scanCwd returns the first non-empty "cwd" value in a JSONL transcript. Lines +// are parsed as JSON (not regex-matched) so escaped paths — notably Windows +// backslashes — decode correctly. +func scanCwd(path string) (string, bool) { + f, err := os.Open(path) + if err != nil { + return "", false + } + defer f.Close() + + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 0, 64*1024), maxJSONLLine) + needle := []byte(`"cwd"`) + for sc.Scan() { + line := sc.Bytes() + if !bytes.Contains(line, needle) { + continue + } + var rec struct { + Cwd string `json:"cwd"` + } + if err := json.Unmarshal(line, &rec); err != nil { + continue + } + if rec.Cwd != "" { + return rec.Cwd, true + } + } + return "", false +} + +// newestJSONLFirst lists *.jsonl transcripts in dir ordered most-recently- +// modified first, so the current checkout's path wins if a project has moved. +func newestJSONLFirst(dir string) []string { + entries, err := os.ReadDir(dir) + if err != nil { + return nil + } + type item struct { + path string + mod int64 + } + var items []item + for _, e := range entries { + if e.IsDir() || !strings.HasSuffix(e.Name(), ".jsonl") { + continue + } + info, err := e.Info() + if err != nil { + continue + } + items = append(items, item{filepath.Join(dir, e.Name()), info.ModTime().UnixNano()}) + } + sort.Slice(items, func(i, j int) bool { return items[i].mod > items[j].mod }) + paths := make([]string, len(items)) + for i, it := range items { + paths[i] = it.path + } + return paths +} diff --git a/internal/project/resolve_test.go b/internal/project/resolve_test.go new file mode 100644 index 0000000..5f33bef --- /dev/null +++ b/internal/project/resolve_test.go @@ -0,0 +1,76 @@ +package project + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestResolvePath_ExtractsCwd(t *testing.T) { + dir := t.TempDir() + line := `{"parentUuid":null,"type":"user","cwd":"/Users/mattartist/Code/repos/claude-utils","message":{"role":"user"}}` + if err := os.WriteFile(filepath.Join(dir, "session.jsonl"), []byte(line+"\n"), 0600); err != nil { + t.Fatal(err) + } + got, ok := ResolvePath(dir) + if !ok || got != "/Users/mattartist/Code/repos/claude-utils" { + t.Fatalf("ResolvePath = %q, %v; want the cwd", got, ok) + } +} + +func TestResolvePath_WindowsPathUnescaped(t *testing.T) { + dir := t.TempDir() + // Backslashes are JSON-escaped in the transcript; JSON decoding must restore them. + line := `{"type":"user","cwd":"C:\\Users\\rocky\\src\\app"}` + if err := os.WriteFile(filepath.Join(dir, "s.jsonl"), []byte(line+"\n"), 0600); err != nil { + t.Fatal(err) + } + got, ok := ResolvePath(dir) + if !ok || got != `C:\Users\rocky\src\app` { + t.Fatalf("ResolvePath = %q, %v; want unescaped Windows path", got, ok) + } +} + +func TestResolvePath_SkipsLinesWithoutCwd(t *testing.T) { + dir := t.TempDir() + content := `{"type":"mode","mode":"normal"} +{"type":"permission-mode","permissionMode":"plan"} +{"type":"user","cwd":"/home/matta/repos/foo"} +` + if err := os.WriteFile(filepath.Join(dir, "s.jsonl"), []byte(content), 0600); err != nil { + t.Fatal(err) + } + got, ok := ResolvePath(dir) + if !ok || got != "/home/matta/repos/foo" { + t.Fatalf("ResolvePath = %q, %v; want the cwd from a later line", got, ok) + } +} + +func TestResolvePath_NewestTranscriptWins(t *testing.T) { + dir := t.TempDir() + old := filepath.Join(dir, "old.jsonl") + newer := filepath.Join(dir, "new.jsonl") + if err := os.WriteFile(old, []byte(`{"cwd":"/old/path"}`+"\n"), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(newer, []byte(`{"cwd":"/new/path"}`+"\n"), 0600); err != nil { + t.Fatal(err) + } + // Make "old" genuinely older so ordering is deterministic. + past := time.Now().Add(-time.Hour) + if err := os.Chtimes(old, past, past); err != nil { + t.Fatal(err) + } + got, ok := ResolvePath(dir) + if !ok || got != "/new/path" { + t.Fatalf("ResolvePath = %q, %v; want the newest transcript's cwd", got, ok) + } +} + +func TestResolvePath_NoTranscript(t *testing.T) { + dir := t.TempDir() + if _, ok := ResolvePath(dir); ok { + t.Errorf("expected ok=false for a dir with no transcript") + } +} diff --git a/internal/sync/keyed_test.go b/internal/sync/keyed_test.go new file mode 100644 index 0000000..c3c705e --- /dev/null +++ b/internal/sync/keyed_test.go @@ -0,0 +1,151 @@ +package sync + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/MarimerLLC/claude-utils/internal/project" +) + +func gitRun(t *testing.T, dir string, args ...string) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t", + ) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } +} + +// makeRepo creates a git repo at dir with a commit and the given origin URL. +func makeRepo(t *testing.T, dir, origin string) { + t.Helper() + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + gitRun(t, dir, "init", "-q", "-b", "main") + if err := os.WriteFile(filepath.Join(dir, "f"), []byte("x"), 0600); err != nil { + t.Fatal(err) + } + gitRun(t, dir, "add", "-A") + gitRun(t, dir, "commit", "-q", "-m", "c") + gitRun(t, dir, "remote", "add", "origin", origin) +} + +// makeClaudeProject creates a Claude project dir (localHash) whose transcript +// records cwd, so project.Build can resolve and fingerprint it. +func makeClaudeProject(t *testing.T, claudeRoot, localHash, cwd string) { + t.Helper() + dir := filepath.Join(claudeRoot, localHash) + if err := os.MkdirAll(dir, 0700); err != nil { + t.Fatal(err) + } + line := `{"type":"user","cwd":"` + filepath.ToSlash(cwd) + `"}` + "\n" + if err := os.WriteFile(filepath.Join(dir, "s.jsonl"), []byte(line), 0600); err != nil { + t.Fatal(err) + } +} + +// TestReconcile_TwoPathsSameRepoUnionInMirror simulates the same repo checked +// out at two different paths (as it would be across machines): two Claude +// projects with different localHashes but the same origin URL resolve to one +// mirror key, and their MEMORY.md sections union there. +func TestReconcile_TwoPathsSameRepoUnionInMirror(t *testing.T) { + r, root := reconcileFixture(t) + + repoA := filepath.Join(root, "macos", "app") + repoB := filepath.Join(root, "linux", "app") + makeRepo(t, repoA, "git@github.com:acme/app.git") + makeRepo(t, repoB, "https://github.com/acme/app.git") // equivalent URL + + makeClaudeProject(t, r.Claude, "-Users-me-app", repoA) + makeClaudeProject(t, r.Claude, "-home-matta-app", repoB) + writeMem(t, r.Claude, "-Users-me-app", "MEMORY.md", "# App\n\n## FromMac\nmac\n") + writeMem(t, r.Claude, "-home-matta-app", "MEMORY.md", "# App\n\n## FromLinux\nlinux\n") + + idx, err := project.Build(r.Claude) + if err != nil { + t.Fatal(err) + } + // Both projects must resolve to the same fingerprint key. + ea, _ := idx.Get("-Users-me-app") + eb, _ := idx.Get("-home-matta-app") + if ea.Key == "" || ea.Key != eb.Key { + t.Fatalf("expected one shared key; got %q and %q", ea.Key, eb.Key) + } + + if _, err := Reconcile(r, idx, NewManifest()); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(filepath.Join(r.Mirror, ea.Key, "memory", "MEMORY.md")) + if err != nil { + t.Fatalf("mirror MEMORY.md missing under key %q: %v", ea.Key, err) + } + s := string(got) + if !contains(s, "## FromMac") || !contains(s, "## FromLinux") { + t.Errorf("mirror MEMORY.md did not union both machines' sections; got %q", s) + } + // The legacy per-path dirs must not exist in the mirror. + if _, err := os.Stat(filepath.Join(r.Mirror, "-Users-me-app")); err == nil { + t.Errorf("mirror should be keyed by fingerprint, not the local path") + } +} + +// TestMigrateLayout_CollapsesLegacyDirs seeds a legacy path-keyed mirror dir and +// a colliding fingerprint-keyed dir (as if another machine already migrated), +// then asserts they union under the fingerprint key and the legacy dir is gone. +func TestMigrateLayout_CollapsesLegacyDirs(t *testing.T) { + r, root := reconcileFixture(t) + repo := filepath.Join(root, "app") + makeRepo(t, repo, "git@github.com:acme/app.git") + makeClaudeProject(t, r.Claude, "-home-matta-app", repo) + + idx, err := project.Build(r.Claude) + if err != nil { + t.Fatal(err) + } + entry, _ := idx.Get("-home-matta-app") + key := entry.Key + if !project.IsFingerprintKey(key) { + t.Fatalf("expected a fingerprint key, got %q", key) + } + + // Legacy path-keyed dir from before the upgrade on this machine. + writeMem(t, r.Mirror, "-home-matta-app", "MEMORY.md", "# App\n\n## Legacy\nold\n") + // Fingerprint-keyed dir already present (other machine migrated + pushed). + writeMem(t, r.Mirror, key, "MEMORY.md", "# App\n\n## Other\nnew\n") + + rep, err := MigrateLayout(r, idx) + if err != nil { + t.Fatal(err) + } + if len(rep.Merged) != 1 { + t.Errorf("expected one merged file; got %#v", rep) + } + got, err := os.ReadFile(filepath.Join(r.Mirror, key, "memory", "MEMORY.md")) + if err != nil { + t.Fatal(err) + } + if s := string(got); !contains(s, "## Legacy") || !contains(s, "## Other") { + t.Errorf("migration did not union sections; got %q", s) + } + if _, err := os.Stat(filepath.Join(r.Mirror, "-home-matta-app")); err == nil { + t.Errorf("legacy mirror dir should have been removed") + } + + // Idempotent: a second run has nothing to migrate and must not error. + rep2, err := MigrateLayout(r, idx) + if err != nil { + t.Fatal(err) + } + if len(rep2.Merged)+len(rep2.CopiedToMirror)+len(rep2.BackedUp) != 0 { + t.Errorf("re-running migration should be a no-op; got %#v", rep2) + } +} diff --git a/internal/sync/loop.go b/internal/sync/loop.go index ae74a46..7e8f150 100644 --- a/internal/sync/loop.go +++ b/internal/sync/loop.go @@ -16,6 +16,7 @@ import ( "github.com/MarimerLLC/claude-utils/internal/config" "github.com/MarimerLLC/claude-utils/internal/distill" "github.com/MarimerLLC/claude-utils/internal/gitwt" + "github.com/MarimerLLC/claude-utils/internal/project" ) // Loop is the long-running daemon body. It watches the Claude memory tree, @@ -26,6 +27,11 @@ type Loop struct { Branch string Hostname string OnFlush func(localChanges bool, err error) // optional, for tests/observability + + // idx maps localHash → fingerprint key for the current Claude tree. Rebuilt + // at startup and on the pull tick; filled lazily for projects that appear + // mid-session. The loop is single-goroutine, so no locking is needed. + idx *project.Index } // Run blocks until ctx is canceled. Returns the first fatal error or nil @@ -63,6 +69,11 @@ func (l *Loop) Run(ctx context.Context) error { log.Println("refresh memory watches:", err) } + // A foreground `run` is otherwise silent, which reads as a hang; announce + // startup so it's clear the daemon is alive and what it's doing. + log.Printf("watching %s → %s (branch %s); pull every %ds. this runs in the foreground — Ctrl-C to stop.", + roots.Claude, l.Cfg.RemoteURL, l.Branch, l.Cfg.PullIntervalSec) + debounce := time.NewTimer(time.Hour) if !debounce.Stop() { <-debounce.C @@ -73,6 +84,16 @@ func (l *Loop) Run(ctx context.Context) error { pendingLocal := false + // Build the project index and migrate any legacy path-keyed mirror dirs to + // fingerprint keys before the first sync, so pre-existing per-machine dirs + // collapse to one key (and merge via the driver on the next push/pull). + l.rebuildIndex(roots) + if rep, err := MigrateLayout(roots, l.idx); err != nil { + log.Println("migrate layout:", err) + } else if n := len(rep.CopiedToMirror) + len(rep.Merged) + len(rep.BackedUp); n > 0 { + log.Printf("migrated legacy mirror dirs to fingerprint keys (%d files)", n) + } + // Initial sync at startup: reconcile local with mirror (using manifest // to detect deletes that happened while the daemon was off), then flush. manifest, err := LoadManifest(l.Cfg.SyncDir) @@ -80,7 +101,7 @@ func (l *Loop) Run(ctx context.Context) error { log.Println("load manifest (continuing without delete detection):", err) manifest = NewManifest() } - if _, err := Reconcile(roots, manifest); err != nil { + if _, err := Reconcile(roots, l.idx, manifest); err != nil { log.Println("initial reconcile:", err) } if err := l.flush(repo, roots, true); err != nil { @@ -93,6 +114,7 @@ func (l *Loop) Run(ctx context.Context) error { } l.refreshManifest(roots) l.rebuildDistilledIndex() + log.Printf("initial sync complete; tracking %d project(s). idle until a memory changes.", len(l.idx.All())) for { select { @@ -124,6 +146,8 @@ func (l *Loop) Run(ctx context.Context) error { err := l.flush(repo, roots, true) if err != nil { log.Println("flush (local):", err) + } else { + log.Println("synced local memory changes") } l.refreshManifest(roots) l.rebuildDistilledIndex() @@ -146,7 +170,10 @@ func (l *Loop) Run(ctx context.Context) error { log.Println("load manifest:", err) m = NewManifest() } - if _, err := Reconcile(roots, m); err != nil { + // Refresh the index so projects opened since startup (and any repo + // whose remote changed) resolve to the right key before reconciling. + l.rebuildIndex(roots) + if _, err := Reconcile(roots, l.idx, m); err != nil { log.Println("reconcile:", err) } err = l.flush(repo, roots, true) @@ -193,17 +220,17 @@ func (l *Loop) handleEvent(watcher *fsnotify.Watcher, roots Roots, ev fsnotify.E switch { case ev.Op&(fsnotify.Create|fsnotify.Write) != 0: - if err := CopyToMirror(roots, hash, name); err != nil { + if err := CopyToMirror(roots, l.idx, hash, name); err != nil { if errors.Is(err, fs.ErrNotExist) { // Race: file vanished between event and read. Treat as remove. - return true, RemoveFromMirror(roots, hash, name) + return true, RemoveFromMirror(roots, l.idx, hash, name) } return false, fmt.Errorf("copy to mirror %s/%s: %w", hash, name, err) } return true, nil case ev.Op&(fsnotify.Remove|fsnotify.Rename) != 0: - if err := RemoveFromMirror(roots, hash, name); err != nil { + if err := RemoveFromMirror(roots, l.idx, hash, name); err != nil { return false, fmt.Errorf("remove from mirror %s/%s: %w", hash, name, err) } return true, nil @@ -240,7 +267,7 @@ func (l *Loop) flush(repo *gitwt.Repo, roots Roots, includeLocal bool) error { } postRev, _ := repo.Run("rev-parse", "HEAD") if strings.TrimSpace(preRev) != strings.TrimSpace(postRev) && strings.TrimSpace(preRev) != "" { - if err := propagateChanges(repo, roots, strings.TrimSpace(preRev), strings.TrimSpace(postRev)); err != nil { + if err := propagateChanges(repo, roots, l.idx, strings.TrimSpace(preRev), strings.TrimSpace(postRev)); err != nil { return fmt.Errorf("propagate: %w", err) } } @@ -274,13 +301,17 @@ func remoteAndLocalIdle(repo *gitwt.Repo, branch string) (bool, error) { return strings.TrimSpace(unpushed) == "0", nil } -// propagateChanges replays the file changes between pre and post revisions -// from the mirror back into Claude's tree. -func propagateChanges(repo *gitwt.Repo, roots Roots, pre, post string) error { +// propagateChanges replays the file changes between pre and post revisions from +// the mirror back into Claude's tree. Mirror paths are keyed by fingerprint, so +// each key is reverse-resolved via idx to the local project dir(s) it belongs to; +// a key with no local project is skipped (it materializes on next reconcile once +// that project is opened here). +func propagateChanges(repo *gitwt.Repo, roots Roots, idx *project.Index, pre, post string) error { out, err := repo.Run("diff", "--name-status", pre+".."+post) if err != nil { return err } + applied := 0 for _, line := range strings.Split(out, "\n") { line = strings.TrimSpace(line) if line == "" { @@ -297,31 +328,44 @@ func propagateChanges(repo *gitwt.Repo, roots Roots, pre, post string) error { if strings.HasPrefix(status, "R") && len(fields) >= 3 { oldPath := fields[1] path = fields[2] - if hash, name, ok := splitMirrorPath(oldPath); ok { - _ = os.Remove(filepath.Join(roots.Claude, hash, "memory", name)) + if key, name, ok := splitMirrorPath(oldPath); ok { + removeFromClaude(roots, idx, key, name) } } else { path = fields[1] } - hash, name, ok := splitMirrorPath(path) + key, name, ok := splitMirrorPath(path) if !ok { continue } switch { case strings.HasPrefix(status, "D"): - _ = os.Remove(filepath.Join(roots.Claude, hash, "memory", name)) + removeFromClaude(roots, idx, key, name) + applied++ default: - if err := CopyToClaude(roots, hash, name); err != nil { + if err := CopyToClaude(roots, idx, key, name); err != nil { log.Printf("propagate %s: %v", path, err) + } else { + applied++ } } } + if applied > 0 { + log.Printf("applied %d inbound memory change(s) from remote", applied) + } return nil } -// splitMirrorPath converts a path of the form "projects//memory/" +// removeFromClaude deletes a memory file from every local project mapping to key. +func removeFromClaude(roots Roots, idx *project.Index, key, name string) { + for _, localHash := range idx.LocalsFor(key) { + _ = os.Remove(filepath.Join(roots.Claude, localHash, "memory", name)) + } +} + +// splitMirrorPath converts a path of the form "projects//memory/" // (as it appears in `git diff --name-only`) into its parts. -func splitMirrorPath(p string) (hash, name string, ok bool) { +func splitMirrorPath(p string) (key, name string, ok bool) { parts := strings.Split(filepath.ToSlash(p), "/") if len(parts) != 4 || parts[0] != "projects" || parts[2] != "memory" { return "", "", false @@ -351,6 +395,24 @@ func refreshMemoryWatches(w *fsnotify.Watcher, claudeRoot string) error { return nil } +// rebuildIndex refreshes the localHash → fingerprint-key mapping from the +// current Claude tree and persists it for observability. Best-effort: on error +// it keeps any existing index (or an empty one) so the loop can proceed. +func (l *Loop) rebuildIndex(roots Roots) { + idx, err := project.Build(roots.Claude) + if err != nil { + log.Println("build project index:", err) + if l.idx == nil { + l.idx = project.NewIndex() + } + return + } + l.idx = idx + if err := idx.Save(l.Cfg.SyncDir); err != nil { + log.Println("save project index:", err) + } +} + // refreshManifest rebuilds the manifest from the current Claude tree state // and saves it. Called after every successful flush so that the next // Reconcile reasons against an up-to-date "as of last sync" snapshot. diff --git a/internal/sync/mirror.go b/internal/sync/mirror.go index 897c333..b5f05e8 100644 --- a/internal/sync/mirror.go +++ b/internal/sync/mirror.go @@ -17,6 +17,7 @@ import ( "time" "github.com/MarimerLLC/claude-utils/internal/merge" + "github.com/MarimerLLC/claude-utils/internal/project" ) // Roots names a pair of (Claude authoritative, mirror) parents. @@ -26,38 +27,15 @@ import ( // Claude: C:\Users\rocky\.claude\projects // Mirror: C:\Users\rocky\.claudesync\projects // -// Each child directory under either root represents one project (named by -// Claude's hash of the absolute project path). +// Claude child dirs are named by Claude's dash-encoding of the absolute project +// path (localHash); mirror child dirs are named by the project's machine- +// independent fingerprint key, so the same repo lines up across machines that +// check it out at different paths. The project.Index maps between the two. type Roots struct { Claude string // ~/.claude/projects Mirror string // ~/.claudesync/projects } -// MemoryDirs lists every /memory directory found under either root. -func (r Roots) MemoryDirs() ([]string, error) { - seen := map[string]struct{}{} - for _, root := range []string{r.Claude, r.Mirror} { - entries, err := os.ReadDir(root) - if errors.Is(err, fs.ErrNotExist) { - continue - } - if err != nil { - return nil, fmt.Errorf("read %s: %w", root, err) - } - for _, e := range entries { - if !e.IsDir() { - continue - } - seen[e.Name()] = struct{}{} - } - } - dirs := make([]string, 0, len(seen)) - for hash := range seen { - dirs = append(dirs, hash) - } - return dirs, nil -} - // SyncReport is returned by reconciliation operations for diagnostics. type SyncReport struct { CopiedToMirror []string // relative paths copied Claude → mirror @@ -71,7 +49,14 @@ type SyncReport struct { // manifest to distinguish "user deleted this" from "this is new from // another PC." Pass a nil or empty manifest to disable delete detection // (the safe behavior for first-ever sync, since with no prior state we -// can't know what to delete). +// can't know what to delete). Pass a nil index to build one from the Claude +// tree. +// +// The Claude side stays keyed by localHash (Claude's dash-encoded dir name); +// the mirror side is keyed by the project's machine-independent fingerprint, +// resolved through idx. Only projects present locally are reconciled — a mirror +// key with no local project (opened only on another PC) is left pending and +// materializes when that project is first opened here. // // Decisions for each file path: // @@ -88,20 +73,46 @@ type SyncReport struct { // This function is used during `init` and as a periodic safety net on the // pull tick. The watcher-driven sync loop also handles incremental events // directly via CopyToMirror / RemoveFromMirror. -func Reconcile(r Roots, manifest *Manifest) (SyncReport, error) { +func Reconcile(r Roots, idx *project.Index, manifest *Manifest) (SyncReport, error) { rep := SyncReport{} if manifest == nil { manifest = NewManifest() } + if idx == nil { + var err error + idx, err = project.Build(r.Claude) + if err != nil { + return rep, err + } + } - hashes, err := r.MemoryDirs() - if err != nil { - return rep, err + // Reconcile every local project against its key, plus any mirror-only dir + // whose key is a legacy path key (== a localHash) so inbound files for + // identical-path / non-git projects still materialize into Claude. A + // fingerprint-keyed mirror dir with no local project can't be reversed to a + // path, so it waits (materializes when the project is first opened here). + type pair struct{ localHash, key string } + var pairs []pair + seenKey := map[string]bool{} + for localHash, entry := range idx.All() { + pairs = append(pairs, pair{localHash, entry.Key}) + seenKey[entry.Key] = true + } + if mirrorEntries, err := os.ReadDir(r.Mirror); err == nil { + for _, e := range mirrorEntries { + if !e.IsDir() || seenKey[e.Name()] || project.IsFingerprintKey(e.Name()) { + continue + } + pairs = append(pairs, pair{e.Name(), e.Name()}) + } + } else if !errors.Is(err, fs.ErrNotExist) { + return rep, fmt.Errorf("read mirror %s: %w", r.Mirror, err) } - for _, hash := range hashes { - claudeMem := filepath.Join(r.Claude, hash, "memory") - mirrorMem := filepath.Join(r.Mirror, hash, "memory") + for _, p := range pairs { + localHash := p.localHash + claudeMem := filepath.Join(r.Claude, localHash, "memory") + mirrorMem := filepath.Join(r.Mirror, p.key, "memory") if err := os.MkdirAll(mirrorMem, 0700); err != nil { return rep, err } @@ -134,7 +145,9 @@ func Reconcile(r Roots, manifest *Manifest) (SyncReport, error) { for name := range files { cp := filepath.Join(claudeMem, name) mp := filepath.Join(mirrorMem, name) - rel := filepath.ToSlash(filepath.Join(hash, "memory", name)) + // Manifest keys are Claude-side (localHash), independent of the mirror + // key, so fingerprinting doesn't disturb delete detection. + rel := filepath.ToSlash(filepath.Join(localHash, "memory", name)) cExists, cBytes := readIfExists(cp) mExists, mBytes := readIfExists(mp) @@ -195,11 +208,13 @@ func Reconcile(r Roots, manifest *Manifest) (SyncReport, error) { return rep, nil } -// CopyToMirror copies a single Claude-side file into the mirror, creating -// parent dirs as needed. Used by the watcher loop on a file change. -func CopyToMirror(r Roots, hash, name string) error { - src := filepath.Join(r.Claude, hash, "memory", name) - dst := filepath.Join(r.Mirror, hash, "memory", name) +// CopyToMirror copies a single Claude-side file into the mirror under the +// project's fingerprint key, creating parent dirs as needed. Used by the watcher +// loop on a file change. +func CopyToMirror(r Roots, idx *project.Index, localHash, name string) error { + key := idx.KeyFor(r.Claude, localHash) + src := filepath.Join(r.Claude, localHash, "memory", name) + dst := filepath.Join(r.Mirror, key, "memory", name) b, err := os.ReadFile(src) if err != nil { return err @@ -210,25 +225,34 @@ func CopyToMirror(r Roots, hash, name string) error { return writeAtomic(dst, b) } -// CopyToClaude copies a single mirror-side file back into Claude's tree. -// Used after a `git pull` brings in remote changes. -func CopyToClaude(r Roots, hash, name string) error { - src := filepath.Join(r.Mirror, hash, "memory", name) - dst := filepath.Join(r.Claude, hash, "memory", name) +// CopyToClaude copies a single mirror-side file (addressed by fingerprint key) +// back into every local project that maps to that key. A key with no local +// project is skipped — it materializes later when the project is first opened +// here. Used after a `git pull` brings in remote changes. +func CopyToClaude(r Roots, idx *project.Index, key, name string) error { + src := filepath.Join(r.Mirror, key, "memory", name) b, err := os.ReadFile(src) if err != nil { return err } - if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil { - return err + for _, localHash := range idx.LocalsFor(key) { + dst := filepath.Join(r.Claude, localHash, "memory", name) + if err := os.MkdirAll(filepath.Dir(dst), 0700); err != nil { + return err + } + if err := writeAtomic(dst, b); err != nil { + return err + } } - return writeAtomic(dst, b) + return nil } -// RemoveFromMirror handles a Claude-side delete: remove the corresponding -// mirror file so the next commit propagates the deletion. -func RemoveFromMirror(r Roots, hash, name string) error { - target := filepath.Join(r.Mirror, hash, "memory", name) +// RemoveFromMirror handles a Claude-side delete: remove the corresponding mirror +// file (under the project's fingerprint key) so the next commit propagates the +// deletion. +func RemoveFromMirror(r Roots, idx *project.Index, localHash, name string) error { + key := idx.KeyFor(r.Claude, localHash) + target := filepath.Join(r.Mirror, key, "memory", name) err := os.Remove(target) if errors.Is(err, fs.ErrNotExist) { return nil @@ -236,6 +260,78 @@ func RemoveFromMirror(r Roots, hash, name string) error { return err } +// MigrateLayout moves legacy path-keyed mirror directories to their fingerprint +// keys, so a project previously split by path (a separate dir per machine) +// collapses to one key. When the target key dir already exists — because another +// machine already migrated and pushed it, or a prior run did — the two are +// unioned: MEMORY.md via the semantic merge driver, other files copied when +// absent, and genuine conflicts preserved as .from-remote-. +// +// Idempotent: once a project's mirror dir is its fingerprint key there is no +// legacy dir left to move. Projects that can't be fingerprinted keep their path +// key and are left untouched. +func MigrateLayout(r Roots, idx *project.Index) (SyncReport, error) { + rep := SyncReport{} + for localHash, entry := range idx.All() { + if entry.Key == localHash { + continue // path fallback — already at its final key + } + oldProj := filepath.Join(r.Mirror, localHash) + oldMem := filepath.Join(oldProj, "memory") + if _, err := os.Stat(oldMem); err != nil { + continue // nothing legacy to migrate for this project + } + newMem := filepath.Join(r.Mirror, entry.Key, "memory") + if err := os.MkdirAll(newMem, 0700); err != nil { + return rep, err + } + + entries, err := os.ReadDir(oldMem) + if err != nil { + return rep, err + } + for _, e := range entries { + if e.IsDir() || strings.HasPrefix(e.Name(), ".") { + continue + } + name := e.Name() + rel := filepath.ToSlash(filepath.Join(entry.Key, "memory", name)) + oldBytes, err := os.ReadFile(filepath.Join(oldMem, name)) + if err != nil { + return rep, err + } + newPath := filepath.Join(newMem, name) + newExists, newBytes := readIfExists(newPath) + switch { + case !newExists: + if err := writeAtomic(newPath, oldBytes); err != nil { + return rep, err + } + rep.CopiedToMirror = append(rep.CopiedToMirror, rel) + case bytes.Equal(oldBytes, newBytes): + // identical — nothing to do + case name == "MEMORY.md": + merged, _ := merge.Merge("", string(newBytes), string(oldBytes)) + if err := writeAtomic(newPath, []byte(merged)); err != nil { + return rep, err + } + rep.Merged = append(rep.Merged, rel) + default: + backup := fmt.Sprintf("%s.from-remote-%s", newPath, randSuffix()) + if err := writeAtomic(backup, oldBytes); err != nil { + return rep, err + } + rep.BackedUp = append(rep.BackedUp, rel) + } + } + + if err := os.RemoveAll(oldProj); err != nil { + return rep, err + } + } + return rep, nil +} + func readIfExists(p string) (bool, []byte) { b, err := os.ReadFile(p) if err != nil { diff --git a/internal/sync/reconcile_test.go b/internal/sync/reconcile_test.go index 11de4a7..bba5b15 100644 --- a/internal/sync/reconcile_test.go +++ b/internal/sync/reconcile_test.go @@ -47,7 +47,7 @@ func TestReconcile_DeleteOnThisPC_PropagatesToMirror(t *testing.T) { manifest := NewManifest() manifest.Add("proj1/memory/feedback_x.md") - rep, err := Reconcile(r, manifest) + rep, err := Reconcile(r, nil, manifest) if err != nil { t.Fatal(err) } @@ -69,7 +69,7 @@ func TestReconcile_NewFromAnotherPC_CopiesToClaude(t *testing.T) { writeMem(t, r.Mirror, "proj1", "feedback_x.md", "from PC2") manifest := NewManifest() - rep, err := Reconcile(r, manifest) + rep, err := Reconcile(r, nil, manifest) if err != nil { t.Fatal(err) } @@ -90,7 +90,7 @@ func TestReconcile_NilManifest_NeverDeletes(t *testing.T) { // (first-run / missing manifest case). writeMem(t, r.Mirror, "proj1", "feedback_x.md", "old content") - rep, err := Reconcile(r, nil) + rep, err := Reconcile(r, nil, nil) if err != nil { t.Fatal(err) } @@ -112,7 +112,7 @@ func TestReconcile_NewLocalFile_CopiesToMirror(t *testing.T) { writeMem(t, r.Claude, "proj1", "MEMORY.md", "# new on this PC\n") manifest := NewManifest() - rep, err := Reconcile(r, manifest) + rep, err := Reconcile(r, nil, manifest) if err != nil { t.Fatal(err) } @@ -130,7 +130,7 @@ func TestReconcile_BothPresentSemanticMerge(t *testing.T) { writeMem(t, r.Mirror, "proj1", "MEMORY.md", "# Project\n\n## Remote\nfrom mirror\n") manifest := NewManifest() - rep, err := Reconcile(r, manifest) + rep, err := Reconcile(r, nil, manifest) if err != nil { t.Fatal(err) }