diff --git a/cmd/pull.go b/cmd/pull.go new file mode 100644 index 0000000..04e0bcc --- /dev/null +++ b/cmd/pull.go @@ -0,0 +1,469 @@ +package cmd + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/dataplanelabs/gcplane/internal/manifest" + "github.com/dataplanelabs/gcplane/internal/provider/goclaw" + "github.com/dataplanelabs/gcplane/internal/reconciler" + "github.com/dataplanelabs/gcplane/internal/skillpkg" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" +) + +var ( + pullKinds []string + pullAll bool + pullDryRun bool + pullPruneFiles bool +) + +var pullCmd = &cobra.Command{ + Use: "pull", + Short: "Reverse-sync evolved skills & agent context files from GoClaw into the repo", + Long: `Pulls the current state of skills and agent context files from a live GoClaw +instance into the local manifest directory. Intended for reviewing self-evolved +artifacts before committing them back to git. + +Default behavior (without --all) pulls only skills with source=gcplane or +source=evolution, and only agents already present in the local manifest. +Use --dry-run to preview changes without writing anything.`, + RunE: func(cmd *cobra.Command, args []string) error { + m, err := loadAndValidateManifest() + if err != nil { + return err + } + + ep, tok, err := resolveConnection(m) + if err != nil { + return err + } + provOpts, err := resolveProviderOpts(m) + if err != nil { + return err + } + + p := goclaw.New(ep, tok, provOpts...) + defer p.Close() + + ctx := cmd.Context() + + if wantKind(pullKinds, "skill") { + if err := pullSkills(ctx, p, m, pullAll, pullDryRun, pullPruneFiles); err != nil { + return err + } + } + if wantKind(pullKinds, "context-files") { + if err := pullContextFiles(ctx, p, m, pullDryRun); err != nil { + return err + } + } + return nil + }, +} + +func init() { + pullCmd.Flags().StringArrayVar(&pullKinds, "kind", nil, "restrict to kind(s): skill, context-files (default: both)") + pullCmd.Flags().BoolVar(&pullAll, "all", false, "pull all skills regardless of source (default: gcplane+evolution only)") + pullCmd.Flags().BoolVar(&pullDryRun, "dry-run", false, "print what would change without writing") + pullCmd.Flags().BoolVar(&pullPruneFiles, "prune-skill-files", false, "delete local skill files absent from server (default: off)") +} + +// wantKind returns true when kinds is empty (= all) or contains k. +func wantKind(kinds []string, k string) bool { + if len(kinds) == 0 { + return true + } + for _, v := range kinds { + if strings.EqualFold(v, k) { + return true + } + } + return false +} + +// shouldPullSkill decides whether a server-side skill is reverse-synced. +// Default: only source=gcplane|evolution skills already declared locally. +// --all: everything the token can see except system/bundled skills. +func shouldPullSkill(info reconciler.ResourceInfo, all, localKnown bool) bool { + if all { + return !info.IsSystem && info.Source != "bundled" + } + if info.Source != "gcplane" && info.Source != "evolution" { + return false + } + return localKnown +} + +// pullSkills downloads skill file trees for gcplane/evolution-sourced skills +// and writes them into the manifest's sourceDir layout. +func pullSkills(ctx context.Context, p *goclaw.Provider, m *manifest.Manifest, all, dryRun, pruneFiles bool) error { + infos, err := p.ListAll(ctx, manifest.KindSkill) + if err != nil { + return fmt.Errorf("list skills: %w", err) + } + + // Build set of slugs declared in local manifest so we don't invent new skills. + localSlugs := make(map[string]string) // slug → sourceDir + for _, r := range m.Resources { + if r.Kind != manifest.KindSkill { + continue + } + if sd, _ := r.Spec["sourceDir"].(string); sd != "" { + localSlugs[r.Name] = sd + } + } + + pulled := 0 + for _, info := range infos { + slug := info.Name + + _, localKnown := localSlugs[slug] + if !shouldPullSkill(info, all, localKnown) { + continue + } + + sourceDir, known := localSlugs[slug] + if !known { + // --all mode: infer sourceDir from the manifest config path. + sourceDir = inferSourceDir(configFile, slug) + } + + files, grantees, err := p.DownloadSkillSource(ctx, slug) + if err != nil { + return fmt.Errorf("download skill %s: %w", slug, err) + } + + if dryRun { + printSkillDiff(slug, sourceDir, files) + } else { + changed, err := skillpkg.UnpackTo(sourceDir, files, pruneFiles) + if err != nil { + return fmt.Errorf("unpack skill %s: %w", slug, err) + } + if err := writeFrontmatter(sourceDir, grantees); err != nil { + return fmt.Errorf("write frontmatter for %s: %w", slug, err) + } + if len(changed) > 0 { + fmt.Printf("skill %s: wrote %d file(s)\n", slug, len(changed)) + for _, f := range changed { + fmt.Printf(" %s\n", f) + } + } else { + fmt.Printf("skill %s: no changes\n", slug) + } + } + pulled++ + } + + if pulled == 0 { + fmt.Println("pull skills: no matching skills found") + } + return nil +} + +// pullContextFiles downloads context files for all Agent resources in the manifest +// and patches them in-place in the agents.yaml files. +func pullContextFiles(ctx context.Context, p *goclaw.Provider, m *manifest.Manifest, dryRun bool) error { + // Collect agent keys from manifest. + var agentKeys []string + for _, r := range m.Resources { + if r.Kind == manifest.KindAgent { + agentKeys = append(agentKeys, r.Name) + } + } + if len(agentKeys) == 0 { + fmt.Println("pull context-files: no Agent resources in manifest") + return nil + } + + // Group agents by the YAML file they live in so we can patch per-file. + // When the manifest is a directory, agents are in agents.yaml inside that dir. + agentsFile := resolveAgentsFile(configFile) + + for _, agentKey := range agentKeys { + files, err := p.DownloadAgentContextFiles(ctx, agentKey) + if err != nil { + return fmt.Errorf("download context files for %s: %w", agentKey, err) + } + if len(files) == 0 { + fmt.Printf("agent %s: no context files\n", agentKey) + continue + } + + if dryRun { + printContextFileDiff(agentKey, agentsFile, files) + } else { + changed, err := patchAgentsYAML(agentsFile, agentKey, files) + if err != nil { + return fmt.Errorf("patch agents.yaml for %s: %w", agentKey, err) + } + if changed { + fmt.Printf("agent %s: context files updated in %s\n", agentKey, agentsFile) + } else { + fmt.Printf("agent %s: context files unchanged\n", agentKey) + } + } + } + return nil +} + +// resolveAgentsFile returns the agents.yaml path relative to the manifest dir. +func resolveAgentsFile(cfgPath string) string { + if cfgPath == "" { + return "agents.yaml" + } + info, err := os.Stat(cfgPath) + if err == nil && info.IsDir() { + return filepath.Join(cfgPath, "agents.yaml") + } + return filepath.Join(filepath.Dir(cfgPath), "agents.yaml") +} + +// inferSourceDir builds a sourceDir for a skill not already in the local manifest. +// Falls back to /skills//. +func inferSourceDir(cfgPath, slug string) string { + base := cfgPath + if cfgPath != "" { + info, err := os.Stat(cfgPath) + if err == nil && !info.IsDir() { + base = filepath.Dir(cfgPath) + } + } + return filepath.Join(base, "skills", slug) +} + +// writeFrontmatter writes a frontmatter.yaml in sourceDir with grants.agents sorted. +// Skips writing if grantees is empty (avoids creating an empty file). +func writeFrontmatter(sourceDir string, grantees []string) error { + outPath := filepath.Join(sourceDir, "frontmatter.yaml") + if len(grantees) == 0 { + // Grants revoked server-side: drop stale frontmatter so apply won't re-grant. + if err := os.Remove(outPath); err != nil && !os.IsNotExist(err) { + return err + } + return nil + } + sort.Strings(grantees) + + type grantsOverlay struct { + Grants struct { + Agents []string `yaml:"agents"` + } `yaml:"grants"` + } + var doc grantsOverlay + doc.Grants.Agents = grantees + + data, err := yaml.Marshal(&doc) + if err != nil { + return err + } + + existing, _ := os.ReadFile(outPath) + if bytes.Equal(existing, data) { + return nil + } + return os.WriteFile(outPath, data, 0o644) +} + +// patchAgentsYAML updates the contextFiles for a named agent in agents.yaml in-place. +// Only rewrites the file when content actually differs to avoid comment churn. +// Returns true if the file was modified. +func patchAgentsYAML(path, agentKey string, files []map[string]string) (bool, error) { + raw, err := os.ReadFile(path) + if err != nil { + return false, fmt.Errorf("read %s: %w", path, err) + } + + var doc yaml.Node + if err := yaml.Unmarshal(raw, &doc); err != nil { + return false, fmt.Errorf("parse %s: %w", path, err) + } + + // Navigate: doc is a document node whose Content[0] is the mapping. + root := docRoot(&doc) + if root == nil { + return false, fmt.Errorf("empty or invalid YAML in %s", path) + } + + // Find the resources sequence. + resources := mappingValue(root, "resources") + if resources == nil || resources.Kind != yaml.SequenceNode { + return false, fmt.Errorf("no 'resources' sequence in %s", path) + } + + // Find the Agent node matching agentKey. + agentNode := findAgent(resources, agentKey) + if agentNode == nil { + // Agent not in this file — silently skip. + return false, nil + } + + // Check if existing content already matches (hash compare). + if !contextFilesChanged(agentNode, files) { + return false, nil + } + + // Build the new contextFiles YAML node. + newCF := buildContextFilesNode(files) + + // Replace or insert contextFiles in the agent's spec. + specNode := mappingValue(agentNode, "spec") + if specNode == nil { + return false, fmt.Errorf("agent %s has no spec in %s", agentKey, path) + } + setMappingValue(specNode, "contextFiles", newCF) + + out, err := yaml.Marshal(&doc) + if err != nil { + return false, fmt.Errorf("marshal %s: %w", path, err) + } + if err := os.WriteFile(path, out, 0o644); err != nil { + return false, fmt.Errorf("write %s: %w", path, err) + } + return true, nil +} + +// contextFilesChanged returns true when the agent's current contextFiles in the +// YAML node differ from the server-fetched files (by SHA-256 of content). +func contextFilesChanged(agentNode *yaml.Node, files []map[string]string) bool { + specNode := mappingValue(agentNode, "spec") + if specNode == nil { + return true + } + cfNode := mappingValue(specNode, "contextFiles") + if cfNode == nil || cfNode.Kind != yaml.SequenceNode { + return true + } + + // Extract current name→contentHash from YAML. + current := make(map[string][32]byte) + for _, item := range cfNode.Content { + if item.Kind != yaml.MappingNode { + continue + } + name := mappingStringValue(item, "name") + content := mappingStringValue(item, "content") + if name != "" { + current[name] = sha256.Sum256([]byte(content)) + } + } + + if len(current) != len(files) { + return true + } + for _, f := range files { + want := sha256.Sum256([]byte(f["content"])) + if got, ok := current[f["name"]]; !ok || got != want { + return true + } + } + return false +} + +// buildContextFilesNode creates a YAML sequence node for contextFiles using +// literal block scalars for content to match the repo's authoring style. +func buildContextFilesNode(files []map[string]string) *yaml.Node { + seq := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + for _, f := range files { + item := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} + nameKey := &yaml.Node{Kind: yaml.ScalarNode, Value: "name"} + nameVal := &yaml.Node{Kind: yaml.ScalarNode, Value: f["name"]} + contentKey := &yaml.Node{Kind: yaml.ScalarNode, Value: "content"} + contentVal := &yaml.Node{ + Kind: yaml.ScalarNode, + Value: f["content"], + Style: yaml.LiteralStyle, + } + item.Content = append(item.Content, nameKey, nameVal, contentKey, contentVal) + seq.Content = append(seq.Content, item) + } + return seq +} + +// docRoot unwraps a yaml.Node document to its root mapping. +func docRoot(n *yaml.Node) *yaml.Node { + if n == nil { + return nil + } + if n.Kind == yaml.DocumentNode && len(n.Content) > 0 { + return n.Content[0] + } + return n +} + +// mappingValue returns the value node for key in a mapping node. +func mappingValue(m *yaml.Node, key string) *yaml.Node { + if m == nil || m.Kind != yaml.MappingNode { + return nil + } + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + return m.Content[i+1] + } + } + return nil +} + +// mappingStringValue returns the string value of key in a mapping node, or "". +func mappingStringValue(m *yaml.Node, key string) string { + v := mappingValue(m, key) + if v == nil { + return "" + } + return v.Value +} + +// setMappingValue sets (or replaces) key→val in a mapping node. +func setMappingValue(m *yaml.Node, key string, val *yaml.Node) { + for i := 0; i+1 < len(m.Content); i += 2 { + if m.Content[i].Value == key { + m.Content[i+1] = val + return + } + } + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} + m.Content = append(m.Content, keyNode, val) +} + +// findAgent returns the mapping node for the Agent resource with agentKey. +func findAgent(resources *yaml.Node, agentKey string) *yaml.Node { + for _, item := range resources.Content { + if item.Kind != yaml.MappingNode { + continue + } + kind := mappingStringValue(item, "kind") + name := mappingStringValue(item, "name") + if kind == "Agent" && name == agentKey { + return item + } + } + return nil +} + +// printSkillDiff prints a summary of what would change for a skill. +func printSkillDiff(slug, sourceDir string, files []skillpkg.SkillFile) { + fmt.Printf("[dry-run] skill %s → %s (%d files)\n", slug, sourceDir, len(files)) + for _, f := range files { + abs := filepath.Join(sourceDir, filepath.FromSlash(f.Path)) + existing, err := os.ReadFile(abs) + if err != nil || !bytes.Equal(existing, f.Data) { + fmt.Printf(" ~ %s\n", f.Path) + } + } +} + +// printContextFileDiff prints a summary of what would change for an agent's context files. +func printContextFileDiff(agentKey, agentsFile string, files []map[string]string) { + fmt.Printf("[dry-run] agent %s context-files in %s:\n", agentKey, agentsFile) + for _, f := range files { + fmt.Printf(" ~ %s (%d bytes)\n", f["name"], len(f["content"])) + } +} diff --git a/cmd/pull_test.go b/cmd/pull_test.go new file mode 100644 index 0000000..8647746 --- /dev/null +++ b/cmd/pull_test.go @@ -0,0 +1,119 @@ +package cmd + +import ( + "os" + "path/filepath" + "testing" + + "github.com/dataplanelabs/gcplane/internal/manifest" + "github.com/dataplanelabs/gcplane/internal/reconciler" +) + +func skillInfo(source string, system bool) reconciler.ResourceInfo { + return reconciler.ResourceInfo{ + Kind: manifest.KindSkill, + Name: "demo-skill", + Source: source, + IsSystem: system, + } +} + +func TestShouldPullSkill_DefaultSourceFilter(t *testing.T) { + tests := []struct { + name string + source string + system bool + localKnown bool + want bool + }{ + {"evolution known", "evolution", false, true, true}, + {"gcplane known", "gcplane", false, true, true}, + {"evolution unknown locally", "evolution", false, false, false}, + {"gcplane unknown locally", "gcplane", false, false, false}, + {"cli source skipped", "cli", false, true, false}, + {"unknown source skipped", "unknown", false, true, false}, + {"bundled source skipped", "bundled", false, true, false}, + {"empty source skipped", "", false, true, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := shouldPullSkill(skillInfo(tt.source, tt.system), false, tt.localKnown) + if got != tt.want { + t.Fatalf("shouldPullSkill(source=%q, all=false, local=%v) = %v, want %v", + tt.source, tt.localKnown, got, tt.want) + } + }) + } +} + +func TestShouldPullSkill_AllMode(t *testing.T) { + tests := []struct { + name string + source string + system bool + want bool + }{ + {"evolution pulled", "evolution", false, true}, + {"gcplane pulled", "gcplane", false, true}, + {"cli pulled under all", "cli", false, true}, + {"unknown pulled under all", "unknown", false, true}, + {"bundled excluded", "bundled", false, false}, + {"is_system excluded", "evolution", true, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // localKnown is irrelevant under --all. + got := shouldPullSkill(skillInfo(tt.source, tt.system), true, false) + if got != tt.want { + t.Fatalf("shouldPullSkill(source=%q, system=%v, all=true) = %v, want %v", + tt.source, tt.system, got, tt.want) + } + }) + } +} + +func TestWriteFrontmatter_WritesGrants(t *testing.T) { + dir := t.TempDir() + if err := writeFrontmatter(dir, []string{"van-anh", "annhien"}); err != nil { + t.Fatalf("writeFrontmatter: %v", err) + } + data, err := os.ReadFile(filepath.Join(dir, "frontmatter.yaml")) + if err != nil { + t.Fatalf("read frontmatter: %v", err) + } + got := string(data) + // Sorted grantees. + if want := "grants:\n agents:\n - annhien\n - van-anh\n"; got != want { + t.Fatalf("frontmatter content = %q, want %q", got, want) + } +} + +func TestWriteFrontmatter_RemovesStaleWhenGrantsRevoked(t *testing.T) { + dir := t.TempDir() + outPath := filepath.Join(dir, "frontmatter.yaml") + + if err := writeFrontmatter(dir, []string{"van-anh"}); err != nil { + t.Fatalf("seed frontmatter: %v", err) + } + if _, err := os.Stat(outPath); err != nil { + t.Fatalf("frontmatter should exist after seed: %v", err) + } + + // Grants revoked server-side → stale file must be removed. + if err := writeFrontmatter(dir, nil); err != nil { + t.Fatalf("writeFrontmatter(empty): %v", err) + } + if _, err := os.Stat(outPath); !os.IsNotExist(err) { + t.Fatalf("stale frontmatter still present, stat err = %v", err) + } +} + +func TestWriteFrontmatter_NoGrantsNoFileIsNoop(t *testing.T) { + dir := t.TempDir() + if err := writeFrontmatter(dir, nil); err != nil { + t.Fatalf("writeFrontmatter(empty) on clean dir: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "frontmatter.yaml")); !os.IsNotExist(err) { + t.Fatalf("no frontmatter should be created, stat err = %v", err) + } +} diff --git a/cmd/root.go b/cmd/root.go index 9418340..6486b15 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -87,6 +87,7 @@ func init() { rootCmd.AddCommand(destroyCmd) rootCmd.AddCommand(validateCmd) rootCmd.AddCommand(exportCmd) + rootCmd.AddCommand(pullCmd) rootCmd.AddCommand(serveCmd) rootCmd.AddCommand(statusCmd) rootCmd.AddCommand(topCmd) diff --git a/internal/provider/goclaw/context_files.go b/internal/provider/goclaw/context_files.go index 01a7bb3..b79d9ca 100644 --- a/internal/provider/goclaw/context_files.go +++ b/internal/provider/goclaw/context_files.go @@ -9,9 +9,13 @@ import ( "io" "mime/multipart" "net/http" + "strings" "time" ) +// maxContextFileSize caps a single decompressed context-file tar entry (gzip-bomb guard). +const maxContextFileSize = 8 << 20 + // syncContextFiles upserts agent context files via the merge import API. // Builds a tar.gz archive with context_files/{name} entries and POSTs it // to /v1/agents/{id}/import?include=context_files as multipart form data. @@ -67,6 +71,66 @@ func (p *Provider) syncContextFiles(ctx context.Context, agentID string, files [ return nil } +// DownloadAgentContextFiles fetches context files for an agent from goclaw. +// Calls GET /v1/agents/{id}/export?sections=context_files&stream=false +// and untars the returned gzip archive into [{name, content}] pairs. +func (p *Provider) DownloadAgentContextFiles(ctx context.Context, agentKey string) ([]map[string]string, error) { + id, err := p.resolveAgentID(ctx, agentKey) + if err != nil { + return nil, err + } + + gz, err := p.http.GetRaw(ctx, "/v1/agents/"+id+"/export?sections=context_files&stream=false") + if err != nil { + return nil, fmt.Errorf("export context files for agent %s: %w", agentKey, err) + } + + return parseContextFilesArchive(gz) +} + +// parseContextFilesArchive untars a gzip archive and returns entries under +// the "context_files/" prefix as [{name, content}] pairs. +func parseContextFilesArchive(gz []byte) ([]map[string]string, error) { + gr, err := gzip.NewReader(bytes.NewReader(gz)) + if err != nil { + return nil, fmt.Errorf("gzip open: %w", err) + } + defer gr.Close() + + tr := tar.NewReader(gr) + const prefix = "context_files/" + var out []map[string]string + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("tar read: %w", err) + } + if hdr.FileInfo().IsDir() { + continue + } + name := hdr.Name + if !strings.HasPrefix(name, prefix) { + continue + } + name = strings.TrimPrefix(name, prefix) + if name == "" { + continue + } + data, err := io.ReadAll(io.LimitReader(tr, maxContextFileSize+1)) + if err != nil { + return nil, fmt.Errorf("read tar entry %s: %w", hdr.Name, err) + } + if int64(len(data)) > maxContextFileSize { + return nil, fmt.Errorf("context file %s exceeds %d bytes", name, maxContextFileSize) + } + out = append(out, map[string]string{"name": name, "content": string(data)}) + } + return out, nil +} + // buildContextFilesArchive creates a tar.gz archive with context_files/{name} entries. func buildContextFilesArchive(files []any) (*bytes.Buffer, error) { buf := &bytes.Buffer{} diff --git a/internal/provider/goclaw/context_files_download_test.go b/internal/provider/goclaw/context_files_download_test.go new file mode 100644 index 0000000..1c4823c --- /dev/null +++ b/internal/provider/goclaw/context_files_download_test.go @@ -0,0 +1,150 @@ +package goclaw + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "io" + "net/http" + "testing" + "time" +) + +// buildTestArchive creates a tar.gz with context_files/{name} entries for testing. +func buildTestArchive(t *testing.T, entries map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + for name, content := range entries { + hdr := &tar.Header{ + Name: "context_files/" + name, + Size: int64(len(content)), + Mode: 0644, + ModTime: time.Now(), + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("tar header: %v", err) + } + if _, err := io.WriteString(tw, content); err != nil { + t.Fatalf("tar write: %v", err) + } + } + if err := tw.Close(); err != nil { + t.Fatalf("tar close: %v", err) + } + if err := gw.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + return buf.Bytes() +} + +func TestDownloadAgentContextFiles_HappyPath(t *testing.T) { + archive := buildTestArchive(t, map[string]string{ + "IDENTITY.md": "# Agent\nName: Bot", + "SOUL.md": "## Personality", + }) + + p, cleanup := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1/agents": + json.NewEncoder(w).Encode(map[string]any{ + "agents": []map[string]any{ + {"id": "agent-uuid", "agent_key": "my-bot"}, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/v1/agents/agent-uuid/export": + w.Header().Set("Content-Type", "application/gzip") + w.Write(archive) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer cleanup() + + files, err := p.DownloadAgentContextFiles(context.Background(), "my-bot") + if err != nil { + t.Fatalf("DownloadAgentContextFiles: %v", err) + } + if len(files) != 2 { + t.Fatalf("expected 2 files, got %d", len(files)) + } + + byName := make(map[string]string) + for _, f := range files { + byName[f["name"]] = f["content"] + } + if byName["IDENTITY.md"] != "# Agent\nName: Bot" { + t.Errorf("IDENTITY.md mismatch: %q", byName["IDENTITY.md"]) + } + if byName["SOUL.md"] != "## Personality" { + t.Errorf("SOUL.md mismatch: %q", byName["SOUL.md"]) + } +} + +func TestDownloadAgentContextFiles_AgentNotFound(t *testing.T) { + p, cleanup := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"agents": []map[string]any{}}) + })) + defer cleanup() + + _, err := p.DownloadAgentContextFiles(context.Background(), "nonexistent") + if err == nil { + t.Fatal("expected error for missing agent") + } +} + +func TestParseContextFilesArchive_SkipsNonPrefixed(t *testing.T) { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + + for _, entry := range []struct{ name, content string }{ + {"context_files/IDENTITY.md", "# Hi"}, + {"other/file.txt", "should be skipped"}, + } { + hdr := &tar.Header{Name: entry.name, Size: int64(len(entry.content)), Mode: 0644, ModTime: time.Now()} + tw.WriteHeader(hdr) //nolint:errcheck + io.WriteString(tw, entry.content) //nolint:errcheck + } + tw.Close() //nolint:errcheck + gw.Close() //nolint:errcheck + + out, err := parseContextFilesArchive(buf.Bytes()) + if err != nil { + t.Fatalf("parseContextFilesArchive: %v", err) + } + if len(out) != 1 { + t.Fatalf("expected 1 entry, got %d: %v", len(out), out) + } + if out[0]["name"] != "IDENTITY.md" { + t.Errorf("expected IDENTITY.md, got %q", out[0]["name"]) + } +} + +func TestParseContextFilesArchive_EmptyArchive(t *testing.T) { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gw) + tw.Close() //nolint:errcheck + gw.Close() //nolint:errcheck + + out, err := parseContextFilesArchive(buf.Bytes()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(out) != 0 { + t.Errorf("expected 0 entries for empty archive, got %d", len(out)) + } +} + +func TestParseContextFilesArchive_RejectsOversizeEntry(t *testing.T) { + archive := buildTestArchive(t, map[string]string{ + "huge.md": string(bytes.Repeat([]byte("x"), maxContextFileSize+1)), + }) + if _, err := parseContextFilesArchive(archive); err == nil { + t.Fatal("expected error for oversize context file, got nil") + } +} diff --git a/internal/provider/goclaw/helpers.go b/internal/provider/goclaw/helpers.go index a89236b..4d64706 100644 --- a/internal/provider/goclaw/helpers.go +++ b/internal/provider/goclaw/helpers.go @@ -108,6 +108,12 @@ func strVal(m map[string]any, key string) string { return s } +// boolVal safely extracts a bool value from a map. +func boolVal(m map[string]any, key string) bool { + b, _ := m[key].(bool) + return b +} + // copyMap creates a shallow copy of a map. func copyMap(m map[string]any) map[string]any { out := make(map[string]any, len(m)) diff --git a/internal/provider/goclaw/http_client.go b/internal/provider/goclaw/http_client.go index 653092e..0a72762 100644 --- a/internal/provider/goclaw/http_client.go +++ b/internal/provider/goclaw/http_client.go @@ -80,6 +80,12 @@ func (c *HTTPClient) Delete(ctx context.Context, path string) error { return err } +// GetRaw performs an authenticated GET and returns the raw response body bytes. +// Used for non-JSON responses: tar.gz archives and ?raw=true file downloads. +func (c *HTTPClient) GetRaw(ctx context.Context, path string) ([]byte, error) { + return c.doRaw(ctx, http.MethodGet, path, nil, "") +} + // PostMultipart performs an authenticated POST with a multipart body. // Used for skill ZIP uploads. The default Content-Type set by do() (application/json) // is overridden via the contentType argument. diff --git a/internal/provider/goclaw/list_all.go b/internal/provider/goclaw/list_all.go index 60b684f..98ae7fc 100644 --- a/internal/provider/goclaw/list_all.go +++ b/internal/provider/goclaw/list_all.go @@ -134,6 +134,8 @@ func (p *Provider) listAllSkills(ctx context.Context) ([]reconciler.ResourceInfo Kind: manifest.KindSkill, Name: strVal(s, "slug"), CreatedBy: strVal(s, "created_by"), + Source: strVal(s, "source"), + IsSystem: boolVal(s, "is_system"), }) } return infos, nil diff --git a/internal/provider/goclaw/skill_download_test.go b/internal/provider/goclaw/skill_download_test.go new file mode 100644 index 0000000..7186cc8 --- /dev/null +++ b/internal/provider/goclaw/skill_download_test.go @@ -0,0 +1,136 @@ +package goclaw + +import ( + "context" + "encoding/json" + "net/http" + "testing" +) + +func TestDownloadSkill_HappyPath(t *testing.T) { + p, cleanup := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1/skills": + json.NewEncoder(w).Encode(map[string]any{ + "skills": []map[string]any{ + {"id": "skill-123", "slug": "my-skill"}, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/v1/skills/skill-123/files": + json.NewEncoder(w).Encode(map[string]any{ + "files": []map[string]any{ + {"path": "SKILL.md", "name": "SKILL.md", "isDir": false, "size": 20}, + {"path": "scripts/run.sh", "name": "run.sh", "isDir": false, "size": 10}, + {"path": "scripts", "name": "scripts", "isDir": true}, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/v1/skills/skill-123/files/SKILL.md": + w.Write([]byte("---\nname: My Skill\n---\nbody\n")) + case r.Method == http.MethodGet && r.URL.Path == "/v1/skills/skill-123/files/scripts/run.sh": + w.Write([]byte("#!/bin/sh\necho hi\n")) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer cleanup() + + files, err := p.DownloadSkill(context.Background(), "my-skill") + if err != nil { + t.Fatalf("DownloadSkill: %v", err) + } + if len(files) != 2 { + t.Fatalf("expected 2 files, got %d", len(files)) + } + + byPath := make(map[string][]byte) + for _, f := range files { + byPath[f.Path] = f.Data + } + if string(byPath["SKILL.md"]) != "---\nname: My Skill\n---\nbody\n" { + t.Errorf("SKILL.md content mismatch: %q", byPath["SKILL.md"]) + } + if string(byPath["scripts/run.sh"]) != "#!/bin/sh\necho hi\n" { + t.Errorf("run.sh content mismatch: %q", byPath["scripts/run.sh"]) + } +} + +func TestDownloadSkill_NotFound(t *testing.T) { + p, cleanup := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"skills": []map[string]any{}}) + })) + defer cleanup() + + _, err := p.DownloadSkill(context.Background(), "missing-skill") + if err == nil { + t.Fatal("expected error for missing skill") + } +} + +func TestDownloadSkill_EmptyFileTree(t *testing.T) { + p, cleanup := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1/skills": + json.NewEncoder(w).Encode(map[string]any{ + "skills": []map[string]any{{"id": "sid", "slug": "empty-skill"}}, + }) + case r.Method == http.MethodGet && r.URL.Path == "/v1/skills/sid/files": + json.NewEncoder(w).Encode(map[string]any{"files": []map[string]any{}}) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer cleanup() + + files, err := p.DownloadSkill(context.Background(), "empty-skill") + if err != nil { + t.Fatalf("DownloadSkill: %v", err) + } + if len(files) != 0 { + t.Errorf("expected 0 files for empty tree, got %d", len(files)) + } +} + +func TestDownloadSkillSource_IncludesGrants(t *testing.T) { + p, cleanup := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/v1/skills": + json.NewEncoder(w).Encode(map[string]any{ + "skills": []map[string]any{{"id": "sk1", "slug": "granted-skill"}}, + }) + case r.Method == http.MethodGet && r.URL.Path == "/v1/skills/sk1/files": + json.NewEncoder(w).Encode(map[string]any{ + "files": []map[string]any{ + {"path": "SKILL.md", "name": "SKILL.md", "isDir": false}, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/v1/skills/sk1/files/SKILL.md": + w.Write([]byte("---\nname: Granted Skill\n---\n")) + case r.Method == http.MethodGet && r.URL.Path == "/v1/agents": + json.NewEncoder(w).Encode(map[string]any{ + "agents": []map[string]any{ + {"id": "a1", "agent_key": "bot-alpha"}, + }, + }) + case r.Method == http.MethodGet && r.URL.Path == "/v1/agents/a1/skills": + json.NewEncoder(w).Encode(map[string]any{ + "skills": []map[string]any{ + {"id": "sk1", "slug": "granted-skill", "granted": true}, + }, + }) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer cleanup() + + files, grantees, err := p.DownloadSkillSource(context.Background(), "granted-skill") + if err != nil { + t.Fatalf("DownloadSkillSource: %v", err) + } + if len(files) != 1 { + t.Errorf("expected 1 file, got %d", len(files)) + } + if len(grantees) != 1 || grantees[0] != "bot-alpha" { + t.Errorf("expected grantees=[bot-alpha], got %v", grantees) + } +} diff --git a/internal/provider/goclaw/skills.go b/internal/provider/goclaw/skills.go index 94586db..670bc4b 100644 --- a/internal/provider/goclaw/skills.go +++ b/internal/provider/goclaw/skills.go @@ -371,6 +371,63 @@ func (p *Provider) agentHasSkillGrant(ctx context.Context, agentID, skillID stri return false, nil } +// DownloadSkill fetches the full file tree for a skill from goclaw. +// Uses GET /v1/skills/{id}/files for the tree, then per-file +// GET /v1/skills/{id}/files/{path}?raw=true for raw bytes. +func (p *Provider) DownloadSkill(ctx context.Context, slug string) ([]skillpkg.SkillFile, error) { + id, err := p.resolveSkillIDFromList(ctx, slug) + if err != nil { + return nil, err + } + + treeData, err := p.http.Get(ctx, "/v1/skills/"+id+"/files") + if err != nil { + return nil, fmt.Errorf("list skill files %s: %w", slug, err) + } + var treeResp struct { + Files []struct { + Path string `json:"path"` + IsDir bool `json:"isDir"` + } `json:"files"` + } + if err := json.Unmarshal(treeData, &treeResp); err != nil { + return nil, fmt.Errorf("parse skill file tree %s: %w", slug, err) + } + + var out []skillpkg.SkillFile + for _, entry := range treeResp.Files { + if entry.IsDir || entry.Path == "" { + continue + } + raw, err := p.http.GetRaw(ctx, "/v1/skills/"+id+"/files/"+entry.Path+"?raw=true") + if err != nil { + return nil, fmt.Errorf("download skill file %s/%s: %w", slug, entry.Path, err) + } + out = append(out, skillpkg.SkillFile{Path: entry.Path, Data: raw}) + } + return out, nil +} + +// DownloadSkillSource fetches skill files plus the grants overlay. +// Returns files and a sorted list of grantee agent keys for frontmatter.yaml. +func (p *Provider) DownloadSkillSource(ctx context.Context, slug string) ([]skillpkg.SkillFile, []string, error) { + files, err := p.DownloadSkill(ctx, slug) + if err != nil { + return nil, nil, err + } + + id, err := p.resolveSkillIDFromList(ctx, slug) + if err != nil { + return nil, nil, err + } + + grantees, err := p.listSkillGrantAgentKeys(ctx, id) + if err != nil { + return nil, nil, fmt.Errorf("list skill grants %s: %w", slug, err) + } + return files, grantees, nil +} + // isSystemSourceDir reports whether the given filesystem path is under a // `_system/` directory anywhere in its ancestry. Used by createSkill to mark // uploads with is_system=true so goclaw exposes them cross-tenant without a diff --git a/internal/reconciler/types.go b/internal/reconciler/types.go index 491fda9..d7cd17a 100644 --- a/internal/reconciler/types.go +++ b/internal/reconciler/types.go @@ -15,13 +15,13 @@ const ( // Change describes a single planned resource change. type Change struct { - Kind manifest.ResourceKind `json:"kind"` - Name string `json:"name"` - Action Action `json:"action"` - Diff map[string]FieldDiff `json:"diff,omitempty"` - Error string `json:"error,omitempty"` - Forced bool `json:"forced,omitempty"` // true when update triggered by --force with no diff - WriteOnlyHash string `json:"writeOnlyHash,omitempty"` // pre-computed hash for execute phase + Kind manifest.ResourceKind `json:"kind"` + Name string `json:"name"` + Action Action `json:"action"` + Diff map[string]FieldDiff `json:"diff,omitempty"` + Error string `json:"error,omitempty"` + Forced bool `json:"forced,omitempty"` // true when update triggered by --force with no diff + WriteOnlyHash string `json:"writeOnlyHash,omitempty"` // pre-computed hash for execute phase } // FieldDiff shows the before/after for a single field. @@ -60,4 +60,6 @@ type ResourceInfo struct { Kind manifest.ResourceKind Name string CreatedBy string + Source string // skill ownership marker: gcplane|evolution|cli|bundled|unknown + IsSystem bool // skill is a system/bundled skill (excluded from --all reverse-sync) } diff --git a/internal/skillpkg/unpack.go b/internal/skillpkg/unpack.go new file mode 100644 index 0000000..af15c0d --- /dev/null +++ b/internal/skillpkg/unpack.go @@ -0,0 +1,114 @@ +package skillpkg + +import ( + "bytes" + "crypto/sha256" + "fmt" + "os" + "path/filepath" + "strings" +) + +// SkillFile is a single file retrieved from the goclaw skill file API. +type SkillFile struct { + Path string // relative, slash-separated + Data []byte +} + +// UnpackTo writes files into outDir, creating parent dirs as needed. +// Only writes a file when its content differs from what's on disk (sha256 compare). +// Rejects ".." traversal and absolute paths; skips hidden/system artifacts. +// prune=true deletes local files absent from the server response. +// Returns the relative paths of files actually written. +func UnpackTo(outDir string, files []SkillFile, prune bool) ([]string, error) { + if outDir == "" { + return nil, fmt.Errorf("outDir must not be empty") + } + + written := make(map[string]bool, len(files)) + var changed []string + + for _, f := range files { + rel := filepath.FromSlash(f.Path) + + // Reject absolute paths and ".." traversal. + if filepath.IsAbs(rel) || strings.Contains(rel, "..") { + return nil, fmt.Errorf("rejected unsafe path %q", f.Path) + } + + name := filepath.Base(rel) + if isSkippedFile(name) || isSkippedDir(strings.Split(rel, string(filepath.Separator))[0]) { + continue + } + + if int64(len(f.Data)) > MaxFileSize { + return nil, fmt.Errorf("file %q exceeds max size %d bytes", f.Path, MaxFileSize) + } + + abs := filepath.Join(outDir, rel) + // Double-check the resolved path is within outDir. + if !strings.HasPrefix(abs, outDir+string(filepath.Separator)) && abs != outDir { + return nil, fmt.Errorf("rejected path escaping outDir: %q", f.Path) + } + + if !needsWrite(abs, f.Data) { + written[rel] = true + continue + } + + if err := os.MkdirAll(filepath.Dir(abs), 0o755); err != nil { + return nil, fmt.Errorf("mkdir %s: %w", filepath.Dir(abs), err) + } + if err := os.WriteFile(abs, f.Data, 0o644); err != nil { + return nil, fmt.Errorf("write %s: %w", rel, err) + } + changed = append(changed, rel) + written[rel] = true + } + + if prune { + pruned, err := pruneLocal(outDir, written) + if err != nil { + return nil, err + } + changed = append(changed, pruned...) + } + + return changed, nil +} + +// needsWrite returns true when the file is absent or its content hash differs. +func needsWrite(abs string, data []byte) bool { + existing, err := os.ReadFile(abs) + if err != nil { + return true + } + want := sha256.Sum256(data) + got := sha256.Sum256(existing) + return !bytes.Equal(want[:], got[:]) +} + +// pruneLocal deletes files under outDir not in the keep set. +func pruneLocal(outDir string, keep map[string]bool) ([]string, error) { + var deleted []string + err := filepath.WalkDir(outDir, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(outDir, path) + if err != nil { + return err + } + if !keep[rel] { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("prune %s: %w", rel, err) + } + deleted = append(deleted, rel) + } + return nil + }) + return deleted, err +} diff --git a/internal/skillpkg/unpack_test.go b/internal/skillpkg/unpack_test.go new file mode 100644 index 0000000..61129db --- /dev/null +++ b/internal/skillpkg/unpack_test.go @@ -0,0 +1,139 @@ +package skillpkg + +import ( + "os" + "path/filepath" + "testing" +) + +func TestUnpackTo_WritesFiles(t *testing.T) { + out := t.TempDir() + files := []SkillFile{ + {Path: "SKILL.md", Data: []byte("---\nname: foo\n---\nbody\n")}, + {Path: "scripts/run.sh", Data: []byte("#!/bin/sh\necho hi\n")}, + } + changed, err := UnpackTo(out, files, false) + if err != nil { + t.Fatalf("UnpackTo: %v", err) + } + if len(changed) != 2 { + t.Fatalf("expected 2 changed files, got %d: %v", len(changed), changed) + } + for _, f := range files { + got, err := os.ReadFile(filepath.Join(out, filepath.FromSlash(f.Path))) + if err != nil { + t.Errorf("read %s: %v", f.Path, err) + } + if string(got) != string(f.Data) { + t.Errorf("%s: content mismatch", f.Path) + } + } +} + +func TestUnpackTo_IdempotentNoDiff(t *testing.T) { + out := t.TempDir() + files := []SkillFile{ + {Path: "SKILL.md", Data: []byte("---\nname: bar\n---\n")}, + } + // First write. + if _, err := UnpackTo(out, files, false); err != nil { + t.Fatalf("first unpack: %v", err) + } + // Second write: identical content → nothing changed. + changed, err := UnpackTo(out, files, false) + if err != nil { + t.Fatalf("second unpack: %v", err) + } + if len(changed) != 0 { + t.Errorf("expected 0 changes on second identical unpack, got %v", changed) + } +} + +func TestUnpackTo_RejectsTraversal(t *testing.T) { + out := t.TempDir() + files := []SkillFile{ + {Path: "../escape.txt", Data: []byte("evil")}, + } + _, err := UnpackTo(out, files, false) + if err == nil { + t.Fatal("expected error for '..' traversal") + } +} + +func TestUnpackTo_RejectsAbsolutePath(t *testing.T) { + out := t.TempDir() + files := []SkillFile{ + {Path: "/etc/passwd", Data: []byte("evil")}, + } + _, err := UnpackTo(out, files, false) + if err == nil { + t.Fatal("expected error for absolute path") + } +} + +func TestUnpackTo_SkipsHiddenFiles(t *testing.T) { + out := t.TempDir() + files := []SkillFile{ + {Path: "SKILL.md", Data: []byte("---\nname: test\n---\n")}, + {Path: ".DS_Store", Data: []byte("junk")}, + } + changed, err := UnpackTo(out, files, false) + if err != nil { + t.Fatalf("UnpackTo: %v", err) + } + // Only SKILL.md should be written. + if len(changed) != 1 { + t.Errorf("expected 1 changed file, got %d: %v", len(changed), changed) + } + if _, err := os.Stat(filepath.Join(out, ".DS_Store")); !os.IsNotExist(err) { + t.Error(".DS_Store should not exist in output") + } +} + +func TestUnpackTo_MaxFileSizeRejected(t *testing.T) { + out := t.TempDir() + big := make([]byte, MaxFileSize+1) + files := []SkillFile{ + {Path: "big.bin", Data: big}, + } + _, err := UnpackTo(out, files, false) + if err == nil { + t.Fatal("expected error for oversized file") + } +} + +func TestUnpackTo_PruneDeletesLocalOnly(t *testing.T) { + out := t.TempDir() + // Pre-create a local file not in server response. + localOnly := filepath.Join(out, "local-only.txt") + if err := os.WriteFile(localOnly, []byte("local"), 0o644); err != nil { + t.Fatalf("setup: %v", err) + } + + files := []SkillFile{ + {Path: "SKILL.md", Data: []byte("---\nname: x\n---\n")}, + } + + // prune=false: local-only file preserved. + if _, err := UnpackTo(out, files, false); err != nil { + t.Fatalf("unpack (no prune): %v", err) + } + if _, err := os.Stat(localOnly); err != nil { + t.Error("local-only file should be preserved when prune=false") + } + + // prune=true: local-only file deleted. + if _, err := UnpackTo(out, files, true); err != nil { + t.Fatalf("unpack (prune): %v", err) + } + if _, err := os.Stat(localOnly); !os.IsNotExist(err) { + t.Error("local-only file should be deleted when prune=true") + } +} + +func TestUnpackTo_EmptyOutDir(t *testing.T) { + _, err := UnpackTo("", []SkillFile{{Path: "x.txt", Data: []byte("x")}}, false) + if err == nil { + t.Fatal("expected error for empty outDir") + } +}