feat(pull): reverse-sync evolved skills + agent context files

cmd/pull_test.go

@@ -0,0 +1,119 @@
+package cmd
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/dataplanelabs/gcplane/internal/manifest"
+	"github.com/dataplanelabs/gcplane/internal/reconciler"
+)
+
+func skillInfo(source string, system bool) reconciler.ResourceInfo {
+	return reconciler.ResourceInfo{
+		Kind:     manifest.KindSkill,
+		Name:     "demo-skill",
+		Source:   source,
+		IsSystem: system,
+	}
+}
+
+func TestShouldPullSkill_DefaultSourceFilter(t *testing.T) {
+	tests := []struct {
+		name       string
+		source     string
+		system     bool
+		localKnown bool
+		want       bool
+	}{
+		{"evolution known", "evolution", false, true, true},
+		{"gcplane known", "gcplane", false, true, true},
+		{"evolution unknown locally", "evolution", false, false, false},
+		{"gcplane unknown locally", "gcplane", false, false, false},
+		{"cli source skipped", "cli", false, true, false},
+		{"unknown source skipped", "unknown", false, true, false},
+		{"bundled source skipped", "bundled", false, true, false},
+		{"empty source skipped", "", false, true, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := shouldPullSkill(skillInfo(tt.source, tt.system), false, tt.localKnown)
+			if got != tt.want {
+				t.Fatalf("shouldPullSkill(source=%q, all=false, local=%v) = %v, want %v",
+					tt.source, tt.localKnown, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestShouldPullSkill_AllMode(t *testing.T) {
+	tests := []struct {
+		name   string
+		source string
+		system bool
+		want   bool
+	}{
+		{"evolution pulled", "evolution", false, true},
+		{"gcplane pulled", "gcplane", false, true},
+		{"cli pulled under all", "cli", false, true},
+		{"unknown pulled under all", "unknown", false, true},
+		{"bundled excluded", "bundled", false, false},
+		{"is_system excluded", "evolution", true, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// localKnown is irrelevant under --all.
+			got := shouldPullSkill(skillInfo(tt.source, tt.system), true, false)
+			if got != tt.want {
+				t.Fatalf("shouldPullSkill(source=%q, system=%v, all=true) = %v, want %v",
+					tt.source, tt.system, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestWriteFrontmatter_WritesGrants(t *testing.T) {
+	dir := t.TempDir()
+	if err := writeFrontmatter(dir, []string{"van-anh", "annhien"}); err != nil {
+		t.Fatalf("writeFrontmatter: %v", err)
+	}
+	data, err := os.ReadFile(filepath.Join(dir, "frontmatter.yaml"))
+	if err != nil {
+		t.Fatalf("read frontmatter: %v", err)
+	}
+	got := string(data)
+	// Sorted grantees.
+	if want := "grants:\n    agents:\n        - annhien\n        - van-anh\n"; got != want {
+		t.Fatalf("frontmatter content = %q, want %q", got, want)
+	}
+}
+
+func TestWriteFrontmatter_RemovesStaleWhenGrantsRevoked(t *testing.T) {
+	dir := t.TempDir()
+	outPath := filepath.Join(dir, "frontmatter.yaml")
+
+	if err := writeFrontmatter(dir, []string{"van-anh"}); err != nil {
+		t.Fatalf("seed frontmatter: %v", err)
+	}
+	if _, err := os.Stat(outPath); err != nil {
+		t.Fatalf("frontmatter should exist after seed: %v", err)
+	}
+
+	// Grants revoked server-side → stale file must be removed.
+	if err := writeFrontmatter(dir, nil); err != nil {
+		t.Fatalf("writeFrontmatter(empty): %v", err)
+	}
+	if _, err := os.Stat(outPath); !os.IsNotExist(err) {
+		t.Fatalf("stale frontmatter still present, stat err = %v", err)
+	}
+}
+
+func TestWriteFrontmatter_NoGrantsNoFileIsNoop(t *testing.T) {
+	dir := t.TempDir()
+	if err := writeFrontmatter(dir, nil); err != nil {
+		t.Fatalf("writeFrontmatter(empty) on clean dir: %v", err)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "frontmatter.yaml")); !os.IsNotExist(err) {
+		t.Fatalf("no frontmatter should be created, stat err = %v", err)
+	}
+}

cmd/root.go

@@ -87,6 +87,7 @@ func init() {
 	rootCmd.AddCommand(destroyCmd)
 	rootCmd.AddCommand(validateCmd)
 	rootCmd.AddCommand(exportCmd)
+	rootCmd.AddCommand(pullCmd)
 	rootCmd.AddCommand(serveCmd)
 	rootCmd.AddCommand(statusCmd)
 	rootCmd.AddCommand(topCmd)

internal/provider/goclaw/context_files.go

@@ -9,9 +9,13 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
+	"strings"
 	"time"
 )
 
+// maxContextFileSize caps a single decompressed context-file tar entry (gzip-bomb guard).
+const maxContextFileSize = 8 << 20
+
 // syncContextFiles upserts agent context files via the merge import API.
 // Builds a tar.gz archive with context_files/{name} entries and POSTs it
 // to /v1/agents/{id}/import?include=context_files as multipart form data.
@@ -67,6 +71,66 @@ func (p *Provider) syncContextFiles(ctx context.Context, agentID string, files [
 	return nil
 }
 
+// DownloadAgentContextFiles fetches context files for an agent from goclaw.
+// Calls GET /v1/agents/{id}/export?sections=context_files&stream=false
+// and untars the returned gzip archive into [{name, content}] pairs.
+func (p *Provider) DownloadAgentContextFiles(ctx context.Context, agentKey string) ([]map[string]string, error) {
+	id, err := p.resolveAgentID(ctx, agentKey)
+	if err != nil {
+		return nil, err
+	}
+
+	gz, err := p.http.GetRaw(ctx, "/v1/agents/"+id+"/export?sections=context_files&stream=false")
+	if err != nil {
+		return nil, fmt.Errorf("export context files for agent %s: %w", agentKey, err)
+	}
+
+	return parseContextFilesArchive(gz)
+}
+
+// parseContextFilesArchive untars a gzip archive and returns entries under
+// the "context_files/" prefix as [{name, content}] pairs.
+func parseContextFilesArchive(gz []byte) ([]map[string]string, error) {
+	gr, err := gzip.NewReader(bytes.NewReader(gz))
+	if err != nil {
+		return nil, fmt.Errorf("gzip open: %w", err)
+	}
+	defer gr.Close()
+
+	tr := tar.NewReader(gr)
+	const prefix = "context_files/"
+	var out []map[string]string
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return nil, fmt.Errorf("tar read: %w", err)
+		}
+		if hdr.FileInfo().IsDir() {
+			continue
+		}
+		name := hdr.Name
+		if !strings.HasPrefix(name, prefix) {
+			continue
+		}
+		name = strings.TrimPrefix(name, prefix)
+		if name == "" {
+			continue
+		}
+		data, err := io.ReadAll(io.LimitReader(tr, maxContextFileSize+1))
+		if err != nil {
+			return nil, fmt.Errorf("read tar entry %s: %w", hdr.Name, err)
+		}
+		if int64(len(data)) > maxContextFileSize {
+			return nil, fmt.Errorf("context file %s exceeds %d bytes", name, maxContextFileSize)
+		}
+		out = append(out, map[string]string{"name": name, "content": string(data)})
+	}
+	return out, nil
+}
+
 // buildContextFilesArchive creates a tar.gz archive with context_files/{name} entries.
 func buildContextFilesArchive(files []any) (*bytes.Buffer, error) {
 	buf := &bytes.Buffer{}

internal/provider/goclaw/context_files_download_test.go

@@ -0,0 +1,150 @@
+package goclaw
+
+import (
+	"archive/tar"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"testing"
+	"time"
+)
+
+// buildTestArchive creates a tar.gz with context_files/{name} entries for testing.
+func buildTestArchive(t *testing.T, entries map[string]string) []byte {
+	t.Helper()
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gw)
+	for name, content := range entries {
+		hdr := &tar.Header{
+			Name:    "context_files/" + name,
+			Size:    int64(len(content)),
+			Mode:    0644,
+			ModTime: time.Now(),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			t.Fatalf("tar header: %v", err)
+		}
+		if _, err := io.WriteString(tw, content); err != nil {
+			t.Fatalf("tar write: %v", err)
+		}
+	}
+	if err := tw.Close(); err != nil {
+		t.Fatalf("tar close: %v", err)
+	}
+	if err := gw.Close(); err != nil {
+		t.Fatalf("gzip close: %v", err)
+	}
+	return buf.Bytes()
+}
+
+func TestDownloadAgentContextFiles_HappyPath(t *testing.T) {
+	archive := buildTestArchive(t, map[string]string{
+		"IDENTITY.md": "# Agent\nName: Bot",
+		"SOUL.md":     "## Personality",
+	})
+
+	p, cleanup := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch {
+		case r.Method == http.MethodGet && r.URL.Path == "/v1/agents":
+			json.NewEncoder(w).Encode(map[string]any{
+				"agents": []map[string]any{
+					{"id": "agent-uuid", "agent_key": "my-bot"},
+				},
+			})
+		case r.Method == http.MethodGet && r.URL.Path == "/v1/agents/agent-uuid/export":
+			w.Header().Set("Content-Type", "application/gzip")
+			w.Write(archive)
+		default:
+			w.WriteHeader(http.StatusNotFound)
+		}
+	}))
+	defer cleanup()
+
+	files, err := p.DownloadAgentContextFiles(context.Background(), "my-bot")
+	if err != nil {
+		t.Fatalf("DownloadAgentContextFiles: %v", err)
+	}
+	if len(files) != 2 {
+		t.Fatalf("expected 2 files, got %d", len(files))
+	}
+
+	byName := make(map[string]string)
+	for _, f := range files {
+		byName[f["name"]] = f["content"]
+	}
+	if byName["IDENTITY.md"] != "# Agent\nName: Bot" {
+		t.Errorf("IDENTITY.md mismatch: %q", byName["IDENTITY.md"])
+	}
+	if byName["SOUL.md"] != "## Personality" {
+		t.Errorf("SOUL.md mismatch: %q", byName["SOUL.md"])
+	}
+}
+
+func TestDownloadAgentContextFiles_AgentNotFound(t *testing.T) {
+	p, cleanup := newTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		json.NewEncoder(w).Encode(map[string]any{"agents": []map[string]any{}})
+	}))
+	defer cleanup()
+
+	_, err := p.DownloadAgentContextFiles(context.Background(), "nonexistent")
+	if err == nil {
+		t.Fatal("expected error for missing agent")
+	}
+}
+
+func TestParseContextFilesArchive_SkipsNonPrefixed(t *testing.T) {
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gw)
+
+	for _, entry := range []struct{ name, content string }{
+		{"context_files/IDENTITY.md", "# Hi"},
+		{"other/file.txt", "should be skipped"},
+	} {
+		hdr := &tar.Header{Name: entry.name, Size: int64(len(entry.content)), Mode: 0644, ModTime: time.Now()}
+		tw.WriteHeader(hdr)               //nolint:errcheck
+		io.WriteString(tw, entry.content) //nolint:errcheck
+	}
+	tw.Close() //nolint:errcheck
+	gw.Close() //nolint:errcheck
+
+	out, err := parseContextFilesArchive(buf.Bytes())
+	if err != nil {
+		t.Fatalf("parseContextFilesArchive: %v", err)
+	}
+	if len(out) != 1 {
+		t.Fatalf("expected 1 entry, got %d: %v", len(out), out)
+	}
+	if out[0]["name"] != "IDENTITY.md" {
+		t.Errorf("expected IDENTITY.md, got %q", out[0]["name"])
+	}
+}
+
+func TestParseContextFilesArchive_EmptyArchive(t *testing.T) {
+	var buf bytes.Buffer
+	gw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(gw)
+	tw.Close() //nolint:errcheck
+	gw.Close() //nolint:errcheck
+
+	out, err := parseContextFilesArchive(buf.Bytes())
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(out) != 0 {
+		t.Errorf("expected 0 entries for empty archive, got %d", len(out))
+	}
+}
+
+func TestParseContextFilesArchive_RejectsOversizeEntry(t *testing.T) {
+	archive := buildTestArchive(t, map[string]string{
+		"huge.md": string(bytes.Repeat([]byte("x"), maxContextFileSize+1)),
+	})
+	if _, err := parseContextFilesArchive(archive); err == nil {
+		t.Fatal("expected error for oversize context file, got nil")
+	}
+}

internal/provider/goclaw/helpers.go

@@ -108,6 +108,12 @@ func strVal(m map[string]any, key string) string {
 	return s
 }
 
+// boolVal safely extracts a bool value from a map.
+func boolVal(m map[string]any, key string) bool {
+	b, _ := m[key].(bool)
+	return b
+}
+
 // copyMap creates a shallow copy of a map.
 func copyMap(m map[string]any) map[string]any {
 	out := make(map[string]any, len(m))

internal/provider/goclaw/http_client.go

@@ -80,6 +80,12 @@ func (c *HTTPClient) Delete(ctx context.Context, path string) error {
 	return err
 }
 
+// GetRaw performs an authenticated GET and returns the raw response body bytes.
+// Used for non-JSON responses: tar.gz archives and ?raw=true file downloads.
+func (c *HTTPClient) GetRaw(ctx context.Context, path string) ([]byte, error) {
+	return c.doRaw(ctx, http.MethodGet, path, nil, "")
+}
+
 // PostMultipart performs an authenticated POST with a multipart body.
 // Used for skill ZIP uploads. The default Content-Type set by do() (application/json)
 // is overridden via the contentType argument.

internal/provider/goclaw/list_all.go

@@ -134,6 +134,8 @@ func (p *Provider) listAllSkills(ctx context.Context) ([]reconciler.ResourceInfo
 			Kind:      manifest.KindSkill,
 			Name:      strVal(s, "slug"),
 			CreatedBy: strVal(s, "created_by"),
+			Source:    strVal(s, "source"),
+			IsSystem:  boolVal(s, "is_system"),
 		})
 	}
 	return infos, nil