Add mirror command and API for selective package mirroring

Add a `proxy mirror` CLI command and `/api/mirror` API endpoints that
pre-populate the cache from various input sources: individual PURLs,
SBOM files (CycloneDX and SPDX), or full registry enumeration.

The mirror reuses the existing handler.Proxy.GetOrFetchArtifact()
pipeline so cached artifacts are identical to those fetched on demand.
A bounded worker pool controls download parallelism.

Metadata caching is opt-in via `cache_metadata: true` in config (or
PROXY_CACHE_METADATA=true). The mirror command always enables it. When
enabled, upstream metadata responses are stored for offline fallback
with ETag-based conditional revalidation.

New internal/mirror package with Source interface, PURLSource,
SBOMSource, RegistrySource, and async JobStore. New metadata_cache
database table for offline metadata serving.

Author: andrew

README.md

@@ -460,6 +460,47 @@ proxy serve [flags]
 proxy [flags]  # same as 'proxy serve'
 ```
 
+### mirror
+
+Pre-populate the cache from PURLs, SBOM files, or entire registries. Useful for ensuring offline availability or warming the cache before deployments.
+
+```bash
+# Mirror specific package versions
+proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
+
+# Mirror all versions of a package
+proxy mirror pkg:npm/lodash
+
+# Mirror from a CycloneDX or SPDX SBOM
+proxy mirror --sbom sbom.cdx.json
+
+# Full registry mirror (npm, pypi, cargo supported)
+proxy mirror --registry npm
+
+# Preview what would be mirrored
+proxy mirror --dry-run pkg:npm/lodash
+
+# Control parallelism
+proxy mirror --concurrency 8 pkg:npm/lodash@4.17.21
+```
+
+The mirror command accepts the same storage and database flags as `serve`. Already-cached artifacts are skipped.
+
+A mirror API is also available when the server is running:
+
+```bash
+# Start a mirror job
+curl -X POST http://localhost:8080/api/mirror \
+  -H "Content-Type: application/json" \
+  -d '{"purls": ["pkg:npm/lodash@4.17.21"]}'
+
+# Check job status
+curl http://localhost:8080/api/mirror/mirror-1
+
+# Cancel a running job
+curl -X DELETE http://localhost:8080/api/mirror/mirror-1
+```
+
 ### stats
 
 Show cache statistics without running the server.
@@ -534,6 +575,14 @@ Recently cached:
 | `GET /debian/*` | Debian/APT repository protocol |
 | `GET /rpm/*` | RPM/Yum repository protocol |
 
+### Mirror API
+
+| Endpoint | Description |
+|----------|-------------|
+| `POST /api/mirror` | Start a mirror job (JSON body with `purls` or `registry`) |
+| `GET /api/mirror/{id}` | Get job status and progress |
+| `DELETE /api/mirror/{id}` | Cancel a running job |
+
 ### Enrichment API
 
 The proxy provides REST endpoints for package metadata enrichment, vulnerability scanning, and outdated detection.

cmd/proxy/main.go

@@ -16,6 +16,7 @@
 //
 //	serve    Start the proxy server (default if no command given)
 //	stats    Show cache statistics
+//	mirror   Pre-populate cache from PURLs, SBOMs, or registries
 //
 // Serve Flags:
 //
@@ -100,7 +101,11 @@ import (
 
 	"github.com/git-pkgs/proxy/internal/config"
 	"github.com/git-pkgs/proxy/internal/database"
+	"github.com/git-pkgs/proxy/internal/handler"
+	"github.com/git-pkgs/proxy/internal/mirror"
 	"github.com/git-pkgs/proxy/internal/server"
+	"github.com/git-pkgs/proxy/internal/storage"
+	"github.com/git-pkgs/registries/fetch"
 )
 
 const defaultTopN = 10
@@ -124,6 +129,10 @@ func main() {
 			os.Args = append(os.Args[:1], os.Args[2:]...)
 			runStats()
 			return
+		case "mirror":
+			os.Args = append(os.Args[:1], os.Args[2:]...)
+			runMirror()
+			return
 		case "-version", "--version":
 			fmt.Printf("proxy %s (%s)\n", Version, Commit)
 			os.Exit(0)
@@ -145,6 +154,7 @@ Usage: proxy [command] [flags]
 Commands:
   serve    Start the proxy server (default)
   stats    Show cache statistics
+  mirror   Pre-populate cache from PURLs, SBOMs, or registries
 
 Run 'proxy <command> -help' for more information on a command.
 
@@ -340,6 +350,158 @@ func runStats() {
 	}
 }
 
+func runMirror() {
+	fs := flag.NewFlagSet("mirror", flag.ExitOnError)
+	configPath := fs.String("config", "", "Path to configuration file")
+	storageURL := fs.String("storage-url", "", "Storage URL (file:// or s3://)")
+	databaseDriver := fs.String("database-driver", "", "Database driver: sqlite or postgres")
+	databasePath := fs.String("database-path", "", "Path to SQLite database file")
+	databaseURL := fs.String("database-url", "", "PostgreSQL connection URL")
+	sbomPath := fs.String("sbom", "", "Path to CycloneDX or SPDX SBOM file")
+	registry := fs.String("registry", "", "Ecosystem name for full registry mirror")
+	concurrency := fs.Int("concurrency", 4, "Number of parallel downloads") //nolint:mnd // default concurrency
+	dryRun := fs.Bool("dry-run", false, "Show what would be mirrored without downloading")
+
+	fs.Usage = func() {
+		fmt.Fprintf(os.Stderr, "git-pkgs proxy - Pre-populate cache\n\n")
+		fmt.Fprintf(os.Stderr, "Usage: proxy mirror [flags] [purl...]\n\n")
+		fmt.Fprintf(os.Stderr, "Examples:\n")
+		fmt.Fprintf(os.Stderr, "  proxy mirror pkg:npm/lodash@4.17.21\n")
+		fmt.Fprintf(os.Stderr, "  proxy mirror --sbom sbom.cdx.json\n")
+		fmt.Fprintf(os.Stderr, "  proxy mirror pkg:npm/lodash  # all versions\n")
+		fmt.Fprintf(os.Stderr, "  proxy mirror --registry npm\n\n")
+		fmt.Fprintf(os.Stderr, "Flags:\n")
+		fs.PrintDefaults()
+	}
+
+	_ = fs.Parse(os.Args[1:])
+	purls := fs.Args()
+
+	// Determine source
+	var source mirror.Source
+	switch {
+	case *sbomPath != "":
+		source = &mirror.SBOMSource{Path: *sbomPath}
+	case *registry != "":
+		source = &mirror.RegistrySource{Ecosystem: *registry}
+	case len(purls) > 0:
+		source = &mirror.PURLSource{PURLs: purls}
+	default:
+		fmt.Fprintf(os.Stderr, "error: provide PURLs, --sbom, or --registry\n")
+		fs.Usage()
+		os.Exit(1)
+	}
+
+	// Load config
+	cfg, err := loadConfig(*configPath)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error loading config: %v\n", err)
+		os.Exit(1)
+	}
+	cfg.LoadFromEnv()
+
+	if *storageURL != "" {
+		cfg.Storage.URL = *storageURL
+	}
+	if *databaseDriver != "" {
+		cfg.Database.Driver = *databaseDriver
+	}
+	if *databasePath != "" {
+		cfg.Database.Path = *databasePath
+	}
+	if *databaseURL != "" {
+		cfg.Database.URL = *databaseURL
+	}
+
+	if err := cfg.Validate(); err != nil {
+		fmt.Fprintf(os.Stderr, "invalid configuration: %v\n", err)
+		os.Exit(1)
+	}
+
+	logger := setupLogger("info", "text")
+
+	// Open database
+	var db *database.DB
+	switch cfg.Database.Driver {
+	case "postgres":
+		db, err = database.OpenPostgresOrCreate(cfg.Database.URL)
+	default:
+		db, err = database.OpenOrCreate(cfg.Database.Path)
+	}
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error opening database: %v\n", err)
+		os.Exit(1)
+	}
+
+	if err := db.MigrateSchema(); err != nil {
+		_ = db.Close()
+		fmt.Fprintf(os.Stderr, "error migrating schema: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Open storage
+	sURL := cfg.Storage.URL
+	if sURL == "" {
+		sURL = "file://" + cfg.Storage.Path //nolint:staticcheck // backwards compat
+	}
+	store, err := storage.OpenBucket(context.Background(), sURL)
+	if err != nil {
+		_ = db.Close()
+		fmt.Fprintf(os.Stderr, "error opening storage: %v\n", err)
+		os.Exit(1)
+	}
+
+	// Build proxy (reuses same pipeline as serve)
+	fetcher := fetch.NewFetcher()
+	resolver := fetch.NewResolver()
+	proxy := handler.NewProxy(db, store, fetcher, resolver, logger)
+	proxy.CacheMetadata = true // mirror always caches metadata
+
+	m := mirror.New(proxy, db, store, logger, *concurrency)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		sigCh := make(chan os.Signal, 1)
+		signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM)
+		<-sigCh
+		cancel()
+	}()
+
+	if *dryRun {
+		items, err := m.RunDryRun(ctx, source)
+		if err != nil {
+			_ = db.Close()
+			fmt.Fprintf(os.Stderr, "error: %v\n", err)
+			os.Exit(1)
+		}
+		fmt.Printf("Would mirror %d package versions:\n", len(items))
+		for _, item := range items {
+			fmt.Printf("  %s\n", item)
+		}
+		_ = db.Close()
+		return
+	}
+
+	progress, err := m.Run(ctx, source)
+	if err != nil {
+		_ = db.Close()
+		fmt.Fprintf(os.Stderr, "error: %v\n", err)
+		os.Exit(1)
+	}
+
+	_ = db.Close()
+
+	fmt.Printf("Mirror complete: %d downloaded, %d skipped (cached), %d failed, %s total\n",
+		progress.Completed, progress.Skipped, progress.Failed, formatSize(progress.Bytes))
+
+	if len(progress.Errors) > 0 {
+		fmt.Fprintf(os.Stderr, "\nErrors:\n")
+		for _, e := range progress.Errors {
+			fmt.Fprintf(os.Stderr, "  %s/%s@%s: %s\n", e.Ecosystem, e.Name, e.Version, e.Error)
+		}
+	}
+}
+
 func printStats(db *database.DB, popular, recent int, asJSON bool) error {
 	defer func() { _ = db.Close() }()
 

docs/architecture.md

@@ -161,6 +161,20 @@ vulnerabilities (
     updated_at    DATETIME
 )
 -- indexes: (vuln_id, ecosystem, package_name) unique, (ecosystem, package_name)
+
+metadata_cache (
+    id            INTEGER PRIMARY KEY,
+    ecosystem     TEXT NOT NULL,
+    name          TEXT NOT NULL,
+    storage_path  TEXT NOT NULL,
+    etag          TEXT,
+    content_type  TEXT,
+    size          INTEGER,           -- BIGINT on Postgres
+    fetched_at    DATETIME,
+    created_at    DATETIME,
+    updated_at    DATETIME
+)
+-- indexes: (ecosystem, name) unique
 ```
 
 On PostgreSQL, `INTEGER PRIMARY KEY` becomes `SERIAL`, `DATETIME` becomes `TIMESTAMP`, `INTEGER DEFAULT 0` booleans become `BOOLEAN DEFAULT FALSE`, and size/count columns use `BIGINT`.
@@ -277,6 +291,12 @@ Version age filtering for supply chain attack mitigation. Configurable at global
 
 Package metadata enrichment. Fetches license, description, homepage, repository URL, and vulnerability data from upstream registries. Powers the `/api/` endpoints and the web UI's package detail pages.
 
+### `internal/mirror`
+
+Selective package mirroring for pre-populating the proxy cache. Supports multiple input sources: individual PURLs (versioned or unversioned), CycloneDX/SPDX SBOM files, and full registry enumeration. Uses a bounded worker pool backed by `errgroup` to download artifacts in parallel, reusing `handler.Proxy.GetOrFetchArtifact()` for the actual fetch-and-cache work.
+
+The package also provides a `MetadataCache` for storing raw upstream metadata blobs so the proxy can serve metadata responses offline. The `JobStore` manages async mirror jobs exposed via the `/api/mirror` endpoints.
+
 ### `internal/config`
 
 Configuration loading.
@@ -326,10 +346,11 @@ Eviction can be implemented as:
 - Ensures clients fetch artifacts through proxy
 - Alternative: Let clients fetch directly, miss cache opportunity
 
-**Why not cache metadata?**
+**Why not cache metadata (by default)?**
 - Simplicity - no invalidation logic needed
 - Fresh data - new versions visible immediately
 - Metadata is small, upstream fetch is fast
+- Set `cache_metadata: true` or use the mirror command to enable metadata caching for offline use via the `metadata_cache` table
 
 **Why stream artifacts?**
 - Memory efficient - don't load large files into RAM

docs/configuration.md

@@ -211,6 +211,40 @@ Resolution order: package override, then ecosystem override, then global default
 
 Currently supported for npm, PyPI, pub.dev, and Composer. These ecosystems include publish timestamps in their metadata. Other ecosystems (Go, Cargo, RubyGems) would require extra API calls and are not yet supported.
 
+## Metadata Caching
+
+By default the proxy fetches metadata fresh from upstream on every request. Enable `cache_metadata` to store metadata responses in the database and storage backend for offline fallback. When upstream is unreachable, the proxy serves the last cached copy. ETag-based revalidation avoids re-downloading unchanged metadata.
+
+```yaml
+cache_metadata: true
+```
+
+Or via environment variable: `PROXY_CACHE_METADATA=true`.
+
+The `proxy mirror` command always enables metadata caching regardless of this setting.
+
+## Mirror Command
+
+The `proxy mirror` command pre-populates the cache from various sources. It accepts the same storage and database flags as `serve`.
+
+| Flag | Default | Description |
+|------|---------|-------------|
+| `--sbom` | | Path to CycloneDX or SPDX SBOM file |
+| `--registry` | | Ecosystem name for full registry mirror |
+| `--concurrency` | `4` | Number of parallel downloads |
+| `--dry-run` | `false` | Show what would be mirrored without downloading |
+| `--config` | | Path to configuration file |
+| `--storage-url` | | Storage URL |
+| `--database-driver` | | Database driver |
+| `--database-path` | | SQLite database file |
+| `--database-url` | | PostgreSQL connection URL |
+
+Positional arguments are treated as PURLs:
+
+```bash
+proxy mirror pkg:npm/lodash@4.17.21 pkg:cargo/serde@1.0.0
+```
+
 ## Docker
 
 ### SQLite with Local Storage

go.mod

@@ -36,6 +36,7 @@ require (
 	github.com/Antonboom/nilnil v1.1.1 // indirect
 	github.com/Antonboom/testifylint v1.6.4 // indirect
 	github.com/BurntSushi/toml v1.6.0 // indirect
+	github.com/CycloneDX/cyclonedx-go v0.10.0 // indirect
 	github.com/Djarvur/go-err113 v0.1.1 // indirect
 	github.com/KyleBanks/depth v1.2.1 // indirect
 	github.com/Masterminds/semver/v3 v3.4.0 // indirect
@@ -50,6 +51,7 @@ require (
 	github.com/alfatraining/structtag v1.0.0 // indirect
 	github.com/alingse/asasalint v0.0.11 // indirect
 	github.com/alingse/nilnesserr v0.2.0 // indirect
+	github.com/anchore/go-struct-converter v0.1.0 // indirect
 	github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
 	github.com/ashanbrown/forbidigo/v2 v2.3.0 // indirect
 	github.com/ashanbrown/makezero/v2 v2.1.0 // indirect
@@ -227,6 +229,7 @@ require (
 	github.com/sivchari/containedctx v1.0.3 // indirect
 	github.com/sonatard/noctx v0.4.0 // indirect
 	github.com/sourcegraph/go-diff v0.7.0 // indirect
+	github.com/spdx/tools-golang v0.5.7 // indirect
 	github.com/spf13/afero v1.15.0 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/cobra v1.10.2 // indirect

go.sum

@@ -45,6 +45,8 @@ github.com/Antonboom/testifylint v1.6.4 h1:gs9fUEy+egzxkEbq9P4cpcMB6/G0DYdMeiFS8
 github.com/Antonboom/testifylint v1.6.4/go.mod h1:YO33FROXX2OoUfwjz8g+gUxQXio5i9qpVy7nXGbxDD4=
 github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk=
 github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/CycloneDX/cyclonedx-go v0.10.0 h1:7xyklU7YD+CUyGzSFIARG18NYLsKVn4QFg04qSsu+7Y=
+github.com/CycloneDX/cyclonedx-go v0.10.0/go.mod h1:vUvbCXQsEm48OI6oOlanxstwNByXjCZ2wuleUlwGEO8=
 github.com/Djarvur/go-err113 v0.1.1 h1:eHfopDqXRwAi+YmCUas75ZE0+hoBHJ2GQNLYRSxao4g=
 github.com/Djarvur/go-err113 v0.1.1/go.mod h1:IaWJdYFLg76t2ihfflPZnM1LIQszWOsFDh2hhhAVF6k=
 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 h1:sBEjpZlNHzK1voKq9695PJSX2o5NEXl7/OL3coiIY0c=
@@ -84,6 +86,8 @@ github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQ
 github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I=
 github.com/alingse/nilnesserr v0.2.0 h1:raLem5KG7EFVb4UIDAXgrv3N2JIaffeKNtcEXkEWd/w=
 github.com/alingse/nilnesserr v0.2.0/go.mod h1:1xJPrXonEtX7wyTq8Dytns5P2hNzoWymVUIaKm4HNFg=
+github.com/anchore/go-struct-converter v0.1.0 h1:2rDRssAl6mgKBSLNiVCMADgZRhoqtw9dedlWa0OhD30=
+github.com/anchore/go-struct-converter v0.1.0/go.mod h1:rYqSE9HbjzpHTI74vwPvae4ZVYZd1lue2ta6xHPdblA=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0 h1:axGnT1gRIfimI7gJifB699GoE/oq+F2MU7Dml6nw9rQ=
 github.com/apapsch/go-jsonmerge/v2 v2.0.0/go.mod h1:lvDnEdqiQrp0O42VQGgmlKpxL1AP2+08jFMw88y4klk=
 github.com/ashanbrown/forbidigo/v2 v2.3.0 h1:OZZDOchCgsX5gvToVtEBoV2UWbFfI6RKQTir2UZzSxo=
@@ -562,6 +566,8 @@ github.com/sonatard/noctx v0.4.0 h1:7MC/5Gg4SQ4lhLYR6mvOP6mQVSxCrdyiExo7atBs27o=
 github.com/sonatard/noctx v0.4.0/go.mod h1:64XdbzFb18XL4LporKXp8poqZtPKbCrqQ402CV+kJas=
 github.com/sourcegraph/go-diff v0.7.0 h1:9uLlrd5T46OXs5qpp8L/MTltk0zikUGi0sNNyCpA8G0=
 github.com/sourcegraph/go-diff v0.7.0/go.mod h1:iBszgVvyxdc8SFZ7gm69go2KDdt3ag071iBaWPF6cjs=
+github.com/spdx/tools-golang v0.5.7 h1:+sWcKGnhwp3vLdMqPcLdA6QK679vd86cK9hQWH3AwCg=
+github.com/spdx/tools-golang v0.5.7/go.mod h1:jg7w0LOpoNAw6OxKEzCoqPC2GCTj45LyTlVmXubDsYw=
 github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
 github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
 github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w=