diff --git a/go.mod b/go.mod index 71af776..65782e8 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( charm.land/log/v2 v2.0.0 cuelang.org/go v0.16.1 github.com/charmbracelet/colorprofile v0.4.2 + github.com/gofrs/flock v0.13.0 github.com/lxc/incus-os/incus-osd v0.0.0-20260505023852-d32ba1f13f6f github.com/meigma/imgcli/schemas v0.0.0-20260504225557-fa97d8c3fe0c github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index 7fd3c53..db121ef 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7 github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gofrs/flock v0.13.0 h1:95JolYOvGMqeH31+FC7D2+uULf6mG61mEZ/A8dRYMzw= +github.com/gofrs/flock v0.13.0/go.mod h1:jxeyy9R1auM5S6JYDBhDt+E2TCo7DkratH4Pgi8P+Z0= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -138,7 +140,7 @@ golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/go.work.sum b/go.work.sum index cd764bf..fec80be 100644 --- a/go.work.sum +++ b/go.work.sum @@ -55,6 +55,7 @@ github.com/containerd/platforms v1.0.0-rc.4/go.mod h1:lKlMXyLybmBedS/JJm11uDofzI github.com/cowsql/go-cowsql v1.22.0/go.mod h1:+QzPcM7QRPIBI8XhsKJ47iUtxGY53lsYGX51G1WQ/4s= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/cpuguy83/go-md2man/v2 v2.0.7/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyphar/filepath-securejoin v0.6.1/go.mod h1:A8hd4EnAeyujCJRrICiOWqjS1AX0a9kM5XL+NwKoYSc= github.com/dgryski/go-farm v0.0.0-20240924180020-3414d57e47da/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e/go.mod h1:YTIHhz/QFSYnu/EhlF2SpU2Uk+32abacUYA5ZPljz1A= @@ -152,6 +153,9 @@ github.com/knadh/koanf/providers/posflag v1.0.1/go.mod h1:3Wn3+YG3f4ljzRyCUgIwH7 github.com/knadh/koanf/providers/structs v1.0.0/go.mod h1:kjo5TFtgpaZORlpoJqcbeLowM2cINodv8kX+oFAeQ1w= github.com/knadh/koanf/v2 v2.3.2/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lmittmann/tint v1.1.3/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE= github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= @@ -212,6 +216,7 @@ github.com/osrg/gobgp/v4 v4.5.0/go.mod h1:pgu8waqTvZUYl4eQuPrKNOaVwhHv7Zt9YymuzC github.com/ovn-kubernetes/libovsdb v0.8.1/go.mod h1:ZlnHLzagmLOSvyd9qfxBIZp6wOSOw0IsRsc+6lNUGbU= github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= github.com/pires/go-proxyproto v0.12.0/go.mod h1:qUvfqUMEoX7T8g0q7TQLDnhMjdTrxnG0hvpMn+7ePNI= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= github.com/pkg/xattr v0.4.12/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= @@ -223,6 +228,7 @@ github.com/prometheus/procfs v0.20.1/go.mod h1:o9EMBZGRyvDrSPH1RqdxhojkuXstoe4Ul github.com/quasilyte/go-ruleguard/dsl v0.3.23/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU= github.com/rivo/tview v0.42.0/go.mod h1:cSfIYfhpSGCjp3r/ECJb+GKS7cGJnqV8vfjQPwoXyfY= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rootless-containers/proto/go-proto v0.0.0-20260207013450-f6ee952d53d9/go.mod h1:LLjEAc6zmycfeN7/1fxIphWQPjHpTt7ElqT7eVf8e4A= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ= @@ -282,13 +288,17 @@ go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go4.org/mem v0.0.0-20240501181205-ae6ca9944745/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/telemetry v0.0.0-20260311193753-579e4da9a98c/go.mod h1:TpUTTEp9frx7rTdLpC9gFG9kdI7zVLFTFFlqaH2Cncw= golang.org/x/telemetry v0.0.0-20260409153401-be6f6cb8b1fa/go.mod h1:kHjTxDEnAu6/Nl9lDkzjWpR+bmKfxeiRuSDlsMb70gE= golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0= golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -297,6 +307,7 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260223185530-2f722ef697dc/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20260427160629-7cedc36a6bc4/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= k8s.io/utils v0.0.0-20260319190234-28399d86e0b5/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= libguestfs.org/libnbd v1.20.0/go.mod h1:pSICAuDOpSGplmGmaZ8QettnBAT3IUJFcBU1bVgWgk4= diff --git a/internal/cache/cache.go b/internal/cache/cache.go index a7e97c0..f801e84 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -16,17 +16,21 @@ import ( const ( cacheDirName = "imgcli" + lockFileName = "cache.lock" bytesPerKiB int64 = 1024 bytesPerMiB = bytesPerKiB * bytesPerKiB bytesPerGiB = bytesPerMiB * bytesPerKiB defaultMaxUnknownSizeGiB = 64 defaultMaxUnknownSizeBytes = defaultMaxUnknownSizeGiB * bytesPerGiB + defaultMaxSizeGiB = 10 + defaultMaxSizeBytes = defaultMaxSizeGiB * bytesPerGiB blobPerm = 0o400 hexCharsPerByte = 2 digestShardLength = 2 dirPerm = 0o750 + metadataPerm = 0o600 sha256HexLength = sha256.Size * hexCharsPerByte tmpPerm = 0o700 writePermMask = 0o222 @@ -69,6 +73,7 @@ type Option func(*DiskStore) type DiskStore struct { root string httpClient *http.Client + maxSizeBytes int64 maxUnknownSizeBytes int64 } @@ -88,6 +93,15 @@ func WithMaxUnknownSizeBytes(size int64) Option { } } +// WithMaxSizeBytes configures the maximum cache size for LRU pruning. +// +// Zero disables cache-size pruning. +func WithMaxSizeBytes(size int64) Option { + return func(store *DiskStore) { + store.maxSizeBytes = size + } +} + // NewDiskStore constructs a disk-backed cache store. func NewDiskStore(options ...Option) (*DiskStore, error) { root, err := defaultRoot() @@ -98,6 +112,7 @@ func NewDiskStore(options ...Option) (*DiskStore, error) { store := &DiskStore{ root: root, httpClient: http.DefaultClient, + maxSizeBytes: defaultMaxSizeBytes, maxUnknownSizeBytes: defaultMaxUnknownSizeBytes, } for _, option := range options { @@ -110,6 +125,9 @@ func NewDiskStore(options ...Option) (*DiskStore, error) { if store.maxUnknownSizeBytes < 0 { return nil, errors.New("cache max unknown-size bytes must be non-negative") } + if store.maxSizeBytes < 0 { + return nil, errors.New("cache max size bytes must be non-negative") + } return store, nil } @@ -122,20 +140,13 @@ func (s *DiskStore) Fetch(ctx context.Context, req FetchRequest) (Blob, error) { } if normalized.ExpectedSHA256 != "" { - path := s.blobPath(normalized.ExpectedSHA256) - if pathErr := s.ensureExpectedDigestPath(path); pathErr != nil { - return Blob{}, pathErr - } - blob, ok, verifyErr := verifyExistingBlob(path, normalized.ExpectedSHA256, normalized.ExpectedSize) - if verifyErr != nil { - return Blob{}, verifyErr + blob, ok, cacheErr := s.verifiedCacheHit(normalized) + if cacheErr != nil { + return Blob{}, cacheErr } if ok { return blob, nil } - if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { - return Blob{}, fmt.Errorf("remove corrupt cached blob: %w", removeErr) - } } tmpPath, downloaded, err := s.download(ctx, normalized) @@ -150,10 +161,37 @@ func (s *DiskStore) Fetch(ctx context.Context, req FetchRequest) (Blob, error) { if err != nil { return Blob{}, err } + if touchErr := s.touchBlob(blob); touchErr != nil { + return Blob{}, touchErr + } return blob, nil } +func (s *DiskStore) verifiedCacheHit(req FetchRequest) (Blob, bool, error) { + path := s.blobPath(req.ExpectedSHA256) + if pathErr := s.ensureExpectedDigestPath(path); pathErr != nil { + return Blob{}, false, pathErr + } + + blob, ok, verifyErr := verifyExistingBlob(path, req.ExpectedSHA256, req.ExpectedSize) + if verifyErr != nil { + return Blob{}, false, verifyErr + } + if !ok { + if removeErr := os.Remove(path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + return Blob{}, false, fmt.Errorf("remove corrupt cached blob: %w", removeErr) + } + return Blob{}, false, nil + } + + if touchErr := s.touchBlob(blob); touchErr != nil { + return Blob{}, false, touchErr + } + + return blob, true, nil +} + func (s *DiskStore) ensureExpectedDigestPath(path string) error { if err := s.ensureDirs(); err != nil { return err @@ -344,6 +382,8 @@ func (s *DiskStore) ensureDirs() error { {path: s.root, perm: dirPerm}, {path: filepath.Join(s.root, "blobs"), perm: dirPerm}, {path: filepath.Join(s.root, "blobs", "sha256"), perm: dirPerm}, + {path: filepath.Join(s.root, "metadata"), perm: dirPerm}, + {path: filepath.Join(s.root, "metadata", "sha256"), perm: dirPerm}, {path: s.tmpDir(), perm: tmpPerm}, } { if err := ensureCacheDir(dir.path, dir.perm); err != nil { diff --git a/internal/cache/cache_test.go b/internal/cache/cache_test.go index 428c676..abeccd8 100644 --- a/internal/cache/cache_test.go +++ b/internal/cache/cache_test.go @@ -4,12 +4,14 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "net/http" "net/http/httptest" "os" "path/filepath" "sync/atomic" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -63,6 +65,10 @@ func TestDiskStoreFetchUsesVerifiedCacheHit(t *testing.T) { assert.Equal(t, int64(len(body)), blob.Size) assert.Equal(t, int64(0), requests.Load()) assertReadOnlyFile(t, blob.Path) + metadata := readCachedMetadata(t, root, digest) + assert.Equal(t, digest, metadata.SHA256) + assert.Equal(t, int64(len(body)), metadata.Size) + assert.False(t, metadata.LastUsed.IsZero()) } func TestDiskStoreFetchRejectsCacheHitWithConflictingSize(t *testing.T) { @@ -106,6 +112,197 @@ func TestDiskStoreFetchUnknownDigest(t *testing.T) { assert.Equal(t, int64(1), requests.Load()) assertFileContent(t, blob.Path, body) assertReadOnlyFile(t, blob.Path) + metadata := readCachedMetadata(t, root, digest) + assert.Equal(t, digest, metadata.SHA256) + assert.Equal(t, int64(len(body)), metadata.Size) + assert.False(t, metadata.LastUsed.IsZero()) +} + +func TestDiskStoreFetchUpdatesMetadataOnCacheHit(t *testing.T) { + body := []byte("already cached bytes") + server, requests := newBlobServer(t, http.StatusInternalServerError, []byte("should not be requested")) + root := t.TempDir() + store := newDiskStore(t, root) + digest := sha256Hex(body) + writeCachedBlob(t, root, digest, body) + writeCachedMetadata(t, root, blobMetadataForTest{ + SHA256: digest, + Size: int64(len(body)), + LastUsed: time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC), + }) + beforeFetch := time.Now().UTC() + + blob, err := store.Fetch(context.Background(), cache.FetchRequest{ + URL: server.URL + "/blob", + ExpectedSHA256: digest, + ExpectedSize: int64(len(body)), + }) + + require.NoError(t, err) + assert.Equal(t, cachedBlobPath(root, digest), blob.Path) + assert.Equal(t, int64(0), requests.Load()) + metadata := readCachedMetadata(t, root, digest) + assert.False(t, metadata.LastUsed.Before(beforeFetch)) +} + +func TestDiskStoreFetchDoesNotPruneAutomatically(t *testing.T) { + root := t.TempDir() + store := newDiskStoreWithOptions(t, cache.WithRoot(root), cache.WithMaxSizeBytes(1)) + oldDigest := writeCachedBlobWithMetadata( + t, + root, + []byte("old cached bytes"), + time.Date(2026, 5, 1, 10, 0, 0, 0, time.UTC), + ) + body := []byte("new cached bytes") + server, _ := newBlobServer(t, http.StatusOK, body) + keepDigest := sha256Hex(body) + + _, err := store.Fetch(context.Background(), cache.FetchRequest{ + URL: server.URL + "/blob", + ExpectedSHA256: keepDigest, + ExpectedSize: int64(len(body)), + }) + + require.NoError(t, err) + assert.FileExists(t, cachedBlobPath(root, oldDigest)) + assert.FileExists(t, cachedBlobPath(root, keepDigest)) +} + +func TestDiskStorePruneRemovesLeastRecentlyUsedBlobs(t *testing.T) { + root := t.TempDir() + store := newDiskStoreWithOptions(t, cache.WithRoot(root), cache.WithMaxSizeBytes(10)) + oldDigest := writeCachedBlobWithMetadata(t, root, []byte("old!"), time.Date(2026, 5, 1, 10, 0, 0, 0, time.UTC)) + middleDigest := writeCachedBlobWithMetadata(t, root, []byte("mid!"), time.Date(2026, 5, 1, 11, 0, 0, 0, time.UTC)) + recentDigest := writeCachedBlobWithMetadata(t, root, []byte("new?"), time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)) + body := []byte("keep") + server, _ := newBlobServer(t, http.StatusOK, body) + keepDigest := sha256Hex(body) + + blob, err := store.Fetch(context.Background(), cache.FetchRequest{ + URL: server.URL + "/blob", + ExpectedSHA256: keepDigest, + ExpectedSize: int64(len(body)), + }) + + require.NoError(t, err) + assert.Equal(t, cachedBlobPath(root, keepDigest), blob.Path) + require.NoError(t, store.Prune(context.Background())) + assert.NoFileExists(t, cachedBlobPath(root, oldDigest)) + assert.NoFileExists(t, cachedBlobPath(root, middleDigest)) + assert.FileExists(t, cachedBlobPath(root, recentDigest)) + assert.FileExists(t, cachedBlobPath(root, keepDigest)) +} + +func TestDiskStorePruneKeepsOnlyBlobWhenItExceedsCacheLimit(t *testing.T) { + body := []byte("larger than cache limit") + server, _ := newBlobServer(t, http.StatusOK, body) + root := t.TempDir() + store := newDiskStoreWithOptions(t, cache.WithRoot(root), cache.WithMaxSizeBytes(1)) + digest := sha256Hex(body) + + blob, err := store.Fetch(context.Background(), cache.FetchRequest{ + URL: server.URL + "/blob", + ExpectedSHA256: digest, + ExpectedSize: int64(len(body)), + }) + + require.NoError(t, err) + assert.Equal(t, cachedBlobPath(root, digest), blob.Path) + require.NoError(t, store.Prune(context.Background())) + assert.FileExists(t, cachedBlobPath(root, digest)) + assert.FileExists(t, cachedMetadataPath(root, digest)) +} + +func TestDiskStorePruneSkipsLRUPruningWhenDisabled(t *testing.T) { + root := t.TempDir() + store := newDiskStoreWithOptions(t, cache.WithRoot(root), cache.WithMaxSizeBytes(0)) + oldDigest := writeCachedBlobWithMetadata( + t, + root, + []byte("old cached bytes"), + time.Date(2026, 5, 1, 10, 0, 0, 0, time.UTC), + ) + body := []byte("new cached bytes") + server, _ := newBlobServer(t, http.StatusOK, body) + keepDigest := sha256Hex(body) + + _, err := store.Fetch(context.Background(), cache.FetchRequest{ + URL: server.URL + "/blob", + ExpectedSHA256: keepDigest, + ExpectedSize: int64(len(body)), + }) + + require.NoError(t, err) + require.NoError(t, store.Prune(context.Background())) + assert.FileExists(t, cachedBlobPath(root, oldDigest)) + assert.FileExists(t, cachedBlobPath(root, keepDigest)) +} + +func TestDiskStorePruneHandlesMissingOrCorruptMetadata(t *testing.T) { + root := t.TempDir() + store := newDiskStoreWithOptions(t, cache.WithRoot(root), cache.WithMaxSizeBytes(8)) + oldTime := time.Date(2026, 5, 1, 8, 0, 0, 0, time.UTC) + missingDigest := writeCachedBlobAt(t, root, []byte("miss"), oldTime) + corruptDigest := writeCachedBlobAt(t, root, []byte("bad!"), oldTime) + writeCorruptCachedMetadata(t, root, corruptDigest) + recentDigest := writeCachedBlobWithMetadata(t, root, []byte("stay"), time.Date(2026, 5, 1, 12, 0, 0, 0, time.UTC)) + body := []byte("keep") + server, _ := newBlobServer(t, http.StatusOK, body) + keepDigest := sha256Hex(body) + + _, err := store.Fetch(context.Background(), cache.FetchRequest{ + URL: server.URL + "/blob", + ExpectedSHA256: keepDigest, + ExpectedSize: int64(len(body)), + }) + + require.NoError(t, err) + require.NoError(t, store.Prune(context.Background())) + assert.NoFileExists(t, cachedBlobPath(root, missingDigest)) + assert.NoFileExists(t, cachedBlobPath(root, corruptDigest)) + assert.FileExists(t, cachedBlobPath(root, recentDigest)) + assert.FileExists(t, cachedBlobPath(root, keepDigest)) +} + +func TestDiskStoreLockCreatesCacheLockFile(t *testing.T) { + root := t.TempDir() + store := newDiskStore(t, root) + + lock, err := store.Lock(context.Background()) + + require.NoError(t, err) + assert.FileExists(t, filepath.Join(root, "cache.lock")) + require.NoError(t, lock.Unlock()) +} + +func TestDiskStoreLockHonorsContextCancellation(t *testing.T) { + root := t.TempDir() + store := newDiskStore(t, root) + firstLock, err := store.Lock(context.Background()) + require.NoError(t, err) + defer firstLock.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + secondLock, err := store.Lock(ctx) + + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Nil(t, secondLock) +} + +func TestDiskStoreLockReleasesOnUnlock(t *testing.T) { + root := t.TempDir() + store := newDiskStore(t, root) + firstLock, err := store.Lock(context.Background()) + require.NoError(t, err) + + require.NoError(t, firstLock.Unlock()) + require.NoError(t, firstLock.Unlock()) + + secondLock, err := store.Lock(context.Background()) + require.NoError(t, err) + require.NoError(t, secondLock.Unlock()) } func TestDiskStoreFetchReplacesCorruptCacheEntry(t *testing.T) { @@ -352,6 +549,8 @@ func TestDiskStoreRepairsUnsafeCacheDirectoryPermissions(t *testing.T) { root, filepath.Join(root, "blobs"), filepath.Join(root, "blobs", "sha256"), + filepath.Join(root, "metadata"), + filepath.Join(root, "metadata", "sha256"), filepath.Join(root, "tmp"), } { require.NoError(t, os.MkdirAll(dir, 0o750)) @@ -367,6 +566,8 @@ func TestDiskStoreRepairsUnsafeCacheDirectoryPermissions(t *testing.T) { assertDirPerm(t, root, 0o750) assertDirPerm(t, filepath.Join(root, "blobs"), 0o750) assertDirPerm(t, filepath.Join(root, "blobs", "sha256"), 0o750) + assertDirPerm(t, filepath.Join(root, "metadata"), 0o750) + assertDirPerm(t, filepath.Join(root, "metadata", "sha256"), 0o750) assertDirPerm(t, filepath.Join(root, "tmp"), 0o700) } @@ -403,18 +604,44 @@ func TestDiskStoreCreatesRestrictiveCacheDirectories(t *testing.T) { assertDirPerm(t, root, 0o750) assertDirPerm(t, filepath.Join(root, "blobs"), 0o750) assertDirPerm(t, filepath.Join(root, "blobs", "sha256"), 0o750) + assertDirPerm(t, filepath.Join(root, "metadata"), 0o750) + assertDirPerm(t, filepath.Join(root, "metadata", "sha256"), 0o750) assertDirPerm(t, filepath.Join(root, "tmp"), 0o700) } func TestNewDiskStoreValidatesOptions(t *testing.T) { - store, err := cache.NewDiskStore( - cache.WithRoot(t.TempDir()), - cache.WithMaxUnknownSizeBytes(-1), - ) + tests := []struct { + name string + options []cache.Option + wantErr string + }{ + { + name: "negative unknown size cap", + options: []cache.Option{ + cache.WithRoot(t.TempDir()), + cache.WithMaxUnknownSizeBytes(-1), + }, + wantErr: "cache max unknown-size bytes must be non-negative", + }, + { + name: "negative max size", + options: []cache.Option{ + cache.WithRoot(t.TempDir()), + cache.WithMaxSizeBytes(-1), + }, + wantErr: "cache max size bytes must be non-negative", + }, + } - require.Error(t, err) - assert.Contains(t, err.Error(), "cache max unknown-size bytes must be non-negative") - assert.Nil(t, store) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store, err := cache.NewDiskStore(tt.options...) + + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + assert.Nil(t, store) + }) + } } func newDiskStore(t *testing.T, root string) *cache.DiskStore { @@ -455,10 +682,70 @@ func writeCachedBlob(t *testing.T, root string, digest string, body []byte) { require.NoError(t, os.WriteFile(path, body, 0o600)) } +func writeCachedBlobAt(t *testing.T, root string, body []byte, modTime time.Time) string { + t.Helper() + + digest := sha256Hex(body) + writeCachedBlob(t, root, digest, body) + path := cachedBlobPath(root, digest) + require.NoError(t, os.Chtimes(path, modTime, modTime)) + return digest +} + +func writeCachedBlobWithMetadata(t *testing.T, root string, body []byte, lastUsed time.Time) string { + t.Helper() + + digest := writeCachedBlobAt(t, root, body, lastUsed) + writeCachedMetadata(t, root, blobMetadataForTest{ + SHA256: digest, + Size: int64(len(body)), + LastUsed: lastUsed, + }) + return digest +} + +type blobMetadataForTest struct { + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + LastUsed time.Time `json:"lastUsed"` +} + +func writeCachedMetadata(t *testing.T, root string, metadata blobMetadataForTest) { + t.Helper() + + path := cachedMetadataPath(root, metadata.SHA256) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750)) + data, err := json.Marshal(metadata) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, data, 0o600)) +} + +func writeCorruptCachedMetadata(t *testing.T, root string, digest string) { + t.Helper() + + path := cachedMetadataPath(root, digest) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750)) + require.NoError(t, os.WriteFile(path, []byte("not-json"), 0o600)) +} + +func readCachedMetadata(t *testing.T, root string, digest string) blobMetadataForTest { + t.Helper() + + data, err := os.ReadFile(cachedMetadataPath(root, digest)) + require.NoError(t, err) + var metadata blobMetadataForTest + require.NoError(t, json.Unmarshal(data, &metadata)) + return metadata +} + func cachedBlobPath(root string, digest string) string { return filepath.Join(root, "blobs", "sha256", digest[:2], digest) } +func cachedMetadataPath(root string, digest string) string { + return filepath.Join(root, "metadata", "sha256", digest[:2], digest+".json") +} + func sha256Hex(data []byte) string { sum := sha256.Sum256(data) return hex.EncodeToString(sum[:]) diff --git a/internal/cache/lock.go b/internal/cache/lock.go new file mode 100644 index 0000000..b23874c --- /dev/null +++ b/internal/cache/lock.go @@ -0,0 +1,53 @@ +package cache + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "time" + + "github.com/gofrs/flock" +) + +const lockRetryDelay = 50 * time.Millisecond + +// Lock is a held advisory cache lock. +type Lock struct { + file *flock.Flock + unlocked bool +} + +// Lock acquires the cache-wide advisory lock. +func (s *DiskStore) Lock(ctx context.Context) (*Lock, error) { + if err := s.ensureDirs(); err != nil { + return nil, err + } + + fileLock := flock.New(filepath.Join(s.root, lockFileName), flock.SetPermissions(metadataPerm)) + locked, err := fileLock.TryLockContext(ctx, lockRetryDelay) + if err != nil { + return nil, fmt.Errorf("acquire cache lock: %w", err) + } + if !locked { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("acquire cache lock: %w", err) + } + return nil, errors.New("acquire cache lock") + } + + return &Lock{file: fileLock}, nil +} + +// Unlock releases the cache-wide advisory lock. +func (l *Lock) Unlock() error { + if l == nil || l.unlocked { + return nil + } + l.unlocked = true + if err := l.file.Unlock(); err != nil { + return fmt.Errorf("release cache lock: %w", err) + } + + return nil +} diff --git a/internal/cache/metadata.go b/internal/cache/metadata.go new file mode 100644 index 0000000..bb12d6b --- /dev/null +++ b/internal/cache/metadata.go @@ -0,0 +1,213 @@ +package cache + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "time" +) + +type blobMetadata struct { + SHA256 string `json:"sha256"` + Size int64 `json:"size"` + LastUsed time.Time `json:"lastUsed"` +} + +type pruneCandidate struct { + path string + sha256 string + size int64 + lastUsed time.Time +} + +func (s *DiskStore) touchBlob(blob Blob) error { + metadata := blobMetadata{ + SHA256: blob.SHA256, + Size: blob.Size, + LastUsed: time.Now().UTC(), + } + if err := s.writeMetadata(metadata); err != nil { + return fmt.Errorf("write cache metadata: %w", err) + } + + return nil +} + +func (s *DiskStore) writeMetadata(metadata blobMetadata) error { + path := s.metadataPath(metadata.SHA256) + if err := ensureCacheDir(filepath.Dir(path), dirPerm); err != nil { + return err + } + + tmp, err := os.CreateTemp(filepath.Dir(path), "."+metadata.SHA256+"-*.tmp") + if err != nil { + return fmt.Errorf("create cache metadata temp file: %w", err) + } + tmpPath := tmp.Name() + cleanup := true + defer func() { + if cleanup { + _ = os.Remove(tmpPath) + } + }() + + encoder := json.NewEncoder(tmp) + encoder.SetIndent("", " ") + if err := encoder.Encode(metadata); err != nil { + _ = tmp.Close() + return fmt.Errorf("encode cache metadata: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close cache metadata temp file: %w", err) + } + if err := os.Chmod(tmpPath, metadataPerm); err != nil { + return fmt.Errorf("set cache metadata permissions: %w", err) + } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("publish cache metadata: %w", err) + } + cleanup = false + + return nil +} + +// Prune removes least-recently-used cached blobs until the store is at or below +// the configured maximum size. +// +// Prune is explicit maintenance. Fetch does not prune because it returns cache +// paths directly, and automatic eviction could invalidate paths held by another +// caller or process. +func (s *DiskStore) Prune(ctx context.Context) error { + if s.maxSizeBytes == 0 { + return nil + } + + candidates, totalSize, err := s.pruneCandidates() + if err != nil || totalSize <= s.maxSizeBytes { + return err + } + + slices.SortFunc(candidates, func(a pruneCandidate, b pruneCandidate) int { + if cmp := a.lastUsed.Compare(b.lastUsed); cmp != 0 { + return cmp + } + if a.sha256 < b.sha256 { + return -1 + } + if a.sha256 > b.sha256 { + return 1 + } + return 0 + }) + + remaining := len(candidates) + for _, candidate := range candidates { + if err := ctx.Err(); err != nil { + return err + } + if totalSize <= s.maxSizeBytes { + return nil + } + if remaining <= 1 { + return nil + } + + if err := os.Remove(candidate.path); err != nil && !errors.Is(err, os.ErrNotExist) { + continue + } + _ = os.Remove(s.metadataPath(candidate.sha256)) + totalSize -= candidate.size + remaining-- + } + + return nil +} + +func (s *DiskStore) pruneCandidates() ([]pruneCandidate, int64, error) { + root := filepath.Join(s.root, "blobs", "sha256") + var candidates []pruneCandidate + var totalSize int64 + + err := filepath.WalkDir(root, func(path string, entry os.DirEntry, err error) error { + if err != nil { + return err + } + if entry.IsDir() { + return nil + } + + digest := entry.Name() + if !isSHA256Hex(digest) { + return nil + } + + info, err := os.Lstat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } + if !info.Mode().IsRegular() { + return nil + } + + size := info.Size() + totalSize += size + metadata, ok := s.readMetadata(digest, size) + lastUsed := info.ModTime().UTC() + if ok { + lastUsed = metadata.LastUsed + } + + candidates = append(candidates, pruneCandidate{ + path: path, + sha256: digest, + size: size, + lastUsed: lastUsed, + }) + return nil + }) + if errors.Is(err, os.ErrNotExist) { + return nil, 0, nil + } + if err != nil { + return nil, 0, err + } + + return candidates, totalSize, nil +} + +func (s *DiskStore) readMetadata(sha256Digest string, size int64) (blobMetadata, bool) { + data, err := os.ReadFile(s.metadataPath(sha256Digest)) + if err != nil { + return blobMetadata{}, false + } + + var metadata blobMetadata + if err := json.Unmarshal(data, &metadata); err != nil { + return blobMetadata{}, false + } + if metadata.SHA256 != sha256Digest || metadata.Size != size || metadata.LastUsed.IsZero() { + return blobMetadata{}, false + } + + return metadata, true +} + +func (s *DiskStore) metadataPath(sha256Digest string) string { + return filepath.Join(s.root, "metadata", "sha256", sha256Digest[:digestShardLength], sha256Digest+".json") +} + +func isSHA256Hex(value string) bool { + if len(value) != sha256HexLength { + return false + } + _, err := hex.DecodeString(value) + return err == nil +} diff --git a/internal/cli/build.go b/internal/cli/build.go index 2fb4203..c374beb 100644 --- a/internal/cli/build.go +++ b/internal/cli/build.go @@ -1,7 +1,10 @@ package cli import ( + "context" + "errors" "fmt" + "io" "github.com/spf13/cobra" @@ -10,6 +13,7 @@ import ( incusosprovider "github.com/meigma/imgcli/internal/providers/incusos" "github.com/meigma/imgcli/internal/providers/incusos/cdn" "github.com/meigma/imgcli/internal/providers/incusos/imagefile" + imgschemas "github.com/meigma/imgcli/schemas" "github.com/meigma/imgcli/schemas/core" ) @@ -26,35 +30,24 @@ func newBuildCommand(rt *runtime) *cobra.Command { return err } - ports, err := rt.incusOSBuildPorts() - if err != nil { - return err + if rt.usesDefaultIncusOSCache() { + return withLockedCache(cmd.Context(), rt.config, func( + catalog incusosprovider.Catalog, + downloader incusosprovider.Downloader, + ) error { + ports, portsErr := rt.incusOSBuildPorts(catalog, downloader) + if portsErr != nil { + return portsErr + } + return runIncusOSBuild(cmd.Context(), config, ports, rt.opts.stdout()) + }) } - provider := incusosprovider.New(*config.Incusos, incusosprovider.Options{ - Catalog: ports.catalog, - Downloader: ports.downloader, - SeedBuilder: ports.seedBuilder, - ImageInjector: ports.imageInjector, - }) - - result, err := provider.Build(cmd.Context(), providers.BuildRequest{ - Plan: providers.Plan{ - Image: config.Image, - }, - OutputDir: buildOutputDir(config.Output), - }) + ports, err := rt.incusOSBuildPorts(nil, nil) if err != nil { return err } - - for _, artifact := range result.Artifacts { - if _, err := fmt.Fprintln(rt.opts.stdout(), artifact.Path); err != nil { - return fmt.Errorf("write build artifact path: %w", err) - } - } - - return nil + return runIncusOSBuild(cmd.Context(), config, ports, rt.opts.stdout()) }, } } @@ -66,7 +59,14 @@ type incusOSBuildPorts struct { imageInjector incusosprovider.ImageInjector } -func (rt *runtime) incusOSBuildPorts() (incusOSBuildPorts, error) { +func (rt *runtime) usesDefaultIncusOSCache() bool { + return rt.opts.IncusOSCatalog == nil && rt.opts.IncusOSDownloader == nil +} + +func (rt *runtime) incusOSBuildPorts( + defaultCatalog incusosprovider.Catalog, + defaultDownloader incusosprovider.Downloader, +) (incusOSBuildPorts, error) { ports := incusOSBuildPorts{ catalog: rt.opts.IncusOSCatalog, downloader: rt.opts.IncusOSDownloader, @@ -74,20 +74,22 @@ func (rt *runtime) incusOSBuildPorts() (incusOSBuildPorts, error) { imageInjector: rt.opts.IncusOSImageInjector, } - if ports.catalog == nil || ports.downloader == nil { - cacheStore, err := cache.NewDiskStore() - if err != nil { - return incusOSBuildPorts{}, fmt.Errorf("configure incusos cache: %w", err) - } - - client := cdn.NewClient(cdn.WithCacheService(cacheStore)) - if ports.catalog == nil { - ports.catalog = client - } - if ports.downloader == nil { - ports.downloader = client + if ports.catalog == nil { + ports.catalog = defaultCatalog + } + if ports.downloader == nil { + if typedDownloader, ok := ports.catalog.(incusosprovider.Downloader); ok { + ports.downloader = typedDownloader + } else { + ports.downloader = defaultDownloader } } + if ports.catalog == nil { + return incusOSBuildPorts{}, errors.New("configure incusos catalog: catalog is required") + } + if ports.downloader == nil { + return incusOSBuildPorts{}, errors.New("configure incusos downloader: downloader is required") + } if ports.seedBuilder == nil { ports.seedBuilder = incusosprovider.SeedArchiveBuilder{} @@ -99,6 +101,80 @@ func (rt *runtime) incusOSBuildPorts() (incusOSBuildPorts, error) { return ports, nil } +func runIncusOSBuild( + ctx context.Context, + config imgschemas.Config, + ports incusOSBuildPorts, + output io.Writer, +) error { + provider := incusosprovider.New(*config.Incusos, incusosprovider.Options{ + Catalog: ports.catalog, + Downloader: ports.downloader, + SeedBuilder: ports.seedBuilder, + ImageInjector: ports.imageInjector, + }) + + result, err := provider.Build(ctx, providers.BuildRequest{ + Plan: providers.Plan{ + Image: config.Image, + }, + OutputDir: buildOutputDir(config.Output), + }) + if err != nil { + return err + } + + for _, artifact := range result.Artifacts { + if _, err := fmt.Fprintln(output, artifact.Path); err != nil { + return fmt.Errorf("write build artifact path: %w", err) + } + } + + return nil +} + +func newCacheStore(cfg Config) (*cache.DiskStore, error) { + options := []cache.Option{ + cache.WithMaxSizeBytes(cfg.CacheMaxSizeBytes), + } + if cfg.CacheDir != "" { + options = append(options, cache.WithRoot(cfg.CacheDir)) + } + + return cache.NewDiskStore(options...) +} + +func withLockedCache( + ctx context.Context, + cfg Config, + run func(catalog incusosprovider.Catalog, downloader incusosprovider.Downloader) error, +) (err error) { + cacheStore, err := newCacheStore(cfg) + if err != nil { + return err + } + + cacheLock, err := cacheStore.Lock(ctx) + if err != nil { + return err + } + defer func() { + if unlockErr := cacheLock.Unlock(); err == nil && unlockErr != nil { + err = unlockErr + } + }() + + client := cdn.NewClient(cdn.WithCacheService(cacheStore)) + if err := run(client, client); err != nil { + return err + } + if err := cacheStore.Prune(ctx); err != nil { + return err + } + + return nil +} + func buildOutputDir(output *core.OutputDefaults) string { if output == nil || output.Dir == "" { return defaultBuildOutputDir diff --git a/internal/cli/build_test.go b/internal/cli/build_test.go new file mode 100644 index 0000000..83c4b43 --- /dev/null +++ b/internal/cli/build_test.go @@ -0,0 +1,105 @@ +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/meigma/imgcli/internal/cache" + incusosprovider "github.com/meigma/imgcli/internal/providers/incusos" +) + +func TestWithLockedCachePrunesAfterSuccess(t *testing.T) { + clearIMGCLIEnv(t) + root := t.TempDir() + oldDigest := sha256HexForBuildTest([]byte("old!")) + recentDigest := sha256HexForBuildTest([]byte("new?")) + + err := withLockedCache(context.Background(), Config{ + CacheDir: root, + CacheMaxSizeBytes: 4, + }, func(catalog incusosprovider.Catalog, downloader incusosprovider.Downloader) error { + require.NotNil(t, catalog) + require.NotNil(t, downloader) + assertCacheLocked(t, root) + writeBuildTestCachedBlob(t, root, []byte("old!"), time.Date(2026, 5, 1, 10, 0, 0, 0, time.UTC)) + writeBuildTestCachedBlob(t, root, []byte("new?"), time.Date(2026, 5, 1, 11, 0, 0, 0, time.UTC)) + return nil + }) + + require.NoError(t, err) + assert.NoFileExists(t, buildTestCachedBlobPath(root, oldDigest)) + assert.FileExists(t, buildTestCachedBlobPath(root, recentDigest)) + assertCacheUnlocked(t, root) +} + +func TestWithLockedCacheSkipsPruneAfterBuildError(t *testing.T) { + clearIMGCLIEnv(t) + root := t.TempDir() + buildErr := errors.New("build failed") + oldDigest := sha256HexForBuildTest([]byte("old!")) + + err := withLockedCache(context.Background(), Config{ + CacheDir: root, + CacheMaxSizeBytes: 1, + }, func(_ incusosprovider.Catalog, _ incusosprovider.Downloader) error { + assertCacheLocked(t, root) + writeBuildTestCachedBlob(t, root, []byte("old!"), time.Date(2026, 5, 1, 10, 0, 0, 0, time.UTC)) + return buildErr + }) + + require.ErrorIs(t, err, buildErr) + assert.FileExists(t, buildTestCachedBlobPath(root, oldDigest)) + assertCacheUnlocked(t, root) +} + +func assertCacheLocked(t *testing.T, root string) { + t.Helper() + + store, err := cache.NewDiskStore(cache.WithRoot(root)) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + + lock, err := store.Lock(ctx) + + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Nil(t, lock) +} + +func assertCacheUnlocked(t *testing.T, root string) { + t.Helper() + + store, err := cache.NewDiskStore(cache.WithRoot(root)) + require.NoError(t, err) + lock, err := store.Lock(context.Background()) + require.NoError(t, err) + require.NoError(t, lock.Unlock()) +} + +func writeBuildTestCachedBlob(t *testing.T, root string, body []byte, modTime time.Time) { + t.Helper() + + digest := sha256HexForBuildTest(body) + path := buildTestCachedBlobPath(root, digest) + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750)) + require.NoError(t, os.WriteFile(path, body, 0o400)) + require.NoError(t, os.Chtimes(path, modTime, modTime)) +} + +func buildTestCachedBlobPath(root string, digest string) string { + return filepath.Join(root, "blobs", "sha256", digest[:2], digest) +} + +func sha256HexForBuildTest(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} diff --git a/internal/cli/config.go b/internal/cli/config.go index c3312d2..78c0039 100644 --- a/internal/cli/config.go +++ b/internal/cli/config.go @@ -1,7 +1,12 @@ package cli import ( + "errors" "fmt" + "math" + "os" + "path/filepath" + "strconv" "strings" "github.com/spf13/cobra" @@ -20,18 +25,31 @@ const ( KeyLogFormat = "log-format" // KeyNoColor is the Viper key for disabling styled terminal output. KeyNoColor = "no-color" + // KeyCacheDir is the Viper key for the cache root directory. + KeyCacheDir = "cache.dir" + // KeyCacheMaxSize is the Viper key for the maximum cache size before LRU pruning. + KeyCacheMaxSize = "cache.max-size" ) const ( - flagConfig = "config" - flagLogLevel = "log-level" - flagLogFormat = "log-format" - flagNoColor = "no-color" + flagConfig = "config" + flagLogLevel = "log-level" + flagLogFormat = "log-format" + flagNoColor = "no-color" + flagCacheDir = "cache-dir" + flagCacheMaxSize = "cache-max-size" ) const ( - defaultLogLevel = "info" - defaultLogFormat = "text" + defaultConfigDirName = "imgcli" + defaultConfigFileName = "config.yaml" + defaultLogLevel = "info" + defaultLogFormat = "text" + defaultCacheMaxSize = "10GB" + + cacheSizeKiBShift = 10 + cacheSizeMiBShift = 20 + cacheSizeGiBShift = 30 ) // Config is the CLI edge configuration resolved from flags, environment, config file, and defaults. @@ -47,6 +65,12 @@ type Config struct { // NoColor disables styled terminal output when true. NoColor bool + + // CacheDir is the optional cache root directory. Empty selects the platform cache directory. + CacheDir string + + // CacheMaxSizeBytes is the maximum cache size used by LRU pruning. Zero disables pruning. + CacheMaxSizeBytes int64 } func configureViper(vp *viper.Viper) { @@ -57,6 +81,8 @@ func configureViper(vp *viper.Viper) { vp.SetDefault(KeyLogLevel, defaultLogLevel) vp.SetDefault(KeyLogFormat, defaultLogFormat) vp.SetDefault(KeyNoColor, false) + vp.SetDefault(KeyCacheDir, "") + vp.SetDefault(KeyCacheMaxSize, defaultCacheMaxSize) } func (rt *runtime) registerGlobalFlags(root *cobra.Command) error { @@ -65,6 +91,12 @@ func (rt *runtime) registerGlobalFlags(root *cobra.Command) error { flags.String(flagLogLevel, defaultLogLevel, "Minimum log level: debug, info, warn, or error") flags.String(flagLogFormat, defaultLogFormat, "Log format: text, json, or logfmt") flags.Bool(flagNoColor, false, "Disable styled terminal output") + flags.String(flagCacheDir, "", "Cache directory") + flags.String( + flagCacheMaxSize, + defaultCacheMaxSize, + "Maximum cache size used by LRU pruning, or 0 to disable", + ) if err := bindConfigFlag(rt.viper, flags, KeyConfig, flagConfig); err != nil { return err @@ -78,6 +110,12 @@ func (rt *runtime) registerGlobalFlags(root *cobra.Command) error { if err := bindConfigFlag(rt.viper, flags, KeyNoColor, flagNoColor); err != nil { return err } + if err := bindConfigFlag(rt.viper, flags, KeyCacheDir, flagCacheDir); err != nil { + return err + } + if err := bindConfigFlag(rt.viper, flags, KeyCacheMaxSize, flagCacheMaxSize); err != nil { + return err + } return nil } @@ -97,18 +135,23 @@ func bindConfigFlag(vp *viper.Viper, flags *pflag.FlagSet, key string, flagName } func loadConfig(vp *viper.Viper) (Config, error) { - if configFile := vp.GetString(KeyConfig); configFile != "" { - vp.SetConfigFile(configFile) - if err := vp.ReadInConfig(); err != nil { - return Config{}, fmt.Errorf("read config file %q: %w", configFile, err) - } + configFile, err := readConfigFile(vp) + if err != nil { + return Config{}, err + } + + cacheMaxSize, err := parseSizeConfig(vp, KeyCacheMaxSize) + if err != nil { + return Config{}, err } cfg := Config{ - ConfigFile: vp.GetString(KeyConfig), - LogLevel: strings.ToLower(strings.TrimSpace(vp.GetString(KeyLogLevel))), - LogFormat: strings.ToLower(strings.TrimSpace(vp.GetString(KeyLogFormat))), - NoColor: vp.GetBool(KeyNoColor), + ConfigFile: configFile, + LogLevel: strings.ToLower(strings.TrimSpace(vp.GetString(KeyLogLevel))), + LogFormat: strings.ToLower(strings.TrimSpace(vp.GetString(KeyLogFormat))), + NoColor: vp.GetBool(KeyNoColor), + CacheDir: strings.TrimSpace(vp.GetString(KeyCacheDir)), + CacheMaxSizeBytes: cacheMaxSize, } if err := validateConfig(cfg); err != nil { @@ -118,6 +161,115 @@ func loadConfig(vp *viper.Viper) (Config, error) { return cfg, nil } +func readConfigFile(vp *viper.Viper) (string, error) { + configFile := vp.GetString(KeyConfig) + if configFile != "" { + return readNamedConfigFile(vp, configFile) + } + + defaultConfig, err := defaultConfigFile() + if err != nil { + return "", err + } + discoveredConfig, err := discoverExistingConfig(defaultConfig) + if err != nil || discoveredConfig == "" { + return "", err + } + + return readNamedConfigFile(vp, discoveredConfig) +} + +func readNamedConfigFile(vp *viper.Viper, configFile string) (string, error) { + vp.SetConfigFile(configFile) + if err := vp.ReadInConfig(); err != nil { + return "", fmt.Errorf("read config file %q: %w", configFile, err) + } + + return configFile, nil +} + +func discoverExistingConfig(path string) (string, error) { + if path == "" { + return "", nil + } + + if _, err := os.Stat(path); err == nil { + return path, nil + } else if !os.IsNotExist(err) { + return "", fmt.Errorf("inspect config file %q: %w", path, err) + } + + return "", nil +} + +func defaultConfigFile() (string, error) { + if xdgConfigHome := strings.TrimSpace(os.Getenv("XDG_CONFIG_HOME")); xdgConfigHome != "" { + if !filepath.IsAbs(xdgConfigHome) { + return "", errors.New("resolve config directory: XDG_CONFIG_HOME must be an absolute path") + } + return filepath.Join(xdgConfigHome, defaultConfigDirName, defaultConfigFileName), nil + } + + userConfigDir, err := os.UserConfigDir() + if err != nil { + return "", fmt.Errorf("resolve user config directory: %w", err) + } + + return filepath.Join(userConfigDir, defaultConfigDirName, defaultConfigFileName), nil +} + +func parseSizeConfig(vp *viper.Viper, key string) (int64, error) { + raw := strings.TrimSpace(vp.GetString(key)) + if _, err := parseSizeLiteral(raw); err != nil { + return 0, fmt.Errorf("invalid %s %q: %w", key, raw, err) + } + + size := vp.GetSizeInBytes(key) + if uint64(size) > uint64(math.MaxInt64) { + return 0, fmt.Errorf("invalid %s %q: size is too large", key, raw) + } + + return int64(size), nil +} + +func parseSizeLiteral(raw string) (int64, error) { + if raw == "" { + return 0, errors.New("size is required") + } + + lower := strings.ToLower(strings.TrimSpace(raw)) + multiplier := int64(1) + number := lower + for _, suffix := range []struct { + unit string + multiplier int64 + }{ + {unit: "gb", multiplier: 1 << cacheSizeGiBShift}, + {unit: "mb", multiplier: 1 << cacheSizeMiBShift}, + {unit: "kb", multiplier: 1 << cacheSizeKiBShift}, + {unit: "b", multiplier: 1}, + } { + if strings.HasSuffix(lower, suffix.unit) { + multiplier = suffix.multiplier + number = strings.TrimSpace(lower[:len(lower)-len(suffix.unit)]) + break + } + } + + value, err := strconv.ParseInt(number, 10, 64) + if err != nil { + return 0, errors.New("must be an integer byte size with optional B, KB, MB, or GB suffix") + } + if value < 0 { + return 0, errors.New("must be non-negative") + } + if value > math.MaxInt64/multiplier { + return 0, errors.New("size is too large") + } + + return value * multiplier, nil +} + func validateConfig(cfg Config) error { if _, err := parseLogLevel(cfg.LogLevel); err != nil { return err diff --git a/internal/cli/config_test.go b/internal/cli/config_test.go index e3c5e23..8b7ac86 100644 --- a/internal/cli/config_test.go +++ b/internal/cli/config_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "testing" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -97,11 +98,122 @@ func TestConfigFlagOverridesConfigEnvironment(t *testing.T) { assert.Equal(t, "dev\n", result.stdout) } +func TestDefaultConfigFileLoadsFromXDGConfigHome(t *testing.T) { + clearIMGCLIEnv(t) + xdgConfigHome := t.TempDir() + t.Setenv("XDG_CONFIG_HOME", xdgConfigHome) + configPath := filepath.Join(xdgConfigHome, "imgcli", "config.yaml") + writeConfigContent(t, configPath, fmt.Sprintf("%s: %q\n", KeyLogFormat, "yaml")) + + result := executeCommand(t, Options{}, "version") + + require.Error(t, result.err) + assert.ErrorContains(t, result.err, `invalid log format "yaml"`) +} + +func TestConfigFlagOverridesDefaultXDGConfigFile(t *testing.T) { + clearIMGCLIEnv(t) + xdgConfigHome := t.TempDir() + t.Setenv("XDG_CONFIG_HOME", xdgConfigHome) + defaultConfigPath := filepath.Join(xdgConfigHome, "imgcli", "config.yaml") + writeConfigContent(t, defaultConfigPath, fmt.Sprintf("%s: %q\n", KeyLogFormat, "yaml")) + flagConfigPath := writeConfig(t, KeyLogFormat, "logfmt") + + result := executeCommand(t, Options{}, "--config", flagConfigPath, "version") + + require.NoError(t, result.err) + assert.Equal(t, "dev\n", result.stdout) +} + +func TestCacheConfigDefaults(t *testing.T) { + clearIMGCLIEnv(t) + cfg, err := loadConfig(newConfigViper()) + + require.NoError(t, err) + assert.Empty(t, cfg.CacheDir) + assert.Equal(t, int64(10*(1<<30)), cfg.CacheMaxSizeBytes) +} + +func TestCacheConfigFileValues(t *testing.T) { + clearIMGCLIEnv(t) + cacheDir := filepath.Join(t.TempDir(), "cache") + configPath := filepath.Join(t.TempDir(), "imgcli.yaml") + writeConfigContent(t, configPath, fmt.Sprintf(` +cache: + dir: %q + max-size: "0" +`, cacheDir)) + vp := newConfigViper() + vp.Set(KeyConfig, configPath) + + cfg, err := loadConfig(vp) + + require.NoError(t, err) + assert.Equal(t, configPath, cfg.ConfigFile) + assert.Equal(t, cacheDir, cfg.CacheDir) + assert.Equal(t, int64(0), cfg.CacheMaxSizeBytes) +} + +func TestCacheConfigPrecedence(t *testing.T) { + t.Run("invalid config file fails", func(t *testing.T) { + clearIMGCLIEnv(t) + configPath := writeConfig(t, KeyCacheMaxSize, "nope") + + result := executeCommand(t, Options{}, "--config", configPath, "version") + + require.Error(t, result.err) + assert.ErrorContains(t, result.err, `invalid cache.max-size "nope"`) + }) + + t.Run("env overrides config file", func(t *testing.T) { + clearIMGCLIEnv(t) + configPath := writeConfig(t, KeyCacheMaxSize, "nope") + t.Setenv("IMGCLI_CACHE_MAX_SIZE", "1GB") + + result := executeCommand(t, Options{}, "--config", configPath, "version") + + require.NoError(t, result.err) + assert.Equal(t, "dev\n", result.stdout) + }) + + t.Run("flag overrides env", func(t *testing.T) { + clearIMGCLIEnv(t) + t.Setenv("IMGCLI_CACHE_MAX_SIZE", "nope") + + result := executeCommand(t, Options{}, "--cache-max-size", "1GB", "version") + + require.NoError(t, result.err) + assert.Equal(t, "dev\n", result.stdout) + }) +} + +func TestCacheConfigRejectsUnsupportedSizeUnits(t *testing.T) { + clearIMGCLIEnv(t) + configPath := writeConfig(t, KeyCacheMaxSize, "10GiB") + + result := executeCommand(t, Options{}, "--config", configPath, "version") + + require.Error(t, result.err) + assert.ErrorContains(t, result.err, `invalid cache.max-size "10GiB"`) +} + func writeConfig(t *testing.T, key string, value string) string { t.Helper() path := filepath.Join(t.TempDir(), "imgcli.yaml") - content := fmt.Sprintf("%s: %q\n", key, value) - require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) + writeConfigContent(t, path, fmt.Sprintf("%s: %q\n", key, value)) return path } + +func writeConfigContent(t *testing.T, path string, content string) { + t.Helper() + + require.NoError(t, os.MkdirAll(filepath.Dir(path), 0o750)) + require.NoError(t, os.WriteFile(path, []byte(content), 0o600)) +} + +func newConfigViper() *viper.Viper { + vp := viper.New() + configureViper(vp) + return vp +} diff --git a/internal/cli/root_test.go b/internal/cli/root_test.go index 4dd8af0..e93fe47 100644 --- a/internal/cli/root_test.go +++ b/internal/cli/root_test.go @@ -230,18 +230,20 @@ incusos: { seed: incusos.SeedArchive{Data: []byte("seed")}, } injector := &testImageInjector{} + cacheDir := filepath.Join(t.TempDir(), "cache") result := executeCommand(t, Options{ IncusOSCatalog: catalog, IncusOSDownloader: downloader, IncusOSSeedBuilder: seedBuilder, IncusOSImageInjector: injector, - }, "build", configPath) + }, "--cache-dir", cacheDir, "build", configPath) require.NoError(t, result.err) wantOutputPath := filepath.Join(outputDir, "test-image-default-amd64.raw.gz") assert.Equal(t, wantOutputPath+"\n", result.stdout) assert.Empty(t, result.stderr) + assert.NoDirExists(t, cacheDir) require.Len(t, catalog.queries, 1) assert.Equal(t, incusos.ImageQuery{ Channel: incusos.ChannelTesting, @@ -282,6 +284,8 @@ func clearIMGCLIEnv(t *testing.T) { t.Helper() for _, key := range []string{ + "IMGCLI_CACHE_DIR", + "IMGCLI_CACHE_MAX_SIZE", "IMGCLI_CONFIG", "IMGCLI_LOG_LEVEL", "IMGCLI_LOG_FORMAT", @@ -289,6 +293,7 @@ func clearIMGCLIEnv(t *testing.T) { } { t.Setenv(key, "") } + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) } func writeImageConfig(t *testing.T, content string) string {