Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 14 additions & 7 deletions images/hooks/vyos-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -144,18 +144,25 @@ wait_for_vyos() {
sudo docker exec "${container}" modprobe br_netfilter 2>/dev/null || true
sudo docker exec "${container}" modprobe 8021q 2>/dev/null || true

# Wait for VyOS config to be applied
# Config migration takes ~100 seconds in container environments
for i in {1..90}; do
if docker logs "${container}" 2>&1 | grep -q "migrate.*configure"; then
echo " VyOS config migration detected"
sleep 15
# Wait for container to become healthy
# The Dockerfile healthcheck uses: systemctl is-system-running --quiet
echo " Waiting for container to become healthy..."
for i in {1..60}; do
health=$(docker inspect --format='{{.State.Health.Status}}' "${container}" 2>/dev/null || echo "unknown")
if [[ "${health}" == "healthy" ]]; then
echo " Container is healthy"
break
fi
echo " Waiting for VyOS config... ($i/90)"
if [[ $i -eq 60 ]]; then
echo " WARNING: Container did not become healthy within timeout"
fi
sleep 2
done

# Wait briefly for VyOS services to fully initialize after systemd reports ready
echo " Waiting for VyOS services to initialize..."
sleep 5

# Verify configuration loaded
echo " Verifying configuration..."
docker exec "${container}" /opt/vyatta/bin/vyatta-op-cmd-wrapper show configuration commands | head -5
Expand Down
5 changes: 5 additions & 0 deletions images/images.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,8 @@ spec:
command: ./images/hooks/talos-embed-config.sh
args: ["cp-1"]
timeout: 10m
# Declare input files that affect the transform output.
# Changes to these files will trigger a re-sync.
inputs:
- "infrastructure/compute/talos/talconfig.yaml"
- "infrastructure/compute/talos/talsecret.sops.yaml"
26 changes: 21 additions & 5 deletions tools/labctl/cmd/images/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"io"
"net/http"
"os"
"path/filepath"
"strings"
"time"

Expand Down Expand Up @@ -121,9 +122,19 @@ func runSync(_ *cobra.Command, _ []string) error {
// Track if any files were changed (for GitHub Actions output)
filesChanged := false

// Determine base directory for resolving hook input paths.
// Use the manifest's parent directory as the base.
manifestDir := filepath.Dir(syncManifest)
baseDir, err := filepath.Abs(manifestDir)
if err != nil {
return fmt.Errorf("resolve manifest directory: %w", err)
}
// Go up one level from images/ to repo root for input path resolution
baseDir = filepath.Dir(baseDir)

// Process each image
for _, img := range manifest.Spec.Images {
changed, err := syncImageWithHTTP(ctx, client, http.DefaultClient, hookExecutor, cacheManager, img, syncDryRun, syncForce, syncNoUpload, syncSkipTransformHooks)
changed, err := syncImageWithHTTP(ctx, client, http.DefaultClient, hookExecutor, cacheManager, img, baseDir, syncDryRun, syncForce, syncNoUpload, syncSkipTransformHooks)
if err != nil {
return fmt.Errorf("sync image %q: %w", img.Name, err)
}
Expand All @@ -148,16 +159,21 @@ func runSync(_ *cobra.Command, _ []string) error {

// syncImage syncs a single image using the default HTTP client.
// This is a convenience wrapper for syncImageWithHTTP.
func syncImage(ctx context.Context, client store.Client, hookExecutor *hooks.Executor, cacheManager *cache.Manager, img config.Image, dryRun, force, noUpload, skipTransformHooks bool) (bool, error) {
return syncImageWithHTTP(ctx, client, http.DefaultClient, hookExecutor, cacheManager, img, dryRun, force, noUpload, skipTransformHooks)
func syncImage(ctx context.Context, client store.Client, hookExecutor *hooks.Executor, cacheManager *cache.Manager, img config.Image, baseDir string, dryRun, force, noUpload, skipTransformHooks bool) (bool, error) {
return syncImageWithHTTP(ctx, client, http.DefaultClient, hookExecutor, cacheManager, img, baseDir, dryRun, force, noUpload, skipTransformHooks)
}

// syncImageWithHTTP syncs an image using the provided HTTP and store clients.
// This function enables dependency injection for testing.
func syncImageWithHTTP(ctx context.Context, client store.Client, httpClient HTTPClient, hookExecutor *hooks.Executor, cacheManager *cache.Manager, img config.Image, dryRun, force, noUpload, skipTransformHooks bool) (bool, error) {
// The baseDir parameter specifies the directory for resolving hook input paths.
func syncImageWithHTTP(ctx context.Context, client store.Client, httpClient HTTPClient, hookExecutor *hooks.Executor, cacheManager *cache.Manager, img config.Image, baseDir string, dryRun, force, noUpload, skipTransformHooks bool) (bool, error) {
fmt.Printf("Processing: %s\n", img.Name)

effectiveChecksum := img.EffectiveChecksum()
// Compute effective checksum including hook input files
effectiveChecksum, err := img.EffectiveChecksumWithInputs(baseDir)
if err != nil {
return false, fmt.Errorf("compute effective checksum: %w", err)
}

// Check if image already exists with matching checksum (skip in no-upload mode)
if !dryRun && !force && !noUpload {
Expand Down
28 changes: 14 additions & 14 deletions tools/labctl/cmd/images/sync_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,7 @@ func TestSyncImage(t *testing.T) {
},
}

changed, err := syncImage(context.Background(), client, nil, nil, img, false, false, false, false)
changed, err := syncImage(context.Background(), client, nil, nil, img, t.TempDir(), false, false, false, false)

require.NoError(t, err)
assert.False(t, changed)
Expand All @@ -245,7 +245,7 @@ func TestSyncImage(t *testing.T) {
},
}

changed, err := syncImage(context.Background(), client, nil, nil, img, true, false, false, false)
changed, err := syncImage(context.Background(), client, nil, nil, img, t.TempDir(), true, false, false, false)

require.NoError(t, err)
assert.False(t, changed)
Expand Down Expand Up @@ -274,7 +274,7 @@ func TestSyncImage(t *testing.T) {

// With force=true and dryRun=true, it should show what would be done
// without checking checksum
_, err := syncImage(context.Background(), client, nil, nil, img, true, true, false, false)
_, err := syncImage(context.Background(), client, nil, nil, img, t.TempDir(), true, true, false, false)

require.NoError(t, err)
assert.False(t, checksumChecked) // Should not check checksum with force
Expand All @@ -296,7 +296,7 @@ func TestSyncImage(t *testing.T) {
},
}

_, err := syncImage(context.Background(), client, nil, nil, img, false, false, false, false)
_, err := syncImage(context.Background(), client, nil, nil, img, t.TempDir(), false, false, false, false)

assert.Error(t, err)
assert.Contains(t, err.Error(), "check existing image")
Expand All @@ -322,7 +322,7 @@ func TestSyncImage(t *testing.T) {

// With noUpload=true, should skip checksum check (client is nil for noUpload)
// and also skip upload - this test verifies the skip behavior
changed, err := syncImage(context.Background(), client, nil, nil, img, true, false, false, false)
changed, err := syncImage(context.Background(), client, nil, nil, img, t.TempDir(), true, false, false, false)

require.NoError(t, err)
assert.False(t, changed)
Expand Down Expand Up @@ -380,7 +380,7 @@ func TestSyncImageWithHTTP(t *testing.T) {
},
}

changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, false, false, false, false)
changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, t.TempDir(), false, false, false, false)

require.NoError(t, err)
assert.False(t, changed) // No updateFile, so no file changes
Expand Down Expand Up @@ -450,7 +450,7 @@ func TestSyncImageWithHTTP(t *testing.T) {
},
}

changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, false, false, false, false)
changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, t.TempDir(), false, false, false, false)

require.NoError(t, err)
assert.False(t, changed)
Expand Down Expand Up @@ -485,7 +485,7 @@ func TestSyncImageWithHTTP(t *testing.T) {
},
}

_, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, false, false, false, false)
_, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, t.TempDir(), false, false, false, false)

assert.Error(t, err)
assert.Contains(t, err.Error(), "download")
Expand Down Expand Up @@ -515,7 +515,7 @@ func TestSyncImageWithHTTP(t *testing.T) {
},
}

_, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, false, false, false, false)
_, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, t.TempDir(), false, false, false, false)

assert.Error(t, err)
assert.Contains(t, err.Error(), "source checksum verification")
Expand Down Expand Up @@ -549,7 +549,7 @@ func TestSyncImageWithHTTP(t *testing.T) {
},
}

_, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, false, false, false, false)
_, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, t.TempDir(), false, false, false, false)

assert.Error(t, err)
assert.Contains(t, err.Error(), "upload")
Expand Down Expand Up @@ -586,7 +586,7 @@ func TestSyncImageWithHTTP(t *testing.T) {
},
}

_, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, false, false, false, false)
_, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, t.TempDir(), false, false, false, false)

assert.Error(t, err)
assert.Contains(t, err.Error(), "write metadata")
Expand Down Expand Up @@ -629,7 +629,7 @@ func TestSyncImageWithHTTP(t *testing.T) {
},
}

_, err = syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, false, false, false, false)
_, err = syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, t.TempDir(), false, false, false, false)

assert.Error(t, err)
assert.Contains(t, err.Error(), "decompressed checksum verification")
Expand Down Expand Up @@ -677,7 +677,7 @@ func TestSyncImageWithHTTP(t *testing.T) {
}

// noUpload=true should download, verify, but skip upload
changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, false, false, true, false)
changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), nil, nil, img, t.TempDir(), false, false, true, false)

require.NoError(t, err)
assert.False(t, changed)
Expand Down Expand Up @@ -741,7 +741,7 @@ func TestSyncImageWithHTTP(t *testing.T) {
},
}

changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), hookExecutor, nil, img, false, false, false, false)
changed, err := syncImageWithHTTP(context.Background(), client, server.Client(), hookExecutor, nil, img, dir, false, false, false, false)

require.NoError(t, err)
assert.False(t, changed)
Expand Down
104 changes: 103 additions & 1 deletion tools/labctl/internal/config/manifest.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,14 @@
package config

import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"sort"
"strings"
"time"

Expand Down Expand Up @@ -69,6 +74,11 @@ type Hook struct {
// WorkDir is the working directory for the command.
// If not specified, uses the current working directory.
WorkDir string `yaml:"workDir,omitempty"`
// Inputs declares files/globs that affect the hook's output.
// When specified, changes to these files will trigger a re-sync
// even if the source checksum matches. Paths are relative to the
// repository root. Supports glob patterns (e.g., "config/*.yaml").
Inputs []string `yaml:"inputs,omitempty"`
}

// Source defines where to download the image from.
Expand Down Expand Up @@ -96,15 +106,107 @@ type Replacement struct {
Value string `yaml:"value"` // Template: {{ .Source.URL }}, {{ .Source.Checksum }}
}

// EffectiveChecksum returns the checksum to use for idempotency checks.
// EffectiveChecksum returns the base checksum to use for idempotency checks.
// If validation.expected is set, use that; otherwise use source.checksum.
// Note: This does not include hook input files. Use EffectiveChecksumWithInputs
// for a checksum that incorporates transform hook input file changes.
func (i *Image) EffectiveChecksum() string {
if i.Validation != nil && i.Validation.Expected != "" {
return i.Validation.Expected
}
return i.Source.Checksum
}

// EffectiveChecksumWithInputs returns a checksum that incorporates both the
// base checksum and the hash of any input files declared by transform hooks.
// This ensures that changes to transform hook inputs trigger a re-sync.
// If no inputs are declared, returns the base EffectiveChecksum().
// The baseDir parameter specifies the directory relative to which input
// paths are resolved (typically the repository root or manifest directory).
func (i *Image) EffectiveChecksumWithInputs(baseDir string) (string, error) {
baseChecksum := i.EffectiveChecksum()

// Collect all inputs from transform hooks
var allInputs []string
if i.Hooks != nil {
for _, h := range i.Hooks.Transform {
allInputs = append(allInputs, h.Inputs...)
}
}

// If no inputs, return base checksum
if len(allInputs) == 0 {
return baseChecksum, nil
}

// Compute hash of all input files
inputsHash, err := hashInputFiles(baseDir, allInputs)
if err != nil {
return "", fmt.Errorf("hash input files: %w", err)
}

// Combine base checksum with inputs hash
// Format: "base_checksum+inputs:hash"
return baseChecksum + "+inputs:" + inputsHash, nil
}

// hashInputFiles computes a combined SHA256 hash of all files matching the
// given glob patterns. Files are processed in sorted order for determinism.
func hashInputFiles(baseDir string, patterns []string) (string, error) {
// Expand all globs and collect unique file paths
fileSet := make(map[string]struct{})
for _, pattern := range patterns {
fullPattern := filepath.Join(baseDir, pattern)
matches, err := filepath.Glob(fullPattern)
if err != nil {
return "", fmt.Errorf("invalid glob pattern %q: %w", pattern, err)
}
for _, match := range matches {
// Only include regular files, not directories
info, err := os.Stat(match)
if err != nil {
return "", fmt.Errorf("stat %q: %w", match, err)
}
if info.Mode().IsRegular() {
fileSet[match] = struct{}{}
}
}
}

// Sort file paths for deterministic hashing
var files []string
for f := range fileSet {
files = append(files, f)
}
sort.Strings(files)

// Compute combined hash
h := sha256.New()
for _, file := range files {
// Include relative path in hash (so renames are detected)
relPath, err := filepath.Rel(baseDir, file)
if err != nil {
relPath = file
}
h.Write([]byte(relPath))
h.Write([]byte{0}) // Separator

// Hash file contents
f, err := os.Open(file) //nolint:gosec // G304: Paths come from trusted manifest
if err != nil {
return "", fmt.Errorf("open %q: %w", file, err)
}
if _, err := io.Copy(h, f); err != nil {
_ = f.Close()
return "", fmt.Errorf("read %q: %w", file, err)
}
_ = f.Close()
h.Write([]byte{0}) // Separator between files
}

return hex.EncodeToString(h.Sum(nil)), nil
}

// FindImageByName returns the image with the given name, or nil if not found.
func (m *ImageManifest) FindImageByName(name string) *Image {
for i := range m.Spec.Images {
Expand Down
Loading