add oci pull to images

.github/workflows/build.yml

@@ -29,12 +29,21 @@ jobs:
       - name: Set up Cloud SDK
         uses: google-github-actions/setup-gcloud@v2
 
+      - name: Setup Go
+        uses: actions/setup-go@v6
+        with:
+          go-version-file: 'go.mod'
+          cache: false
+
       - name: Make tag
         run: |
           [ "${GITHUB_EVENT_NAME}" == 'pull_request' ] && echo "TARGET_BINARY_LOCATION=pull-requests/$(echo $GITHUB_REF | awk -F / '{print $3}')-${GITHUB_HEAD_REF##*/}" >> $GITHUB_ENV || true
           [ "${GITHUB_EVENT_NAME}" == 'release' ] && echo "TARGET_BINARY_LOCATION=${GITHUB_REF##*/}" >> $GITHUB_ENV || true
           [ "${GITHUB_EVENT_NAME}" == 'push' ] && echo "TARGET_BINARY_LOCATION=latest" >> $GITHUB_ENV || true
 
+      - name: Run integration test
+        run: make test-integration
+
       - name: Build image
         run: |
           make metal-hammer-initrd.img.lz4

Makefile

@@ -29,7 +29,7 @@ LINKMODE := -linkmode external -extldflags '-static -s -w' \
 		 -X 'github.com/metal-stack/v.GitSHA1=$(SHA)' \
 		 -X 'github.com/metal-stack/v.BuildDate=$(BUILDDATE)'
 
-bin/$(BINARY): test $(GOSRC)
+bin/$(BINARY): test-unit $(GOSRC)
 	$(info CGO_ENABLED="$(CGO_ENABLED)")
 	$(GO) build \
 		-tags netgo \
@@ -39,10 +39,14 @@ bin/$(BINARY): test $(GOSRC)
 		$(MAINMODULE)
 	strip bin/$(BINARY)
 
-.PHONY: test
-test:
+.PHONY: test-unit
+test-unit:
 	CGO_ENABLED=1 $(GO) test -cover ./...
 
+.PHONY: test-integration
+test-integration:
+	CGO_ENABLED=1 find . -name '*_integration_test.go' -type f -printf '%h\n' | sort -u | xargs -r $(GO) test -cover -tags=integration
+
 .PHONY: clean
 clean::
 	rm -f ${INITRD} ${INITRD_COMPRESSED}
@@ -138,4 +142,4 @@ start:
 		--cmdline "console=ttyS0" \
 		--cpus boot=4 \
 		--memory size=1024M \
-		--net "tap=,mac=,ip=,mask="
+		--net "tap=,mac=,ip=,mask="

cmd/image/image.go

@@ -1,13 +1,22 @@
 package image
 
 import (
+	"archive/tar"
+	"context"
+	"errors"
 	"fmt"
 	"log/slog"
+	"path/filepath"
 
 	pb "github.com/cheggaaa/pb/v3"
 	"github.com/mholt/archiver"
 	lz4 "github.com/pierrec/lz4/v4"
 
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+
 	//nolint:gosec
 	"crypto/md5"
 	"io"
@@ -25,7 +34,44 @@ func NewImage(log *slog.Logger) *Image {
 	return &Image{log: log}
 }
 
-// Pull a image from s3
+func (i *Image) OciPull(ctx context.Context, imageRef, mountDir, username, password string) error {
+	imageRefWithoutOciPrefix := strings.TrimPrefix(imageRef, "oci://")
+
+	// TODO: just for testing - remove log statement before merging
+	i.log.Info("log image ref without oci prefix", "imageRefWithoutOciPrefix", imageRefWithoutOciPrefix)
+	ref, err := name.ParseReference(imageRefWithoutOciPrefix)
+	if err != nil {
+		return fmt.Errorf("parsing image reference: %w", err)
+	}
+
+	var auth = authn.Anonymous
+	if username != "" || password != "" {
+		auth = &authn.Basic{
+			Username: username,
+			Password: password,
+		}
+	}
+
+	i.log.Info("pull oci image", "image", imageRef)
+	img, err := remote.Image(ref, remote.WithAuth(auth))
+	if err != nil {
+		return fmt.Errorf("fetching remote image: %w", err)
+	}
+
+	// Flatten layers and create a tar stream
+	rc := mutate.Extract(img)
+	defer rc.Close()
+
+	i.log.Info(fmt.Sprintf("untar oci image into %s", mountDir), "image", imageRef)
+	if err := i.untar(rc, mountDir); err != nil {
+		return fmt.Errorf("extracting tar: %w", err)
+	}
+
+	i.log.Info("pulled oci image successfully", "image", imageRef)
+	return nil
+}
+
+// Pull an image from s3
 func (i *Image) Pull(image, destination string) error {
 	i.log.Info("pull image", "image", image)
 	md5destination := destination + ".md5"
@@ -49,7 +95,7 @@ func (i *Image) Pull(image, destination string) error {
 	return nil
 }
 
-// Burn a image pulling a tarball and unpack to a specific directory
+// Burn an image pulling a tarball and unpack to a specific directory
 func (i *Image) Burn(prefix, image, source string) error {
 	i.log.Info("burn image", "image", image)
 	begin := time.Now()
@@ -171,3 +217,74 @@ func (i *Image) download(source, dest string) error {
 
 	return nil
 }
+
+func (i *Image) untar(r io.Reader, dest string) error {
+	tr := tar.NewReader(r)
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("reading tar: %w", err)
+		}
+
+		target := filepath.Join(dest, hdr.Name)
+
+		i.log.Debug("untar oci image", "image", fmt.Sprintf("extracting:%s\n", target))
+
+		if strings.HasSuffix(target, ".log") {
+			i.log.Debug("untar oci image", "image", fmt.Sprintf("skipping:%s\n", target))
+			continue
+		}
+
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil {
+				return fmt.Errorf("creating dir: %w", err)
+			}
+			if err := os.Lchown(target, hdr.Uid, hdr.Gid); err != nil && !errors.Is(err, os.ErrPermission) {
+				return fmt.Errorf("chown dir: %w", err)
+			}
+
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+				return fmt.Errorf("creating parent dir: %w", err)
+			}
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode))
+			if err != nil {
+				return fmt.Errorf("creating file: %w", err)
+			}
+			if _, err := io.Copy(f, tr); err != nil {
+				f.Close()
+				return fmt.Errorf("copying file: %w", err)
+			}
+			f.Close()
+
+			if err := os.Chmod(target, os.FileMode(hdr.Mode)); err != nil {
+				return fmt.Errorf("chmod file: %w", err)
+			}
+			if err := os.Lchown(target, hdr.Uid, hdr.Gid); err != nil && !errors.Is(err, os.ErrPermission) {
+				return fmt.Errorf("chown file: %w", err)
+			}
+
+		case tar.TypeSymlink:
+			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+				return fmt.Errorf("creating parent dir: %w", err)
+			}
+			if err := os.Symlink(hdr.Linkname, target); err != nil {
+				return fmt.Errorf("creating symlink: %w", err)
+			}
+			if err := os.Lchown(target, hdr.Uid, hdr.Gid); err != nil && !errors.Is(err, os.ErrPermission) {
+				return fmt.Errorf("chown symlink: %w", err)
+			}
+
+		default:
+			// skip unsupported or special files, but log them
+			i.log.Debug("untar oci image", "image", fmt.Sprintf("skipping unsupported file type:%s\n", target))
+			continue
+		}
+	}
+
+	return nil
+}

cmd/image/image_integration_test.go

@@ -0,0 +1,183 @@
+//go:build integration
+package image
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/foomo/htpasswd"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/metal-stack/metal-lib/pkg/pointer"
+	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
+)
+
+func TestOciPull(t *testing.T) {
+	var (
+		mountDir     = "/tmp/oci-pull-mount-dir"
+		extractedBin = "a"
+
+		anonymousUsername = ""
+		anonymousPassword = ""
+	)
+
+	t.Run("successful anonymous pull", func(t *testing.T) {
+		regIP, regPort, err := startRegistry(nil, nil, nil)
+		require.NoError(t, err)
+		registry := fmt.Sprintf("%s:%d", regIP, regPort)
+
+		imageRef := fmt.Sprintf("%s/library/image", registry)
+		err = createImage(imageRef, "", "")
+		require.NoError(t, err)
+
+		err = os.MkdirAll(mountDir, 0777)
+		require.NoError(t, err)
+		defer os.RemoveAll(mountDir)
+
+		i := NewImage(slog.Default())
+		if err = i.OciPull(t.Context(), imageRef, mountDir, anonymousUsername, anonymousPassword); err != nil {
+			require.NoError(t, err)
+		}
+
+		extractedBinFullPath := filepath.Join(mountDir, extractedBin)
+		require.FileExists(t, extractedBinFullPath)
+	})
+
+	t.Run("successful authenticated pull", func(t *testing.T) {
+		var (
+			username = "test-user"
+			password = "test-password"
+		)
+
+		f, err := os.CreateTemp("", "htpasswd")
+		require.NoError(t, err)
+		defer func() {
+			_ = os.Remove(f.Name())
+		}()
+
+		err = htpasswd.SetPassword(f.Name(), username, password, htpasswd.HashBCrypt)
+		require.NoError(t, err)
+
+		env := map[string]string{
+			"REGISTRY_AUTH":                "htpasswd",
+			"REGISTRY_AUTH_HTPASSWD_REALM": "registry-login",
+			"REGISTRY_AUTH_HTPASSWD_PATH":  "/htpasswd",
+		}
+		regIP, regPort, err := startRegistry(env, pointer.Pointer(f.Name()), pointer.Pointer("/htpasswd"))
+		require.NoError(t, err)
+		registry := fmt.Sprintf("%s:%d", regIP, regPort)
+
+		imageRefBehindAuth := fmt.Sprintf("%s/library/image", registry)
+		err = createImage(imageRefBehindAuth, username, password)
+		require.NoError(t, err)
+
+		err = os.MkdirAll(mountDir, 0777)
+		require.NoError(t, err)
+		defer os.RemoveAll(mountDir)
+
+		i := NewImage(slog.Default())
+		if err = i.OciPull(t.Context(), imageRefBehindAuth, mountDir, username, password); err != nil {
+			require.NoError(t, err)
+		}
+
+		extractedBinFullPath := filepath.Join(mountDir, extractedBin)
+		require.FileExists(t, extractedBinFullPath)
+	})
+
+	t.Run("parsing of image refs fails", func(t *testing.T) {
+		invalidImageRef := "invalid://"
+		i := NewImage(slog.Default())
+		err := i.OciPull(t.Context(), invalidImageRef, mountDir, anonymousUsername, anonymousPassword)
+		require.EqualError(t, err, "parsing image reference: could not parse reference: invalid://")
+	})
+
+	t.Run("pulling remote image fails", func(t *testing.T) {
+		imageRefDoesNotExist := "oci://does/not/exist:tag"
+		i := NewImage(slog.Default())
+		err := i.OciPull(t.Context(), imageRefDoesNotExist, mountDir, anonymousUsername, anonymousPassword)
+		require.Error(t, err)
+	})
+}
+
+// HELPER FUNCTIONS
+func startRegistry(env map[string]string, src, dst *string) (string, int, error) {
+	ctx := context.Background()
+	var (
+		c   testcontainers.Container
+		err error
+	)
+
+	req := testcontainers.ContainerRequest{
+		Image:        "registry:3",
+		ExposedPorts: []string{"5000/tcp"},
+		Env:          env,
+		WaitingFor: wait.ForAll(
+			wait.ForLog("listening on"),
+			wait.ForListeningPort("5000/tcp"),
+		),
+	}
+	c, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		return "", 0, err
+	}
+	if src != nil && dst != nil {
+		err = c.CopyFileToContainer(ctx, *src, *dst, 0o777)
+		if err != nil {
+			return "", 0, err
+		}
+	}
+
+	ip, err := c.Host(ctx)
+	if err != nil {
+		return ip, 0, err
+	}
+	port, err := c.MappedPort(ctx, "5000")
+	if err != nil {
+		return ip, port.Int(), err
+	}
+
+	return ip, port.Int(), nil
+}
+
+func createImage(imageName, username, password string, tags ...string) error {
+	// ensure every image has distinct content
+	buf := make([]byte, 128)
+	_, err := rand.Read(buf)
+	if err != nil {
+		return err
+	}
+	img, err := crane.Image(map[string][]byte{"a": buf})
+	if err != nil {
+		return err
+	}
+
+	var auth = authn.Anonymous
+	if username != "" || password != "" {
+		auth = &authn.Basic{
+			Username: username,
+			Password: password,
+		}
+	}
+	err = crane.Push(img, imageName, crane.WithAuth(auth))
+	if err != nil {
+		return err
+	}
+	for _, tag := range tags {
+		err := crane.Push(img, imageName+":"+tag, crane.WithAuth(auth))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}