feat: initial commit for erofs support

.github/workflows/build.yaml

@@ -20,7 +20,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install bats fuse3 make libcryptsetup-dev libgpgme-dev \
           libcap-dev lxc libdevmapper-dev libacl1-dev libarchive-tools \
-          squashfuse squashfs-tools
+          squashfuse squashfs-tools erofs-utils
       - name: setup lxc
         run: |
           chmod ugo+x $HOME

Makefile

@@ -12,7 +12,7 @@ VERSION_LDFLAGS=-X main.Version=$(MAIN_VERSION)
 BATS = $(TOOLS_D)/bin/bats
 BATS_VERSION := v1.10.0
 STACKER = $(TOOLS_D)/bin/stacker
-STACKER_VERSION := v1.0.0
+STACKER_VERSION := v1.1.0-erofs
 TOOLS_D := $(ROOT)/tools
 GOCOVERDIR ?= $(ROOT)
 
@@ -36,7 +36,7 @@ gotest: $(GO_SRC)
 
 $(STACKER):
 	mkdir -p $(TOOLS_D)/bin
-	wget --progress=dot:giga https://github.com/project-stacker/stacker/releases/download/$(STACKER_VERSION)/stacker
+	wget --progress=dot:giga https://github.com/rchincha/stacker/releases/download/$(STACKER_VERSION)/stacker
 	chmod +x stacker
 	cp stacker $(TOOLS_D)/bin/
 

atomfs.go

@@ -0,0 +1 @@
+package atomfs

cmd/atomfs/mount.go

@@ -9,9 +9,8 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
-
-	"machinerun.io/atomfs"
-	"machinerun.io/atomfs/squashfs"
+	"machinerun.io/atomfs/pkg/common"
+	"machinerun.io/atomfs/pkg/molecule"
 )
 
 var mountCmd = cli.Command{
@@ -51,7 +50,7 @@ func findImage(ctx *cli.Context) (string, string, error) {
 	}
 	ocidir := r[0]
 	tag := r[1]
-	if !atomfs.PathExists(ocidir) {
+	if !common.PathExists(ocidir) {
 		return "", "", fmt.Errorf("oci directory %s does not exist: %w", ocidir, mountUsage(ctx.App.Name))
 	}
 	return ocidir, tag, nil
@@ -94,7 +93,7 @@ func doMount(ctx *cli.Context) error {
 			return fmt.Errorf("--persist requires an argument")
 		}
 	}
-	opts := atomfs.MountOCIOpts{
+	opts := molecule.MountOCIOpts{
 		OCIDir:                 absOCIDir,
 		Tag:                    tag,
 		Target:                 absTarget,
@@ -104,7 +103,7 @@ func doMount(ctx *cli.Context) error {
 		MetadataDir:            ctx.String("metadir"), // nil here means /run/atomfs
 	}
 
-	mol, err := atomfs.BuildMoleculeFromOCI(opts)
+	mol, err := molecule.BuildMoleculeFromOCI(opts)
 	if err != nil {
 		return errors.Wrapf(err, "couldn't build molecule with opts %+v", opts)
 	}
@@ -132,7 +131,7 @@ func amPrivileged() bool {
 
 func squashUmount(p string) error {
 	if amPrivileged() {
-		return squashfs.Umount(p)
+		return common.Umount(p)
 	}
 	return RunCommand("fusermount", "-u", p)
 }

cmd/atomfs/umount.go

@@ -5,8 +5,7 @@ import (
 	"path/filepath"
 
 	"github.com/urfave/cli"
-	"machinerun.io/atomfs"
-	"machinerun.io/atomfs/mount"
+	"machinerun.io/atomfs/pkg/molecule"
 )
 
 var umountCmd = cli.Command{
@@ -26,11 +25,6 @@ func umountUsage(me string) error {
 	return fmt.Errorf("Usage: %s umount mountpoint", me)
 }
 
-func isMountpoint(p string) bool {
-	mounted, err := mount.IsMountpoint(p)
-	return err == nil && mounted
-}
-
 func doUmount(ctx *cli.Context) error {
 	if ctx.NArg() < 1 {
 		return umountUsage(ctx.App.Name)
@@ -46,5 +40,5 @@ func doUmount(ctx *cli.Context) error {
 		}
 	}
 
-	return atomfs.UmountWithMetadir(mountpoint, ctx.String("metadir"))
+	return molecule.UmountWithMetadir(mountpoint, ctx.String("metadir"))
 }

cmd/atomfs/verify.go

@@ -6,10 +6,10 @@ import (
 	"strings"
 
 	"github.com/urfave/cli"
-	"machinerun.io/atomfs"
-	"machinerun.io/atomfs/log"
-	"machinerun.io/atomfs/mount"
-	"machinerun.io/atomfs/squashfs"
+	"machinerun.io/atomfs/pkg/common"
+	"machinerun.io/atomfs/pkg/log"
+	"machinerun.io/atomfs/pkg/mount"
+	"machinerun.io/atomfs/pkg/verity"
 )
 
 var verifyCmd = cli.Command{
@@ -45,16 +45,16 @@ func doVerify(ctx *cli.Context) error {
 		}
 	}
 
-	if !isMountpoint(mountpoint) {
+	if !common.IsMountpoint(mountpoint) {
 		return fmt.Errorf("%s is not a mountpoint", mountpoint)
 	}
 
-	mountNSName, err := atomfs.GetMountNSName()
+	mountNSName, err := common.GetMountNSName()
 	if err != nil {
 		return err
 	}
 
-	metadir := filepath.Join(atomfs.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, atomfs.ReplacePathSeparators(mountpoint))
+	metadir := filepath.Join(common.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, common.ReplacePathSeparators(mountpoint))
 	mountsdir := filepath.Join(metadir, "mounts")
 
 	mounts, err := mount.ParseMounts("/proc/self/mountinfo")
@@ -83,7 +83,7 @@ func doVerify(ctx *cli.Context) error {
 			continue
 		}
 		checkedCount = checkedCount + 1
-		err = squashfs.ConfirmExistingVerityDeviceCurrentValidity(m.Source)
+		err = verity.ConfirmExistingVerityDeviceCurrentValidity(m.Source)
 		if err != nil {
 			fmt.Printf("%s: CORRUPTION FOUND\n", m.Source)
 			allOK = false

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db
 	github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6
 	github.com/opencontainers/go-digest v1.0.0
-	github.com/opencontainers/image-spec v1.1.0-rc2
+	github.com/opencontainers/image-spec v1.1.0
 	github.com/opencontainers/runc v1.2.3 // indirect
 	github.com/opencontainers/umoci v0.4.8-0.20220412065115-12453f247749
 	github.com/pkg/errors v0.9.1
@@ -22,7 +22,6 @@ require (
 	github.com/cyphar/filepath-securejoin v0.3.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
 	github.com/klauspost/compress v1.15.15 // indirect
 	github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect
 	github.com/moby/sys/user v0.3.0 // indirect

go.sum

@@ -43,9 +43,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -88,8 +87,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
-github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
+github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
+github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
 github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc=
 github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80=
 github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM=

pkg/common/common_test.go

@@ -0,0 +1,69 @@
+package common
+
+import (
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type uidmapTestcase struct {
+	uidmap   string
+	expected bool
+}
+
+var uidmapTests = []uidmapTestcase{
+	{
+		uidmap:   `         0          0 4294967295`,
+		expected: true,
+	},
+	{
+		uidmap: `         0          0 1000
+2000 2000 1`,
+		expected: false,
+	},
+	{
+		uidmap:   `         0          0 1000`,
+		expected: false,
+	},
+	{
+		uidmap:   `         10          0 4294967295`,
+		expected: false,
+	},
+	{
+		uidmap:   `         0          10 4294967295`,
+		expected: false,
+	},
+	{
+		uidmap:   `         0          0 1`,
+		expected: false,
+	},
+}
+
+func TestAmHostRoot(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+	for _, testcase := range uidmapTests {
+		v := uidmapIsHost(testcase.uidmap)
+		assert.Equal(v, testcase.expected)
+	}
+}
+
+func TestIsEmpytDir(t *testing.T) {
+	t.Parallel()
+	assert := assert.New(t)
+	v, e := IsEmptyDir("/")
+	assert.NoError(e)
+	assert.False(v)
+
+	v, e = IsEmptyDir("/root")
+	assert.Error(e)
+
+	dname, err := os.MkdirTemp("", "squashfs_empty_test_dir")
+	assert.NoError(err)
+	defer os.RemoveAll(dname)
+
+	v, e = IsEmptyDir(dname)
+	assert.NoError(e)
+	assert.True(v)
+}

pkg/common/exclude.go

@@ -0,0 +1,84 @@
+package common
+
+import (
+	"bytes"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+// ExcludePaths represents a list of paths to exclude in a filesystem listing.
+// Users should do something like filepath.Walk() over the whole filesystem,
+// calling AddExclude() or AddInclude() based on whether they want to include
+// or exclude a particular file. Note that if e.g. /usr is excluded, then
+// everyting underneath is also implicitly excluded. The
+// AddExclude()/AddInclude() methods do the math to figure out what is the
+// correct set of things to exclude or include based on what paths have been
+// previously included or excluded.
+type ExcludePaths struct {
+	exclude map[string]bool
+	include []string
+}
+
+func NewExcludePaths() *ExcludePaths {
+	return &ExcludePaths{
+		exclude: map[string]bool{},
+		include: []string{},
+	}
+}
+
+func (eps *ExcludePaths) AddExclude(p string) {
+	for _, inc := range eps.include {
+		// If /usr/bin/ls has changed but /usr hasn't, we don't want to list
+		// /usr in the include paths any more, so let's be sure to only
+		// add things which aren't prefixes.
+		if strings.HasPrefix(inc, p) {
+			return
+		}
+	}
+	eps.exclude[p] = true
+}
+
+func (eps *ExcludePaths) AddInclude(orig string, isDir bool) {
+	// First, remove this thing and all its parents from exclude.
+	p := orig
+
+	// normalize to the first dir
+	if !isDir {
+		p = path.Dir(p)
+	}
+	for {
+		// our paths are all absolute, so this is a base case
+		if p == "/" {
+			break
+		}
+
+		delete(eps.exclude, p)
+		p = filepath.Dir(p)
+	}
+
+	// now add it to the list of includes, so we don't accidentally re-add
+	// anything above.
+	eps.include = append(eps.include, orig)
+}
+
+func (eps *ExcludePaths) String() (string, error) {
+	var buf bytes.Buffer
+	for p := range eps.exclude {
+		_, err := buf.WriteString(p)
+		if err != nil {
+			return "", err
+		}
+		_, err = buf.WriteString("\n")
+		if err != nil {
+			return "", err
+		}
+	}
+
+	_, err := buf.WriteString("\n")
+	if err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
+}

pkg/common/fuse.go

@@ -0,0 +1,5 @@
+package common
+
+import "os/exec"
+
+type FuseCmd func(fsImgFile, extractDir string) (*exec.Cmd, error)