diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3ed285f..95c1868 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,7 +20,7 @@ jobs: sudo apt-get update sudo apt-get install bats fuse3 make libcryptsetup-dev libgpgme-dev \ libcap-dev lxc libdevmapper-dev libacl1-dev libarchive-tools \ - squashfuse squashfs-tools + squashfuse squashfs-tools erofs-utils - name: setup lxc run: | chmod ugo+x $HOME diff --git a/Makefile b/Makefile index cc824b6..5f71eab 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION_LDFLAGS=-X main.Version=$(MAIN_VERSION) BATS = $(TOOLS_D)/bin/bats BATS_VERSION := v1.10.0 STACKER = $(TOOLS_D)/bin/stacker -STACKER_VERSION := v1.0.0 +STACKER_VERSION := v1.1.0-erofs TOOLS_D := $(ROOT)/tools GOCOVERDIR ?= $(ROOT) @@ -36,7 +36,7 @@ gotest: $(GO_SRC) $(STACKER): mkdir -p $(TOOLS_D)/bin - wget --progress=dot:giga https://github.com/project-stacker/stacker/releases/download/$(STACKER_VERSION)/stacker + wget --progress=dot:giga https://github.com/rchincha/stacker/releases/download/$(STACKER_VERSION)/stacker chmod +x stacker cp stacker $(TOOLS_D)/bin/ diff --git a/atomfs.go b/atomfs.go new file mode 100644 index 0000000..2769f4a --- /dev/null +++ b/atomfs.go @@ -0,0 +1 @@ +package atomfs diff --git a/cmd/atomfs/mount.go b/cmd/atomfs/mount.go index 4eec79d..ec97a10 100644 --- a/cmd/atomfs/mount.go +++ b/cmd/atomfs/mount.go @@ -9,9 +9,8 @@ import ( "github.com/pkg/errors" "github.com/urfave/cli" - - "machinerun.io/atomfs" - "machinerun.io/atomfs/squashfs" + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/molecule" ) var mountCmd = cli.Command{ @@ -51,7 +50,7 @@ func findImage(ctx *cli.Context) (string, string, error) { } ocidir := r[0] tag := r[1] - if !atomfs.PathExists(ocidir) { + if !common.PathExists(ocidir) { return "", "", fmt.Errorf("oci directory %s does not exist: %w", ocidir, mountUsage(ctx.App.Name)) } return ocidir, tag, nil @@ -94,7 +93,7 @@ func doMount(ctx *cli.Context) error { return fmt.Errorf("--persist requires an argument") } } - opts := atomfs.MountOCIOpts{ + opts := molecule.MountOCIOpts{ OCIDir: absOCIDir, Tag: tag, Target: absTarget, @@ -104,7 +103,7 @@ func doMount(ctx *cli.Context) error { MetadataDir: ctx.String("metadir"), // nil here means /run/atomfs } - mol, err := atomfs.BuildMoleculeFromOCI(opts) + mol, err := molecule.BuildMoleculeFromOCI(opts) if err != nil { return errors.Wrapf(err, "couldn't build molecule with opts %+v", opts) } @@ -132,7 +131,7 @@ func amPrivileged() bool { func squashUmount(p string) error { if amPrivileged() { - return squashfs.Umount(p) + return common.Umount(p) } return RunCommand("fusermount", "-u", p) } diff --git a/cmd/atomfs/umount.go b/cmd/atomfs/umount.go index 2e123d7..129e5be 100644 --- a/cmd/atomfs/umount.go +++ b/cmd/atomfs/umount.go @@ -5,8 +5,7 @@ import ( "path/filepath" "github.com/urfave/cli" - "machinerun.io/atomfs" - "machinerun.io/atomfs/mount" + "machinerun.io/atomfs/pkg/molecule" ) var umountCmd = cli.Command{ @@ -26,11 +25,6 @@ func umountUsage(me string) error { return fmt.Errorf("Usage: %s umount mountpoint", me) } -func isMountpoint(p string) bool { - mounted, err := mount.IsMountpoint(p) - return err == nil && mounted -} - func doUmount(ctx *cli.Context) error { if ctx.NArg() < 1 { return umountUsage(ctx.App.Name) @@ -46,5 +40,5 @@ func doUmount(ctx *cli.Context) error { } } - return atomfs.UmountWithMetadir(mountpoint, ctx.String("metadir")) + return molecule.UmountWithMetadir(mountpoint, ctx.String("metadir")) } diff --git a/cmd/atomfs/verify.go b/cmd/atomfs/verify.go index c587405..212bf4e 100644 --- a/cmd/atomfs/verify.go +++ b/cmd/atomfs/verify.go @@ -6,10 +6,10 @@ import ( "strings" "github.com/urfave/cli" - "machinerun.io/atomfs" - "machinerun.io/atomfs/log" - "machinerun.io/atomfs/mount" - "machinerun.io/atomfs/squashfs" + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/log" + "machinerun.io/atomfs/pkg/mount" + "machinerun.io/atomfs/pkg/verity" ) var verifyCmd = cli.Command{ @@ -45,16 +45,16 @@ func doVerify(ctx *cli.Context) error { } } - if !isMountpoint(mountpoint) { + if !common.IsMountpoint(mountpoint) { return fmt.Errorf("%s is not a mountpoint", mountpoint) } - mountNSName, err := atomfs.GetMountNSName() + mountNSName, err := common.GetMountNSName() if err != nil { return err } - metadir := filepath.Join(atomfs.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, atomfs.ReplacePathSeparators(mountpoint)) + metadir := filepath.Join(common.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, common.ReplacePathSeparators(mountpoint)) mountsdir := filepath.Join(metadir, "mounts") mounts, err := mount.ParseMounts("/proc/self/mountinfo") @@ -83,7 +83,7 @@ func doVerify(ctx *cli.Context) error { continue } checkedCount = checkedCount + 1 - err = squashfs.ConfirmExistingVerityDeviceCurrentValidity(m.Source) + err = verity.ConfirmExistingVerityDeviceCurrentValidity(m.Source) if err != nil { fmt.Printf("%s: CORRUPTION FOUND\n", m.Source) allOK = false diff --git a/go.mod b/go.mod index a7a003e..be11d9f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc2 + github.com/opencontainers/image-spec v1.1.0 github.com/opencontainers/runc v1.2.3 // indirect github.com/opencontainers/umoci v0.4.8-0.20220412065115-12453f247749 github.com/pkg/errors v0.9.1 @@ -22,7 +22,6 @@ require ( github.com/cyphar/filepath-securejoin v0.3.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/google/go-cmp v0.5.6 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect github.com/moby/sys/user v0.3.0 // indirect diff --git a/go.sum b/go.sum index 1eae085..463f799 100644 --- a/go.sum +++ b/go.sum @@ -43,9 +43,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -88,8 +87,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= diff --git a/pkg/common/common_test.go b/pkg/common/common_test.go new file mode 100644 index 0000000..c74b2da --- /dev/null +++ b/pkg/common/common_test.go @@ -0,0 +1,69 @@ +package common + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +type uidmapTestcase struct { + uidmap string + expected bool +} + +var uidmapTests = []uidmapTestcase{ + { + uidmap: ` 0 0 4294967295`, + expected: true, + }, + { + uidmap: ` 0 0 1000 +2000 2000 1`, + expected: false, + }, + { + uidmap: ` 0 0 1000`, + expected: false, + }, + { + uidmap: ` 10 0 4294967295`, + expected: false, + }, + { + uidmap: ` 0 10 4294967295`, + expected: false, + }, + { + uidmap: ` 0 0 1`, + expected: false, + }, +} + +func TestAmHostRoot(t *testing.T) { + t.Parallel() + assert := assert.New(t) + for _, testcase := range uidmapTests { + v := uidmapIsHost(testcase.uidmap) + assert.Equal(v, testcase.expected) + } +} + +func TestIsEmpytDir(t *testing.T) { + t.Parallel() + assert := assert.New(t) + v, e := IsEmptyDir("/") + assert.NoError(e) + assert.False(v) + + v, e = IsEmptyDir("/root") + assert.Error(e) + + dname, err := os.MkdirTemp("", "squashfs_empty_test_dir") + assert.NoError(err) + defer os.RemoveAll(dname) + + v, e = IsEmptyDir(dname) + assert.NoError(e) + assert.True(v) +} diff --git a/pkg/common/exclude.go b/pkg/common/exclude.go new file mode 100644 index 0000000..afc4503 --- /dev/null +++ b/pkg/common/exclude.go @@ -0,0 +1,84 @@ +package common + +import ( + "bytes" + "path" + "path/filepath" + "strings" +) + +// ExcludePaths represents a list of paths to exclude in a filesystem listing. +// Users should do something like filepath.Walk() over the whole filesystem, +// calling AddExclude() or AddInclude() based on whether they want to include +// or exclude a particular file. Note that if e.g. /usr is excluded, then +// everyting underneath is also implicitly excluded. The +// AddExclude()/AddInclude() methods do the math to figure out what is the +// correct set of things to exclude or include based on what paths have been +// previously included or excluded. +type ExcludePaths struct { + exclude map[string]bool + include []string +} + +func NewExcludePaths() *ExcludePaths { + return &ExcludePaths{ + exclude: map[string]bool{}, + include: []string{}, + } +} + +func (eps *ExcludePaths) AddExclude(p string) { + for _, inc := range eps.include { + // If /usr/bin/ls has changed but /usr hasn't, we don't want to list + // /usr in the include paths any more, so let's be sure to only + // add things which aren't prefixes. + if strings.HasPrefix(inc, p) { + return + } + } + eps.exclude[p] = true +} + +func (eps *ExcludePaths) AddInclude(orig string, isDir bool) { + // First, remove this thing and all its parents from exclude. + p := orig + + // normalize to the first dir + if !isDir { + p = path.Dir(p) + } + for { + // our paths are all absolute, so this is a base case + if p == "/" { + break + } + + delete(eps.exclude, p) + p = filepath.Dir(p) + } + + // now add it to the list of includes, so we don't accidentally re-add + // anything above. + eps.include = append(eps.include, orig) +} + +func (eps *ExcludePaths) String() (string, error) { + var buf bytes.Buffer + for p := range eps.exclude { + _, err := buf.WriteString(p) + if err != nil { + return "", err + } + _, err = buf.WriteString("\n") + if err != nil { + return "", err + } + } + + _, err := buf.WriteString("\n") + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/pkg/common/fuse.go b/pkg/common/fuse.go new file mode 100644 index 0000000..034bbbf --- /dev/null +++ b/pkg/common/fuse.go @@ -0,0 +1,5 @@ +package common + +import "os/exec" + +type FuseCmd func(fsImgFile, extractDir string) (*exec.Cmd, error) diff --git a/pkg/common/mount.go b/pkg/common/mount.go new file mode 100644 index 0000000..600a4cd --- /dev/null +++ b/pkg/common/mount.go @@ -0,0 +1,131 @@ +package common + +import ( + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "golang.org/x/sys/unix" + "machinerun.io/atomfs/pkg/mount" + "machinerun.io/atomfs/pkg/verity" +) + +func HostMount(fsImgFile string, fsType string, mountpoint string, rootHash string, veritySize int64, verityOffset uint64) error { + return verity.VerityHostMount(fsImgFile, fsType, mountpoint, rootHash, veritySize, verityOffset) +} + +// Mount a filesystem as container root, without host root +// privileges. We do this using fuse "cmd" which is passed in from actual filesystem backends. +func GuestMount(fsImgFile string, mountpoint string, fuseCmd FuseCmd) error { + if IsMountpoint(mountpoint) { + return errors.Errorf("%s is already mounted", mountpoint) + } + + abs, err := filepath.Abs(fsImgFile) + if err != nil { + return errors.Errorf("Failed to get absolute path for %s: %v", fsImgFile, err) + } + fsImgFile = abs + + abs, err = filepath.Abs(mountpoint) + if err != nil { + return errors.Errorf("Failed to get absolute path for %s: %v", mountpoint, err) + } + mountpoint = abs + + cmd, err := fuseCmd(fsImgFile, mountpoint) + if err != nil { + return err + } + + if err := cmd.Process.Release(); err != nil { + return errors.Errorf("Failed to release process after guestmount %s: %v", fsImgFile, err) + } + return nil +} + +// unmounts a squash mountpoint and detaches any verity / loop devices that back +// it. Only use this if you are sure the underlying devices aren't in use by +// other mount points. +func Umount(mountpoint string) error { + devPath, err := GetBackingDevice(mountpoint) + + err = unix.Unmount(mountpoint, 0) + if err != nil { + return errors.Wrapf(err, "failed unmounting %v", mountpoint) + } + + return MaybeCleanupBackingDevice(devPath) +} + +func GetBackingDevice(mountpoint string) (string, error) { + mounts, err := mount.ParseMounts("/proc/self/mountinfo") + if err != nil { + return "", err + } + + theMount, found := mounts.FindMount(mountpoint) + if !found { + return "", errors.Errorf("%s is not a mountpoint", mountpoint) + } + return theMount.Source, nil +} + +// given a device path that had been used as the backing device for a squash +// mountpoint, cleans up and detaches verity device if it still exists. +// +// If the device path does not exist, that is OK - this happens if the device +// was a regular loopback and not -verity. +func MaybeCleanupBackingDevice(devPath string) error { + if _, err := os.Stat(devPath); err != nil { + if os.IsNotExist(err) { + // It's been detached, this is fine. + return nil + } + return errors.WithStack(err) + } + + // was this a verity mount or a regular loopback mount? (if it's a + // regular loopback mount, we detached it above, so need to do anything + // special here; verity doesn't play as nicely) + if strings.HasSuffix(devPath, verity.VeritySuffix) { + err := verity.VerityUnmount(devPath) + if err != nil { + return errors.Wrapf(err, "failed verity-unmounting %v", devPath) + } + } + + return nil +} + +func IsMountpoint(dest string) bool { + mounted, err := mount.IsMountpoint(dest) + return err == nil && mounted +} + +func IsMountedAtDir(_, dest string) (bool, error) { + dstat, err := os.Stat(dest) + if os.IsNotExist(err) { + return false, nil + } + if !dstat.IsDir() { + return false, nil + } + mounts, err := mount.ParseMounts("/proc/self/mountinfo") + if err != nil { + return false, err + } + + fdest, err := filepath.Abs(dest) + if err != nil { + return false, err + } + for _, m := range mounts { + if m.Target == fdest { + return true, nil + } + } + + return false, nil +} diff --git a/pkg/common/utils.go b/pkg/common/utils.go new file mode 100644 index 0000000..06ee998 --- /dev/null +++ b/pkg/common/utils.go @@ -0,0 +1,147 @@ +package common + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "golang.org/x/sys/unix" +) + +var TestOverrideRuntimeDirKey = "ATOMFS_TEST_RUN_DIR" + +func EnsureDir(dir string) error { + err := os.MkdirAll(dir, 0755) + if err != nil { + return fmt.Errorf("Failed creating directory %s: %w", dir, err) + } + return nil +} + +func PathExists(d string) bool { + _, err := os.Stat(d) + if err != nil && os.IsNotExist(err) { + return false + } + return true +} + +// remove dir separators to make one dir name. It is OK that this can't be +// cleanly backed out, we don't need it to +func ReplacePathSeparators(p string) string { + if p[0] == '/' { + p = p[1:] + } + return strings.ReplaceAll(p, "/", "-") +} + +func GetMountNSName() (string, error) { + val, err := os.Readlink("/proc/self/ns/mnt") + if err != nil { + return "", err + } + // link target looks like 'mnt:[NUMBER]', we just want NUMBER + return val[5 : len(val)-1], nil +} + +// Allow overriding runtime dir for tests so we can assert empty dirs, etc. +func RuntimeDir(metadir string) string { + testOverrideDir := os.Getenv(TestOverrideRuntimeDirKey) + if testOverrideDir == "" { + if metadir == "" { + return "/run/atomfs" + } else { + return metadir + } + } + return testOverrideDir +} + +func IsEmptyDir(path string) (bool, error) { + fh, err := os.Open(path) + if err != nil { + return false, err + } + defer fh.Close() + + _, err = fh.ReadDir(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +// Which - like the unix utility, return empty string for not-found. +// this might fit well in lib/, but currently lib's test imports +// squashfs creating a import loop. +func Which(name string) string { + return whichSearch(name, strings.Split(os.Getenv("PATH"), ":")) +} + +func whichSearch(name string, paths []string) string { + var search []string + + if strings.ContainsRune(name, os.PathSeparator) { + if filepath.IsAbs(name) { + search = []string{name} + } else { + search = []string{"./" + name} + } + } else { + search = []string{} + for _, p := range paths { + search = append(search, filepath.Join(p, name)) + } + } + + for _, fPath := range search { + if err := unix.Access(fPath, unix.X_OK); err == nil { + return fPath + } + } + + return "" +} + +func FileChanged(a os.FileInfo, path string) bool { + b, err := os.Lstat(path) + if err != nil { + return true + } + return !os.SameFile(a, b) +} + +// Takes /proc/self/uid_map contents as one string +// Returns true if this is a uidmap representing the whole host +// uid range. +func uidmapIsHost(oneline string) bool { + oneline = strings.TrimSuffix(oneline, "\n") + if len(oneline) == 0 { + return false + } + lines := strings.Split(oneline, "\n") + if len(lines) != 1 { + return false + } + words := strings.Fields(lines[0]) + if len(words) != 3 || words[0] != "0" || words[1] != "0" || words[2] != "4294967295" { + return false + } + + return true +} + +func AmHostRoot() bool { + // if not uid 0, not host root + if os.Geteuid() != 0 { + return false + } + // if uid_map doesn't map 0 to 0, not host root + bytes, err := os.ReadFile("/proc/self/uid_map") + if err != nil { + return false + } + return uidmapIsHost(string(bytes)) +} diff --git a/pkg/erofs/erofs.go b/pkg/erofs/erofs.go new file mode 100644 index 0000000..b8e7533 --- /dev/null +++ b/pkg/erofs/erofs.go @@ -0,0 +1,547 @@ +// This package is a small go "library" (read: exec wrapper) around the +// mkfs.erofs binary that provides some useful primitives. +package erofs + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + "sync" + "syscall" + "time" + + "github.com/pkg/errors" + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/log" + types "machinerun.io/atomfs/pkg/types" + vrty "machinerun.io/atomfs/pkg/verity" +) + +type erofsFuseInfoStruct struct { + Path string + Version string +} + +var once sync.Once +var erofsFuseInfo = erofsFuseInfoStruct{"", ""} + +func MakeErofs(tempdir string, rootfs string, eps *common.ExcludePaths, verity vrty.VerityMetadata) (io.ReadCloser, string, string, error) { + var excludesFile string + var err error + var toExclude string + var rootHash string + + if eps != nil { + toExclude, err = eps.String() + if err != nil { + return nil, "", rootHash, errors.Wrapf(err, "couldn't create exclude path list") + } + } + + if len(toExclude) != 0 { + excludes, err := os.CreateTemp(tempdir, "stacker-erofs-exclude-") + if err != nil { + return nil, "", rootHash, err + } + defer os.Remove(excludes.Name()) + + excludesFile = excludes.Name() + _, err = excludes.WriteString(toExclude) + excludes.Close() + if err != nil { + return nil, "", rootHash, err + } + } + + tmpErofs, err := os.CreateTemp(tempdir, "stacker-erofs-img-") + if err != nil { + return nil, "", rootHash, err + } + // the following achieves the effect of creating a temporary file name + // without actually creating the file;the goal being to provide a temporary + // filename to provide to `mkfs.XXX` tool so we have a predictable name to + // consume after `mkfs.XXX` has completed its task. + // + // NB: there's a TOCTOU here; something else can predict and produce + // output in the tempfile name we created after we delete it and before + // `mkfs.XXX` runs. + tmpErofs.Close() + os.Remove(tmpErofs.Name()) + + defer os.Remove(tmpErofs.Name()) + + args := []string{tmpErofs.Name(), rootfs} + compression := LZ4HCCompression + if false { // FIXME: following features are experimental, disabling for now + zstdOk, parallelOk := mkerofsSupportsFeature() + if zstdOk { + args = append(args, "-z", "zstd") + compression = ZstdCompression + } + if parallelOk { + args = append(args, "--workers", fmt.Sprintf("%d", runtime.NumCPU())) + } + } + if len(toExclude) != 0 { + args = append(args, "--exclude-path", excludesFile) + } + cmd := exec.Command("mkfs.erofs", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err = cmd.Run(); err != nil { + return nil, "", rootHash, errors.Wrap(err, "couldn't build erofs") + } + + if verity { + rootHash, err = vrty.AppendVerityData(tmpErofs.Name()) + if err != nil { + return nil, "", rootHash, err + } + } + + blob, err := os.Open(tmpErofs.Name()) + if err != nil { + return nil, "", rootHash, errors.WithStack(err) + } + + return blob, GenerateErofsMediaType(compression), rootHash, nil +} + +func findErofsFuseInfo() { + var erofsPath string + if p := common.Which("erofsfuse"); p != "" { + erofsPath = p + } + if erofsPath == "" { + return + } + version := erofsfuseVersion(erofsPath) + log.Infof("Found erofsfuse at %s (version=%s)", erofsPath, version) + erofsFuseInfo = erofsFuseInfoStruct{erofsPath, version} +} + +// erofsfuseVersion - returns true if erofsfuse supports mount +// notification, false otherwise +// erofsfuse is the path to the erofsfuse binary +func erofsfuseVersion(erofsfuse string) string { + cmd := exec.Command(erofsfuse) + + // `erofsfuse` always returns an error... so we ignore it. + out, _ := cmd.CombinedOutput() + + firstLine := strings.Split(string(out[:]), "\n")[0] + version := strings.Split(firstLine, " ")[1] + + return version +} + +var erofsFuseNotFound = errors.Errorf("erofsfuse program not found") + +// erofsFuse - mount erofsFile to extractDir +// return a pointer to the erofsfuse cmd. +// The caller of the this is responsible for the process created. +func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { + var cmd *exec.Cmd + + once.Do(findErofsFuseInfo) + if erofsFuseInfo.Path == "" { + return cmd, erofsFuseNotFound + } + + // given extractDir of path/to/some/dir[/], log to path/to/some/.dir-erofs.log + extractDir = strings.TrimSuffix(extractDir, "/") + + var cmdOut io.Writer + var err error + + logf := filepath.Join(path.Dir(extractDir), "."+filepath.Base(extractDir)+"-erofsfuse.log") + if cmdOut, err = os.OpenFile(logf, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644); err != nil { + log.Errorf("Failed to open %s for write: %v", logf, err) + return cmd, err + } + + fiPre, err := os.Lstat(extractDir) + if err != nil { + return cmd, errors.Wrapf(err, "Failed stat'ing %q", extractDir) + } + if fiPre.Mode()&os.ModeSymlink != 0 { + return cmd, errors.Errorf("Refusing to mount onto a symbolic link %q", extractDir) + } + + // It would be nice to only enable debug (or maybe to only log to file at all) + // if 'stacker --debug', but we do not have access to that info here. + // to debug erofsfuse, use "allow_other,debug" + optionArgs := "debug" + cmd = exec.Command(erofsFuseInfo.Path, "-f", "-o", optionArgs, erofsFile, extractDir) + cmd.Stdin = nil + cmd.Stdout = cmdOut + cmd.Stderr = cmdOut + cmdOut.Write([]byte(fmt.Sprintf("# %s\n", strings.Join(cmd.Args, " ")))) + if err != nil { + return cmd, errors.Wrapf(err, "Failed writing to %s", logf) + } + log.Debugf("Extracting %s -> %s with %s [%s]", erofsFile, extractDir, erofsFuseInfo.Path, logf) + err = cmd.Start() + if err != nil { + return cmd, err + } + + // now poll/wait for one of 3 things to happen + // a. child process exits - if it did, then some error has occurred. + // b. the directory Entry is different than it was before the call + // to erofsfuse. We have to do this because we do not have another + // way to know when the mount has been populated. + // https://github.com/vasi/squashfuse/issues/49 + // c. a timeout (timeLimit) was hit + // + // FIXME: this has been borrowed from squashfs code, may not be needed? + startTime := time.Now() + timeLimit := 30 * time.Second + alarmCh := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(alarmCh) + }() + for count := 0; !common.FileChanged(fiPre, extractDir); count++ { + if cmd.ProcessState != nil { + // process exited, the Wait() call in the goroutine above + // caused ProcessState to be populated. + return cmd, errors.Errorf("erofsfuse mount of %s with %s exited unexpectedly with %d", erofsFile, erofsFuseInfo.Path, cmd.ProcessState.ExitCode()) + } + if time.Since(startTime) > timeLimit { + cmd.Process.Kill() + return cmd, errors.Wrapf(err, "Gave up on erofsfuse mount of %s with %s after %s", erofsFile, erofsFuseInfo.Path, timeLimit) + } + if count%10 == 1 { + log.Debugf("%s is not yet mounted...(%s)", extractDir, time.Since(startTime)) + } + time.Sleep(time.Duration(50 * time.Millisecond)) + } + + return cmd, nil +} + +type ExtractPolicy struct { + Extractors []types.FsExtractor + Extractor types.FsExtractor + Excuses map[string]error + initialized bool + mutex sync.Mutex +} + +var exPolInfo struct { + once sync.Once + err error + policy *ExtractPolicy +} + +func NewExtractPolicy(args ...string) (*ExtractPolicy, error) { + p := &ExtractPolicy{ + Extractors: []types.FsExtractor{}, + Excuses: map[string]error{}, + } + + allEx := []types.FsExtractor{ + &KernelExtractor{}, + &ErofsFuseExtractor{}, + &FsckErofsExtractor{}, + } + byName := map[string]types.FsExtractor{} + for _, i := range allEx { + byName[i.Name()] = i + } + + for _, i := range args { + extractor, ok := byName[i] + if !ok { + return nil, errors.Errorf("Unknown extractor: '%s'", i) + } + excuse := extractor.IsAvailable() + if excuse != nil { + p.Excuses[i] = excuse + continue + } + p.Extractors = append(p.Extractors, extractor) + } + return p, nil +} + +type FsckErofsExtractor struct { + mutex sync.Mutex +} + +func (k *FsckErofsExtractor) Name() string { + return "fsck.erofs" +} + +func (k *FsckErofsExtractor) IsAvailable() error { + if common.Which("fsck.erofs") == "" { + return errors.Errorf("no 'fsck.erofs' in PATH") + } + return nil +} + +func (k *FsckErofsExtractor) Mount(erofsFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + // check if already extracted + empty, err := common.IsEmptyDir(extractDir) + if err != nil { + return errors.Wrapf(err, "Error checking for empty dir") + } + if !empty { + return nil + } + + log.Debugf("fsck.erofs %s -> %s", erofsFile, extractDir) + cmd := exec.Command("fsck.erofs", "-d", "--extract", extractDir, erofsFile) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = nil + err = cmd.Run() + + // on failure, remove the directory + if err != nil { + if rmErr := os.RemoveAll(extractDir); rmErr != nil { + log.Errorf("Failed to remove %s after failed extraction of %s: %v", extractDir, erofsFile, rmErr) + } + return err + } + + // assert that extraction must create files. This way we can assume non-empty dir above + // was populated by fsck.erofs. + empty, err = common.IsEmptyDir(extractDir) + if err != nil { + return errors.Errorf("Failed to read %s after successful extraction of %s: %v", + extractDir, erofsFile, err) + } + if empty { + return errors.Errorf("%s was an empty fs image", erofsFile) + } + + return nil +} + +type KernelExtractor struct { + mutex sync.Mutex +} + +func (k *KernelExtractor) Name() string { + return "kmount" +} + +func (k *KernelExtractor) IsAvailable() error { + if !common.AmHostRoot() { + return errors.Errorf("not host root") + } + return nil +} + +func (k *KernelExtractor) Mount(erofsFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + if mounted, err := common.IsMountedAtDir(erofsFile, extractDir); err != nil { + return err + } else if mounted { + return nil + } + + ecmd := []string{"mount", "-terofs", "-oloop,ro", erofsFile, extractDir} + var output bytes.Buffer + cmd := exec.Command(ecmd[0], ecmd[1:]...) + cmd.Stdin = nil + cmd.Stdout = &output + cmd.Stderr = cmd.Stdout + err := cmd.Run() + if err == nil { + return nil + } + + var retErr error + + exitError, ok := err.(*exec.ExitError) + if !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-rc), in exec (%v): %v", + erofsFile, ecmd, err) + } else if status, ok := exitError.Sys().(syscall.WaitStatus); !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-status), in exec (%v): %v", + erofsFile, ecmd, err) + } else { + retErr = errors.Errorf("kmount(%s) exited %d: %v", erofsFile, status.ExitStatus(), output.String()) + } + + return retErr +} + +type ErofsFuseExtractor struct { + mutex sync.Mutex +} + +func (f *ErofsFuseExtractor) Name() string { + return "erofsfuse" +} + +func (f *ErofsFuseExtractor) IsAvailable() error { + once.Do(findErofsFuseInfo) + if erofsFuseInfo.Path == "" { + return errors.Errorf("no 'erofsfuse' in PATH") + } + return nil +} + +func (f *ErofsFuseExtractor) Mount(erofsFile, extractDir string) error { + f.mutex.Lock() + defer f.mutex.Unlock() + + if mounted, err := common.IsMountedAtDir(erofsFile, extractDir); mounted && err == nil { + log.Debugf("[%s] %s already mounted -> %s", f.Name(), erofsFile, extractDir) + return nil + } else if err != nil { + return err + } + + cmd, err := erofsFuse(erofsFile, extractDir) + if err != nil { + return err + } + + log.Debugf("erofsfuse mounted (%d) %s -> %s", cmd.Process.Pid, erofsFile, extractDir) + if err := cmd.Process.Release(); err != nil { + return errors.Errorf("Failed to release process %s: %v", cmd, err) + } + return nil +} + +// ExtractSingleErofsPolicy - extract erofsfile to extractDir +func ExtractSingleErofsPolicy(erofsFile, extractDir string, policy *ExtractPolicy) error { + const initName = "init" + if policy == nil { + return errors.Errorf("policy cannot be nil") + } + + // avoid taking a lock if already initialized (possibly premature optimization) + if !policy.initialized { + policy.mutex.Lock() + // We may have been waiting on the initializer. If so, then the policy will now be initialized. + // if not, then we are the initializer. + if !policy.initialized { + defer policy.mutex.Unlock() + defer func() { + policy.initialized = true + }() + } else { + policy.mutex.Unlock() + } + } + + err := os.MkdirAll(extractDir, 0755) + if err != nil { + return err + } + + fdest, err := filepath.Abs(extractDir) + if err != nil { + return err + } + + if policy.initialized { + if err, ok := policy.Excuses[initName]; ok { + return err + } + return policy.Extractor.Mount(erofsFile, fdest) + } + + // At this point we are the initialzer + if policy.Excuses == nil { + policy.Excuses = map[string]error{} + } + + if len(policy.Extractors) == 0 { + policy.Excuses[initName] = errors.Errorf("policy had no extractors") + return policy.Excuses[initName] + } + + var extractor types.FsExtractor + allExcuses := []string{} + for _, extractor = range policy.Extractors { + err = extractor.Mount(erofsFile, fdest) + if err == nil { + policy.Extractor = extractor + log.Debugf("Selected erofs extractor %s", extractor.Name()) + return nil + } + policy.Excuses[extractor.Name()] = err + } + + for n, exc := range policy.Excuses { + allExcuses = append(allExcuses, fmt.Sprintf("%s: %v", n, exc)) + } + + // nothing worked. populate Excuses[initName] + policy.Excuses[initName] = errors.Errorf("No suitable extractor found:\n %s", strings.Join(allExcuses, "\n ")) + return policy.Excuses[initName] +} + +// ExtractSingleErofs - extract the erofsFile to extractDir +// Initialize a extractPolicy struct and then call ExtractSingleErofsPolicy +// wik()th that. +func ExtractSingleErofs(erofsFile string, extractDir string) error { + exPolInfo.once.Do(func() { + const envName = "STACKER_EROFS_EXTRACT_POLICY" + const defPolicy = "kmount erofsfuse fsc.erofs" + val := os.Getenv(envName) + if val == "" { + val = defPolicy + } + exPolInfo.policy, exPolInfo.err = NewExtractPolicy(strings.Fields(val)...) + if exPolInfo.err == nil { + for k, v := range exPolInfo.policy.Excuses { + log.Debugf(" erofs extractor %s is not available: %v", k, v) + } + } + }) + + if exPolInfo.err != nil { + return exPolInfo.err + } + + return ExtractSingleErofsPolicy(erofsFile, extractDir, exPolInfo.policy) +} + +var checkSupported sync.Once +var zstdIsSuspported bool +var parallelIsSupported bool + +func mkerofsSupportsFeature() (bool, bool) { + checkSupported.Do(func() { + var stdoutBuffer strings.Builder + var stderrBuffer strings.Builder + + cmd := exec.Command("mkfs.erofs", "--help") + cmd.Stdout = &stdoutBuffer + cmd.Stderr = &stderrBuffer + + // Ignore errs here as `mkerofs --help` exit status code is 1 + _ = cmd.Run() + + if strings.Contains(stdoutBuffer.String(), "zstd") || + strings.Contains(stderrBuffer.String(), "zstd") { + zstdIsSuspported = true + } + + if strings.Contains(stdoutBuffer.String(), "workers") || + strings.Contains(stderrBuffer.String(), "workers") { + parallelIsSupported = true + } + }) + + return zstdIsSuspported, parallelIsSupported +} diff --git a/pkg/erofs/fs.go b/pkg/erofs/fs.go new file mode 100644 index 0000000..8f39df4 --- /dev/null +++ b/pkg/erofs/fs.go @@ -0,0 +1,72 @@ +package erofs + +import ( + "io" + "os" + + "github.com/pkg/errors" + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/verity" +) + +type erofs struct { +} + +func New() *erofs { + return &erofs{} +} + +func (er *erofs) Make(tempdir string, rootfs string, eps *common.ExcludePaths, verity verity.VerityMetadata) (io.ReadCloser, string, string, error) { + return MakeErofs(tempdir, rootfs, eps, verity) +} + +func (er *erofs) ExtractSingle(fsImgFile string, extractDir string) error { + return ExtractSingleErofs(fsImgFile, extractDir) +} + +func (er *erofs) Mount(fsImgFile, mountpoint, rootHash string) error { + if !common.AmHostRoot() { + return er.guestMount(fsImgFile, mountpoint) + } + err := er.hostMount(fsImgFile, mountpoint, rootHash) + if err == nil || rootHash != "" { + return err + } + return er.guestMount(fsImgFile, mountpoint) +} + +func fsImgVerityLocation(fsImgFile string) (int64, uint64, error) { + fi, err := os.Stat(fsImgFile) + if err != nil { + return -1, 0, errors.WithStack(err) + } + + sblock, err := readSuperblock(fsImgFile) + if err != nil { + return -1, 0, err + } + + verityOffset, err := verityDataLocation(sblock) + if err != nil { + return -1, 0, err + } + + return fi.Size(), verityOffset, nil +} + +func (er *erofs) hostMount(fsImgFile string, mountpoint string, rootHash string) error { + veritySize, verityOffset, err := fsImgVerityLocation(fsImgFile) + if err != nil { + return err + } + + return common.HostMount(fsImgFile, "erofs", mountpoint, rootHash, veritySize, verityOffset) +} + +func (er *erofs) guestMount(fsImgFile string, mountpoint string) error { + return common.GuestMount(fsImgFile, mountpoint, erofsFuse) +} + +func (er *erofs) Umount(mountpoint string) error { + return common.Umount(mountpoint) +} diff --git a/pkg/erofs/mediatype.go b/pkg/erofs/mediatype.go new file mode 100644 index 0000000..289e32e --- /dev/null +++ b/pkg/erofs/mediatype.go @@ -0,0 +1,24 @@ +package erofs + +import ( + "fmt" + "strings" +) + +type ErofsCompression string + +const ( + BaseMediaTypeLayerErofs = "application/vnd.stacker.image.layer.erofs" + + LZ4HCCompression ErofsCompression = "lz4hc" + LZ4Compression ErofsCompression = "lz4" + ZstdCompression ErofsCompression = "zstd" +) + +func IsErofsMediaType(mediaType string) bool { + return strings.HasPrefix(mediaType, BaseMediaTypeLayerErofs) +} + +func GenerateErofsMediaType(comp ErofsCompression) string { + return fmt.Sprintf("%s+%s", BaseMediaTypeLayerErofs, comp) +} diff --git a/pkg/erofs/superblock.go b/pkg/erofs/superblock.go new file mode 100644 index 0000000..0736e3f --- /dev/null +++ b/pkg/erofs/superblock.go @@ -0,0 +1,216 @@ +package erofs + +import ( + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "os" + + "github.com/pkg/errors" +) + +/* + +https://docs.kernel.org/filesystems/erofs.html + +On-disk details + + |-> aligned with the block size + ____________________________________________________________ +| |SB| | ... | Metadata | ... | Data | Metadata | ... | Data | +|_|__|_|_____|__________|_____|______|__________|_____|______| +0 +1K + +*/ + +const ( + // Definitions for superblock. + superblockMagicV1 = 0xe0f5e1e2 + superblockMagic = superblockMagicV1 + superblockOffset = 1024 + blockSize = 4096 + + // Inode slot size in bit shift. + InodeSlotBits = 5 + + // Max file name length. + MaxNameLen = 255 +) + +// Bit definitions for Inode*::Format. +const ( + InodeLayoutBit = 0 + InodeLayoutBits = 1 + + InodeDataLayoutBit = 1 + InodeDataLayoutBits = 3 +) + +// Inode layouts. +const ( + InodeLayoutCompact = 0 + InodeLayoutExtended = 1 +) + +// Inode data layouts. +const ( + InodeDataLayoutFlatPlain = iota + InodeDataLayoutFlatCompressionLegacy + InodeDataLayoutFlatInline + InodeDataLayoutFlatCompression + InodeDataLayoutChunkBased + InodeDataLayoutMax +) + +// Features w/ backward compatibility. +// This is not exhaustive, unused features are not listed. +const ( + FeatureCompatSuperBlockChecksum = 0x00000001 +) + +// Features w/o backward compatibility. +// +// Any features that aren't in FeatureIncompatSupported are incompatible +// with this implementation. +// +// This is not exhaustive, unused features are not listed. +const ( + FeatureIncompatSupported = 0x0 +) + +// Sizes of on-disk structures in bytes. +const ( + superblockSize = 128 + InodeCompactSize = 32 + InodeExtendedSize = 64 + DirentSize = 12 +) + +type superblock struct { + Magic uint32 + Checksum uint32 + FeatureCompat uint32 + BlockSizeBits uint8 + ExtSlots uint8 + RootNid uint16 + Inodes uint64 + BuildTime uint64 + BuildTimeNsec uint32 + Blocks uint32 + MetaBlockAddr uint32 + XattrBlockAddr uint32 + UUID [16]uint8 + VolumeName [16]uint8 + FeatureIncompat uint32 + Union1 uint16 + ExtraDevices uint16 + DevTableSlotOff uint16 + Reserved [38]uint8 +} + +func verifyChecksum(sb *superblock, sbBlock []byte) error { + if sb.FeatureCompat&FeatureCompatSuperBlockChecksum == 0 { + return nil + } + + sbsum := sb.Checksum + + // zero out Checksum field + sbBlock[superblockOffset+4] = 0 + sbBlock[superblockOffset+5] = 0 + sbBlock[superblockOffset+6] = 0 + sbBlock[superblockOffset+7] = 0 + + table := crc32.MakeTable(crc32.Castagnoli) + + checksum := crc32.Checksum(sbBlock[superblockOffset:superblockOffset+superblockSize], table) + checksum = ^crc32.Update(checksum, table, sbBlock[superblockOffset+superblockSize:]) + if checksum != sbsum { + return fmt.Errorf("invalid checksum: 0x%x, expected: 0x%x", checksum, sbsum) + } + + sb.Checksum = sbsum + + return nil +} + +func parseSuperblock(b []byte) (*superblock, error) { + if len(b) != superblockSize { + return nil, errors.Errorf("superblock had %d bytes instead of expected %d", len(b), superblockSize) + } + + magic := binary.LittleEndian.Uint32(b[0:4]) + if magic != superblockMagic { + return nil, errors.Errorf("superblock had magic of %d instead of expected %d", magic, superblockMagic) + } + + sb := &superblock{ + Magic: magic, // b[0:4] + Checksum: binary.LittleEndian.Uint32(b[4:8]), + FeatureCompat: binary.LittleEndian.Uint32(b[8:12]), + BlockSizeBits: b[12], // b[12:13] + ExtSlots: b[13], // b[13:14] + RootNid: binary.LittleEndian.Uint16(b[14:16]), + Inodes: binary.LittleEndian.Uint64(b[16:24]), + BuildTime: binary.LittleEndian.Uint64(b[24:32]), + BuildTimeNsec: binary.LittleEndian.Uint32(b[32:36]), + Blocks: binary.LittleEndian.Uint32(b[36:40]), + MetaBlockAddr: binary.LittleEndian.Uint32(b[40:44]), + XattrBlockAddr: binary.LittleEndian.Uint32(b[44:48]), + UUID: [16]byte(b[48:64]), + VolumeName: [16]byte(b[64:80]), + FeatureIncompat: binary.LittleEndian.Uint32(b[80:84]), + Union1: binary.LittleEndian.Uint16(b[84:86]), + ExtraDevices: binary.LittleEndian.Uint16(b[86:88]), + DevTableSlotOff: binary.LittleEndian.Uint16(b[88:90]), + Reserved: [38]byte(b[90:128]), + } + + if featureIncompat := sb.FeatureIncompat & ^uint32(FeatureIncompatSupported); featureIncompat != 0 { + return nil, errors.Errorf("unsupported incompatible features detected: 0x%x", featureIncompat) + } + + if (1< %s", k.Name(), squashFile, extractDir) return nil } else if err != nil { @@ -631,7 +527,7 @@ func ExtractSingleSquashPolicy(squashFile, extractDir string, policy *ExtractPol return policy.Excuses[initName] } - var extractor SquashExtractor + var extractor types.FsExtractor allExcuses := []string{} for _, extractor = range policy.Extractors { err = extractor.Mount(squashFile, fdest) @@ -678,6 +574,9 @@ func ExtractSingleSquash(squashFile string, extractDir string) error { return ExtractSingleSquashPolicy(squashFile, extractDir, exPolInfo.policy) } +var checkZstdSupported sync.Once +var zstdIsSuspported bool + func mksquashfsSupportsZstd() bool { checkZstdSupported.Do(func() { var stdoutBuffer strings.Builder @@ -698,49 +597,3 @@ func mksquashfsSupportsZstd() bool { return zstdIsSuspported } - -func isEmptyDir(path string) (bool, error) { - fh, err := os.Open(path) - if err != nil { - return false, err - } - defer fh.Close() - - _, err = fh.ReadDir(1) - if err == io.EOF { - return true, nil - } - return false, err -} - -// which - like the unix utility, return empty string for not-found. -// this might fit well in lib/, but currently lib's test imports -// squashfs creating a import loop. -func which(name string) string { - return whichSearch(name, strings.Split(os.Getenv("PATH"), ":")) -} - -func whichSearch(name string, paths []string) string { - var search []string - - if strings.ContainsRune(name, os.PathSeparator) { - if filepath.IsAbs(name) { - search = []string{name} - } else { - search = []string{"./" + name} - } - } else { - search = []string{} - for _, p := range paths { - search = append(search, filepath.Join(p, name)) - } - } - - for _, fPath := range search { - if err := unix.Access(fPath, unix.X_OK); err == nil { - return fPath - } - } - - return "" -} diff --git a/squashfs/superblock.go b/pkg/squashfs/superblock.go similarity index 100% rename from squashfs/superblock.go rename to pkg/squashfs/superblock.go diff --git a/pkg/squashfs/verity.go b/pkg/squashfs/verity.go new file mode 100644 index 0000000..ba99840 --- /dev/null +++ b/pkg/squashfs/verity.go @@ -0,0 +1,15 @@ +package squashfs + +// verityDataLocation returns the end of filesystem image where the verity data +// can be appended. +// squashfs image must be padded to be 4K aligned. +func verityDataLocation(sblock *superblock) (uint64, error) { + squashLen := sblock.size + + // squashfs is padded out to the nearest 4k + if squashLen%4096 != 0 { + squashLen = squashLen + (4096 - squashLen%4096) + } + + return squashLen, nil +} diff --git a/squashfs/verity_test.go b/pkg/squashfs/verity_test.go similarity index 74% rename from squashfs/verity_test.go rename to pkg/squashfs/verity_test.go index 1ee55b7..017c80b 100644 --- a/squashfs/verity_test.go +++ b/pkg/squashfs/verity_test.go @@ -9,50 +9,9 @@ import ( "testing" "github.com/stretchr/testify/assert" + "machinerun.io/atomfs/pkg/verity" ) -type uidmapTestcase struct { - uidmap string - expected bool -} - -var uidmapTests = []uidmapTestcase{ - { - uidmap: ` 0 0 4294967295`, - expected: true, - }, - { - uidmap: ` 0 0 1000 -2000 2000 1`, - expected: false, - }, - { - uidmap: ` 0 0 1000`, - expected: false, - }, - { - uidmap: ` 10 0 4294967295`, - expected: false, - }, - { - uidmap: ` 0 10 4294967295`, - expected: false, - }, - { - uidmap: ` 0 0 1`, - expected: false, - }, -} - -func TestAmHostRoot(t *testing.T) { - t.Parallel() - assert := assert.New(t) - for _, testcase := range uidmapTests { - v := uidmapIsHost(testcase.uidmap) - assert.Equal(v, testcase.expected) - } -} - func TestVerityMetadata(t *testing.T) { assert := assert.New(t) @@ -67,8 +26,8 @@ func TestVerityMetadata(t *testing.T) { err = os.WriteFile(path.Join(rootfs, "foo"), []byte("bar"), 0644) assert.NoError(err) - reader, _, rootHash, err := MakeSquashfs(tempdir, rootfs, nil, VerityMetadataPresent) - if err == cryptsetupTooOld { + reader, _, rootHash, err := MakeSquashfs(tempdir, rootfs, nil, verity.VerityMetadataPresent) + if err == verity.CryptsetupTooOld { t.Skip("libcryptsetup too old") } assert.NoError(err) diff --git a/pkg/types/fs.go b/pkg/types/fs.go new file mode 100644 index 0000000..c27c333 --- /dev/null +++ b/pkg/types/fs.go @@ -0,0 +1,30 @@ +package fs + +import ( + "io" + + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/verity" +) + +type Filesystem interface { + // Make creates a new filesystem image. + Make(tempdir string, rootfs string, eps *common.ExcludePaths, verity verity.VerityMetadata) (io.ReadCloser, string, string, error) + // ExtractSingle extracts a filesystem image. + ExtractSingle(fsImgFile string, extractDir string) error + // Mount mounts a filesystem image on a given mountpoint. + Mount(fsImgFile, mountpoint, rootHash string) error + // Unmount umounts a filesystem image. + Umount(mountpoint string) error +} + +type FilesystemType string + +type FsExtractor interface { + Name() string + IsAvailable() error + // Mount - Mount or extract path to dest. + // Return nil on "already extracted" + // Return error on failure. + Mount(path, dest string) error +} diff --git a/pkg/verity/metadata.go b/pkg/verity/metadata.go new file mode 100644 index 0000000..6333e79 --- /dev/null +++ b/pkg/verity/metadata.go @@ -0,0 +1,10 @@ +package verity + +type VerityMetadata bool + +const ( + VeritySuffix = "verity" + + VerityMetadataPresent VerityMetadata = true + VerityMetadataMissing VerityMetadata = false +) diff --git a/squashfs/verity.go b/pkg/verity/verity.go similarity index 64% rename from squashfs/verity.go rename to pkg/verity/verity.go index c99fd68..7ae9b81 100644 --- a/squashfs/verity.go +++ b/pkg/verity/verity.go @@ -1,4 +1,4 @@ -package squashfs +package verity // #cgo pkg-config: libcryptsetup devmapper --static // #include @@ -81,17 +81,16 @@ import ( "strconv" "strings" "syscall" + "time" "unsafe" "github.com/freddierice/go-losetup" "github.com/martinjungblut/go-cryptsetup" "github.com/pkg/errors" "golang.org/x/sys/unix" - - "machinerun.io/atomfs/mount" ) -const VerityRootHashAnnotation = "io.stackeroci.stacker.squashfs_verity_root_hash" +const VerityRootHashAnnotation = "io.stackeroci.stacker.atomfs_verity_root_hash" type verityDeviceType struct { Flags uint @@ -139,9 +138,9 @@ func isCryptsetupEINVAL(err error) bool { return ok && cse.Code() == -22 } -var cryptsetupTooOld = errors.Errorf("libcryptsetup not new enough, need >= 2.3.0") +var CryptsetupTooOld = errors.Errorf("libcryptsetup not new enough, need >= 2.3.0") -func appendVerityData(file string) (string, error) { +func AppendVerityData(file string) (string, error) { fi, err := os.Lstat(file) if err != nil { return "", errors.WithStack(err) @@ -149,7 +148,7 @@ func appendVerityData(file string) (string, error) { verityOffset := fi.Size() - // we expect mksquashfs to have padded the file to the nearest 4k + // we expect make fs to have padded the file to the nearest 4k // (dm-verity requires device block size, which is 512 for loopback, // which is a multiple of 4k), let's check that here if verityOffset%512 != 0 { @@ -182,7 +181,7 @@ func appendVerityData(file string) (string, error) { // render a special error message. rootHash, _, err := verityDevice.VolumeKeyGet(cryptsetup.CRYPT_ANY_SLOT, "") if isCryptsetupEINVAL(err) { - return "", cryptsetupTooOld + return "", CryptsetupTooOld } else if err != nil { return "", err } @@ -190,138 +189,27 @@ func appendVerityData(file string) (string, error) { return fmt.Sprintf("%x", rootHash), errors.WithStack(err) } -func verityDataLocation(sblock *superblock) (uint64, error) { - squashLen := sblock.size - - // squashfs is padded out to the nearest 4k - if squashLen%4096 != 0 { - squashLen = squashLen + (4096 - squashLen%4096) - } - - return squashLen, nil -} - func verityName(p string) string { - return fmt.Sprintf("%s-%s", p, veritySuffix) -} - -func fileChanged(a os.FileInfo, path string) bool { - b, err := os.Lstat(path) - if err != nil { - return true - } - return !os.SameFile(a, b) -} - -// Mount a filesystem as container root, without host root -// privileges. We do this using squashfuse. -func GuestMount(squashFile string, mountpoint string) error { - if isMountpoint(mountpoint) { - return errors.Errorf("%s is already mounted", mountpoint) - } - - abs, err := filepath.Abs(squashFile) - if err != nil { - return errors.Errorf("Failed to get absolute path for %s: %v", squashFile, err) - } - squashFile = abs - - abs, err = filepath.Abs(mountpoint) - if err != nil { - return errors.Errorf("Failed to get absolute path for %s: %v", mountpoint, err) - } - mountpoint = abs - - cmd, err := squashFuse(squashFile, mountpoint) - if err != nil { - return err - } - if err := cmd.Process.Release(); err != nil { - return errors.Errorf("Failed to release process after guestmount %s: %v", squashFile, err) - } - return nil -} - -func isMountpoint(dest string) bool { - mounted, err := mount.IsMountpoint(dest) - return err == nil && mounted -} - -// Takes /proc/self/uid_map contents as one string -// Returns true if this is a uidmap representing the whole host -// uid range. -func uidmapIsHost(oneline string) bool { - oneline = strings.TrimSuffix(oneline, "\n") - if len(oneline) == 0 { - return false - } - lines := strings.Split(oneline, "\n") - if len(lines) != 1 { - return false - } - words := strings.Fields(lines[0]) - if len(words) != 3 || words[0] != "0" || words[1] != "0" || words[2] != "4294967295" { - return false - } - - return true -} - -func AmHostRoot() bool { - // if not uid 0, not host root - if os.Geteuid() != 0 { - return false - } - // if uid_map doesn't map 0 to 0, not host root - bytes, err := os.ReadFile("/proc/self/uid_map") - if err != nil { - return false - } - return uidmapIsHost(string(bytes)) -} - -func Mount(squashfs, mountpoint, rootHash string) error { - if !AmHostRoot() { - return GuestMount(squashfs, mountpoint) - } - err := HostMount(squashfs, mountpoint, rootHash) - if err == nil || rootHash != "" { - return err - } - return GuestMount(squashfs, mountpoint) + return fmt.Sprintf("%s-%s", p, VeritySuffix) } -func HostMount(squashfs string, mountpoint string, rootHash string) error { - fi, err := os.Stat(squashfs) - if err != nil { - return errors.WithStack(err) - } - - sblock, err := readSuperblock(squashfs) - if err != nil { - return err - } - - verityOffset, err := verityDataLocation(sblock) - if err != nil { - return err - } - - if verityOffset == uint64(fi.Size()) && rootHash != "" { +func VerityHostMount(fsImgFile string, fsType string, mountpoint string, rootHash string, veritySize int64, verityOffset uint64) error { + if verityOffset == uint64(veritySize) && rootHash != "" { return errors.Errorf("asked for verity but no data present") } - if rootHash == "" && verityOffset != uint64(fi.Size()) { + if rootHash == "" && verityOffset != uint64(veritySize) { return errors.Errorf("verity data present but no root hash specified") } mountSourcePath := "" var verityDevice *cryptsetup.Device - name := verityName(path.Base(squashfs)) + name := verityName(path.Base(fsImgFile)) loopDevNeedsClosedOnErr := false var loopDev losetup.Device + var err error // set up the verity device if necessary if rootHash != "" { @@ -333,7 +221,7 @@ func HostMount(squashfs string, mountpoint string, rootHash string) error { return errors.WithStack(err) } - loopDev, err = losetup.Attach(squashfs, 0, true) + loopDev, err = losetup.Attach(fsImgFile, 0, true) if err != nil { return errors.WithStack(err) } @@ -392,7 +280,7 @@ func HostMount(squashfs string, mountpoint string, rootHash string) error { } } else { - loopDev, err = losetup.Attach(squashfs, 0, true) + loopDev, err = losetup.Attach(fsImgFile, 0, true) if err != nil { return errors.WithStack(err) } @@ -401,7 +289,7 @@ func HostMount(squashfs string, mountpoint string, rootHash string) error { } - err = errors.WithStack(unix.Mount(mountSourcePath, mountpoint, "squashfs", unix.MS_RDONLY, "")) + err = errors.WithStack(unix.Mount(mountSourcePath, mountpoint, fsType, unix.MS_RDONLY, "")) if err != nil { if verityDevice != nil { _ = verityDevice.Deactivate(name) @@ -449,72 +337,58 @@ func findLoopBackingVerity(device string) (int64, error) { return deviceNo, nil } -// unmounts a squash mountpoint and detaches any verity / loop devices that back -// it. Only use this if you are sure the underlying devices aren't in use by -// other mount points. -func Umount(mountpoint string) error { - devPath, err := GetBackingDevice(mountpoint) +func retryIfBusy(fn func() error) error { + const maxRetries = 5 + var err error - err = unix.Unmount(mountpoint, 0) - if err != nil { - return errors.Wrapf(err, "failed unmounting %v", mountpoint) - } + for retry := 0; retry < maxRetries; retry++ { + err = fn() + if err == nil { + return nil + } - return MaybeCleanupBackingDevice(devPath) -} + var errno syscall.Errno + if errors.As(err, &errno) && errno != syscall.EBUSY { + return err + } -func GetBackingDevice(mountpoint string) (string, error) { - mounts, err := mount.ParseMounts("/proc/self/mountinfo") - if err != nil { - return "", err + // sleep between retries + time.Sleep(1 * time.Second) } - theMount, found := mounts.FindMount(mountpoint) - if !found { - return "", errors.Errorf("%s is not a mountpoint", mountpoint) - } - return theMount.Source, nil + return err } -// given a device path that had been used as the backing device for a squash -// mountpoint, cleans up and detaches verity device if it still exists. -// -// If the device path does not exist, that is OK - this happens if the device -// was a regular loopback and not -verity. -func MaybeCleanupBackingDevice(devPath string) error { - if _, err := os.Stat(devPath); err != nil { - if os.IsNotExist(err) { - // It's been detached, this is fine. - return nil - } - return errors.WithStack(err) +func VerityUnmount(mountPath string) error { + // find the loop device that backs the verity device + deviceNo, err := findLoopBackingVerity(mountPath) + if err != nil { + return err } - // was this a verity mount or a regular loopback mount? (if it's a - // regular loopback mount, we detached it above, so need to do anything - // special here; verity doesn't play as nicely) - if strings.HasSuffix(devPath, veritySuffix) { - // find the loop device that backs the verity device - deviceNo, err := findLoopBackingVerity(devPath) + loopDev := losetup.New(uint64(deviceNo), 0) + // here, we don't have the loopback device any more (we detached it + // above). the cryptsetup API allows us to pass NULL for the crypt + // device, but go-cryptsetup doesn't have a way to initialize a NULL + // crypt device short of making the struct by hand like this. + + // erofs may temporarily EBUSY, so retry a few times + err = retryIfBusy(func() error { + err = (&cryptsetup.Device{}).Deactivate(mountPath) if err != nil { return err } - loopDev := losetup.New(uint64(deviceNo), 0) - // here, we don't have the loopback device any more (we detached it - // above). the cryptsetup API allows us to pass NULL for the crypt - // device, but go-cryptsetup doesn't have a way to initialize a NULL - // crypt device short of making the struct by hand like this. - err = (&cryptsetup.Device{}).Deactivate(devPath) - if err != nil { - return errors.WithStack(err) - } + return nil + }) + if err != nil { + return errors.WithStack(err) + } - // finally, kill the loop dev - err = loopDev.Detach() - if err != nil { - return errors.Wrapf(err, "failed to detach loop dev for %v", devPath) - } + // finally, kill the loop dev + err = loopDev.Detach() + if err != nil { + return errors.Wrapf(err, "failed to detach loop dev for %v", mountPath) } // NOTE: because of lazy device destruction, it is possible for a non-verity @@ -524,7 +398,7 @@ func MaybeCleanupBackingDevice(devPath string) error { return nil } -// If we are using squashfuse, then we will be unable to get verity has from +// If we are using fuse, then we will be unable to get verity has from // the mount device. This is not a safe thing, we we only allow it when the // device was mounted originally with AllowMissingVerityData. diff --git a/squashfs/verity_static.go b/pkg/verity/verity_static.go similarity index 94% rename from squashfs/verity_static.go rename to pkg/verity/verity_static.go index feac991..d95456c 100644 --- a/squashfs/verity_static.go +++ b/pkg/verity/verity_static.go @@ -1,7 +1,7 @@ //go:build static_build // +build static_build -package squashfs +package verity // cryptsetup's pkgconfig is broken (it does not set Requires.private or // Libs.private at all), so we do the LDLIBS for it by hand. diff --git a/squashfs/mediatype.go b/squashfs/mediatype.go deleted file mode 100644 index 051fe9b..0000000 --- a/squashfs/mediatype.go +++ /dev/null @@ -1,37 +0,0 @@ -package squashfs - -import ( - "fmt" - "strings" -) - -type SquashfsCompression string -type VerityMetadata bool - -const ( - BaseMediaTypeLayerSquashfs = "application/vnd.stacker.image.layer.squashfs" - - GzipCompression SquashfsCompression = "gzip" - ZstdCompression SquashfsCompression = "zstd" - - veritySuffix = "verity" - - VerityMetadataPresent VerityMetadata = true - VerityMetadataMissing VerityMetadata = false -) - -func IsSquashfsMediaType(mediaType string) bool { - return strings.HasPrefix(mediaType, BaseMediaTypeLayerSquashfs) -} - -func GenerateSquashfsMediaType(comp SquashfsCompression, verity VerityMetadata) string { - verityString := "" - if verity { - verityString = fmt.Sprintf("+%s", veritySuffix) - } - return fmt.Sprintf("%s+%s%s", BaseMediaTypeLayerSquashfs, comp, verityString) -} - -func HasVerityMetadata(mediaType string) VerityMetadata { - return VerityMetadata(strings.HasSuffix(mediaType, veritySuffix)) -} diff --git a/squashfs/squashfs_test.go b/squashfs/squashfs_test.go deleted file mode 100644 index c7b25e2..0000000 --- a/squashfs/squashfs_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package squashfs - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestIsEmpytDir(t *testing.T) { - t.Parallel() - assert := assert.New(t) - v, e := isEmptyDir("/") - assert.NoError(e) - assert.False(v) - - v, e = isEmptyDir("/root") - assert.Error(e) - - dname, err := os.MkdirTemp("", "squashfs_empty_test_dir") - assert.NoError(err) - defer os.RemoveAll(dname) - - v, e = isEmptyDir(dname) - assert.NoError(e) - assert.True(v) -} diff --git a/test/lxc.conf b/test/lxc.conf new file mode 100644 index 0000000..7db3c06 --- /dev/null +++ b/test/lxc.conf @@ -0,0 +1,46 @@ +# Template used to create this container: /usr/share/lxc/templates/lxc-download +# Parameters passed to the template: --dist alpine --release 3.19 --arch amd64 +# For additional config options, please look at lxc.container.conf(5) + +# Uncomment the following line to support nesting containers: +#lxc.include = /usr/share/lxc/config/nesting.conf +# (Be aware this has security implications) + + +# Distribution configuration +lxc.include = /usr/share/lxc/config/common.conf +lxc.include = /usr/share/lxc/config/userns.conf +lxc.arch = linux64 + +# Container specific configuration +lxc.idmap = u 0 165536 65536 +lxc.idmap = g 0 165536 65536 +lxc.rootfs.path = dir://rootfs +lxc.uts.name = mycontainer + +# Network configuration +lxc.net.0.type = veth +lxc.net.0.flags = up +lxc.net.0.link = lxcbr0 + +# mounts +lxc.mount.auto = proc:mixed sys:mixed cgroup:mixed +lxc.mount.entry = tmpfs tmp tmpfs size=1M 0 0 +lxc.mount.entry = /dev/null dev/null none bind,optional,create=file 0 0 +lxc.mount.entry = /dev/urandom dev/urandom none bind,optional,create=file 0 0 +lxc.mount.entry = /dev/random dev/random none bind,optional,create=file 0 0 +#lxc.mount.entry = /bin bin none ro,bind 0 0 +#lxc.mount.entry = /lib lib none ro,rbind 0 0 +#lxc.mount.entry = /sbin sbin none ro,bind 0 0 +#lxc.mount.entry = /usr/bin usr/bin none ro,bind 0 0 +#lxc.mount.entry = /usr/lib usr/lib none ro,bind 0 0 +#lxc.mount.entry = /usr/sbin usr/sbin none ro,bind 0 0 +#lxc.mount.entry = /etc/alternatives etc/alternatives none ro,bind 0 0 +#lxc.mount.entry = /sys sys none ro,bind 0 0 +lxc.mount.entry = /etc/resolv.conf etc/resolv.conf none bind,ro,optional,create=file 0 0 + +lxc.console.path = none + +lxc.environment = PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +#lxc.cap.keep = setuid setgid diff --git a/test/stacker.yaml b/test/stacker.yaml new file mode 100644 index 0000000..4126771 --- /dev/null +++ b/test/stacker.yaml @@ -0,0 +1,22 @@ +base: + from: + type: oci + url: /tmp/oci/busybox:1.37-glibc + run: | + touch /base + truncate -s 10M /test.model +a: + from: + type: built + tag: base + run: touch /a +b: + from: + type: built + tag: base + run: touch /b +c: + from: + type: built + tag: base + run: touch /c diff --git a/utils.go b/utils.go deleted file mode 100644 index 3788018..0000000 --- a/utils.go +++ /dev/null @@ -1,56 +0,0 @@ -package atomfs - -import ( - "fmt" - "os" - "strings" -) - -var TestOverrideRuntimeDirKey = "ATOMFS_TEST_RUN_DIR" - -func EnsureDir(dir string) error { - err := os.MkdirAll(dir, 0755) - if err != nil { - return fmt.Errorf("Failed creating directory %s: %w", dir, err) - } - return nil -} - -func PathExists(d string) bool { - _, err := os.Stat(d) - if err != nil && os.IsNotExist(err) { - return false - } - return true -} - -// remove dir separators to make one dir name. It is OK that this can't be -// cleanly backed out, we don't need it to -func ReplacePathSeparators(p string) string { - if p[0] == '/' { - p = p[1:] - } - return strings.ReplaceAll(p, "/", "-") -} - -func GetMountNSName() (string, error) { - val, err := os.Readlink("/proc/self/ns/mnt") - if err != nil { - return "", err - } - // link target looks like 'mnt:[NUMBER]', we just want NUMBER - return val[5 : len(val)-1], nil -} - -// Allow overriding runtime dir for tests so we can assert empty dirs, etc. -func RuntimeDir(metadir string) string { - testOverrideDir := os.Getenv(TestOverrideRuntimeDirKey) - if testOverrideDir == "" { - if metadir == "" { - return "/run/atomfs" - } else { - return metadir - } - } - return testOverrideDir -}