From 9983a1e6f99d07d425856d5aa3ae9725a5384cd0 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Sat, 2 Nov 2024 06:07:40 +0000 Subject: [PATCH 1/8] feat: initial commit for erofs support Before this commit, only squashfs was supported. However, there are other filesystems such as erofs that fit the same theme, and additional filesystem support requires refactoring and exposing a more generic filesystem interface. pkg/fs/fs.go - Filesystem interface pkg/squashfs - squashfs pkg/erofs - erofs pkg/common - filesystem-agnostic common routines pkg/verity - verity routines Signed-off-by: Ramkumar Chinchani --- .github/workflows/build.yaml | 2 +- atomfs.go | 1 + cmd/atomfs/mount.go | 13 +- cmd/atomfs/umount.go | 8 +- cmd/atomfs/verify.go | 16 +- go.mod | 3 +- go.sum | 7 +- pkg/common/common_test.go | 49 ++ pkg/common/exclude.go | 84 +++ pkg/common/fs.go | 47 ++ pkg/common/fuse.go | 5 + pkg/common/mount.go | 114 +++ utils.go => pkg/common/utils.go | 2 +- pkg/erofs/erofs.go | 652 ++++++++++++++++++ pkg/erofs/fs.go | 72 ++ pkg/erofs/mediatype.go | 30 + pkg/erofs/superblock.go | 216 ++++++ pkg/erofs/verity.go | 8 + pkg/erofs/verity_test.go | 94 +++ pkg/fs/fs.go | 49 ++ {log => pkg/log}/log.go | 0 molecule.go => pkg/molecule/molecule.go | 46 +- .../molecule/molecule_test.go | 2 +- oci.go => pkg/molecule/oci.go | 4 +- {mount => pkg/mount}/mountinfo.go | 0 {oci => pkg/oci}/oci.go | 0 pkg/squashfs/fs.go | 72 ++ {squashfs => pkg/squashfs}/mediatype.go | 16 +- {squashfs => pkg/squashfs}/squashfs.go | 154 +---- {squashfs => pkg/squashfs}/squashfs_test.go | 0 {squashfs => pkg/squashfs}/superblock.go | 0 pkg/squashfs/verity.go | 15 + {squashfs => pkg/squashfs}/verity_test.go | 47 +- pkg/verity/metadata.go | 16 + {squashfs => pkg/verity}/verity.go | 218 +----- {squashfs => pkg/verity}/verity_static.go | 2 +- test/lxc.conf | 46 ++ 37 files changed, 1688 insertions(+), 422 deletions(-) create mode 100644 atomfs.go create mode 100644 pkg/common/common_test.go create mode 100644 pkg/common/exclude.go create mode 100644 pkg/common/fs.go create mode 100644 pkg/common/fuse.go create mode 100644 pkg/common/mount.go rename utils.go => pkg/common/utils.go (98%) create mode 100644 pkg/erofs/erofs.go create mode 100644 pkg/erofs/fs.go create mode 100644 pkg/erofs/mediatype.go create mode 100644 pkg/erofs/superblock.go create mode 100644 pkg/erofs/verity.go create mode 100644 pkg/erofs/verity_test.go create mode 100644 pkg/fs/fs.go rename {log => pkg/log}/log.go (100%) rename molecule.go => pkg/molecule/molecule.go (88%) rename molecule_test.go => pkg/molecule/molecule_test.go (97%) rename oci.go => pkg/molecule/oci.go (96%) rename {mount => pkg/mount}/mountinfo.go (100%) rename {oci => pkg/oci}/oci.go (100%) create mode 100644 pkg/squashfs/fs.go rename {squashfs => pkg/squashfs}/mediatype.go (61%) rename {squashfs => pkg/squashfs}/squashfs.go (84%) rename {squashfs => pkg/squashfs}/squashfs_test.go (100%) rename {squashfs => pkg/squashfs}/superblock.go (100%) create mode 100644 pkg/squashfs/verity.go rename {squashfs => pkg/squashfs}/verity_test.go (74%) create mode 100644 pkg/verity/metadata.go rename {squashfs => pkg/verity}/verity.go (63%) rename {squashfs => pkg/verity}/verity_static.go (94%) create mode 100644 test/lxc.conf diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 3ed285f..95c1868 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -20,7 +20,7 @@ jobs: sudo apt-get update sudo apt-get install bats fuse3 make libcryptsetup-dev libgpgme-dev \ libcap-dev lxc libdevmapper-dev libacl1-dev libarchive-tools \ - squashfuse squashfs-tools + squashfuse squashfs-tools erofs-utils - name: setup lxc run: | chmod ugo+x $HOME diff --git a/atomfs.go b/atomfs.go new file mode 100644 index 0000000..2769f4a --- /dev/null +++ b/atomfs.go @@ -0,0 +1 @@ +package atomfs diff --git a/cmd/atomfs/mount.go b/cmd/atomfs/mount.go index 4eec79d..ec97a10 100644 --- a/cmd/atomfs/mount.go +++ b/cmd/atomfs/mount.go @@ -9,9 +9,8 @@ import ( "github.com/pkg/errors" "github.com/urfave/cli" - - "machinerun.io/atomfs" - "machinerun.io/atomfs/squashfs" + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/molecule" ) var mountCmd = cli.Command{ @@ -51,7 +50,7 @@ func findImage(ctx *cli.Context) (string, string, error) { } ocidir := r[0] tag := r[1] - if !atomfs.PathExists(ocidir) { + if !common.PathExists(ocidir) { return "", "", fmt.Errorf("oci directory %s does not exist: %w", ocidir, mountUsage(ctx.App.Name)) } return ocidir, tag, nil @@ -94,7 +93,7 @@ func doMount(ctx *cli.Context) error { return fmt.Errorf("--persist requires an argument") } } - opts := atomfs.MountOCIOpts{ + opts := molecule.MountOCIOpts{ OCIDir: absOCIDir, Tag: tag, Target: absTarget, @@ -104,7 +103,7 @@ func doMount(ctx *cli.Context) error { MetadataDir: ctx.String("metadir"), // nil here means /run/atomfs } - mol, err := atomfs.BuildMoleculeFromOCI(opts) + mol, err := molecule.BuildMoleculeFromOCI(opts) if err != nil { return errors.Wrapf(err, "couldn't build molecule with opts %+v", opts) } @@ -132,7 +131,7 @@ func amPrivileged() bool { func squashUmount(p string) error { if amPrivileged() { - return squashfs.Umount(p) + return common.Umount(p) } return RunCommand("fusermount", "-u", p) } diff --git a/cmd/atomfs/umount.go b/cmd/atomfs/umount.go index 2e123d7..6217c04 100644 --- a/cmd/atomfs/umount.go +++ b/cmd/atomfs/umount.go @@ -5,8 +5,7 @@ import ( "path/filepath" "github.com/urfave/cli" - "machinerun.io/atomfs" - "machinerun.io/atomfs/mount" + "machinerun.io/atomfs/pkg/common" ) var umountCmd = cli.Command{ @@ -26,11 +25,6 @@ func umountUsage(me string) error { return fmt.Errorf("Usage: %s umount mountpoint", me) } -func isMountpoint(p string) bool { - mounted, err := mount.IsMountpoint(p) - return err == nil && mounted -} - func doUmount(ctx *cli.Context) error { if ctx.NArg() < 1 { return umountUsage(ctx.App.Name) diff --git a/cmd/atomfs/verify.go b/cmd/atomfs/verify.go index c587405..212bf4e 100644 --- a/cmd/atomfs/verify.go +++ b/cmd/atomfs/verify.go @@ -6,10 +6,10 @@ import ( "strings" "github.com/urfave/cli" - "machinerun.io/atomfs" - "machinerun.io/atomfs/log" - "machinerun.io/atomfs/mount" - "machinerun.io/atomfs/squashfs" + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/log" + "machinerun.io/atomfs/pkg/mount" + "machinerun.io/atomfs/pkg/verity" ) var verifyCmd = cli.Command{ @@ -45,16 +45,16 @@ func doVerify(ctx *cli.Context) error { } } - if !isMountpoint(mountpoint) { + if !common.IsMountpoint(mountpoint) { return fmt.Errorf("%s is not a mountpoint", mountpoint) } - mountNSName, err := atomfs.GetMountNSName() + mountNSName, err := common.GetMountNSName() if err != nil { return err } - metadir := filepath.Join(atomfs.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, atomfs.ReplacePathSeparators(mountpoint)) + metadir := filepath.Join(common.RuntimeDir(ctx.String("metadir")), "meta", mountNSName, common.ReplacePathSeparators(mountpoint)) mountsdir := filepath.Join(metadir, "mounts") mounts, err := mount.ParseMounts("/proc/self/mountinfo") @@ -83,7 +83,7 @@ func doVerify(ctx *cli.Context) error { continue } checkedCount = checkedCount + 1 - err = squashfs.ConfirmExistingVerityDeviceCurrentValidity(m.Source) + err = verity.ConfirmExistingVerityDeviceCurrentValidity(m.Source) if err != nil { fmt.Printf("%s: CORRUPTION FOUND\n", m.Source) allOK = false diff --git a/go.mod b/go.mod index a7a003e..be11d9f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/freddierice/go-losetup v0.0.0-20220711213114-2a14873012db github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/image-spec v1.1.0-rc2 + github.com/opencontainers/image-spec v1.1.0 github.com/opencontainers/runc v1.2.3 // indirect github.com/opencontainers/umoci v0.4.8-0.20220412065115-12453f247749 github.com/pkg/errors v0.9.1 @@ -22,7 +22,6 @@ require ( github.com/cyphar/filepath-securejoin v0.3.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/google/go-cmp v0.5.6 // indirect github.com/klauspost/compress v1.15.15 // indirect github.com/klauspost/pgzip v1.2.6-0.20220930104621-17e8dac29df8 // indirect github.com/moby/sys/user v0.3.0 // indirect diff --git a/go.sum b/go.sum index 1eae085..463f799 100644 --- a/go.sum +++ b/go.sum @@ -43,9 +43,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -88,8 +87,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= -github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= diff --git a/pkg/common/common_test.go b/pkg/common/common_test.go new file mode 100644 index 0000000..2d99a2f --- /dev/null +++ b/pkg/common/common_test.go @@ -0,0 +1,49 @@ +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type uidmapTestcase struct { + uidmap string + expected bool +} + +var uidmapTests = []uidmapTestcase{ + { + uidmap: ` 0 0 4294967295`, + expected: true, + }, + { + uidmap: ` 0 0 1000 +2000 2000 1`, + expected: false, + }, + { + uidmap: ` 0 0 1000`, + expected: false, + }, + { + uidmap: ` 10 0 4294967295`, + expected: false, + }, + { + uidmap: ` 0 10 4294967295`, + expected: false, + }, + { + uidmap: ` 0 0 1`, + expected: false, + }, +} + +func TestAmHostRoot(t *testing.T) { + t.Parallel() + assert := assert.New(t) + for _, testcase := range uidmapTests { + v := uidmapIsHost(testcase.uidmap) + assert.Equal(v, testcase.expected) + } +} diff --git a/pkg/common/exclude.go b/pkg/common/exclude.go new file mode 100644 index 0000000..afc4503 --- /dev/null +++ b/pkg/common/exclude.go @@ -0,0 +1,84 @@ +package common + +import ( + "bytes" + "path" + "path/filepath" + "strings" +) + +// ExcludePaths represents a list of paths to exclude in a filesystem listing. +// Users should do something like filepath.Walk() over the whole filesystem, +// calling AddExclude() or AddInclude() based on whether they want to include +// or exclude a particular file. Note that if e.g. /usr is excluded, then +// everyting underneath is also implicitly excluded. The +// AddExclude()/AddInclude() methods do the math to figure out what is the +// correct set of things to exclude or include based on what paths have been +// previously included or excluded. +type ExcludePaths struct { + exclude map[string]bool + include []string +} + +func NewExcludePaths() *ExcludePaths { + return &ExcludePaths{ + exclude: map[string]bool{}, + include: []string{}, + } +} + +func (eps *ExcludePaths) AddExclude(p string) { + for _, inc := range eps.include { + // If /usr/bin/ls has changed but /usr hasn't, we don't want to list + // /usr in the include paths any more, so let's be sure to only + // add things which aren't prefixes. + if strings.HasPrefix(inc, p) { + return + } + } + eps.exclude[p] = true +} + +func (eps *ExcludePaths) AddInclude(orig string, isDir bool) { + // First, remove this thing and all its parents from exclude. + p := orig + + // normalize to the first dir + if !isDir { + p = path.Dir(p) + } + for { + // our paths are all absolute, so this is a base case + if p == "/" { + break + } + + delete(eps.exclude, p) + p = filepath.Dir(p) + } + + // now add it to the list of includes, so we don't accidentally re-add + // anything above. + eps.include = append(eps.include, orig) +} + +func (eps *ExcludePaths) String() (string, error) { + var buf bytes.Buffer + for p := range eps.exclude { + _, err := buf.WriteString(p) + if err != nil { + return "", err + } + _, err = buf.WriteString("\n") + if err != nil { + return "", err + } + } + + _, err := buf.WriteString("\n") + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/pkg/common/fs.go b/pkg/common/fs.go new file mode 100644 index 0000000..0d98b5f --- /dev/null +++ b/pkg/common/fs.go @@ -0,0 +1,47 @@ +package common + +import ( + "os" + "strings" +) + +func FileChanged(a os.FileInfo, path string) bool { + b, err := os.Lstat(path) + if err != nil { + return true + } + return !os.SameFile(a, b) +} + +// Takes /proc/self/uid_map contents as one string +// Returns true if this is a uidmap representing the whole host +// uid range. +func uidmapIsHost(oneline string) bool { + oneline = strings.TrimSuffix(oneline, "\n") + if len(oneline) == 0 { + return false + } + lines := strings.Split(oneline, "\n") + if len(lines) != 1 { + return false + } + words := strings.Fields(lines[0]) + if len(words) != 3 || words[0] != "0" || words[1] != "0" || words[2] != "4294967295" { + return false + } + + return true +} + +func AmHostRoot() bool { + // if not uid 0, not host root + if os.Geteuid() != 0 { + return false + } + // if uid_map doesn't map 0 to 0, not host root + bytes, err := os.ReadFile("/proc/self/uid_map") + if err != nil { + return false + } + return uidmapIsHost(string(bytes)) +} diff --git a/pkg/common/fuse.go b/pkg/common/fuse.go new file mode 100644 index 0000000..034bbbf --- /dev/null +++ b/pkg/common/fuse.go @@ -0,0 +1,5 @@ +package common + +import "os/exec" + +type FuseCmd func(fsImgFile, extractDir string) (*exec.Cmd, error) diff --git a/pkg/common/mount.go b/pkg/common/mount.go new file mode 100644 index 0000000..33e9a55 --- /dev/null +++ b/pkg/common/mount.go @@ -0,0 +1,114 @@ +package common + +import ( + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + "golang.org/x/sys/unix" + "machinerun.io/atomfs/pkg/mount" + "machinerun.io/atomfs/pkg/verity" +) + +func HostMount(fsImgFile string, fsType string, mountpoint string, rootHash string, veritySize int64, verityOffset uint64) error { + return verity.VerityHostMount(fsImgFile, fsType, mountpoint, rootHash, veritySize, verityOffset) +} + +// Mount a filesystem as container root, without host root +// privileges. We do this using fuse "cmd" which is passed in from actual filesystem backends. +func GuestMount(fsImgFile string, mountpoint string, fuseCmd FuseCmd) error { + if IsMountpoint(mountpoint) { + return errors.Errorf("%s is already mounted", mountpoint) + } + + abs, err := filepath.Abs(fsImgFile) + if err != nil { + return errors.Errorf("Failed to get absolute path for %s: %v", fsImgFile, err) + } + fsImgFile = abs + + abs, err = filepath.Abs(mountpoint) + if err != nil { + return errors.Errorf("Failed to get absolute path for %s: %v", mountpoint, err) + } + mountpoint = abs + + cmd, err := fuseCmd(fsImgFile, mountpoint) + if err != nil { + return err + } + + if err := cmd.Process.Release(); err != nil { + return errors.Errorf("Failed to release process after guestmount %s: %v", fsImgFile, err) + } + return nil +} + +func Umount(mountpoint string) error { + mounts, err := mount.ParseMounts("/proc/self/mountinfo") + if err != nil { + return err + } + + // first, find the verity device that backs the mount + theMount, found := mounts.FindMount(mountpoint) + if !found { + return errors.Errorf("%s is not a mountpoint", mountpoint) + } + + err = unix.Unmount(mountpoint, 0) + if err != nil { + return errors.Wrapf(err, "failed unmounting %v", mountpoint) + } + + if _, err := os.Stat(theMount.Source); err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.WithStack(err) + } + + // was this a verity mount or a regular loopback mount? (if it's a + // regular loopback mount, we detached it above, so need to do anything + // special here; verity doesn't play as nicely) + if strings.HasSuffix(theMount.Source, verity.VeritySuffix) { + err = verity.VerityUnmount(theMount.Source) + if err != nil { + return errors.Wrapf(err, "failed verity-unmounting %v", theMount.Source) + } + } + + return nil +} + +func IsMountpoint(dest string) bool { + mounted, err := mount.IsMountpoint(dest) + return err == nil && mounted +} + +func IsMountedAtDir(_, dest string) (bool, error) { + dstat, err := os.Stat(dest) + if os.IsNotExist(err) { + return false, nil + } + if !dstat.IsDir() { + return false, nil + } + mounts, err := mount.ParseMounts("/proc/self/mountinfo") + if err != nil { + return false, err + } + + fdest, err := filepath.Abs(dest) + if err != nil { + return false, err + } + for _, m := range mounts { + if m.Target == fdest { + return true, nil + } + } + + return false, nil +} diff --git a/utils.go b/pkg/common/utils.go similarity index 98% rename from utils.go rename to pkg/common/utils.go index 3788018..324eff9 100644 --- a/utils.go +++ b/pkg/common/utils.go @@ -1,4 +1,4 @@ -package atomfs +package common import ( "fmt" diff --git a/pkg/erofs/erofs.go b/pkg/erofs/erofs.go new file mode 100644 index 0000000..191abae --- /dev/null +++ b/pkg/erofs/erofs.go @@ -0,0 +1,652 @@ +// This package is a small go "library" (read: exec wrapper) around the +// mkfs.erofs binary that provides some useful primitives. +package erofs + +import ( + "bytes" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + "sync" + "syscall" + "time" + + "github.com/pkg/errors" + "golang.org/x/sys/unix" + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/log" + vrty "machinerun.io/atomfs/pkg/verity" +) + +type erofsFuseInfoStruct struct { + Path string + Version string + SupportsNotify bool +} + +var once sync.Once +var erofsFuseInfo = erofsFuseInfoStruct{"", "", false} + +func MakeErofs(tempdir string, rootfs string, eps *common.ExcludePaths, verity vrty.VerityMetadata) (io.ReadCloser, string, string, error) { + var excludesFile string + var err error + var toExclude string + var rootHash string + + if eps != nil { + toExclude, err = eps.String() + if err != nil { + return nil, "", rootHash, errors.Wrapf(err, "couldn't create exclude path list") + } + } + + if len(toExclude) != 0 { + excludes, err := os.CreateTemp(tempdir, "stacker-erofs-exclude-") + if err != nil { + return nil, "", rootHash, err + } + defer os.Remove(excludes.Name()) + + excludesFile = excludes.Name() + _, err = excludes.WriteString(toExclude) + excludes.Close() + if err != nil { + return nil, "", rootHash, err + } + } + + tmpErofs, err := os.CreateTemp(tempdir, "stacker-erofs-img-") + if err != nil { + return nil, "", rootHash, err + } + // the following achieves the effect of creating a temporary file name + // without actually creating the file;the goal being to provide a temporary + // filename to provide to `mkfs.XXX` tool so we have a predictable name to + // consume after `mkfs.XXX` has completed its task. + // + // NB: there's a TOCTOU here; something else can predict and produce + // output in the tempfile name we created after we delete it and before + // `mkfs.XXX` runs. + tmpErofs.Close() + os.Remove(tmpErofs.Name()) + + defer os.Remove(tmpErofs.Name()) + + args := []string{tmpErofs.Name(), rootfs} + compression := LZ4HCCompression + zstdOk, parallelOk := mkerofsSupportsFeature() + if zstdOk { + args = append(args, "-z", "zstd") + compression = ZstdCompression + } + if parallelOk { + args = append(args, "--workers", fmt.Sprintf("%d", runtime.NumCPU())) + } + if len(toExclude) != 0 { + args = append(args, "--exclude-path", excludesFile) + } + cmd := exec.Command("mkfs.erofs", args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err = cmd.Run(); err != nil { + return nil, "", rootHash, errors.Wrap(err, "couldn't build erofs") + } + + if verity { + rootHash, err = vrty.AppendVerityData(tmpErofs.Name()) + if err != nil { + return nil, "", rootHash, err + } + } + + blob, err := os.Open(tmpErofs.Name()) + if err != nil { + return nil, "", rootHash, errors.WithStack(err) + } + + return blob, GenerateErofsMediaType(compression, verity), rootHash, nil +} + +func findErofsFuseInfo() { + var erofsPath string + if p := which("erofsfuse"); p != "" { + erofsPath = p + } else { + erofsPath = which("erofsfuse") + } + if erofsPath == "" { + return + } + version, supportsNotify := erofsfuseSupportsMountNotification(erofsPath) + log.Infof("Found erofsfuse at %s (version=%s notify=%t)", erofsPath, version, supportsNotify) + erofsFuseInfo = erofsFuseInfoStruct{erofsPath, version, supportsNotify} +} + +// erofsfuseSupportsMountNotification - returns true if erofsfuse supports mount +// notification, false otherwise +// erofsfuse is the path to the erofsfuse binary +func erofsfuseSupportsMountNotification(erofsfuse string) (string, bool) { + cmd := exec.Command(erofsfuse) + + // `erofsfuse` always returns an error... so we ignore it. + out, _ := cmd.CombinedOutput() + + firstLine := strings.Split(string(out[:]), "\n")[0] + version := strings.Split(firstLine, " ")[1] + + return version, false +} + +var erofsNotFound = errors.Errorf("erofsfuse program not found") + +// erofsFuse - mount erofsFile to extractDir +// return a pointer to the erofsfuse cmd. +// The caller of the this is responsible for the process created. +func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { + var cmd *exec.Cmd + + once.Do(findErofsFuseInfo) + if erofsFuseInfo.Path == "" { + return cmd, erofsNotFound + } + + notifyOpts := "" + notifyPath := "" + if erofsFuseInfo.SupportsNotify { + sockdir, err := os.MkdirTemp("", "sock") + if err != nil { + return cmd, err + } + defer os.RemoveAll(sockdir) + notifyPath = filepath.Join(sockdir, "notifypipe") + if err := syscall.Mkfifo(notifyPath, 0640); err != nil { + return cmd, err + } + notifyOpts = "notify_pipe=" + notifyPath + } + + // given extractDir of path/to/some/dir[/], log to path/to/some/.dir-erofs.log + extractDir = strings.TrimSuffix(extractDir, "/") + + var cmdOut io.Writer + var err error + + logf := filepath.Join(path.Dir(extractDir), "."+filepath.Base(extractDir)+"-erofsfuse.log") + if cmdOut, err = os.OpenFile(logf, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644); err != nil { + log.Infof("Failed to open %s for write: %v", logf, err) + return cmd, err + } + + fiPre, err := os.Lstat(extractDir) + if err != nil { + return cmd, errors.Wrapf(err, "Failed stat'ing %q", extractDir) + } + if fiPre.Mode()&os.ModeSymlink != 0 { + return cmd, errors.Errorf("Refusing to mount onto a symbolic linkd") + } + + // It would be nice to only enable debug (or maybe to only log to file at all) + // if 'stacker --debug', but we do not have access to that info here. + // to debug erofsfuse, use "allow_other,debug" + optionArgs := "debug" + if notifyOpts != "" { + optionArgs += "," + notifyOpts + } + cmd = exec.Command(erofsFuseInfo.Path, "-f", "-o", optionArgs, erofsFile, extractDir) + cmd.Stdin = nil + cmd.Stdout = cmdOut + cmd.Stderr = cmdOut + cmdOut.Write([]byte(fmt.Sprintf("# %s\n", strings.Join(cmd.Args, " ")))) + if err != nil { + return cmd, errors.Wrapf(err, "Failed writing to %s", logf) + } + log.Debugf("Extracting %s -> %s with %s [%s]", erofsFile, extractDir, erofsFuseInfo.Path, logf) + err = cmd.Start() + if err != nil { + return cmd, err + } + + // now poll/wait for one of 3 things to happen + // a. child process exits - if it did, then some error has occurred. + // b. the directory Entry is different than it was before the call + // to erofsfuse. We have to do this because we do not have another + // way to know when the mount has been populated. + // https://github.com/vasi/erofsfuse/issues/49 + // c. a timeout (timeLimit) was hit + startTime := time.Now() + timeLimit := 30 * time.Second + alarmCh := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(alarmCh) + }() + if erofsFuseInfo.SupportsNotify { + notifyCh := make(chan byte) + log.Infof("%s supports notify pipe, watching %q", erofsFuseInfo.Path, notifyPath) + go func() { + f, err := os.Open(notifyPath) + if err != nil { + return + } + defer f.Close() + b1 := make([]byte, 1) + for { + n1, err := f.Read(b1) + if err != nil { + return + } + if err == nil && n1 >= 1 { + break + } + } + notifyCh <- b1[0] + }() + + select { + case <-alarmCh: + cmd.Process.Kill() + return cmd, errors.Wrapf(err, "Gave up on erofsfuse mount of %s with %s after %s", erofsFile, erofsFuseInfo.Path, timeLimit) + case ret := <-notifyCh: + if ret == 's' { + return cmd, nil + } else { + return cmd, errors.Errorf("erofsfuse returned an error, check %s", logf) + } + } + } + for count := 0; !common.FileChanged(fiPre, extractDir); count++ { + if cmd.ProcessState != nil { + // process exited, the Wait() call in the goroutine above + // caused ProcessState to be populated. + return cmd, errors.Errorf("erofsfuse mount of %s with %s exited unexpectedly with %d", erofsFile, erofsFuseInfo.Path, cmd.ProcessState.ExitCode()) + } + if time.Since(startTime) > timeLimit { + cmd.Process.Kill() + return cmd, errors.Wrapf(err, "Gave up on erofsfuse mount of %s with %s after %s", erofsFile, erofsFuseInfo.Path, timeLimit) + } + if count%10 == 1 { + log.Debugf("%s is not yet mounted...(%s)", extractDir, time.Since(startTime)) + } + time.Sleep(time.Duration(50 * time.Millisecond)) + } + + return cmd, nil +} + +type ExtractPolicy struct { + Extractors []ErofsExtractor + Extractor ErofsExtractor + Excuses map[string]error + initialized bool + mutex sync.Mutex +} + +var exPolInfo struct { + once sync.Once + err error + policy *ExtractPolicy +} + +type ErofsExtractor interface { + Name() string + IsAvailable() error + // Mount - Mount or extract path to dest. + // Return nil on "already extracted" + // Return error on failure. + Mount(path, dest string) error +} + +func NewExtractPolicy(args ...string) (*ExtractPolicy, error) { + p := &ExtractPolicy{ + Extractors: []ErofsExtractor{}, + Excuses: map[string]error{}, + } + + allEx := []ErofsExtractor{ + &KernelExtractor{}, + &ErofsFuseExtractor{}, + &FsckErofsExtractor{}, + } + byName := map[string]ErofsExtractor{} + for _, i := range allEx { + byName[i.Name()] = i + } + + for _, i := range args { + extractor, ok := byName[i] + if !ok { + return nil, errors.Errorf("Unknown extractor: '%s'", i) + } + excuse := extractor.IsAvailable() + if excuse != nil { + p.Excuses[i] = excuse + continue + } + p.Extractors = append(p.Extractors, extractor) + } + return p, nil +} + +type FsckErofsExtractor struct { + mutex sync.Mutex +} + +func (k *FsckErofsExtractor) Name() string { + return "fsck.erofs" +} + +func (k *FsckErofsExtractor) IsAvailable() error { + if which("fsck.erofs") == "" { + return errors.Errorf("no 'fsck.erofs' in PATH") + } + return nil +} + +func (k *FsckErofsExtractor) Mount(squashFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + // check if already extracted + empty, err := isEmptyDir(extractDir) + if err != nil { + return errors.Wrapf(err, "Error checking for empty dir") + } + if !empty { + return nil + } + + log.Debugf("fsck.erofs %s -> %s", squashFile, extractDir) + cmd := exec.Command("fsck.erofs", "-d", "--extract", extractDir, squashFile) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = nil + err = cmd.Run() + + // on failure, remove the directory + if err != nil { + if rmErr := os.RemoveAll(extractDir); rmErr != nil { + log.Errorf("Failed to remove %s after failed extraction of %s: %v", extractDir, squashFile, rmErr) + } + return err + } + + // assert that extraction must create files. This way we can assume non-empty dir above + // was populated by fsck.erofs. + empty, err = isEmptyDir(extractDir) + if err != nil { + return errors.Errorf("Failed to read %s after successful extraction of %s: %v", + extractDir, squashFile, err) + } + if empty { + return errors.Errorf("%s was an empty fs image", squashFile) + } + + return nil +} + +type KernelExtractor struct { + mutex sync.Mutex +} + +func (k *KernelExtractor) Name() string { + return "kmount" +} + +func (k *KernelExtractor) IsAvailable() error { + if !common.AmHostRoot() { + return errors.Errorf("not host root") + } + return nil +} + +func (k *KernelExtractor) Mount(squashFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + if mounted, err := common.IsMountedAtDir(squashFile, extractDir); err != nil { + return err + } else if mounted { + return nil + } + + ecmd := []string{"mount", "-terofs", "-oloop,ro", squashFile, extractDir} + var output bytes.Buffer + cmd := exec.Command(ecmd[0], ecmd[1:]...) + cmd.Stdin = nil + cmd.Stdout = &output + cmd.Stderr = cmd.Stdout + err := cmd.Run() + if err == nil { + return nil + } + + var retErr error + + exitError, ok := err.(*exec.ExitError) + if !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-rc), in exec (%v): %v", + squashFile, ecmd, err) + } else if status, ok := exitError.Sys().(syscall.WaitStatus); !ok { + retErr = errors.Errorf("kmount(%s) had unexpected error (no-status), in exec (%v): %v", + squashFile, ecmd, err) + } else { + retErr = errors.Errorf("kmount(%s) exited %d: %v", squashFile, status.ExitStatus(), output.String()) + } + + return retErr +} + +type ErofsFuseExtractor struct { + mutex sync.Mutex +} + +func (k *ErofsFuseExtractor) Name() string { + return "erofsfuse" +} + +func (k *ErofsFuseExtractor) IsAvailable() error { + once.Do(findErofsFuseInfo) + if erofsFuseInfo.Path == "" { + return errors.Errorf("no 'erofsfuse' in PATH") + } + return nil +} + +func (k *ErofsFuseExtractor) Mount(erofsFile, extractDir string) error { + k.mutex.Lock() + defer k.mutex.Unlock() + + if mounted, err := common.IsMountedAtDir(erofsFile, extractDir); mounted && err == nil { + log.Debugf("[%s] %s already mounted -> %s", k.Name(), erofsFile, extractDir) + return nil + } else if err != nil { + return err + } + + cmd, err := erofsFuse(erofsFile, extractDir) + if err != nil { + return err + } + + log.Debugf("erofsfuse mounted (%d) %s -> %s", cmd.Process.Pid, erofsFile, extractDir) + if err := cmd.Process.Release(); err != nil { + return errors.Errorf("Failed to release process %s: %v", cmd, err) + } + return nil +} + +// ExtractSingleErofsPolicy - extract squashfile to extractDir +func ExtractSingleErofsPolicy(squashFile, extractDir string, policy *ExtractPolicy) error { + const initName = "init" + if policy == nil { + return errors.Errorf("policy cannot be nil") + } + + // avoid taking a lock if already initialized (possibly premature optimization) + if !policy.initialized { + policy.mutex.Lock() + // We may have been waiting on the initializer. If so, then the policy will now be initialized. + // if not, then we are the initializer. + if !policy.initialized { + defer policy.mutex.Unlock() + defer func() { + policy.initialized = true + }() + } else { + policy.mutex.Unlock() + } + } + + err := os.MkdirAll(extractDir, 0755) + if err != nil { + return err + } + + fdest, err := filepath.Abs(extractDir) + if err != nil { + return err + } + + if policy.initialized { + if err, ok := policy.Excuses[initName]; ok { + return err + } + return policy.Extractor.Mount(squashFile, fdest) + } + + // At this point we are the initialzer + if policy.Excuses == nil { + policy.Excuses = map[string]error{} + } + + if len(policy.Extractors) == 0 { + policy.Excuses[initName] = errors.Errorf("policy had no extractors") + return policy.Excuses[initName] + } + + var extractor ErofsExtractor + allExcuses := []string{} + for _, extractor = range policy.Extractors { + err = extractor.Mount(squashFile, fdest) + if err == nil { + policy.Extractor = extractor + log.Debugf("Selected erofs extractor %s", extractor.Name()) + return nil + } + policy.Excuses[extractor.Name()] = err + } + + for n, exc := range policy.Excuses { + allExcuses = append(allExcuses, fmt.Sprintf("%s: %v", n, exc)) + } + + // nothing worked. populate Excuses[initName] + policy.Excuses[initName] = errors.Errorf("No suitable extractor found:\n %s", strings.Join(allExcuses, "\n ")) + return policy.Excuses[initName] +} + +// ExtractSingleErofs - extract the squashFile to extractDir +// Initialize a extractPolicy struct and then call ExtractSingleErofsPolicy +// wik()th that. +func ExtractSingleErofs(squashFile string, extractDir string) error { + exPolInfo.once.Do(func() { + const envName = "STACKER_EROFS_EXTRACT_POLICY" + const defPolicy = "kmount erofsfuse fsc.erofs" + val := os.Getenv(envName) + if val == "" { + val = defPolicy + } + exPolInfo.policy, exPolInfo.err = NewExtractPolicy(strings.Fields(val)...) + if exPolInfo.err == nil { + for k, v := range exPolInfo.policy.Excuses { + log.Debugf(" erofs extractor %s is not available: %v", k, v) + } + } + }) + + if exPolInfo.err != nil { + return exPolInfo.err + } + + return ExtractSingleErofsPolicy(squashFile, extractDir, exPolInfo.policy) +} + +var checkSupported sync.Once +var zstdIsSuspported bool +var parallelIsSupported bool + +func mkerofsSupportsFeature() (bool, bool) { + checkSupported.Do(func() { + var stdoutBuffer strings.Builder + var stderrBuffer strings.Builder + + cmd := exec.Command("mkfs.erofs", "--help") + cmd.Stdout = &stdoutBuffer + cmd.Stderr = &stderrBuffer + + // Ignore errs here as `mkerofs --help` exit status code is 1 + _ = cmd.Run() + + if strings.Contains(stdoutBuffer.String(), "zstd") || + strings.Contains(stderrBuffer.String(), "zstd") { + zstdIsSuspported = true + } + + if strings.Contains(stdoutBuffer.String(), "workers") || + strings.Contains(stderrBuffer.String(), "workers") { + parallelIsSupported = true + } + }) + + return zstdIsSuspported, parallelIsSupported +} + +func isEmptyDir(path string) (bool, error) { + fh, err := os.Open(path) + if err != nil { + return false, err + } + + _, err = fh.ReadDir(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +// which - like the unix utility, return empty string for not-found. +// this might fit well in lib/, but currently lib's test imports +// erofs creating a import loop. +func which(name string) string { + return whichSearch(name, strings.Split(os.Getenv("PATH"), ":")) +} + +func whichSearch(name string, paths []string) string { + var search []string + + if strings.ContainsRune(name, os.PathSeparator) { + if filepath.IsAbs(name) { + search = []string{name} + } else { + search = []string{"./" + name} + } + } else { + search = []string{} + for _, p := range paths { + search = append(search, filepath.Join(p, name)) + } + } + + for _, fPath := range search { + if err := unix.Access(fPath, unix.X_OK); err == nil { + return fPath + } + } + + return "" +} diff --git a/pkg/erofs/fs.go b/pkg/erofs/fs.go new file mode 100644 index 0000000..8f39df4 --- /dev/null +++ b/pkg/erofs/fs.go @@ -0,0 +1,72 @@ +package erofs + +import ( + "io" + "os" + + "github.com/pkg/errors" + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/verity" +) + +type erofs struct { +} + +func New() *erofs { + return &erofs{} +} + +func (er *erofs) Make(tempdir string, rootfs string, eps *common.ExcludePaths, verity verity.VerityMetadata) (io.ReadCloser, string, string, error) { + return MakeErofs(tempdir, rootfs, eps, verity) +} + +func (er *erofs) ExtractSingle(fsImgFile string, extractDir string) error { + return ExtractSingleErofs(fsImgFile, extractDir) +} + +func (er *erofs) Mount(fsImgFile, mountpoint, rootHash string) error { + if !common.AmHostRoot() { + return er.guestMount(fsImgFile, mountpoint) + } + err := er.hostMount(fsImgFile, mountpoint, rootHash) + if err == nil || rootHash != "" { + return err + } + return er.guestMount(fsImgFile, mountpoint) +} + +func fsImgVerityLocation(fsImgFile string) (int64, uint64, error) { + fi, err := os.Stat(fsImgFile) + if err != nil { + return -1, 0, errors.WithStack(err) + } + + sblock, err := readSuperblock(fsImgFile) + if err != nil { + return -1, 0, err + } + + verityOffset, err := verityDataLocation(sblock) + if err != nil { + return -1, 0, err + } + + return fi.Size(), verityOffset, nil +} + +func (er *erofs) hostMount(fsImgFile string, mountpoint string, rootHash string) error { + veritySize, verityOffset, err := fsImgVerityLocation(fsImgFile) + if err != nil { + return err + } + + return common.HostMount(fsImgFile, "erofs", mountpoint, rootHash, veritySize, verityOffset) +} + +func (er *erofs) guestMount(fsImgFile string, mountpoint string) error { + return common.GuestMount(fsImgFile, mountpoint, erofsFuse) +} + +func (er *erofs) Umount(mountpoint string) error { + return common.Umount(mountpoint) +} diff --git a/pkg/erofs/mediatype.go b/pkg/erofs/mediatype.go new file mode 100644 index 0000000..e6975a1 --- /dev/null +++ b/pkg/erofs/mediatype.go @@ -0,0 +1,30 @@ +package erofs + +import ( + "fmt" + "strings" + + vrty "machinerun.io/atomfs/pkg/verity" +) + +type ErofsCompression string + +const ( + BaseMediaTypeLayerErofs = "application/vnd.stacker.image.layer.erofs" + + LZ4HCCompression ErofsCompression = "lz4hc" + LZ4Compression ErofsCompression = "lz4" + ZstdCompression ErofsCompression = "zstd" +) + +func IsErofsMediaType(mediaType string) bool { + return strings.HasPrefix(mediaType, BaseMediaTypeLayerErofs) +} + +func GenerateErofsMediaType(comp ErofsCompression, verity vrty.VerityMetadata) string { + verityString := "" + if verity { + verityString = fmt.Sprintf("+%s", vrty.VeritySuffix) + } + return fmt.Sprintf("%s+%s%s", BaseMediaTypeLayerErofs, comp, verityString) +} diff --git a/pkg/erofs/superblock.go b/pkg/erofs/superblock.go new file mode 100644 index 0000000..0736e3f --- /dev/null +++ b/pkg/erofs/superblock.go @@ -0,0 +1,216 @@ +package erofs + +import ( + "encoding/binary" + "fmt" + "hash/crc32" + "io" + "os" + + "github.com/pkg/errors" +) + +/* + +https://docs.kernel.org/filesystems/erofs.html + +On-disk details + + |-> aligned with the block size + ____________________________________________________________ +| |SB| | ... | Metadata | ... | Data | Metadata | ... | Data | +|_|__|_|_____|__________|_____|______|__________|_____|______| +0 +1K + +*/ + +const ( + // Definitions for superblock. + superblockMagicV1 = 0xe0f5e1e2 + superblockMagic = superblockMagicV1 + superblockOffset = 1024 + blockSize = 4096 + + // Inode slot size in bit shift. + InodeSlotBits = 5 + + // Max file name length. + MaxNameLen = 255 +) + +// Bit definitions for Inode*::Format. +const ( + InodeLayoutBit = 0 + InodeLayoutBits = 1 + + InodeDataLayoutBit = 1 + InodeDataLayoutBits = 3 +) + +// Inode layouts. +const ( + InodeLayoutCompact = 0 + InodeLayoutExtended = 1 +) + +// Inode data layouts. +const ( + InodeDataLayoutFlatPlain = iota + InodeDataLayoutFlatCompressionLegacy + InodeDataLayoutFlatInline + InodeDataLayoutFlatCompression + InodeDataLayoutChunkBased + InodeDataLayoutMax +) + +// Features w/ backward compatibility. +// This is not exhaustive, unused features are not listed. +const ( + FeatureCompatSuperBlockChecksum = 0x00000001 +) + +// Features w/o backward compatibility. +// +// Any features that aren't in FeatureIncompatSupported are incompatible +// with this implementation. +// +// This is not exhaustive, unused features are not listed. +const ( + FeatureIncompatSupported = 0x0 +) + +// Sizes of on-disk structures in bytes. +const ( + superblockSize = 128 + InodeCompactSize = 32 + InodeExtendedSize = 64 + DirentSize = 12 +) + +type superblock struct { + Magic uint32 + Checksum uint32 + FeatureCompat uint32 + BlockSizeBits uint8 + ExtSlots uint8 + RootNid uint16 + Inodes uint64 + BuildTime uint64 + BuildTimeNsec uint32 + Blocks uint32 + MetaBlockAddr uint32 + XattrBlockAddr uint32 + UUID [16]uint8 + VolumeName [16]uint8 + FeatureIncompat uint32 + Union1 uint16 + ExtraDevices uint16 + DevTableSlotOff uint16 + Reserved [38]uint8 +} + +func verifyChecksum(sb *superblock, sbBlock []byte) error { + if sb.FeatureCompat&FeatureCompatSuperBlockChecksum == 0 { + return nil + } + + sbsum := sb.Checksum + + // zero out Checksum field + sbBlock[superblockOffset+4] = 0 + sbBlock[superblockOffset+5] = 0 + sbBlock[superblockOffset+6] = 0 + sbBlock[superblockOffset+7] = 0 + + table := crc32.MakeTable(crc32.Castagnoli) + + checksum := crc32.Checksum(sbBlock[superblockOffset:superblockOffset+superblockSize], table) + checksum = ^crc32.Update(checksum, table, sbBlock[superblockOffset+superblockSize:]) + if checksum != sbsum { + return fmt.Errorf("invalid checksum: 0x%x, expected: 0x%x", checksum, sbsum) + } + + sb.Checksum = sbsum + + return nil +} + +func parseSuperblock(b []byte) (*superblock, error) { + if len(b) != superblockSize { + return nil, errors.Errorf("superblock had %d bytes instead of expected %d", len(b), superblockSize) + } + + magic := binary.LittleEndian.Uint32(b[0:4]) + if magic != superblockMagic { + return nil, errors.Errorf("superblock had magic of %d instead of expected %d", magic, superblockMagic) + } + + sb := &superblock{ + Magic: magic, // b[0:4] + Checksum: binary.LittleEndian.Uint32(b[4:8]), + FeatureCompat: binary.LittleEndian.Uint32(b[8:12]), + BlockSizeBits: b[12], // b[12:13] + ExtSlots: b[13], // b[13:14] + RootNid: binary.LittleEndian.Uint16(b[14:16]), + Inodes: binary.LittleEndian.Uint64(b[16:24]), + BuildTime: binary.LittleEndian.Uint64(b[24:32]), + BuildTimeNsec: binary.LittleEndian.Uint32(b[32:36]), + Blocks: binary.LittleEndian.Uint32(b[36:40]), + MetaBlockAddr: binary.LittleEndian.Uint32(b[40:44]), + XattrBlockAddr: binary.LittleEndian.Uint32(b[44:48]), + UUID: [16]byte(b[48:64]), + VolumeName: [16]byte(b[64:80]), + FeatureIncompat: binary.LittleEndian.Uint32(b[80:84]), + Union1: binary.LittleEndian.Uint16(b[84:86]), + ExtraDevices: binary.LittleEndian.Uint16(b[86:88]), + DevTableSlotOff: binary.LittleEndian.Uint16(b[88:90]), + Reserved: [38]byte(b[90:128]), + } + + if featureIncompat := sb.FeatureIncompat & ^uint32(FeatureIncompatSupported); featureIncompat != 0 { + return nil, errors.Errorf("unsupported incompatible features detected: 0x%x", featureIncompat) + } + + if (1< %s", k.Name(), squashFile, extractDir) return nil } else if err != nil { @@ -678,6 +583,9 @@ func ExtractSingleSquash(squashFile string, extractDir string) error { return ExtractSingleSquashPolicy(squashFile, extractDir, exPolInfo.policy) } +var checkZstdSupported sync.Once +var zstdIsSuspported bool + func mksquashfsSupportsZstd() bool { checkZstdSupported.Do(func() { var stdoutBuffer strings.Builder diff --git a/squashfs/squashfs_test.go b/pkg/squashfs/squashfs_test.go similarity index 100% rename from squashfs/squashfs_test.go rename to pkg/squashfs/squashfs_test.go diff --git a/squashfs/superblock.go b/pkg/squashfs/superblock.go similarity index 100% rename from squashfs/superblock.go rename to pkg/squashfs/superblock.go diff --git a/pkg/squashfs/verity.go b/pkg/squashfs/verity.go new file mode 100644 index 0000000..ba99840 --- /dev/null +++ b/pkg/squashfs/verity.go @@ -0,0 +1,15 @@ +package squashfs + +// verityDataLocation returns the end of filesystem image where the verity data +// can be appended. +// squashfs image must be padded to be 4K aligned. +func verityDataLocation(sblock *superblock) (uint64, error) { + squashLen := sblock.size + + // squashfs is padded out to the nearest 4k + if squashLen%4096 != 0 { + squashLen = squashLen + (4096 - squashLen%4096) + } + + return squashLen, nil +} diff --git a/squashfs/verity_test.go b/pkg/squashfs/verity_test.go similarity index 74% rename from squashfs/verity_test.go rename to pkg/squashfs/verity_test.go index 1ee55b7..017c80b 100644 --- a/squashfs/verity_test.go +++ b/pkg/squashfs/verity_test.go @@ -9,50 +9,9 @@ import ( "testing" "github.com/stretchr/testify/assert" + "machinerun.io/atomfs/pkg/verity" ) -type uidmapTestcase struct { - uidmap string - expected bool -} - -var uidmapTests = []uidmapTestcase{ - { - uidmap: ` 0 0 4294967295`, - expected: true, - }, - { - uidmap: ` 0 0 1000 -2000 2000 1`, - expected: false, - }, - { - uidmap: ` 0 0 1000`, - expected: false, - }, - { - uidmap: ` 10 0 4294967295`, - expected: false, - }, - { - uidmap: ` 0 10 4294967295`, - expected: false, - }, - { - uidmap: ` 0 0 1`, - expected: false, - }, -} - -func TestAmHostRoot(t *testing.T) { - t.Parallel() - assert := assert.New(t) - for _, testcase := range uidmapTests { - v := uidmapIsHost(testcase.uidmap) - assert.Equal(v, testcase.expected) - } -} - func TestVerityMetadata(t *testing.T) { assert := assert.New(t) @@ -67,8 +26,8 @@ func TestVerityMetadata(t *testing.T) { err = os.WriteFile(path.Join(rootfs, "foo"), []byte("bar"), 0644) assert.NoError(err) - reader, _, rootHash, err := MakeSquashfs(tempdir, rootfs, nil, VerityMetadataPresent) - if err == cryptsetupTooOld { + reader, _, rootHash, err := MakeSquashfs(tempdir, rootfs, nil, verity.VerityMetadataPresent) + if err == verity.CryptsetupTooOld { t.Skip("libcryptsetup too old") } assert.NoError(err) diff --git a/pkg/verity/metadata.go b/pkg/verity/metadata.go new file mode 100644 index 0000000..6bfabbb --- /dev/null +++ b/pkg/verity/metadata.go @@ -0,0 +1,16 @@ +package verity + +import "strings" + +type VerityMetadata bool + +const ( + VeritySuffix = "verity" + + VerityMetadataPresent VerityMetadata = true + VerityMetadataMissing VerityMetadata = false +) + +func HasVerityMetadata(mediaType string) VerityMetadata { + return VerityMetadata(strings.HasSuffix(mediaType, VeritySuffix)) +} diff --git a/squashfs/verity.go b/pkg/verity/verity.go similarity index 63% rename from squashfs/verity.go rename to pkg/verity/verity.go index c99fd68..6d0fe01 100644 --- a/squashfs/verity.go +++ b/pkg/verity/verity.go @@ -1,4 +1,4 @@ -package squashfs +package verity // #cgo pkg-config: libcryptsetup devmapper --static // #include @@ -87,11 +87,9 @@ import ( "github.com/martinjungblut/go-cryptsetup" "github.com/pkg/errors" "golang.org/x/sys/unix" - - "machinerun.io/atomfs/mount" ) -const VerityRootHashAnnotation = "io.stackeroci.stacker.squashfs_verity_root_hash" +const VerityRootHashAnnotation = "io.stackeroci.stacker.atomfs_verity_root_hash" type verityDeviceType struct { Flags uint @@ -139,9 +137,9 @@ func isCryptsetupEINVAL(err error) bool { return ok && cse.Code() == -22 } -var cryptsetupTooOld = errors.Errorf("libcryptsetup not new enough, need >= 2.3.0") +var CryptsetupTooOld = errors.Errorf("libcryptsetup not new enough, need >= 2.3.0") -func appendVerityData(file string) (string, error) { +func AppendVerityData(file string) (string, error) { fi, err := os.Lstat(file) if err != nil { return "", errors.WithStack(err) @@ -149,7 +147,7 @@ func appendVerityData(file string) (string, error) { verityOffset := fi.Size() - // we expect mksquashfs to have padded the file to the nearest 4k + // we expect make fs to have padded the file to the nearest 4k // (dm-verity requires device block size, which is 512 for loopback, // which is a multiple of 4k), let's check that here if verityOffset%512 != 0 { @@ -182,7 +180,7 @@ func appendVerityData(file string) (string, error) { // render a special error message. rootHash, _, err := verityDevice.VolumeKeyGet(cryptsetup.CRYPT_ANY_SLOT, "") if isCryptsetupEINVAL(err) { - return "", cryptsetupTooOld + return "", CryptsetupTooOld } else if err != nil { return "", err } @@ -190,138 +188,27 @@ func appendVerityData(file string) (string, error) { return fmt.Sprintf("%x", rootHash), errors.WithStack(err) } -func verityDataLocation(sblock *superblock) (uint64, error) { - squashLen := sblock.size - - // squashfs is padded out to the nearest 4k - if squashLen%4096 != 0 { - squashLen = squashLen + (4096 - squashLen%4096) - } - - return squashLen, nil -} - func verityName(p string) string { - return fmt.Sprintf("%s-%s", p, veritySuffix) -} - -func fileChanged(a os.FileInfo, path string) bool { - b, err := os.Lstat(path) - if err != nil { - return true - } - return !os.SameFile(a, b) -} - -// Mount a filesystem as container root, without host root -// privileges. We do this using squashfuse. -func GuestMount(squashFile string, mountpoint string) error { - if isMountpoint(mountpoint) { - return errors.Errorf("%s is already mounted", mountpoint) - } - - abs, err := filepath.Abs(squashFile) - if err != nil { - return errors.Errorf("Failed to get absolute path for %s: %v", squashFile, err) - } - squashFile = abs - - abs, err = filepath.Abs(mountpoint) - if err != nil { - return errors.Errorf("Failed to get absolute path for %s: %v", mountpoint, err) - } - mountpoint = abs - - cmd, err := squashFuse(squashFile, mountpoint) - if err != nil { - return err - } - if err := cmd.Process.Release(); err != nil { - return errors.Errorf("Failed to release process after guestmount %s: %v", squashFile, err) - } - return nil -} - -func isMountpoint(dest string) bool { - mounted, err := mount.IsMountpoint(dest) - return err == nil && mounted -} - -// Takes /proc/self/uid_map contents as one string -// Returns true if this is a uidmap representing the whole host -// uid range. -func uidmapIsHost(oneline string) bool { - oneline = strings.TrimSuffix(oneline, "\n") - if len(oneline) == 0 { - return false - } - lines := strings.Split(oneline, "\n") - if len(lines) != 1 { - return false - } - words := strings.Fields(lines[0]) - if len(words) != 3 || words[0] != "0" || words[1] != "0" || words[2] != "4294967295" { - return false - } - - return true -} - -func AmHostRoot() bool { - // if not uid 0, not host root - if os.Geteuid() != 0 { - return false - } - // if uid_map doesn't map 0 to 0, not host root - bytes, err := os.ReadFile("/proc/self/uid_map") - if err != nil { - return false - } - return uidmapIsHost(string(bytes)) -} - -func Mount(squashfs, mountpoint, rootHash string) error { - if !AmHostRoot() { - return GuestMount(squashfs, mountpoint) - } - err := HostMount(squashfs, mountpoint, rootHash) - if err == nil || rootHash != "" { - return err - } - return GuestMount(squashfs, mountpoint) + return fmt.Sprintf("%s-%s", p, VeritySuffix) } -func HostMount(squashfs string, mountpoint string, rootHash string) error { - fi, err := os.Stat(squashfs) - if err != nil { - return errors.WithStack(err) - } - - sblock, err := readSuperblock(squashfs) - if err != nil { - return err - } - - verityOffset, err := verityDataLocation(sblock) - if err != nil { - return err - } - - if verityOffset == uint64(fi.Size()) && rootHash != "" { +func VerityHostMount(fsImgFile string, fsType string, mountpoint string, rootHash string, veritySize int64, verityOffset uint64) error { + if verityOffset == uint64(veritySize) && rootHash != "" { return errors.Errorf("asked for verity but no data present") } - if rootHash == "" && verityOffset != uint64(fi.Size()) { + if rootHash == "" && verityOffset != uint64(veritySize) { return errors.Errorf("verity data present but no root hash specified") } mountSourcePath := "" var verityDevice *cryptsetup.Device - name := verityName(path.Base(squashfs)) + name := verityName(path.Base(fsImgFile)) loopDevNeedsClosedOnErr := false var loopDev losetup.Device + var err error // set up the verity device if necessary if rootHash != "" { @@ -333,7 +220,7 @@ func HostMount(squashfs string, mountpoint string, rootHash string) error { return errors.WithStack(err) } - loopDev, err = losetup.Attach(squashfs, 0, true) + loopDev, err = losetup.Attach(fsImgFile, 0, true) if err != nil { return errors.WithStack(err) } @@ -392,7 +279,7 @@ func HostMount(squashfs string, mountpoint string, rootHash string) error { } } else { - loopDev, err = losetup.Attach(squashfs, 0, true) + loopDev, err = losetup.Attach(fsImgFile, 0, true) if err != nil { return errors.WithStack(err) } @@ -401,7 +288,7 @@ func HostMount(squashfs string, mountpoint string, rootHash string) error { } - err = errors.WithStack(unix.Mount(mountSourcePath, mountpoint, "squashfs", unix.MS_RDONLY, "")) + err = errors.WithStack(unix.Mount(mountSourcePath, mountpoint, fsType, unix.MS_RDONLY, "")) if err != nil { if verityDevice != nil { _ = verityDevice.Deactivate(name) @@ -449,72 +336,27 @@ func findLoopBackingVerity(device string) (int64, error) { return deviceNo, nil } -// unmounts a squash mountpoint and detaches any verity / loop devices that back -// it. Only use this if you are sure the underlying devices aren't in use by -// other mount points. -func Umount(mountpoint string) error { - devPath, err := GetBackingDevice(mountpoint) - - err = unix.Unmount(mountpoint, 0) +func VerityUnmount(mountPath string) error { + // find the loop device that backs the verity device + deviceNo, err := findLoopBackingVerity(mountPath) if err != nil { - return errors.Wrapf(err, "failed unmounting %v", mountpoint) + return err } - return MaybeCleanupBackingDevice(devPath) -} - -func GetBackingDevice(mountpoint string) (string, error) { - mounts, err := mount.ParseMounts("/proc/self/mountinfo") + loopDev := losetup.New(uint64(deviceNo), 0) + // here, we don't have the loopback device any more (we detached it + // above). the cryptsetup API allows us to pass NULL for the crypt + // device, but go-cryptsetup doesn't have a way to initialize a NULL + // crypt device short of making the struct by hand like this. + err = (&cryptsetup.Device{}).Deactivate(mountPath) if err != nil { - return "", err - } - - theMount, found := mounts.FindMount(mountpoint) - if !found { - return "", errors.Errorf("%s is not a mountpoint", mountpoint) - } - return theMount.Source, nil -} - -// given a device path that had been used as the backing device for a squash -// mountpoint, cleans up and detaches verity device if it still exists. -// -// If the device path does not exist, that is OK - this happens if the device -// was a regular loopback and not -verity. -func MaybeCleanupBackingDevice(devPath string) error { - if _, err := os.Stat(devPath); err != nil { - if os.IsNotExist(err) { - // It's been detached, this is fine. - return nil - } return errors.WithStack(err) } - // was this a verity mount or a regular loopback mount? (if it's a - // regular loopback mount, we detached it above, so need to do anything - // special here; verity doesn't play as nicely) - if strings.HasSuffix(devPath, veritySuffix) { - // find the loop device that backs the verity device - deviceNo, err := findLoopBackingVerity(devPath) - if err != nil { - return err - } - - loopDev := losetup.New(uint64(deviceNo), 0) - // here, we don't have the loopback device any more (we detached it - // above). the cryptsetup API allows us to pass NULL for the crypt - // device, but go-cryptsetup doesn't have a way to initialize a NULL - // crypt device short of making the struct by hand like this. - err = (&cryptsetup.Device{}).Deactivate(devPath) - if err != nil { - return errors.WithStack(err) - } - - // finally, kill the loop dev - err = loopDev.Detach() - if err != nil { - return errors.Wrapf(err, "failed to detach loop dev for %v", devPath) - } + // finally, kill the loop dev + err = loopDev.Detach() + if err != nil { + return errors.Wrapf(err, "failed to detach loop dev for %v", mountPath) } // NOTE: because of lazy device destruction, it is possible for a non-verity @@ -524,7 +366,7 @@ func MaybeCleanupBackingDevice(devPath string) error { return nil } -// If we are using squashfuse, then we will be unable to get verity has from +// If we are using fuse, then we will be unable to get verity has from // the mount device. This is not a safe thing, we we only allow it when the // device was mounted originally with AllowMissingVerityData. diff --git a/squashfs/verity_static.go b/pkg/verity/verity_static.go similarity index 94% rename from squashfs/verity_static.go rename to pkg/verity/verity_static.go index feac991..d95456c 100644 --- a/squashfs/verity_static.go +++ b/pkg/verity/verity_static.go @@ -1,7 +1,7 @@ //go:build static_build // +build static_build -package squashfs +package verity // cryptsetup's pkgconfig is broken (it does not set Requires.private or // Libs.private at all), so we do the LDLIBS for it by hand. diff --git a/test/lxc.conf b/test/lxc.conf new file mode 100644 index 0000000..7db3c06 --- /dev/null +++ b/test/lxc.conf @@ -0,0 +1,46 @@ +# Template used to create this container: /usr/share/lxc/templates/lxc-download +# Parameters passed to the template: --dist alpine --release 3.19 --arch amd64 +# For additional config options, please look at lxc.container.conf(5) + +# Uncomment the following line to support nesting containers: +#lxc.include = /usr/share/lxc/config/nesting.conf +# (Be aware this has security implications) + + +# Distribution configuration +lxc.include = /usr/share/lxc/config/common.conf +lxc.include = /usr/share/lxc/config/userns.conf +lxc.arch = linux64 + +# Container specific configuration +lxc.idmap = u 0 165536 65536 +lxc.idmap = g 0 165536 65536 +lxc.rootfs.path = dir://rootfs +lxc.uts.name = mycontainer + +# Network configuration +lxc.net.0.type = veth +lxc.net.0.flags = up +lxc.net.0.link = lxcbr0 + +# mounts +lxc.mount.auto = proc:mixed sys:mixed cgroup:mixed +lxc.mount.entry = tmpfs tmp tmpfs size=1M 0 0 +lxc.mount.entry = /dev/null dev/null none bind,optional,create=file 0 0 +lxc.mount.entry = /dev/urandom dev/urandom none bind,optional,create=file 0 0 +lxc.mount.entry = /dev/random dev/random none bind,optional,create=file 0 0 +#lxc.mount.entry = /bin bin none ro,bind 0 0 +#lxc.mount.entry = /lib lib none ro,rbind 0 0 +#lxc.mount.entry = /sbin sbin none ro,bind 0 0 +#lxc.mount.entry = /usr/bin usr/bin none ro,bind 0 0 +#lxc.mount.entry = /usr/lib usr/lib none ro,bind 0 0 +#lxc.mount.entry = /usr/sbin usr/sbin none ro,bind 0 0 +#lxc.mount.entry = /etc/alternatives etc/alternatives none ro,bind 0 0 +#lxc.mount.entry = /sys sys none ro,bind 0 0 +lxc.mount.entry = /etc/resolv.conf etc/resolv.conf none bind,ro,optional,create=file 0 0 + +lxc.console.path = none + +lxc.environment = PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +#lxc.cap.keep = setuid setgid From 2a4417389fefcf770818e035a9a164431a86d0c9 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Tue, 3 Dec 2024 03:51:57 +0000 Subject: [PATCH 2/8] fix: disable "advanced/incompatible" features for now Signed-off-by: Ramkumar Chinchani --- pkg/erofs/erofs.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/pkg/erofs/erofs.go b/pkg/erofs/erofs.go index 191abae..e488013 100644 --- a/pkg/erofs/erofs.go +++ b/pkg/erofs/erofs.go @@ -79,13 +79,15 @@ func MakeErofs(tempdir string, rootfs string, eps *common.ExcludePaths, verity v args := []string{tmpErofs.Name(), rootfs} compression := LZ4HCCompression - zstdOk, parallelOk := mkerofsSupportsFeature() - if zstdOk { - args = append(args, "-z", "zstd") - compression = ZstdCompression - } - if parallelOk { - args = append(args, "--workers", fmt.Sprintf("%d", runtime.NumCPU())) + if false { // FIXME: following features are experimental, disabling for now + zstdOk, parallelOk := mkerofsSupportsFeature() + if zstdOk { + args = append(args, "-z", "zstd") + compression = ZstdCompression + } + if parallelOk { + args = append(args, "--workers", fmt.Sprintf("%d", runtime.NumCPU())) + } } if len(toExclude) != 0 { args = append(args, "--exclude-path", excludesFile) From 6057e2bca190990b600c73b6bf9ab4bf6b96088a Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Wed, 4 Dec 2024 02:41:14 +0000 Subject: [PATCH 3/8] fix!: drop the double invalid '+' in layer media-type BREAKING-CHANGE: the layer media-type no longer contains "+verity" For a layer media-type, we add the fstype+compression+verity_present. Only one '+' is allowed as per following RFC. https://datatracker.ietf.org/doc/html/rfc6838#section-4.2 Instead just rely on "root_hash" annotation. Signed-off-by: Ramkumar Chinchani --- pkg/erofs/erofs.go | 2 +- pkg/erofs/mediatype.go | 10 ++-------- pkg/squashfs/mediatype.go | 10 ++-------- pkg/squashfs/squashfs.go | 2 +- 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/pkg/erofs/erofs.go b/pkg/erofs/erofs.go index e488013..340427e 100644 --- a/pkg/erofs/erofs.go +++ b/pkg/erofs/erofs.go @@ -111,7 +111,7 @@ func MakeErofs(tempdir string, rootfs string, eps *common.ExcludePaths, verity v return nil, "", rootHash, errors.WithStack(err) } - return blob, GenerateErofsMediaType(compression, verity), rootHash, nil + return blob, GenerateErofsMediaType(compression), rootHash, nil } func findErofsFuseInfo() { diff --git a/pkg/erofs/mediatype.go b/pkg/erofs/mediatype.go index e6975a1..289e32e 100644 --- a/pkg/erofs/mediatype.go +++ b/pkg/erofs/mediatype.go @@ -3,8 +3,6 @@ package erofs import ( "fmt" "strings" - - vrty "machinerun.io/atomfs/pkg/verity" ) type ErofsCompression string @@ -21,10 +19,6 @@ func IsErofsMediaType(mediaType string) bool { return strings.HasPrefix(mediaType, BaseMediaTypeLayerErofs) } -func GenerateErofsMediaType(comp ErofsCompression, verity vrty.VerityMetadata) string { - verityString := "" - if verity { - verityString = fmt.Sprintf("+%s", vrty.VeritySuffix) - } - return fmt.Sprintf("%s+%s%s", BaseMediaTypeLayerErofs, comp, verityString) +func GenerateErofsMediaType(comp ErofsCompression) string { + return fmt.Sprintf("%s+%s", BaseMediaTypeLayerErofs, comp) } diff --git a/pkg/squashfs/mediatype.go b/pkg/squashfs/mediatype.go index e1e007f..39ae9f8 100644 --- a/pkg/squashfs/mediatype.go +++ b/pkg/squashfs/mediatype.go @@ -3,8 +3,6 @@ package squashfs import ( "fmt" "strings" - - vrty "machinerun.io/atomfs/pkg/verity" ) type SquashfsCompression string @@ -20,10 +18,6 @@ func IsSquashfsMediaType(mediaType string) bool { return strings.HasPrefix(mediaType, BaseMediaTypeLayerSquashfs) } -func GenerateSquashfsMediaType(comp SquashfsCompression, verity vrty.VerityMetadata) string { - verityString := "" - if verity { - verityString = fmt.Sprintf("+%s", vrty.VeritySuffix) - } - return fmt.Sprintf("%s+%s%s", BaseMediaTypeLayerSquashfs, comp, verityString) +func GenerateSquashfsMediaType(comp SquashfsCompression) string { + return fmt.Sprintf("%s+%s", BaseMediaTypeLayerSquashfs, comp) } diff --git a/pkg/squashfs/squashfs.go b/pkg/squashfs/squashfs.go index 966c262..e9592ba 100644 --- a/pkg/squashfs/squashfs.go +++ b/pkg/squashfs/squashfs.go @@ -105,7 +105,7 @@ func MakeSquashfs(tempdir string, rootfs string, eps *common.ExcludePaths, verit return nil, "", rootHash, errors.WithStack(err) } - return blob, GenerateSquashfsMediaType(compression, verity), rootHash, nil + return blob, GenerateSquashfsMediaType(compression), rootHash, nil } func findSquashFuseInfo() { From 21b98ca9e4736ab01ac981798c0c1d6520c793f7 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Wed, 4 Dec 2024 02:56:35 +0000 Subject: [PATCH 4/8] test: add a stacker.yaml for tests --- test/stacker.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 test/stacker.yaml diff --git a/test/stacker.yaml b/test/stacker.yaml new file mode 100644 index 0000000..4126771 --- /dev/null +++ b/test/stacker.yaml @@ -0,0 +1,22 @@ +base: + from: + type: oci + url: /tmp/oci/busybox:1.37-glibc + run: | + touch /base + truncate -s 10M /test.model +a: + from: + type: built + tag: base + run: touch /a +b: + from: + type: built + tag: base + run: touch /b +c: + from: + type: built + tag: base + run: touch /c From 03ba08229270fecac2b5fbb66f0dc03c38556e86 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Fri, 31 Jan 2025 18:25:47 -0800 Subject: [PATCH 5/8] fix: rebasing after Mike's changes Signed-off-by: Ramkumar Chinchani --- cmd/atomfs/umount.go | 4 ++-- pkg/common/mount.go | 41 ++++++++++++++++++++++++---------- pkg/erofs/erofs.go | 48 +++++++++++++++++++--------------------- pkg/molecule/molecule.go | 7 ++++-- pkg/squashfs/squashfs.go | 8 +++---- pkg/verity/metadata.go | 6 ----- pkg/verity/verity.go | 34 +++++++++++++++++++++++++++- 7 files changed, 96 insertions(+), 52 deletions(-) diff --git a/cmd/atomfs/umount.go b/cmd/atomfs/umount.go index 6217c04..129e5be 100644 --- a/cmd/atomfs/umount.go +++ b/cmd/atomfs/umount.go @@ -5,7 +5,7 @@ import ( "path/filepath" "github.com/urfave/cli" - "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/molecule" ) var umountCmd = cli.Command{ @@ -40,5 +40,5 @@ func doUmount(ctx *cli.Context) error { } } - return atomfs.UmountWithMetadir(mountpoint, ctx.String("metadir")) + return molecule.UmountWithMetadir(mountpoint, ctx.String("metadir")) } diff --git a/pkg/common/mount.go b/pkg/common/mount.go index 33e9a55..600a4cd 100644 --- a/pkg/common/mount.go +++ b/pkg/common/mount.go @@ -45,25 +45,42 @@ func GuestMount(fsImgFile string, mountpoint string, fuseCmd FuseCmd) error { return nil } +// unmounts a squash mountpoint and detaches any verity / loop devices that back +// it. Only use this if you are sure the underlying devices aren't in use by +// other mount points. func Umount(mountpoint string) error { + devPath, err := GetBackingDevice(mountpoint) + + err = unix.Unmount(mountpoint, 0) + if err != nil { + return errors.Wrapf(err, "failed unmounting %v", mountpoint) + } + + return MaybeCleanupBackingDevice(devPath) +} + +func GetBackingDevice(mountpoint string) (string, error) { mounts, err := mount.ParseMounts("/proc/self/mountinfo") if err != nil { - return err + return "", err } - // first, find the verity device that backs the mount theMount, found := mounts.FindMount(mountpoint) if !found { - return errors.Errorf("%s is not a mountpoint", mountpoint) - } - - err = unix.Unmount(mountpoint, 0) - if err != nil { - return errors.Wrapf(err, "failed unmounting %v", mountpoint) + return "", errors.Errorf("%s is not a mountpoint", mountpoint) } + return theMount.Source, nil +} - if _, err := os.Stat(theMount.Source); err != nil { +// given a device path that had been used as the backing device for a squash +// mountpoint, cleans up and detaches verity device if it still exists. +// +// If the device path does not exist, that is OK - this happens if the device +// was a regular loopback and not -verity. +func MaybeCleanupBackingDevice(devPath string) error { + if _, err := os.Stat(devPath); err != nil { if os.IsNotExist(err) { + // It's been detached, this is fine. return nil } return errors.WithStack(err) @@ -72,10 +89,10 @@ func Umount(mountpoint string) error { // was this a verity mount or a regular loopback mount? (if it's a // regular loopback mount, we detached it above, so need to do anything // special here; verity doesn't play as nicely) - if strings.HasSuffix(theMount.Source, verity.VeritySuffix) { - err = verity.VerityUnmount(theMount.Source) + if strings.HasSuffix(devPath, verity.VeritySuffix) { + err := verity.VerityUnmount(devPath) if err != nil { - return errors.Wrapf(err, "failed verity-unmounting %v", theMount.Source) + return errors.Wrapf(err, "failed verity-unmounting %v", devPath) } } diff --git a/pkg/erofs/erofs.go b/pkg/erofs/erofs.go index 340427e..3152cd6 100644 --- a/pkg/erofs/erofs.go +++ b/pkg/erofs/erofs.go @@ -118,8 +118,6 @@ func findErofsFuseInfo() { var erofsPath string if p := which("erofsfuse"); p != "" { erofsPath = p - } else { - erofsPath = which("erofsfuse") } if erofsPath == "" { return @@ -144,7 +142,7 @@ func erofsfuseSupportsMountNotification(erofsfuse string) (string, bool) { return version, false } -var erofsNotFound = errors.Errorf("erofsfuse program not found") +var erofsFuseNotFound = errors.Errorf("erofsfuse program not found") // erofsFuse - mount erofsFile to extractDir // return a pointer to the erofsfuse cmd. @@ -154,7 +152,7 @@ func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { once.Do(findErofsFuseInfo) if erofsFuseInfo.Path == "" { - return cmd, erofsNotFound + return cmd, erofsFuseNotFound } notifyOpts := "" @@ -180,7 +178,7 @@ func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { logf := filepath.Join(path.Dir(extractDir), "."+filepath.Base(extractDir)+"-erofsfuse.log") if cmdOut, err = os.OpenFile(logf, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644); err != nil { - log.Infof("Failed to open %s for write: %v", logf, err) + log.Errorf("Failed to open %s for write: %v", logf, err) return cmd, err } @@ -189,7 +187,7 @@ func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { return cmd, errors.Wrapf(err, "Failed stat'ing %q", extractDir) } if fiPre.Mode()&os.ModeSymlink != 0 { - return cmd, errors.Errorf("Refusing to mount onto a symbolic linkd") + return cmd, errors.Errorf("Refusing to mount onto a symbolic link %q", extractDir) } // It would be nice to only enable debug (or maybe to only log to file at all) @@ -349,7 +347,7 @@ func (k *FsckErofsExtractor) IsAvailable() error { return nil } -func (k *FsckErofsExtractor) Mount(squashFile, extractDir string) error { +func (k *FsckErofsExtractor) Mount(erofsFile, extractDir string) error { k.mutex.Lock() defer k.mutex.Unlock() @@ -362,8 +360,8 @@ func (k *FsckErofsExtractor) Mount(squashFile, extractDir string) error { return nil } - log.Debugf("fsck.erofs %s -> %s", squashFile, extractDir) - cmd := exec.Command("fsck.erofs", "-d", "--extract", extractDir, squashFile) + log.Debugf("fsck.erofs %s -> %s", erofsFile, extractDir) + cmd := exec.Command("fsck.erofs", "-d", "--extract", extractDir, erofsFile) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = nil @@ -372,7 +370,7 @@ func (k *FsckErofsExtractor) Mount(squashFile, extractDir string) error { // on failure, remove the directory if err != nil { if rmErr := os.RemoveAll(extractDir); rmErr != nil { - log.Errorf("Failed to remove %s after failed extraction of %s: %v", extractDir, squashFile, rmErr) + log.Errorf("Failed to remove %s after failed extraction of %s: %v", extractDir, erofsFile, rmErr) } return err } @@ -382,10 +380,10 @@ func (k *FsckErofsExtractor) Mount(squashFile, extractDir string) error { empty, err = isEmptyDir(extractDir) if err != nil { return errors.Errorf("Failed to read %s after successful extraction of %s: %v", - extractDir, squashFile, err) + extractDir, erofsFile, err) } if empty { - return errors.Errorf("%s was an empty fs image", squashFile) + return errors.Errorf("%s was an empty fs image", erofsFile) } return nil @@ -406,17 +404,17 @@ func (k *KernelExtractor) IsAvailable() error { return nil } -func (k *KernelExtractor) Mount(squashFile, extractDir string) error { +func (k *KernelExtractor) Mount(erofsFile, extractDir string) error { k.mutex.Lock() defer k.mutex.Unlock() - if mounted, err := common.IsMountedAtDir(squashFile, extractDir); err != nil { + if mounted, err := common.IsMountedAtDir(erofsFile, extractDir); err != nil { return err } else if mounted { return nil } - ecmd := []string{"mount", "-terofs", "-oloop,ro", squashFile, extractDir} + ecmd := []string{"mount", "-terofs", "-oloop,ro", erofsFile, extractDir} var output bytes.Buffer cmd := exec.Command(ecmd[0], ecmd[1:]...) cmd.Stdin = nil @@ -432,12 +430,12 @@ func (k *KernelExtractor) Mount(squashFile, extractDir string) error { exitError, ok := err.(*exec.ExitError) if !ok { retErr = errors.Errorf("kmount(%s) had unexpected error (no-rc), in exec (%v): %v", - squashFile, ecmd, err) + erofsFile, ecmd, err) } else if status, ok := exitError.Sys().(syscall.WaitStatus); !ok { retErr = errors.Errorf("kmount(%s) had unexpected error (no-status), in exec (%v): %v", - squashFile, ecmd, err) + erofsFile, ecmd, err) } else { - retErr = errors.Errorf("kmount(%s) exited %d: %v", squashFile, status.ExitStatus(), output.String()) + retErr = errors.Errorf("kmount(%s) exited %d: %v", erofsFile, status.ExitStatus(), output.String()) } return retErr @@ -482,8 +480,8 @@ func (k *ErofsFuseExtractor) Mount(erofsFile, extractDir string) error { return nil } -// ExtractSingleErofsPolicy - extract squashfile to extractDir -func ExtractSingleErofsPolicy(squashFile, extractDir string, policy *ExtractPolicy) error { +// ExtractSingleErofsPolicy - extract erofsfile to extractDir +func ExtractSingleErofsPolicy(erofsFile, extractDir string, policy *ExtractPolicy) error { const initName = "init" if policy == nil { return errors.Errorf("policy cannot be nil") @@ -518,7 +516,7 @@ func ExtractSingleErofsPolicy(squashFile, extractDir string, policy *ExtractPoli if err, ok := policy.Excuses[initName]; ok { return err } - return policy.Extractor.Mount(squashFile, fdest) + return policy.Extractor.Mount(erofsFile, fdest) } // At this point we are the initialzer @@ -534,7 +532,7 @@ func ExtractSingleErofsPolicy(squashFile, extractDir string, policy *ExtractPoli var extractor ErofsExtractor allExcuses := []string{} for _, extractor = range policy.Extractors { - err = extractor.Mount(squashFile, fdest) + err = extractor.Mount(erofsFile, fdest) if err == nil { policy.Extractor = extractor log.Debugf("Selected erofs extractor %s", extractor.Name()) @@ -552,10 +550,10 @@ func ExtractSingleErofsPolicy(squashFile, extractDir string, policy *ExtractPoli return policy.Excuses[initName] } -// ExtractSingleErofs - extract the squashFile to extractDir +// ExtractSingleErofs - extract the erofsFile to extractDir // Initialize a extractPolicy struct and then call ExtractSingleErofsPolicy // wik()th that. -func ExtractSingleErofs(squashFile string, extractDir string) error { +func ExtractSingleErofs(erofsFile string, extractDir string) error { exPolInfo.once.Do(func() { const envName = "STACKER_EROFS_EXTRACT_POLICY" const defPolicy = "kmount erofsfuse fsc.erofs" @@ -575,7 +573,7 @@ func ExtractSingleErofs(squashFile string, extractDir string) error { return exPolInfo.err } - return ExtractSingleErofsPolicy(squashFile, extractDir, exPolInfo.policy) + return ExtractSingleErofsPolicy(erofsFile, extractDir, exPolInfo.policy) } var checkSupported sync.Once diff --git a/pkg/molecule/molecule.go b/pkg/molecule/molecule.go index f89b816..d8e03bb 100644 --- a/pkg/molecule/molecule.go +++ b/pkg/molecule/molecule.go @@ -71,8 +71,6 @@ func (m Molecule) mountUnderlyingAtoms() (error, func()) { return errors.Wrapf(err, "failed to find mounted atoms path for %+v", a), cleanupAtoms } - fsi := fs.NewFromMediaType(a.MediaType) - rootHash := a.Annotations[verity.VerityRootHashAnnotation] if !m.config.AllowMissingVerityData { @@ -112,6 +110,11 @@ func (m Molecule) mountUnderlyingAtoms() (error, func()) { return err, cleanupAtoms } + fsi := fs.NewFromMediaType(a.MediaType) + if fsi == nil { + return errors.Errorf("unknown media-type %s", a.MediaType), cleanupAtoms + } + err = fsi.Mount(m.config.AtomsPath(a.Digest.Encoded()), target, rootHash) if err != nil { return err, cleanupAtoms diff --git a/pkg/squashfs/squashfs.go b/pkg/squashfs/squashfs.go index e9592ba..0f14f23 100644 --- a/pkg/squashfs/squashfs.go +++ b/pkg/squashfs/squashfs.go @@ -149,7 +149,7 @@ func sqfuseSupportsMountNotification(sqfuse string) (string, bool) { return version, false } -var squashNotFound = errors.Errorf("squashfuse program not found") +var squashFuseNotFound = errors.Errorf("squashfuse program not found") // squashFuse - mount squashFile to extractDir // return a pointer to the squashfuse cmd. @@ -159,7 +159,7 @@ func squashFuse(squashFile, extractDir string) (*exec.Cmd, error) { once.Do(findSquashFuseInfo) if squashFuseInfo.Path == "" { - return cmd, squashNotFound + return cmd, squashFuseNotFound } notifyOpts := "" @@ -185,7 +185,7 @@ func squashFuse(squashFile, extractDir string) (*exec.Cmd, error) { logf := filepath.Join(path.Dir(extractDir), "."+filepath.Base(extractDir)+"-squashfuse.log") if cmdOut, err = os.OpenFile(logf, os.O_RDWR|os.O_TRUNC|os.O_CREATE, 0644); err != nil { - log.Infof("Failed to open %s for write: %v", logf, err) + log.Errorf("Failed to open %s for write: %v", logf, err) return cmd, err } @@ -194,7 +194,7 @@ func squashFuse(squashFile, extractDir string) (*exec.Cmd, error) { return cmd, errors.Wrapf(err, "Failed stat'ing %q", extractDir) } if fiPre.Mode()&os.ModeSymlink != 0 { - return cmd, errors.Errorf("Refusing to mount onto a symbolic linkd") + return cmd, errors.Errorf("Refusing to mount onto a symbolic link %q", extractDir) } // It would be nice to only enable debug (or maybe to only log to file at all) diff --git a/pkg/verity/metadata.go b/pkg/verity/metadata.go index 6bfabbb..6333e79 100644 --- a/pkg/verity/metadata.go +++ b/pkg/verity/metadata.go @@ -1,7 +1,5 @@ package verity -import "strings" - type VerityMetadata bool const ( @@ -10,7 +8,3 @@ const ( VerityMetadataPresent VerityMetadata = true VerityMetadataMissing VerityMetadata = false ) - -func HasVerityMetadata(mediaType string) VerityMetadata { - return VerityMetadata(strings.HasSuffix(mediaType, VeritySuffix)) -} diff --git a/pkg/verity/verity.go b/pkg/verity/verity.go index 6d0fe01..7ae9b81 100644 --- a/pkg/verity/verity.go +++ b/pkg/verity/verity.go @@ -81,6 +81,7 @@ import ( "strconv" "strings" "syscall" + "time" "unsafe" "github.com/freddierice/go-losetup" @@ -336,6 +337,28 @@ func findLoopBackingVerity(device string) (int64, error) { return deviceNo, nil } +func retryIfBusy(fn func() error) error { + const maxRetries = 5 + var err error + + for retry := 0; retry < maxRetries; retry++ { + err = fn() + if err == nil { + return nil + } + + var errno syscall.Errno + if errors.As(err, &errno) && errno != syscall.EBUSY { + return err + } + + // sleep between retries + time.Sleep(1 * time.Second) + } + + return err +} + func VerityUnmount(mountPath string) error { // find the loop device that backs the verity device deviceNo, err := findLoopBackingVerity(mountPath) @@ -348,7 +371,16 @@ func VerityUnmount(mountPath string) error { // above). the cryptsetup API allows us to pass NULL for the crypt // device, but go-cryptsetup doesn't have a way to initialize a NULL // crypt device short of making the struct by hand like this. - err = (&cryptsetup.Device{}).Deactivate(mountPath) + + // erofs may temporarily EBUSY, so retry a few times + err = retryIfBusy(func() error { + err = (&cryptsetup.Device{}).Deactivate(mountPath) + if err != nil { + return err + } + + return nil + }) if err != nil { return errors.WithStack(err) } From 2651a8543f6acc7860fc7d372ca5bfcae4394ad2 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Sat, 22 Feb 2025 22:05:38 -0800 Subject: [PATCH 6/8] fix: some more refactoring --- pkg/erofs/erofs.go | 56 +++++----------------------------------- pkg/fs/fs.go | 26 ++++--------------- pkg/squashfs/squashfs.go | 22 +++++----------- pkg/types/fs.go | 30 +++++++++++++++++++++ 4 files changed, 49 insertions(+), 85 deletions(-) create mode 100644 pkg/types/fs.go diff --git a/pkg/erofs/erofs.go b/pkg/erofs/erofs.go index 3152cd6..daba2f5 100644 --- a/pkg/erofs/erofs.go +++ b/pkg/erofs/erofs.go @@ -20,6 +20,7 @@ import ( "golang.org/x/sys/unix" "machinerun.io/atomfs/pkg/common" "machinerun.io/atomfs/pkg/log" + types "machinerun.io/atomfs/pkg/types" vrty "machinerun.io/atomfs/pkg/verity" ) @@ -225,40 +226,6 @@ func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { _ = cmd.Wait() close(alarmCh) }() - if erofsFuseInfo.SupportsNotify { - notifyCh := make(chan byte) - log.Infof("%s supports notify pipe, watching %q", erofsFuseInfo.Path, notifyPath) - go func() { - f, err := os.Open(notifyPath) - if err != nil { - return - } - defer f.Close() - b1 := make([]byte, 1) - for { - n1, err := f.Read(b1) - if err != nil { - return - } - if err == nil && n1 >= 1 { - break - } - } - notifyCh <- b1[0] - }() - - select { - case <-alarmCh: - cmd.Process.Kill() - return cmd, errors.Wrapf(err, "Gave up on erofsfuse mount of %s with %s after %s", erofsFile, erofsFuseInfo.Path, timeLimit) - case ret := <-notifyCh: - if ret == 's' { - return cmd, nil - } else { - return cmd, errors.Errorf("erofsfuse returned an error, check %s", logf) - } - } - } for count := 0; !common.FileChanged(fiPre, extractDir); count++ { if cmd.ProcessState != nil { // process exited, the Wait() call in the goroutine above @@ -279,8 +246,8 @@ func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { } type ExtractPolicy struct { - Extractors []ErofsExtractor - Extractor ErofsExtractor + Extractors []types.FsExtractor + Extractor types.FsExtractor Excuses map[string]error initialized bool mutex sync.Mutex @@ -292,27 +259,18 @@ var exPolInfo struct { policy *ExtractPolicy } -type ErofsExtractor interface { - Name() string - IsAvailable() error - // Mount - Mount or extract path to dest. - // Return nil on "already extracted" - // Return error on failure. - Mount(path, dest string) error -} - func NewExtractPolicy(args ...string) (*ExtractPolicy, error) { p := &ExtractPolicy{ - Extractors: []ErofsExtractor{}, + Extractors: []types.FsExtractor{}, Excuses: map[string]error{}, } - allEx := []ErofsExtractor{ + allEx := []types.FsExtractor{ &KernelExtractor{}, &ErofsFuseExtractor{}, &FsckErofsExtractor{}, } - byName := map[string]ErofsExtractor{} + byName := map[string]types.FsExtractor{} for _, i := range allEx { byName[i.Name()] = i } @@ -529,7 +487,7 @@ func ExtractSingleErofsPolicy(erofsFile, extractDir string, policy *ExtractPolic return policy.Excuses[initName] } - var extractor ErofsExtractor + var extractor types.FsExtractor allExcuses := []string{} for _, extractor = range policy.Extractors { err = extractor.Mount(erofsFile, fdest) diff --git a/pkg/fs/fs.go b/pkg/fs/fs.go index 28e1c20..4a2959b 100644 --- a/pkg/fs/fs.go +++ b/pkg/fs/fs.go @@ -1,33 +1,17 @@ package fs import ( - "io" - - "machinerun.io/atomfs/pkg/common" "machinerun.io/atomfs/pkg/erofs" "machinerun.io/atomfs/pkg/squashfs" - "machinerun.io/atomfs/pkg/verity" + types "machinerun.io/atomfs/pkg/types" ) -type Filesystem interface { - // Make creates a new filesystem image. - Make(tempdir string, rootfs string, eps *common.ExcludePaths, verity verity.VerityMetadata) (io.ReadCloser, string, string, error) - // ExtractSingle extracts a filesystem image. - ExtractSingle(fsImgFile string, extractDir string) error - // Mount mounts a filesystem image on a given mountpoint. - Mount(fsImgFile, mountpoint, rootHash string) error - // Unmount umounts a filesystem image. - Umount(mountpoint string) error -} - -type FilesystemType string - const ( - SquashfsType FilesystemType = "squashfs" - ErofsType FilesystemType = "erofs" + SquashfsType types.FilesystemType = "squashfs" + ErofsType types.FilesystemType = "erofs" ) -func New(fsType FilesystemType) Filesystem { +func New(fsType types.FilesystemType) types.Filesystem { switch fsType { case SquashfsType: return squashfs.New() @@ -38,7 +22,7 @@ func New(fsType FilesystemType) Filesystem { return nil } -func NewFromMediaType(mediaType string) Filesystem { +func NewFromMediaType(mediaType string) types.Filesystem { if squashfs.IsSquashfsMediaType(mediaType) { return squashfs.New() } else if erofs.IsErofsMediaType(mediaType) { diff --git a/pkg/squashfs/squashfs.go b/pkg/squashfs/squashfs.go index 0f14f23..1d2f49b 100644 --- a/pkg/squashfs/squashfs.go +++ b/pkg/squashfs/squashfs.go @@ -20,6 +20,7 @@ import ( "golang.org/x/sys/unix" "machinerun.io/atomfs/pkg/common" "machinerun.io/atomfs/pkg/log" + types "machinerun.io/atomfs/pkg/types" vrty "machinerun.io/atomfs/pkg/verity" ) @@ -286,8 +287,8 @@ func squashFuse(squashFile, extractDir string) (*exec.Cmd, error) { } type ExtractPolicy struct { - Extractors []SquashExtractor - Extractor SquashExtractor + Extractors []types.FsExtractor + Extractor types.FsExtractor Excuses map[string]error initialized bool mutex sync.Mutex @@ -299,27 +300,18 @@ var exPolInfo struct { policy *ExtractPolicy } -type SquashExtractor interface { - Name() string - IsAvailable() error - // Mount - Mount or extract path to dest. - // Return nil on "already extracted" - // Return error on failure. - Mount(path, dest string) error -} - func NewExtractPolicy(args ...string) (*ExtractPolicy, error) { p := &ExtractPolicy{ - Extractors: []SquashExtractor{}, + Extractors: []types.FsExtractor{}, Excuses: map[string]error{}, } - allEx := []SquashExtractor{ + allEx := []types.FsExtractor{ &KernelExtractor{}, &SquashFuseExtractor{}, &UnsquashfsExtractor{}, } - byName := map[string]SquashExtractor{} + byName := map[string]types.FsExtractor{} for _, i := range allEx { byName[i.Name()] = i } @@ -536,7 +528,7 @@ func ExtractSingleSquashPolicy(squashFile, extractDir string, policy *ExtractPol return policy.Excuses[initName] } - var extractor SquashExtractor + var extractor types.FsExtractor allExcuses := []string{} for _, extractor = range policy.Extractors { err = extractor.Mount(squashFile, fdest) diff --git a/pkg/types/fs.go b/pkg/types/fs.go new file mode 100644 index 0000000..c27c333 --- /dev/null +++ b/pkg/types/fs.go @@ -0,0 +1,30 @@ +package fs + +import ( + "io" + + "machinerun.io/atomfs/pkg/common" + "machinerun.io/atomfs/pkg/verity" +) + +type Filesystem interface { + // Make creates a new filesystem image. + Make(tempdir string, rootfs string, eps *common.ExcludePaths, verity verity.VerityMetadata) (io.ReadCloser, string, string, error) + // ExtractSingle extracts a filesystem image. + ExtractSingle(fsImgFile string, extractDir string) error + // Mount mounts a filesystem image on a given mountpoint. + Mount(fsImgFile, mountpoint, rootHash string) error + // Unmount umounts a filesystem image. + Umount(mountpoint string) error +} + +type FilesystemType string + +type FsExtractor interface { + Name() string + IsAvailable() error + // Mount - Mount or extract path to dest. + // Return nil on "already extracted" + // Return error on failure. + Mount(path, dest string) error +} From 64bc8c29de5f50ccea9f37bf13c2c9d313c03793 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Thu, 27 Feb 2025 22:43:57 -0800 Subject: [PATCH 7/8] fix: remove notify support from erofs --- pkg/common/common_test.go | 20 +++++++ pkg/common/fs.go | 47 --------------- pkg/common/utils.go | 91 +++++++++++++++++++++++++++++ pkg/erofs/erofs.go | 107 +++++++--------------------------- pkg/squashfs/squashfs.go | 57 ++---------------- pkg/squashfs/squashfs_test.go | 27 --------- 6 files changed, 138 insertions(+), 211 deletions(-) delete mode 100644 pkg/common/fs.go delete mode 100644 pkg/squashfs/squashfs_test.go diff --git a/pkg/common/common_test.go b/pkg/common/common_test.go index 2d99a2f..c74b2da 100644 --- a/pkg/common/common_test.go +++ b/pkg/common/common_test.go @@ -1,6 +1,7 @@ package common import ( + "os" "testing" "github.com/stretchr/testify/assert" @@ -47,3 +48,22 @@ func TestAmHostRoot(t *testing.T) { assert.Equal(v, testcase.expected) } } + +func TestIsEmpytDir(t *testing.T) { + t.Parallel() + assert := assert.New(t) + v, e := IsEmptyDir("/") + assert.NoError(e) + assert.False(v) + + v, e = IsEmptyDir("/root") + assert.Error(e) + + dname, err := os.MkdirTemp("", "squashfs_empty_test_dir") + assert.NoError(err) + defer os.RemoveAll(dname) + + v, e = IsEmptyDir(dname) + assert.NoError(e) + assert.True(v) +} diff --git a/pkg/common/fs.go b/pkg/common/fs.go deleted file mode 100644 index 0d98b5f..0000000 --- a/pkg/common/fs.go +++ /dev/null @@ -1,47 +0,0 @@ -package common - -import ( - "os" - "strings" -) - -func FileChanged(a os.FileInfo, path string) bool { - b, err := os.Lstat(path) - if err != nil { - return true - } - return !os.SameFile(a, b) -} - -// Takes /proc/self/uid_map contents as one string -// Returns true if this is a uidmap representing the whole host -// uid range. -func uidmapIsHost(oneline string) bool { - oneline = strings.TrimSuffix(oneline, "\n") - if len(oneline) == 0 { - return false - } - lines := strings.Split(oneline, "\n") - if len(lines) != 1 { - return false - } - words := strings.Fields(lines[0]) - if len(words) != 3 || words[0] != "0" || words[1] != "0" || words[2] != "4294967295" { - return false - } - - return true -} - -func AmHostRoot() bool { - // if not uid 0, not host root - if os.Geteuid() != 0 { - return false - } - // if uid_map doesn't map 0 to 0, not host root - bytes, err := os.ReadFile("/proc/self/uid_map") - if err != nil { - return false - } - return uidmapIsHost(string(bytes)) -} diff --git a/pkg/common/utils.go b/pkg/common/utils.go index 324eff9..06ee998 100644 --- a/pkg/common/utils.go +++ b/pkg/common/utils.go @@ -2,8 +2,12 @@ package common import ( "fmt" + "io" "os" + "path/filepath" "strings" + + "golang.org/x/sys/unix" ) var TestOverrideRuntimeDirKey = "ATOMFS_TEST_RUN_DIR" @@ -54,3 +58,90 @@ func RuntimeDir(metadir string) string { } return testOverrideDir } + +func IsEmptyDir(path string) (bool, error) { + fh, err := os.Open(path) + if err != nil { + return false, err + } + defer fh.Close() + + _, err = fh.ReadDir(1) + if err == io.EOF { + return true, nil + } + return false, err +} + +// Which - like the unix utility, return empty string for not-found. +// this might fit well in lib/, but currently lib's test imports +// squashfs creating a import loop. +func Which(name string) string { + return whichSearch(name, strings.Split(os.Getenv("PATH"), ":")) +} + +func whichSearch(name string, paths []string) string { + var search []string + + if strings.ContainsRune(name, os.PathSeparator) { + if filepath.IsAbs(name) { + search = []string{name} + } else { + search = []string{"./" + name} + } + } else { + search = []string{} + for _, p := range paths { + search = append(search, filepath.Join(p, name)) + } + } + + for _, fPath := range search { + if err := unix.Access(fPath, unix.X_OK); err == nil { + return fPath + } + } + + return "" +} + +func FileChanged(a os.FileInfo, path string) bool { + b, err := os.Lstat(path) + if err != nil { + return true + } + return !os.SameFile(a, b) +} + +// Takes /proc/self/uid_map contents as one string +// Returns true if this is a uidmap representing the whole host +// uid range. +func uidmapIsHost(oneline string) bool { + oneline = strings.TrimSuffix(oneline, "\n") + if len(oneline) == 0 { + return false + } + lines := strings.Split(oneline, "\n") + if len(lines) != 1 { + return false + } + words := strings.Fields(lines[0]) + if len(words) != 3 || words[0] != "0" || words[1] != "0" || words[2] != "4294967295" { + return false + } + + return true +} + +func AmHostRoot() bool { + // if not uid 0, not host root + if os.Geteuid() != 0 { + return false + } + // if uid_map doesn't map 0 to 0, not host root + bytes, err := os.ReadFile("/proc/self/uid_map") + if err != nil { + return false + } + return uidmapIsHost(string(bytes)) +} diff --git a/pkg/erofs/erofs.go b/pkg/erofs/erofs.go index daba2f5..b8e7533 100644 --- a/pkg/erofs/erofs.go +++ b/pkg/erofs/erofs.go @@ -17,7 +17,6 @@ import ( "time" "github.com/pkg/errors" - "golang.org/x/sys/unix" "machinerun.io/atomfs/pkg/common" "machinerun.io/atomfs/pkg/log" types "machinerun.io/atomfs/pkg/types" @@ -25,13 +24,12 @@ import ( ) type erofsFuseInfoStruct struct { - Path string - Version string - SupportsNotify bool + Path string + Version string } var once sync.Once -var erofsFuseInfo = erofsFuseInfoStruct{"", "", false} +var erofsFuseInfo = erofsFuseInfoStruct{"", ""} func MakeErofs(tempdir string, rootfs string, eps *common.ExcludePaths, verity vrty.VerityMetadata) (io.ReadCloser, string, string, error) { var excludesFile string @@ -117,21 +115,21 @@ func MakeErofs(tempdir string, rootfs string, eps *common.ExcludePaths, verity v func findErofsFuseInfo() { var erofsPath string - if p := which("erofsfuse"); p != "" { + if p := common.Which("erofsfuse"); p != "" { erofsPath = p } if erofsPath == "" { return } - version, supportsNotify := erofsfuseSupportsMountNotification(erofsPath) - log.Infof("Found erofsfuse at %s (version=%s notify=%t)", erofsPath, version, supportsNotify) - erofsFuseInfo = erofsFuseInfoStruct{erofsPath, version, supportsNotify} + version := erofsfuseVersion(erofsPath) + log.Infof("Found erofsfuse at %s (version=%s)", erofsPath, version) + erofsFuseInfo = erofsFuseInfoStruct{erofsPath, version} } -// erofsfuseSupportsMountNotification - returns true if erofsfuse supports mount +// erofsfuseVersion - returns true if erofsfuse supports mount // notification, false otherwise // erofsfuse is the path to the erofsfuse binary -func erofsfuseSupportsMountNotification(erofsfuse string) (string, bool) { +func erofsfuseVersion(erofsfuse string) string { cmd := exec.Command(erofsfuse) // `erofsfuse` always returns an error... so we ignore it. @@ -140,7 +138,7 @@ func erofsfuseSupportsMountNotification(erofsfuse string) (string, bool) { firstLine := strings.Split(string(out[:]), "\n")[0] version := strings.Split(firstLine, " ")[1] - return version, false + return version } var erofsFuseNotFound = errors.Errorf("erofsfuse program not found") @@ -156,21 +154,6 @@ func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { return cmd, erofsFuseNotFound } - notifyOpts := "" - notifyPath := "" - if erofsFuseInfo.SupportsNotify { - sockdir, err := os.MkdirTemp("", "sock") - if err != nil { - return cmd, err - } - defer os.RemoveAll(sockdir) - notifyPath = filepath.Join(sockdir, "notifypipe") - if err := syscall.Mkfifo(notifyPath, 0640); err != nil { - return cmd, err - } - notifyOpts = "notify_pipe=" + notifyPath - } - // given extractDir of path/to/some/dir[/], log to path/to/some/.dir-erofs.log extractDir = strings.TrimSuffix(extractDir, "/") @@ -195,9 +178,6 @@ func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { // if 'stacker --debug', but we do not have access to that info here. // to debug erofsfuse, use "allow_other,debug" optionArgs := "debug" - if notifyOpts != "" { - optionArgs += "," + notifyOpts - } cmd = exec.Command(erofsFuseInfo.Path, "-f", "-o", optionArgs, erofsFile, extractDir) cmd.Stdin = nil cmd.Stdout = cmdOut @@ -217,8 +197,10 @@ func erofsFuse(erofsFile, extractDir string) (*exec.Cmd, error) { // b. the directory Entry is different than it was before the call // to erofsfuse. We have to do this because we do not have another // way to know when the mount has been populated. - // https://github.com/vasi/erofsfuse/issues/49 + // https://github.com/vasi/squashfuse/issues/49 // c. a timeout (timeLimit) was hit + // + // FIXME: this has been borrowed from squashfs code, may not be needed? startTime := time.Now() timeLimit := 30 * time.Second alarmCh := make(chan struct{}) @@ -299,7 +281,7 @@ func (k *FsckErofsExtractor) Name() string { } func (k *FsckErofsExtractor) IsAvailable() error { - if which("fsck.erofs") == "" { + if common.Which("fsck.erofs") == "" { return errors.Errorf("no 'fsck.erofs' in PATH") } return nil @@ -310,7 +292,7 @@ func (k *FsckErofsExtractor) Mount(erofsFile, extractDir string) error { defer k.mutex.Unlock() // check if already extracted - empty, err := isEmptyDir(extractDir) + empty, err := common.IsEmptyDir(extractDir) if err != nil { return errors.Wrapf(err, "Error checking for empty dir") } @@ -335,7 +317,7 @@ func (k *FsckErofsExtractor) Mount(erofsFile, extractDir string) error { // assert that extraction must create files. This way we can assume non-empty dir above // was populated by fsck.erofs. - empty, err = isEmptyDir(extractDir) + empty, err = common.IsEmptyDir(extractDir) if err != nil { return errors.Errorf("Failed to read %s after successful extraction of %s: %v", extractDir, erofsFile, err) @@ -403,11 +385,11 @@ type ErofsFuseExtractor struct { mutex sync.Mutex } -func (k *ErofsFuseExtractor) Name() string { +func (f *ErofsFuseExtractor) Name() string { return "erofsfuse" } -func (k *ErofsFuseExtractor) IsAvailable() error { +func (f *ErofsFuseExtractor) IsAvailable() error { once.Do(findErofsFuseInfo) if erofsFuseInfo.Path == "" { return errors.Errorf("no 'erofsfuse' in PATH") @@ -415,12 +397,12 @@ func (k *ErofsFuseExtractor) IsAvailable() error { return nil } -func (k *ErofsFuseExtractor) Mount(erofsFile, extractDir string) error { - k.mutex.Lock() - defer k.mutex.Unlock() +func (f *ErofsFuseExtractor) Mount(erofsFile, extractDir string) error { + f.mutex.Lock() + defer f.mutex.Unlock() if mounted, err := common.IsMountedAtDir(erofsFile, extractDir); mounted && err == nil { - log.Debugf("[%s] %s already mounted -> %s", k.Name(), erofsFile, extractDir) + log.Debugf("[%s] %s already mounted -> %s", f.Name(), erofsFile, extractDir) return nil } else if err != nil { return err @@ -563,48 +545,3 @@ func mkerofsSupportsFeature() (bool, bool) { return zstdIsSuspported, parallelIsSupported } - -func isEmptyDir(path string) (bool, error) { - fh, err := os.Open(path) - if err != nil { - return false, err - } - - _, err = fh.ReadDir(1) - if err == io.EOF { - return true, nil - } - return false, err -} - -// which - like the unix utility, return empty string for not-found. -// this might fit well in lib/, but currently lib's test imports -// erofs creating a import loop. -func which(name string) string { - return whichSearch(name, strings.Split(os.Getenv("PATH"), ":")) -} - -func whichSearch(name string, paths []string) string { - var search []string - - if strings.ContainsRune(name, os.PathSeparator) { - if filepath.IsAbs(name) { - search = []string{name} - } else { - search = []string{"./" + name} - } - } else { - search = []string{} - for _, p := range paths { - search = append(search, filepath.Join(p, name)) - } - } - - for _, fPath := range search { - if err := unix.Access(fPath, unix.X_OK); err == nil { - return fPath - } - } - - return "" -} diff --git a/pkg/squashfs/squashfs.go b/pkg/squashfs/squashfs.go index 1d2f49b..63b9279 100644 --- a/pkg/squashfs/squashfs.go +++ b/pkg/squashfs/squashfs.go @@ -17,7 +17,6 @@ import ( "github.com/Masterminds/semver/v3" "github.com/pkg/errors" - "golang.org/x/sys/unix" "machinerun.io/atomfs/pkg/common" "machinerun.io/atomfs/pkg/log" types "machinerun.io/atomfs/pkg/types" @@ -111,10 +110,10 @@ func MakeSquashfs(tempdir string, rootfs string, eps *common.ExcludePaths, verit func findSquashFuseInfo() { var sqfsPath string - if p := which("squashfuse_ll"); p != "" { + if p := common.Which("squashfuse_ll"); p != "" { sqfsPath = p } else { - sqfsPath = which("squashfuse") + sqfsPath = common.Which("squashfuse") } if sqfsPath == "" { return @@ -340,7 +339,7 @@ func (k *UnsquashfsExtractor) Name() string { } func (k *UnsquashfsExtractor) IsAvailable() error { - if which("unsquashfs") == "" { + if common.Which("unsquashfs") == "" { return errors.Errorf("no 'unsquashfs' in PATH") } return nil @@ -351,7 +350,7 @@ func (k *UnsquashfsExtractor) Mount(squashFile, extractDir string) error { defer k.mutex.Unlock() // check if already extracted - empty, err := isEmptyDir(extractDir) + empty, err := common.IsEmptyDir(extractDir) if err != nil { return errors.Wrapf(err, "Error checking for empty dir") } @@ -376,7 +375,7 @@ func (k *UnsquashfsExtractor) Mount(squashFile, extractDir string) error { // assert that extraction must create files. This way we can assume non-empty dir above // was populated by unsquashfs. - empty, err = isEmptyDir(extractDir) + empty, err = common.IsEmptyDir(extractDir) if err != nil { return errors.Errorf("Failed to read %s after successful extraction of %s: %v", extractDir, squashFile, err) @@ -598,49 +597,3 @@ func mksquashfsSupportsZstd() bool { return zstdIsSuspported } - -func isEmptyDir(path string) (bool, error) { - fh, err := os.Open(path) - if err != nil { - return false, err - } - defer fh.Close() - - _, err = fh.ReadDir(1) - if err == io.EOF { - return true, nil - } - return false, err -} - -// which - like the unix utility, return empty string for not-found. -// this might fit well in lib/, but currently lib's test imports -// squashfs creating a import loop. -func which(name string) string { - return whichSearch(name, strings.Split(os.Getenv("PATH"), ":")) -} - -func whichSearch(name string, paths []string) string { - var search []string - - if strings.ContainsRune(name, os.PathSeparator) { - if filepath.IsAbs(name) { - search = []string{name} - } else { - search = []string{"./" + name} - } - } else { - search = []string{} - for _, p := range paths { - search = append(search, filepath.Join(p, name)) - } - } - - for _, fPath := range search { - if err := unix.Access(fPath, unix.X_OK); err == nil { - return fPath - } - } - - return "" -} diff --git a/pkg/squashfs/squashfs_test.go b/pkg/squashfs/squashfs_test.go deleted file mode 100644 index c7b25e2..0000000 --- a/pkg/squashfs/squashfs_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package squashfs - -import ( - "os" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestIsEmpytDir(t *testing.T) { - t.Parallel() - assert := assert.New(t) - v, e := isEmptyDir("/") - assert.NoError(e) - assert.False(v) - - v, e = isEmptyDir("/root") - assert.Error(e) - - dname, err := os.MkdirTemp("", "squashfs_empty_test_dir") - assert.NoError(err) - defer os.RemoveAll(dname) - - v, e = isEmptyDir(dname) - assert.NoError(e) - assert.True(v) -} From c114b131b6bd55985ea971dc87f9c057e70b48a9 Mon Sep 17 00:00:00 2001 From: Ramkumar Chinchani Date: Sat, 22 Feb 2025 23:16:20 -0800 Subject: [PATCH 8/8] fix: try a released stacker version Signed-off-by: Ramkumar Chinchani --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index cc824b6..5f71eab 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ VERSION_LDFLAGS=-X main.Version=$(MAIN_VERSION) BATS = $(TOOLS_D)/bin/bats BATS_VERSION := v1.10.0 STACKER = $(TOOLS_D)/bin/stacker -STACKER_VERSION := v1.0.0 +STACKER_VERSION := v1.1.0-erofs TOOLS_D := $(ROOT)/tools GOCOVERDIR ?= $(ROOT) @@ -36,7 +36,7 @@ gotest: $(GO_SRC) $(STACKER): mkdir -p $(TOOLS_D)/bin - wget --progress=dot:giga https://github.com/project-stacker/stacker/releases/download/$(STACKER_VERSION)/stacker + wget --progress=dot:giga https://github.com/rchincha/stacker/releases/download/$(STACKER_VERSION)/stacker chmod +x stacker cp stacker $(TOOLS_D)/bin/