support for reproducible builds

.travis.yml

@@ -10,7 +10,8 @@ install:
   - sudo rm -rf /var/lib/apt/lists/*
   - sudo add-apt-repository ppa:duggan/bats --yes
   - sudo apt-get update -qq
-  - sudo apt-get install -qq upx
+  - sudo apt-get install -qq upx coreutils
+  - sudo apt-get install -qq realpath || true
   - sudo ./scripts/travis-install-rkt.sh
   - sudo apt-get install -qq bats
 

aci-builder/bin-run/builder.go

@@ -29,6 +29,7 @@ type Builder struct {
 	aciTargetPath string
 	upperId       string
 	pod           *stage1commontypes.Pod
+	fullManifest  *common.AciManifest // includes non-standard sections such as 'Builder'
 }
 
 func NewBuilder(podRoot string, podUUID *types.UUID) (*Builder, error) {
@@ -123,6 +124,8 @@ func (b *Builder) writeManifest() error {
 		aciManifest.NameAndVersion = *common.NewACFullName(aciManifest.NameAndVersion.Name() + ":" + common.GenerateVersion(b.aciTargetPath))
 	}
 
+	b.fullManifest = aciManifest
+
 	if err := common.WriteAciManifest(aciManifest, target, aciManifest.NameAndVersion.Name(), dgrVersion); err != nil {
 		return errs.WithEF(err, b.fields.WithField("file", target), "Failed to write manifest")
 	}
@@ -136,39 +139,52 @@ func (b *Builder) tarAci() error {
 	}
 
 	upperPath := b.pod.Root + PATH_OVERLAY + "/" + upperId + PATH_UPPER
-	upperNamedRootfs := upperPath + "/" + manifestApp(b.pod).Name.String()
-	upperRootfs := upperPath + common.PathRootfs
-
-	if err := os.RemoveAll(upperNamedRootfs + PATH_TMP); err != nil {
-		logs.WithEF(err, b.fields.WithField("path", upperNamedRootfs+PATH_TMP)).Warn("Failed to clean tmp directory")
-	}
-
-	if err := os.Rename(upperNamedRootfs, upperRootfs); err != nil { // TODO this is dirty and can probably be renamed during tar
-		return errs.WithEF(err, b.fields.WithField("path", upperNamedRootfs), "Failed to rename rootfs")
-	}
-	defer os.Rename(upperRootfs, upperNamedRootfs)
-
-	dir, err := os.Getwd()
-	if err != nil {
-		return errs.WithEF(err, b.fields, "Failed to get current working directory")
+	rootfsAlias := manifestApp(b.pod).Name.String()
+	destination := b.aciTargetPath + common.PathImageAci // absolute dir, outside upperPath (think: /tmp/…)
+
+	params := []string{"--sort=name", "--numeric-owner", "--exclude", rootfsAlias + PATH_TMP + "/*"}
+	params = append(params, "-C", upperPath, "--transform", "s@^"+rootfsAlias+"@rootfs@")
+	for _, expression := range b.fullManifest.Build.Transform {
+		params = append(params, "--transform", expression)
+	}
+	for _, path := range b.fullManifest.Build.Exclude {
+		if strings.HasPrefix(path, "/") {
+			params = append(params, "--exclude", rootfsAlias+path)
+		} else {
+			params = append(params, "--exclude", path)
+		}
 	}
-	defer func() {
-		if err := os.Chdir(dir); err != nil {
-			logs.WithEF(err, b.fields.WithField("path", dir)).Warn("Failed to chdir back")
+	if v, ok := b.fullManifest.Aci.Annotations.Get("build-date"); ok {
+		logs.WithFields(b.fields).WithField("build-date", v).Info("Using the given fixed build-date")
+		params = append(params, "--mtime", v, "--clamp-mtime")
+	} else {
+		logs.WithFields(b.fields).Info("This is no reproducible build: no build-date has been set")
+	}
+	params = append(params, "-cf", destination, common.PathManifest[1:], rootfsAlias)
+
+	logs.WithF(b.fields).Debug("Calling tar to collect all files")
+	if err := common.ExecCmd("tar", params...); err != nil {
+		// In case the host's tar is too old, try the builder's if it exists.
+		stage1Tar := rktcommon.Stage1RootfsPath(b.pod.Root)
+		var buildersTar string
+		for _, t := range []string{"/dgr/usr/bin/tar", "/bin/tar", "/usr/bin/tar"} {
+			buildersTar = filepath.Join(stage1Tar, t)
+			if _, err := os.Stat(buildersTar); err == nil {
+				break
+			}
 		}
-	}()
 
-	if err := os.Chdir(upperPath); err != nil {
-		return errs.WithEF(err, b.fields.WithField("path", upperPath), "Failed to chdir to upper base path")
-	}
-	if err := common.Tar(b.aciTargetPath+common.PathImageAci, common.PathManifest[1:], common.PathRootfs[1:]+"/"); err != nil {
-		return errs.WithEF(err, b.fields, "Failed to tar aci")
+		if err2 := common.ExecCmd(buildersTar, params...); err2 != nil {
+			// If that failed, output the original error nevertheless.
+			logs.WithFields(b.fields).WithField("params", params).Error("Parameters to 'tar' within the builder")
+			return errs.WithEF(err, b.fields, "Failed to tar aci")
+		}
+		logs.WithF(b.fields).Debug("Had to resort to 'tar' from the builder to create the aci file")
 	}
 	// common.ExecCmd sometimes silently fails, hence the redundant check.
-	if _, err := os.Stat(b.aciTargetPath + common.PathImageAci); os.IsNotExist(err) {
+	if _, err := os.Stat(destination); os.IsNotExist(err) {
 		return errs.WithEF(err, b.fields, "Expected aci has not been created")
 	}
-	logs.WithField("path", dir).Debug("chdir")
 	return nil
 }
 
@@ -272,6 +288,7 @@ func (b *Builder) prepareNspawnArgsAndEnv(commandPath string) ([]string, []strin
 	if err != nil {
 		return args, env, errs.WithEF(err, b.fields.WithField("content", string(content)), "Failed to process manifest template")
 	}
+	b.fullManifest = aciManifest
 	for _, mount := range aciManifest.Builder.MountPoints {
 		if strings.HasPrefix(mount.From, "~/") {
 			user, err := user.Current()

aci-builder/build.sh

@@ -16,26 +16,53 @@ mkdir -p ${rootfs}/dgr ${rootfs}/usr/bin
 GOOS=linux GOARCH=amd64 go build --ldflags '-s -w -extldflags "-static"' -o ${rootfs}/dgr/builder/stage1/run ${dir}/bin-run
 upx ${rootfs}/dgr/builder/stage1/run
 
-sudo tar xf ${dir}/rootfs.tar.xz -C ${rootfs}/dgr/
+: ${tar:="$(realpath "${dir}/files/dgr/usr/bin/tar")"}
+if [[ ! -x "${tar}" ]] || ! "${tar}" --help | grep -q -F sort; then
+  # 'core2' is very old, but without setting an option some GCC versions
+  # will pick built-in defaults – which could be as "recent" as 'silvermont',
+  # thus excluding CPUs without SSE4 (like some pre-2007 AMDs still in the wild).
+  : ${CPU_SETTING:="-march=core2 -mtune=intel"}
+  if [[ "$(uname -m)" != "x86_64" ]]; then
+    CPU_SETTING=""
+  fi
+
+  WORKDIR="$(mktemp -d -t aci-builder-tar.XXXXXX)"
+  pushd .
+  cd ${WORKDIR}
+  curl -fLROsS http://ftp.gnu.org/gnu/tar/tar-1.29.tar.xz
+  tar --strip-components=1 -xaf tar-1.29.tar.xz
+
+  ./configure --prefix=/usr --libexecdir=/libexec --disable-rpath \
+    CFLAGS="-Os ${CPU_SETTING} -ffunction-sections -fdata-sections -fstack-protector-strong -fpie -fpic" \
+    LDFLAGS="-Wl,-O1 -Wl,-z,relro -Wl,-znow -Wl,--as-needed -Wl,--strip-all -Wl,--gc-sections" >/dev/null
+  make -j$(nproc) >/dev/null
+
+  popd
+  mkdir -p ${dir}/files/dgr/usr/bin
+  mv ${WORKDIR}/src/tar "${tar}"
+  rm -r ${WORKDIR}
+fi
+
+sudo "${tar}" \
+  --transform "s:usr/sbin/haveged:usr/bin/haveged:" \
+  --exclude "./etc/udev" \
+  --exclude "./usr/share/locale" \
+  --exclude "./usr/libexec" \
+  --exclude "./usr/lib/systemd" \
+  --exclude "./usr/lib/udev" \
+  --exclude "./usr/sbin" \
+  -C ${rootfs}/dgr/ \
+  -xf ${dir}/rootfs.tar.xz
 sudo cp -R ${dir}/files/. ${rootfs}
 sudo chown root: ${rootfs}
 cp ${dir}/manifest.json ${target}/manifest
 sudo cp --no-preserve=ownership ${dist}/templater ${rootfs}/dgr/usr/bin/
 
-# some cleanup
-sudo rm -Rf ${rootfs}/dgr/etc/udev
-sudo rm -Rf ${rootfs}/dgr/usr/share/locale
-sudo rm -Rf ${rootfs}/dgr/usr/libexec
-sudo rm -Rf ${rootfs}/dgr/usr/lib/systemd
-sudo rm -Rf ${rootfs}/dgr/usr/lib/udev
-
-
-sudo mv ${rootfs}/dgr/usr/sbin/haveged ${rootfs}/dgr/usr/bin/haveged
-sudo rm -Rf ${rootfs}/dgr/usr/sbin/
 sudo bash -c "cd ${rootfs}/dgr/usr && ln -s bin sbin && cd -"
 
 cd ${target}
-sudo tar cpfz ../bindata/aci-builder.aci rootfs manifest
+sudo "${tar}" --sort=name --numeric-owner \
+  -cpzf ../bindata/aci-builder.aci manifest rootfs
 sudo chown ${USER}: ../bindata/aci-builder.aci
 sudo rm -Rf rootfs/
 cd -

aci-tester/build.sh

@@ -30,6 +30,9 @@ curl --fail --silent --show-error --location --remote-time --compressed --create
 
 chmod +x ${rootfs}/dgr/usr/bin/*
 
+: ${tar:="$(realpath "${dir}/../aci-builder/files/dgr/usr/bin/tar")"}
 cd ${target}
-tar cpfz ../bindata/aci-tester.aci rootfs manifest
+"${tar}" --sort=name --numeric-owner \
+  --owner=0 --group=0 \
+  -cpzf ../bindata/aci-tester.aci manifest rootfs
 cd -

dgr/common/aci-manifest.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/appc/spec/aci"
@@ -96,20 +97,38 @@ func WriteAciManifest(m *AciManifest, targetFile string, projectName string, dgr
 
 	//dgrBuilderIdentifier, _ := types.NewACIdentifier(ManifestDrgBuilder)
 	dgrVersionIdentifier, _ := types.NewACIdentifier(ManifestDrgVersion)
-	buildDateIdentifier, _ := types.NewACIdentifier("build-date")
 	im.Annotations.Set(*dgrVersionIdentifier, dgrVersion)
-	//im.Annotations.Set(*dgrBuilderIdentifier, m.Builder.Image.String())
-	im.Annotations.Set(*buildDateIdentifier, time.Now().Format(time.RFC3339))
+
+	if _, ok := im.Annotations.Get("build-date"); !ok {
+		buildDateIdentifier, _ := types.NewACIdentifier("build-date")
+		im.Annotations.Set(*buildDateIdentifier, time.Now().Format(time.RFC3339))
+	}
+
 	im.Dependencies, err = ToAppcDependencies(m.Aci.Dependencies)
 	if err != nil {
 		return errs.WithEF(err, fields, "Failed to prepare dependencies for manifest")
 	}
 	im.Name = *name
 	im.Labels = labels
 
+	for _, exclusion := range m.Build.Exclude {
+		if strings.HasPrefix("/dgr/bin", exclusion) {
+			goto collectionIsComplete
+		}
+	}
 	if len(m.Aci.App.Exec) == 0 {
 		m.Aci.App.Exec = []string{"/dgr/bin/busybox", "sh"}
 	}
+	// Set a pre-start handler, to run dgr's scripts.
+	for i := range m.Aci.App.EventHandlers {
+		if m.Aci.App.EventHandlers[i].Name == "pre-start" {
+			goto collectionIsComplete
+		}
+	}
+	m.Aci.App.EventHandlers = append(m.Aci.App.EventHandlers,
+		types.EventHandler{Name: "pre-start", Exec: []string{"/dgr/bin/prestart"}})
+
+collectionIsComplete:
 
 	isolators, err := ToAppcIsolators(m.Aci.App.Isolators)
 	if err != nil {
@@ -118,7 +137,7 @@ func WriteAciManifest(m *AciManifest, targetFile string, projectName string, dgr
 
 	im.App = &types.App{
 		Exec:              m.Aci.App.Exec,
-		EventHandlers:     []types.EventHandler{{Name: "pre-start", Exec: []string{"/dgr/bin/prestart"}}},
+		EventHandlers:     m.Aci.App.EventHandlers,
 		User:              m.Aci.App.User,
 		Group:             m.Aci.App.Group,
 		WorkingDirectory:  m.Aci.App.WorkingDirectory,

dgr/common/dgr-manifest.go

@@ -74,6 +74,8 @@ type BuilderDefinition struct {
 
 type BuildDefinition struct {
 	MountPoints []MountInfo `json:"mountPoints,omitempty" yaml:"mountPoints,omitempty"`
+	Exclude     []string    `json:"exclude,omitempty" yaml:"exclude,omitempty"`
+	Transform   []string    `json:"transform,omitempty" yaml:"transform,omitempty"`
 }
 
 type AciManifest struct {
@@ -97,15 +99,16 @@ type AciDefinition struct {
 }
 
 type DgrApp struct {
-	Exec              types.Exec         `json:"exec,omitempty" yaml:"exec,omitempty"`
-	User              string             `json:"user,omitempty" yaml:"user,omitempty"`
-	Group             string             `json:"group,omitempty" yaml:"group,omitempty"`
-	SupplementaryGIDs []int              `json:"supplementaryGIDs,omitempty" yaml:"supplementaryGIDs,omitempty"`
-	WorkingDirectory  string             `json:"workingDirectory,omitempty" yaml:"workingDirectory,omitempty"`
-	Environment       types.Environment  `json:"environment,omitempty" yaml:"environment,omitempty"`
-	MountPoints       []types.MountPoint `json:"mountPoints,omitempty" yaml:"mountPoints,omitempty"`
-	Ports             []types.Port       `json:"ports,omitempty" yaml:"ports,omitempty"`
-	Isolators         []Isolator         `json:"isolators,omitempty" yaml:"isolators,omitempty"`
+	Exec              types.Exec           `json:"exec,omitempty" yaml:"exec,omitempty"`
+	User              string               `json:"user,omitempty" yaml:"user,omitempty"`
+	Group             string               `json:"group,omitempty" yaml:"group,omitempty"`
+	SupplementaryGIDs []int                `json:"supplementaryGIDs,omitempty" yaml:"supplementaryGIDs,omitempty"`
+	WorkingDirectory  string               `json:"workingDirectory,omitempty" yaml:"workingDirectory,omitempty"`
+	Environment       types.Environment    `json:"environment,omitempty" yaml:"environment,omitempty"`
+	EventHandlers     []types.EventHandler `json:"eventHandlers,omitempty" yaml:"eventHandlers,omitempty"`
+	MountPoints       []types.MountPoint   `json:"mountPoints,omitempty" yaml:"mountPoints,omitempty"`
+	Ports             []types.Port         `json:"ports,omitempty" yaml:"ports,omitempty"`
+	Isolators         []Isolator           `json:"isolators,omitempty" yaml:"isolators,omitempty"`
 }
 
 type LinuxCapabilitiesSetValue struct {