From d8e99a90113574f613519897855fc1bfa8ffe14c Mon Sep 17 00:00:00 2001 From: Jesse Brown Date: Thu, 20 Jan 2022 16:35:51 -0600 Subject: [PATCH 1/7] wip Signed-off-by: Jesse Brown --- exporter.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ extender/main.go | 40 ++++++++++++++++++++++++-------------- test-extender.sh | 41 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 14 deletions(-) diff --git a/exporter.go b/exporter.go index fc5696c10..b3b13e84a 100644 --- a/exporter.go +++ b/exporter.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -91,6 +92,11 @@ func (e *Exporter) Export(opts ExportOptions) (platform.ExportReport, error) { return platform.ExportReport{}, errors.Wrap(err, "read build metadata") } + // extender layers + if err := e.addExtenderLayers(opts, &meta); err != nil { + return platform.ExportReport{}, err + } + // buildpack-provided layers if err := e.addBuildpackLayers(opts, &meta); err != nil { return platform.ExportReport{}, err @@ -284,6 +290,50 @@ func (e *Exporter) addAppLayers(opts ExportOptions, slices []layers.Slice, meta return nil } +func (e *Exporter) addExtenderLayers(opts ExportOptions, meta *platform.LayersMetadata) error { + e.Logger.Info("Adding extender layers") + manifestPath := "/layers-run/kaniko/manifest.json" + var manifest []struct { + Layers []string `json:"Layers"` + } + + f, err := os.Open(manifestPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } + + return err + } + + defer f.Close() + + contents, _ := ioutil.ReadAll(f) + e.Logger.Infof("kaniko manifest: %s", string(contents)) + f.Seek(0, 0) + + err = json.NewDecoder(f).Decode(&manifest) + if err != nil { + return err + } + + if len(manifest) == 0 || len(manifest[0].Layers) == 0 { + return nil + } + + // Iterate layers except first one (the base layer) + for _, layerName := range manifest[0].Layers { + tarPath := "/layers-run/kaniko/" + layerName + e.Logger.Infof("Adding extended layer '%s'", tarPath) + err := opts.WorkingImage.AddLayer(tarPath) + if err != nil { + return errors.Wrapf(err, "creating ext layer") + } + } + + return nil +} + func (e *Exporter) setLabels(opts ExportOptions, meta platform.LayersMetadata, buildMD *platform.BuildMetadata) error { data, err := json.Marshal(meta) if err != nil { diff --git a/extender/main.go b/extender/main.go index 02618ed5a..333915eb0 100644 --- a/extender/main.go +++ b/extender/main.go @@ -3,15 +3,16 @@ package main import ( "encoding/json" "fmt" - "github.com/BurntSushi/toml" - "github.com/GoogleContainerTools/kaniko/pkg/config" - cfg "github.com/redhat-buildpacks/poc/kaniko/buildpackconfig" - "github.com/redhat-buildpacks/poc/kaniko/util" "os" "os/exec" "path" "path/filepath" "syscall" + + "github.com/BurntSushi/toml" + "github.com/GoogleContainerTools/kaniko/pkg/config" + cfg "github.com/redhat-buildpacks/poc/kaniko/buildpackconfig" + "github.com/redhat-buildpacks/poc/kaniko/util" ) type BuildMetadata struct { @@ -108,6 +109,12 @@ func doKaniko(kind, baseimage string) { if err != nil { panic(err) } + + // make sure the child processes are done + err = reapChildProcesses() + if err != nil { + panic(err) + } } // Log the content of the Kaniko dir @@ -132,18 +139,18 @@ func doKaniko(kind, baseimage string) { panic(err) } - fmt.Printf("Extract the content of the tarball file %s under the cache %s\n", b.Opts.TarPath, b.Opts.CacheDir) - err = untar(dstPath, b.CacheDir) - if err != nil { - panic(err) - } + if kind == "build" { + fmt.Printf("Extract the content of the tarball file %s under the cache %s\n", b.Opts.TarPath, b.Opts.CacheDir) + err = untar(dstPath, b.CacheDir) + if err != nil { + panic(err) + } - layerFiles, err := layerTarfiles() - if err != nil { - panic(err) - } + layerFiles, err := layerTarfiles() + if err != nil { + panic(err) + } - if kind == "build" { // We're in "build" mode, untar layers to root filesystem: / for _, layerFile := range layerFiles { workingDirectory := "/" @@ -168,6 +175,11 @@ func doKaniko(kind, baseimage string) { panic(err) } } + + if kind == "run" { + // We're in "run" mode, leave the layer tar files for exporter to collect later. + // TODO: do we do anything else? Or is kaniko snapshot tar output all we need? + } } func untar(tarPath, workingDirectory string) error { diff --git a/test-extender.sh b/test-extender.sh index 189580ed3..e0d7b07c4 100755 --- a/test-extender.sh +++ b/test-extender.sh @@ -21,6 +21,11 @@ rm -rf $SAMPLES_REPO_PATH/kaniko mkdir -p $SAMPLES_REPO_PATH/kaniko rm -rf $SAMPLES_REPO_PATH/layers/kaniko mkdir -p $SAMPLES_REPO_PATH/layers/kaniko +rm -rf $SAMPLES_REPO_PATH/kaniko-run +mkdir -p $SAMPLES_REPO_PATH/kaniko-run +rm -rf $SAMPLES_REPO_PATH/layers-run/kaniko +mkdir -p $SAMPLES_REPO_PATH/layers-run/kaniko + echo ">>>>>>>>>> Running detect..." @@ -44,6 +49,11 @@ docker run \ test-builder \ /cnb/lifecycle/builder -use-extensions -log-level debug +echo ">>>>>>>>>> Copy build extension layers for run extension..." + +# this needs to come along for the ride for extending run image +cp -R $PWD/layers/* $PWD/layers-run + echo ">>>>>>>>>> Running extend on build image followed by build for buildpacks..." docker run \ @@ -57,3 +67,34 @@ docker run \ test-builder \ /cnb/lifecycle/extender kaniko build ubuntu:bionic # args: + +echo ">>>>>>>>>> Running extend on run image..." + +docker run \ + -v $PWD/workspace/:/workspace \ + -v $PWD/kaniko-run/:/kaniko \ + -v $PWD/layers-run/:/layers \ + -v $PWD/platform/:/platform \ + -v $PWD/cnb/ext/:/cnb/ext \ + -v $PWD/cnb/buildpacks/:/cnb/buildpacks \ + -u root \ + test-builder \ + /cnb/lifecycle/extender kaniko run ubuntu:bionic + # args: + +echo ">>>>>>>>>> Exporting final app image..." + +docker run \ + -v $PWD/workspace/:/workspace \ + -v $PWD/layers-run/:/layers-run \ + -v $PWD/layers/:/layers \ + -v $PWD/platform/:/platform \ + -v $PWD/cnb/ext/:/cnb/ext \ + -v $PWD/cnb/buildpacks/:/cnb/buildpacks \ + -u root \ + test-builder \ + /cnb/lifecycle/exporter -run-image ubuntu:bionic -log-level debug $IMAGE_TAG + +echo ">>>>>>>>>> Validate app image..." +# TODO: this fails because curl is not on there +docker run --rm --entrypoint bash -it $IMAGE_TAG -- curl google.com From 87ff4de6e6184028e3a91eb55f9193465a91c995 Mon Sep 17 00:00:00 2001 From: Jesse Brown Date: Thu, 20 Jan 2022 16:57:39 -0600 Subject: [PATCH 2/7] fixup! wip Signed-off-by: Jesse Brown --- extender/main.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/extender/main.go b/extender/main.go index 333915eb0..3f32e938f 100644 --- a/extender/main.go +++ b/extender/main.go @@ -109,12 +109,6 @@ func doKaniko(kind, baseimage string) { if err != nil { panic(err) } - - // make sure the child processes are done - err = reapChildProcesses() - if err != nil { - panic(err) - } } // Log the content of the Kaniko dir From d97defe789f831d784baef4d8ba6cb95ae0cc4ab Mon Sep 17 00:00:00 2001 From: Jesse Brown Date: Thu, 20 Jan 2022 17:57:21 -0600 Subject: [PATCH 3/7] store kaniko snapshots in different directories Signed-off-by: Jesse Brown --- exporter.go | 63 +++++++++++++++++---------- extender/main.go | 110 ++++++++++++++++++++++++++--------------------- test-extender.sh | 3 +- 3 files changed, 103 insertions(+), 73 deletions(-) diff --git a/exporter.go b/exporter.go index b3b13e84a..75bfba0a2 100644 --- a/exporter.go +++ b/exporter.go @@ -292,42 +292,59 @@ func (e *Exporter) addAppLayers(opts ExportOptions, slices []layers.Slice, meta func (e *Exporter) addExtenderLayers(opts ExportOptions, meta *platform.LayersMetadata) error { e.Logger.Info("Adding extender layers") - manifestPath := "/layers-run/kaniko/manifest.json" - var manifest []struct { - Layers []string `json:"Layers"` - } - f, err := os.Open(manifestPath) + path := filepath.Join("/layers-run/kaniko") + dirs, err := ioutil.ReadDir(path) if err != nil { if os.IsNotExist(err) { return nil } - return err } - defer f.Close() + for _, dir := range dirs { + if !dir.IsDir() { + continue + } - contents, _ := ioutil.ReadAll(f) - e.Logger.Infof("kaniko manifest: %s", string(contents)) - f.Seek(0, 0) + e.Logger.Infof("Adding extender layers for %s", dir.Name()) + manifestPath := filepath.Join(path, dir.Name(), "manifest.json") + var manifest []struct { + Layers []string `json:"Layers"` + } - err = json.NewDecoder(f).Decode(&manifest) - if err != nil { - return err - } + f, err := os.Open(manifestPath) + if err != nil { + if os.IsNotExist(err) { + return nil + } - if len(manifest) == 0 || len(manifest[0].Layers) == 0 { - return nil - } + return err + } + + defer f.Close() + + contents, _ := ioutil.ReadAll(f) + e.Logger.Infof("kaniko manifest: %s", string(contents)) + f.Seek(0, 0) - // Iterate layers except first one (the base layer) - for _, layerName := range manifest[0].Layers { - tarPath := "/layers-run/kaniko/" + layerName - e.Logger.Infof("Adding extended layer '%s'", tarPath) - err := opts.WorkingImage.AddLayer(tarPath) + err = json.NewDecoder(f).Decode(&manifest) if err != nil { - return errors.Wrapf(err, "creating ext layer") + return err + } + + if len(manifest) == 0 || len(manifest[0].Layers) == 0 { + return nil + } + + // Iterate layers except first one (the base layer) + for _, layerName := range manifest[0].Layers[1:] { + tarPath := filepath.Join(path, dir.Name(), layerName) + e.Logger.Infof("Adding extended layer '%s'", tarPath) + err := opts.WorkingImage.AddLayer(tarPath) + if err != nil { + return errors.Wrapf(err, "creating ext layer") + } } } diff --git a/extender/main.go b/extender/main.go index 3f32e938f..cb3b1fa1a 100644 --- a/extender/main.go +++ b/extender/main.go @@ -77,6 +77,7 @@ func doKaniko(kind, baseimage string) { panic(err) } + // TODO: make args specific to build/run phase toMultiArg := func(args []DockerfileArg) []string { var result []string for _, arg := range args { @@ -87,7 +88,7 @@ func doKaniko(kind, baseimage string) { return result } - for _, d := range meta.Dockerfiles { + for i, d := range meta.Dockerfiles { // Define opts opts := config.KanikoOptions{ BuildArgs: toMultiArg(d.Args), @@ -109,71 +110,82 @@ func doKaniko(kind, baseimage string) { if err != nil { panic(err) } - } - - // Log the content of the Kaniko dir - fmt.Printf("Reading dir content of: %s\n", b.KanikoDir) - util.ReadFilesFromPath(b.KanikoDir) - - // TODO: caching doesn't seem to be working at the moment. Need to investigate... - // Copy the tgz layer file to the Cache dir - srcPath := path.Join("/", b.LayerTarFileName) - dstPath := path.Join(b.CacheDir, b.LayerTarFileName) - // Ensure cache directory exists - fmt.Printf("Creating %s dir ...\n", b.CacheDir) - err = os.MkdirAll(b.CacheDir, os.ModePerm) - if err != nil { - panic(err) - } + // Log the content of the Kaniko dir + fmt.Printf("Reading dir content of: %s\n", b.KanikoDir) + util.ReadFilesFromPath(b.KanikoDir) - fmt.Printf("Copy the %s file to the %s dir ...\n", srcPath, dstPath) - err = util.File(srcPath, dstPath) - if err != nil { - panic(err) - } + // TODO: caching doesn't seem to be working at the moment. Need to investigate... + // Copy the tgz layer file to the Cache dir + srcPath := path.Join("/", b.LayerTarFileName) + // TODO: use something else besides the index so we can cache even if order changes + dstPath := path.Join(b.CacheDir, fmt.Sprintf("%d.tar", i)) - if kind == "build" { - fmt.Printf("Extract the content of the tarball file %s under the cache %s\n", b.Opts.TarPath, b.Opts.CacheDir) - err = untar(dstPath, b.CacheDir) + // Ensure cache directory exists + fmt.Printf("Creating %s dir ...\n", b.CacheDir) + err = os.MkdirAll(b.CacheDir, os.ModePerm) if err != nil { panic(err) } - layerFiles, err := layerTarfiles() + fmt.Printf("Copy the %s file to the %s dir ...\n", srcPath, dstPath) + err = util.File(srcPath, dstPath) if err != nil { panic(err) } - // We're in "build" mode, untar layers to root filesystem: / - for _, layerFile := range layerFiles { - workingDirectory := "/" - tarPath := "/layers/kaniko/" + layerFile + extractPath := path.Join(b.CacheDir, fmt.Sprintf("%d", i)) + err = os.MkdirAll(extractPath, os.ModePerm) + if err != nil { + panic(err) + } + fmt.Printf("Extract the content of the tarball file %s under the cache %s\n", dstPath, extractPath) + err = untar(dstPath, extractPath) + if err != nil { + panic(err) + } - err = untar(tarPath, workingDirectory) + if kind == "build" { + // TODO: Why are we doing this for build? Executing the commands is enough right? Do we need to extract over the same files? + // + // manifestPath := path.Join(b.CacheDir, fmt.Sprintf("%d", i), "manifest.json") + // fmt.Printf("Untaring layers to root filesystem for %s", manifestPath) + // layerFiles, err := layerTarfiles(manifestPath) + // if err != nil { + // panic(err) + // } + + // // We're in "build" mode, untar layers to root filesystem: / + // for _, layerFile := range layerFiles { + // workingDirectory := "/" + // tarPath := "/layers/kaniko/" + layerFile + + // err = untar(tarPath, workingDirectory) + // if err != nil { + // panic(err) + // } + // } + + // Run the build for buildpacks with lowered privileges. + // We must assume that this extender is run as root. + cmd := exec.Command("/cnb/lifecycle/builder", "-app", "/workspace", "-log-level", "debug") + cmd.Env = append(cmd.Env, "CNB_PLATFORM_API=0.8") + cmd.SysProcAttr = &syscall.SysProcAttr{} + cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 1000, Gid: 1000} + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() if err != nil { panic(err) } } - // Run the build for buildpacks with lowered privileges. - // We must assume that this extender is run as root. - cmd := exec.Command("/cnb/lifecycle/builder", "-app", "/workspace", "-log-level", "debug") - cmd.Env = append(cmd.Env, "CNB_PLATFORM_API=0.8") - cmd.SysProcAttr = &syscall.SysProcAttr{} - cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 1000, Gid: 1000} - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - panic(err) + if kind == "run" { + // We're in "run" mode, leave the layer tar files for exporter to collect later. + // TODO: do we do anything else? Or is kaniko snapshot tar output all we need? + // TODO: I think we can cleanup the 0.tar files } } - - if kind == "run" { - // We're in "run" mode, leave the layer tar files for exporter to collect later. - // TODO: do we do anything else? Or is kaniko snapshot tar output all we need? - } } func untar(tarPath, workingDirectory string) error { @@ -183,8 +195,8 @@ func untar(tarPath, workingDirectory string) error { return command.Run() } -func layerTarfiles() ([]string, error) { - manifestPath := "/layers/kaniko/manifest.json" +func layerTarfiles(manifestPath string) ([]string, error) { + // manifestPath := "/layers/kaniko/manifest.json" var manifest []struct { Layers []string `json:"Layers"` } diff --git a/test-extender.sh b/test-extender.sh index e0d7b07c4..a6de09bda 100755 --- a/test-extender.sh +++ b/test-extender.sh @@ -96,5 +96,6 @@ docker run \ /cnb/lifecycle/exporter -run-image ubuntu:bionic -log-level debug $IMAGE_TAG echo ">>>>>>>>>> Validate app image..." -# TODO: this fails because curl is not on there +docker pull $IMAGE_TAG +# TODO: this fails because "error while loading shared libraries: libnghttp2.so.14: cannot open shared object file: No such file or directory" docker run --rm --entrypoint bash -it $IMAGE_TAG -- curl google.com From 1f2d774ac344be7faeb5bc472e7df308503bc12f Mon Sep 17 00:00:00 2001 From: Jesse Brown Date: Tue, 25 Jan 2022 10:52:06 -0600 Subject: [PATCH 4/7] more wip work Signed-off-by: Jesse Brown --- exporter.go | 7 +- extender/main.go | 178 +++++++++++++++++++++++------------------------ test-extender.sh | 49 +++++++------ 3 files changed, 119 insertions(+), 115 deletions(-) diff --git a/exporter.go b/exporter.go index 75bfba0a2..ab301a1c5 100644 --- a/exporter.go +++ b/exporter.go @@ -93,9 +93,10 @@ func (e *Exporter) Export(opts ExportOptions) (platform.ExportReport, error) { } // extender layers - if err := e.addExtenderLayers(opts, &meta); err != nil { - return platform.ExportReport{}, err - } + // the new base is currently being referenced via run-image + // if err := e.addExtenderLayers(opts, &meta); err != nil { + // return platform.ExportReport{}, err + // } // buildpack-provided layers if err := e.addBuildpackLayers(opts, &meta); err != nil { diff --git a/extender/main.go b/extender/main.go index cb3b1fa1a..6e85c00ee 100644 --- a/extender/main.go +++ b/extender/main.go @@ -5,14 +5,12 @@ import ( "fmt" "os" "os/exec" - "path" "path/filepath" "syscall" "github.com/BurntSushi/toml" "github.com/GoogleContainerTools/kaniko/pkg/config" cfg "github.com/redhat-buildpacks/poc/kaniko/buildpackconfig" - "github.com/redhat-buildpacks/poc/kaniko/util" ) type BuildMetadata struct { @@ -62,6 +60,7 @@ func doKaniko(kind, baseimage string) { fmt.Println("Initialize the BuildPackConfig and set the defaults values ...") b := cfg.NewBuildPackConfig() b.InitDefaults() + b.ExtractLayers = false fmt.Printf("Kaniko dir: %s\n", b.KanikoDir) @@ -77,114 +76,109 @@ func doKaniko(kind, baseimage string) { panic(err) } - // TODO: make args specific to build/run phase - toMultiArg := func(args []DockerfileArg) []string { - var result []string - for _, arg := range args { - result = append(result, fmt.Sprintf("%s=%s", arg.Key, arg.Value)) + // start with the base image given to us by the builder + current_base_image := baseimage + + for _, d := range meta.Dockerfiles { + // skip the extensions not of this type + if d.Type != kind { + continue } - result = append(result, fmt.Sprintf(`base_image=%s`, baseimage)) - return result - } + toMultiArg := func(args []DockerfileArg) []string { + var result []string + for _, arg := range args { + result = append(result, fmt.Sprintf("%s=%s", arg.Key, arg.Value)) + } - for i, d := range meta.Dockerfiles { - // Define opts - opts := config.KanikoOptions{ - BuildArgs: toMultiArg(d.Args), - DockerfilePath: d.Path, - CacheOptions: config.CacheOptions{CacheDir: b.CacheDir}, - IgnoreVarRun: true, - NoPush: true, - SrcContext: b.WorkspaceDir, - SnapshotMode: "full", - IgnorePaths: b.IgnorePaths, - TarPath: b.LayerTarFileName, - Destinations: []string{b.Destination}, - ForceBuildMetadata: true, + result = append(result, fmt.Sprintf(`base_image=%s`, current_base_image)) + return result } - // Build the Dockerfile - fmt.Printf("Building the %s\n", opts.DockerfilePath) - err := b.BuildDockerFile(opts) - if err != nil { - panic(err) + if kind == "build" { + // TODO: execute all but the FROM clauses in the Dockerfile? + + // Define opts + opts := config.KanikoOptions{ + BuildArgs: toMultiArg(d.Args), + DockerfilePath: d.Path, + // Cache: true, + // CacheOptions: config.CacheOptions{ + // CacheDir: b.CacheDir, + // }, + // IgnoreVarRun: true, + NoPush: true, + SrcContext: b.WorkspaceDir, + SnapshotMode: "full", + // IgnorePaths: b.IgnorePaths, + // TODO: we can not output a tar for intermediate images, we only need the last one + // TarPath: "/layers/kaniko/new_base.tar", + // Destinations: []string{b.Destination}, + // ForceBuildMetadata: true, + // Cleanup: true, + } + + // Build the Dockerfile + fmt.Printf("Building the %s\n", opts.DockerfilePath) + err := b.BuildDockerFile(opts) + if err != nil { + panic(err) + } + continue } - // Log the content of the Kaniko dir - fmt.Printf("Reading dir content of: %s\n", b.KanikoDir) - util.ReadFilesFromPath(b.KanikoDir) + // run extensions - // TODO: caching doesn't seem to be working at the moment. Need to investigate... - // Copy the tgz layer file to the Cache dir - srcPath := path.Join("/", b.LayerTarFileName) - // TODO: use something else besides the index so we can cache even if order changes - dstPath := path.Join(b.CacheDir, fmt.Sprintf("%d.tar", i)) + // we need a registry right now, because kaniko is pulling the image to build on top of for subsequent Dockerfile exts + registryHost := os.Getenv("REGISTRY_HOST") + b.Destination = fmt.Sprintf("%s/extended/runimage", registryHost) + fmt.Printf("Destination Image: %s\n", b.Destination) - // Ensure cache directory exists - fmt.Printf("Creating %s dir ...\n", b.CacheDir) - err = os.MkdirAll(b.CacheDir, os.ModePerm) - if err != nil { - panic(err) + // Define opts + opts := config.KanikoOptions{ + BuildArgs: toMultiArg(d.Args), + DockerfilePath: d.Path, + // Cache: true, + // CacheOptions: config.CacheOptions{ + // CacheDir: b.CacheDir, + // }, + // IgnoreVarRun: true, + // NoPush: true, + SrcContext: b.WorkspaceDir, + SnapshotMode: "full", + // IgnorePaths: b.IgnorePaths, + // TODO: we can not output a tar for intermediate images, we only need the last one + TarPath: "/layers/kaniko/new_base.tar", + Destinations: []string{b.Destination}, + // ForceBuildMetadata: true, + // Cleanup: true, } - fmt.Printf("Copy the %s file to the %s dir ...\n", srcPath, dstPath) - err = util.File(srcPath, dstPath) + // Build the Dockerfile + fmt.Printf("Building the %s\n", opts.DockerfilePath) + err := b.BuildDockerFile(opts) if err != nil { panic(err) } - extractPath := path.Join(b.CacheDir, fmt.Sprintf("%d", i)) - err = os.MkdirAll(extractPath, os.ModePerm) - if err != nil { - panic(err) - } - fmt.Printf("Extract the content of the tarball file %s under the cache %s\n", dstPath, extractPath) - err = untar(dstPath, extractPath) + // TODO: next base image is the one we just built, we should use digest instead + current_base_image = b.Destination + } + + // run buildpacks now + if kind == "build" { + // Run the build for buildpacks with lowered privileges. + // We must assume that this extender is run as root. + cmd := exec.Command("/cnb/lifecycle/builder", "-app", "/workspace", "-log-level", "debug") + cmd.Env = append(cmd.Env, "CNB_PLATFORM_API=0.8") + cmd.SysProcAttr = &syscall.SysProcAttr{} + cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 1000, Gid: 1000} + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() if err != nil { panic(err) } - - if kind == "build" { - // TODO: Why are we doing this for build? Executing the commands is enough right? Do we need to extract over the same files? - // - // manifestPath := path.Join(b.CacheDir, fmt.Sprintf("%d", i), "manifest.json") - // fmt.Printf("Untaring layers to root filesystem for %s", manifestPath) - // layerFiles, err := layerTarfiles(manifestPath) - // if err != nil { - // panic(err) - // } - - // // We're in "build" mode, untar layers to root filesystem: / - // for _, layerFile := range layerFiles { - // workingDirectory := "/" - // tarPath := "/layers/kaniko/" + layerFile - - // err = untar(tarPath, workingDirectory) - // if err != nil { - // panic(err) - // } - // } - - // Run the build for buildpacks with lowered privileges. - // We must assume that this extender is run as root. - cmd := exec.Command("/cnb/lifecycle/builder", "-app", "/workspace", "-log-level", "debug") - cmd.Env = append(cmd.Env, "CNB_PLATFORM_API=0.8") - cmd.SysProcAttr = &syscall.SysProcAttr{} - cmd.SysProcAttr.Credential = &syscall.Credential{Uid: 1000, Gid: 1000} - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - err = cmd.Run() - if err != nil { - panic(err) - } - } - - if kind == "run" { - // We're in "run" mode, leave the layer tar files for exporter to collect later. - // TODO: do we do anything else? Or is kaniko snapshot tar output all we need? - // TODO: I think we can cleanup the 0.tar files - } } } diff --git a/test-extender.sh b/test-extender.sh index a6de09bda..617a6751c 100755 --- a/test-extender.sh +++ b/test-extender.sh @@ -1,8 +1,12 @@ +set -e + cd $LIFECYCLE_REPO_PATH echo ">>>>>>>>>> Building lifecycle..." -docker image rm test-builder --force +docker image rm $REGISTRY_HOST/test-builder --force +# remove the $REGISTRY_HOST/extended/runimage from any previous run +docker image rm $REGISTRY_HOST/extended/runimage --force make clean build-linux-amd64 @@ -13,7 +17,8 @@ FROM cnbs/sample-builder:bionic COPY ./lifecycle /cnb/lifecycle EOF -docker build -t test-builder . +docker build -t $REGISTRY_HOST/test-builder . +docker push $REGISTRY_HOST/test-builder cd $SAMPLES_REPO_PATH @@ -26,7 +31,6 @@ mkdir -p $SAMPLES_REPO_PATH/kaniko-run rm -rf $SAMPLES_REPO_PATH/layers-run/kaniko mkdir -p $SAMPLES_REPO_PATH/layers-run/kaniko - echo ">>>>>>>>>> Running detect..." docker run \ @@ -35,7 +39,7 @@ docker run \ -v $PWD/platform/:/platform \ -v $PWD/cnb/ext/:/cnb/ext \ -v $PWD/cnb/buildpacks/:/cnb/buildpacks \ - test-builder \ + $REGISTRY_HOST/test-builder \ /cnb/lifecycle/detector -order /layers/order.toml -log-level debug echo ">>>>>>>>>> Running build for extensions..." @@ -46,7 +50,7 @@ docker run \ -v $PWD/platform/:/platform \ -v $PWD/cnb/ext/:/cnb/ext \ -v $PWD/cnb/buildpacks/:/cnb/buildpacks \ - test-builder \ + $REGISTRY_HOST/test-builder \ /cnb/lifecycle/builder -use-extensions -log-level debug echo ">>>>>>>>>> Copy build extension layers for run extension..." @@ -64,25 +68,29 @@ docker run \ -v $PWD/cnb/ext/:/cnb/ext \ -v $PWD/cnb/buildpacks/:/cnb/buildpacks \ -u root \ - test-builder \ - /cnb/lifecycle/extender kaniko build ubuntu:bionic + $REGISTRY_HOST/test-builder \ + /cnb/lifecycle/extender kaniko build "$REGISTRY_HOST/test-builder" # args: echo ">>>>>>>>>> Running extend on run image..." - + # TODO: we should probably not have to mount /cnb/ext? Can we copy the static Dockerfiles to layers? docker run \ -v $PWD/workspace/:/workspace \ -v $PWD/kaniko-run/:/kaniko \ -v $PWD/layers-run/:/layers \ - -v $PWD/platform/:/platform \ -v $PWD/cnb/ext/:/cnb/ext \ - -v $PWD/cnb/buildpacks/:/cnb/buildpacks \ -u root \ - test-builder \ - /cnb/lifecycle/extender kaniko run ubuntu:bionic + -e REGISTRY_HOST=$REGISTRY_HOST \ + $REGISTRY_HOST/test-builder \ + /cnb/lifecycle/extender kaniko run cnbs/sample-stack-run:bionic # args: -echo ">>>>>>>>>> Exporting final app image..." +echo ">>>>>>>>>> Validate extended run image..." +docker load -i $PWD/layers-run/kaniko/new_base.tar +docker run --rm -it $REGISTRY_HOST/extended/runimage cat /opt/arg.txt +docker run --rm -it $REGISTRY_HOST/extended/runimage curl google.com + +# echo ">>>>>>>>>> Exporting final app image..." docker run \ -v $PWD/workspace/:/workspace \ @@ -92,10 +100,11 @@ docker run \ -v $PWD/cnb/ext/:/cnb/ext \ -v $PWD/cnb/buildpacks/:/cnb/buildpacks \ -u root \ - test-builder \ - /cnb/lifecycle/exporter -run-image ubuntu:bionic -log-level debug $IMAGE_TAG - -echo ">>>>>>>>>> Validate app image..." -docker pull $IMAGE_TAG -# TODO: this fails because "error while loading shared libraries: libnghttp2.so.14: cannot open shared object file: No such file or directory" -docker run --rm --entrypoint bash -it $IMAGE_TAG -- curl google.com + $REGISTRY_HOST/test-builder \ + /cnb/lifecycle/exporter -log-level debug -run-image $REGISTRY_HOST/extended/runimage $REGISTRY_HOST/appimage + +# echo ">>>>>>>>>> Validate app image..." +docker pull $REGISTRY_HOST/appimage +docker run --rm --entrypoint cat -it $REGISTRY_HOST/appimage /opt/arg.txt +# # TODO: this fails because "error while loading shared libraries: libnghttp2.so.14: cannot open shared object file: No such file or directory" +docker run --rm --entrypoint curl -it $REGISTRY_HOST/appimage google.com From 0f6fb26dd58038b93e3a194c8f113a0d37509ea6 Mon Sep 17 00:00:00 2001 From: Jesse Brown Date: Tue, 25 Jan 2022 11:01:21 -0600 Subject: [PATCH 5/7] fixup! more wip work Signed-off-by: Jesse Brown --- test-extender.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test-extender.sh b/test-extender.sh index 617a6751c..c75792167 100755 --- a/test-extender.sh +++ b/test-extender.sh @@ -88,7 +88,7 @@ docker run \ echo ">>>>>>>>>> Validate extended run image..." docker load -i $PWD/layers-run/kaniko/new_base.tar docker run --rm -it $REGISTRY_HOST/extended/runimage cat /opt/arg.txt -docker run --rm -it $REGISTRY_HOST/extended/runimage curl google.com +# docker run --rm -it $REGISTRY_HOST/extended/runimage curl google.com # echo ">>>>>>>>>> Exporting final app image..." @@ -107,4 +107,4 @@ docker run \ docker pull $REGISTRY_HOST/appimage docker run --rm --entrypoint cat -it $REGISTRY_HOST/appimage /opt/arg.txt # # TODO: this fails because "error while loading shared libraries: libnghttp2.so.14: cannot open shared object file: No such file or directory" -docker run --rm --entrypoint curl -it $REGISTRY_HOST/appimage google.com +# docker run --rm --entrypoint curl -it $REGISTRY_HOST/appimage google.com From 03a9503c344caade7aac80a1f270a49c3f1ce1e5 Mon Sep 17 00:00:00 2001 From: Jesse Brown Date: Tue, 25 Jan 2022 13:44:51 -0600 Subject: [PATCH 6/7] even more work Signed-off-by: Jesse Brown --- extender/main.go | 25 +++++++++++++++++-------- test-extender.sh | 25 ++++++++++++++----------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/extender/main.go b/extender/main.go index 6e85c00ee..4872c1fca 100644 --- a/extender/main.go +++ b/extender/main.go @@ -97,25 +97,29 @@ func doKaniko(kind, baseimage string) { if kind == "build" { // TODO: execute all but the FROM clauses in the Dockerfile? - // Define opts opts := config.KanikoOptions{ BuildArgs: toMultiArg(d.Args), DockerfilePath: d.Path, - // Cache: true, + Cache: true, // CacheOptions: config.CacheOptions{ // CacheDir: b.CacheDir, // }, - // IgnoreVarRun: true, + IgnoreVarRun: true, NoPush: true, SrcContext: b.WorkspaceDir, SnapshotMode: "full", + // OCILayoutPath: "/layers/kaniko", + // ImageNameDigestFile: fmt.Sprintf("/layers/kaniko/idk%s", b.Destination), + RegistryOptions: config.RegistryOptions{ + SkipTLSVerify: true, + }, // IgnorePaths: b.IgnorePaths, // TODO: we can not output a tar for intermediate images, we only need the last one // TarPath: "/layers/kaniko/new_base.tar", // Destinations: []string{b.Destination}, // ForceBuildMetadata: true, - // Cleanup: true, + Cleanup: false, } // Build the Dockerfile @@ -138,20 +142,25 @@ func doKaniko(kind, baseimage string) { opts := config.KanikoOptions{ BuildArgs: toMultiArg(d.Args), DockerfilePath: d.Path, - // Cache: true, + Cache: true, // CacheOptions: config.CacheOptions{ // CacheDir: b.CacheDir, // }, - // IgnoreVarRun: true, + IgnoreVarRun: true, // NoPush: true, SrcContext: b.WorkspaceDir, SnapshotMode: "full", + // OCILayoutPath: "/layers/kaniko", + // ImageNameDigestFile: fmt.Sprintf("/layers/kaniko/idk%s", b.Destination), // IgnorePaths: b.IgnorePaths, // TODO: we can not output a tar for intermediate images, we only need the last one - TarPath: "/layers/kaniko/new_base.tar", + // TarPath: "/layers/kaniko/new_base.tar", Destinations: []string{b.Destination}, // ForceBuildMetadata: true, - // Cleanup: true, + Cleanup: true, + RegistryOptions: config.RegistryOptions{ + SkipTLSVerify: true, + }, } // Build the Dockerfile diff --git a/test-extender.sh b/test-extender.sh index c75792167..86e25d798 100755 --- a/test-extender.sh +++ b/test-extender.sh @@ -20,6 +20,15 @@ EOF docker build -t $REGISTRY_HOST/test-builder . docker push $REGISTRY_HOST/test-builder +cat << EOF > Dockerfile.extender +FROM scratch +COPY ./lifecycle /cnb/lifecycle +ENTRYPOINT /cnb/lifecycle/extender +EOF + +docker build -f Dockerfile.extender -t $REGISTRY_HOST/extender . +docker push $REGISTRY_HOST/extender + cd $SAMPLES_REPO_PATH rm -rf $SAMPLES_REPO_PATH/kaniko @@ -79,18 +88,13 @@ docker run \ -v $PWD/kaniko-run/:/kaniko \ -v $PWD/layers-run/:/layers \ -v $PWD/cnb/ext/:/cnb/ext \ - -u root \ -e REGISTRY_HOST=$REGISTRY_HOST \ - $REGISTRY_HOST/test-builder \ - /cnb/lifecycle/extender kaniko run cnbs/sample-stack-run:bionic + --entrypoint /cnb/lifecycle/extender \ + $REGISTRY_HOST/extender \ + kaniko run cnbs/sample-stack-run:bionic # args: -echo ">>>>>>>>>> Validate extended run image..." -docker load -i $PWD/layers-run/kaniko/new_base.tar -docker run --rm -it $REGISTRY_HOST/extended/runimage cat /opt/arg.txt -# docker run --rm -it $REGISTRY_HOST/extended/runimage curl google.com - -# echo ">>>>>>>>>> Exporting final app image..." +echo ">>>>>>>>>> Exporting final app image..." docker run \ -v $PWD/workspace/:/workspace \ @@ -106,5 +110,4 @@ docker run \ # echo ">>>>>>>>>> Validate app image..." docker pull $REGISTRY_HOST/appimage docker run --rm --entrypoint cat -it $REGISTRY_HOST/appimage /opt/arg.txt -# # TODO: this fails because "error while loading shared libraries: libnghttp2.so.14: cannot open shared object file: No such file or directory" -# docker run --rm --entrypoint curl -it $REGISTRY_HOST/appimage google.com +docker run --rm --entrypoint curl -it $REGISTRY_HOST/appimage google.com From 9dd5d4b7284c4373d9a2b752ea5b4de29154cf6d Mon Sep 17 00:00:00 2001 From: Jesse Brown Date: Tue, 25 Jan 2022 13:45:32 -0600 Subject: [PATCH 7/7] turn off cache Signed-off-by: Jesse Brown --- extender/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extender/main.go b/extender/main.go index 4872c1fca..10eea5681 100644 --- a/extender/main.go +++ b/extender/main.go @@ -101,7 +101,7 @@ func doKaniko(kind, baseimage string) { opts := config.KanikoOptions{ BuildArgs: toMultiArg(d.Args), DockerfilePath: d.Path, - Cache: true, + // Cache: true, // CacheOptions: config.CacheOptions{ // CacheDir: b.CacheDir, // }, @@ -142,7 +142,7 @@ func doKaniko(kind, baseimage string) { opts := config.KanikoOptions{ BuildArgs: toMultiArg(d.Args), DockerfilePath: d.Path, - Cache: true, + // Cache: true, // CacheOptions: config.CacheOptions{ // CacheDir: b.CacheDir, // },