public_git/kaniko
1
0
Fork 0
mirror of https://github.com/GoogleContainerTools/kaniko synced 2026-10-18 13:59:03 +02:00
Code Issues Releases Wiki Activity

Switch from containers/image to go-containerregistry (#140)

Browse Source 
* Vendor changes for go-containerregistry switch.

* Manual changes for go-containerregistry switch.

The biggest change is refactoring the tarball unpacking.

* Pull more of container-diff out.

* More vendor removals.

* More unit tests.
This commit is contained in:
dlorenc
2018-04-25 19:21:05 -07:00
committed by GitHub
parent 806555538f
commit cd5b744904
994 changed files with 5363 additions and 141520 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/commands/add.go
+3 -2
View File
@@ -20,8 +20,9 @@ import (
"path/filepath"
"strings"
"github.com/google/go-containerregistry/v1"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/sirupsen/logrus"
)
@@ -40,7 +41,7 @@ type AddCommand struct {
// - If dest doesn't end with a slash, the filepath is inferred to be <dest>/<filename>
// 2. If <src> is a local tar archive:
// -If <src> is a local tar archive, it is unpacked at the dest, as 'tar -x' would
func (a *AddCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (a *AddCommand) ExecuteCommand(config *v1.Config) error {
srcs := a.cmd.SourcesAndDest[:len(a.cmd.SourcesAndDest)-1]
dest := a.cmd.SourcesAndDest[len(a.cmd.SourcesAndDest)-1]
pkg/commands/cmd.go
+5 -4
View File
@@ -17,10 +17,11 @@ limitations under the License.
package commands
import (
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/sirupsen/logrus"
"strings"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
type CmdCommand struct {
@@ -29,7 +30,7 @@ type CmdCommand struct {
// ExecuteCommand executes the CMD command
// Argument handling is the same as RUN.
func (c *CmdCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (c *CmdCommand) ExecuteCommand(config *v1.Config) error {
logrus.Info("cmd: CMD")
var newCommand []string
if c.cmd.PrependShell {
pkg/commands/cmd_test.go
+6 -6
View File
@@ -18,32 +18,32 @@ package commands
import (
"testing"
"github.com/google/go-containerregistry/v1"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/containers/image/manifest"
"github.com/containers/image/pkg/strslice"
"github.com/docker/docker/builder/dockerfile/instructions"
)
var cmdTests = []struct {
prependShell bool
cmdLine []string
expectedCmd strslice.StrSlice
expectedCmd []string
}{
{
prependShell: true,
cmdLine: []string{"echo", "cmd1"},
expectedCmd: strslice.StrSlice{"/bin/sh", "-c", "echo cmd1"},
expectedCmd: []string{"/bin/sh", "-c", "echo cmd1"},
},
{
prependShell: false,
cmdLine: []string{"echo", "cmd2"},
expectedCmd: strslice.StrSlice{"echo", "cmd2"},
expectedCmd: []string{"echo", "cmd2"},
},
}
func TestExecuteCmd(t *testing.T) {
cfg := &manifest.Schema2Config{
cfg := &v1.Config{
Cmd: nil,
}
pkg/commands/commands.go
+2 -2
View File
@@ -17,8 +17,8 @@ limitations under the License.
package commands
import (
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@@ -28,7 +28,7 @@ type DockerCommand interface {
// 1. Making required changes to the filesystem (ex. copying files for ADD/COPY or setting ENV variables)
// 2. Updating metadata fields in the config
// It should not change the config history.
ExecuteCommand(*manifest.Schema2Config) error
ExecuteCommand(*v1.Config) error
// The config history has a "created by" field, should return information about the command
CreatedBy() string
// A list of files to snapshot, empty for metadata commands or nil if we don't know
pkg/commands/copy.go
+6 -5
View File
@@ -17,13 +17,14 @@ limitations under the License.
package commands
import (
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/sirupsen/logrus"
"os"
"path/filepath"
"strings"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
type CopyCommand struct {
@@ -32,7 +33,7 @@ type CopyCommand struct {
snapshotFiles []string
}
func (c *CopyCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (c *CopyCommand) ExecuteCommand(config *v1.Config) error {
srcs := c.cmd.SourcesAndDest[:len(c.cmd.SourcesAndDest)-1]
dest := c.cmd.SourcesAndDest[len(c.cmd.SourcesAndDest)-1]
pkg/commands/entrypoint.go
+5 -4
View File
@@ -17,10 +17,11 @@ limitations under the License.
package commands
import (
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/sirupsen/logrus"
"strings"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
type EntrypointCommand struct {
@@ -28,7 +29,7 @@ type EntrypointCommand struct {
}
// ExecuteCommand handles command processing similar to CMD and RUN,
func (e *EntrypointCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (e *EntrypointCommand) ExecuteCommand(config *v1.Config) error {
logrus.Info("cmd: ENTRYPOINT")
var newCommand []string
if e.cmd.PrependShell {
pkg/commands/entrypoint_test.go
+5 -6
View File
@@ -19,31 +19,30 @@ import (
"testing"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/containers/image/manifest"
"github.com/containers/image/pkg/strslice"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
)
var entrypointTests = []struct {
prependShell bool
cmdLine []string
expectedCmd strslice.StrSlice
expectedCmd []string
}{
{
prependShell: true,
cmdLine: []string{"echo", "cmd1"},
expectedCmd: strslice.StrSlice{"/bin/sh", "-c", "echo cmd1"},
expectedCmd: []string{"/bin/sh", "-c", "echo cmd1"},
},
{
prependShell: false,
cmdLine: []string{"echo", "cmd2"},
expectedCmd: strslice.StrSlice{"echo", "cmd2"},
expectedCmd: []string{"echo", "cmd2"},
},
}
func TestEntrypointExecuteCmd(t *testing.T) {
cfg := &manifest.Schema2Config{
cfg := &v1.Config{
Cmd: nil,
}
pkg/commands/env.go
+3 -3
View File
@@ -20,8 +20,8 @@ import (
"strings"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
@@ -29,7 +29,7 @@ type EnvCommand struct {
cmd *instructions.EnvCommand
}
func (e *EnvCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (e *EnvCommand) ExecuteCommand(config *v1.Config) error {
logrus.Info("cmd: ENV")
newEnvs := e.cmd.Env
for index, pair := range newEnvs {
@@ -49,7 +49,7 @@ func (e *EnvCommand) ExecuteCommand(config *manifest.Schema2Config) error {
return updateConfigEnv(newEnvs, config)
}
func updateConfigEnv(newEnvs []instructions.KeyValuePair, config *manifest.Schema2Config) error {
func updateConfigEnv(newEnvs []instructions.KeyValuePair, config *v1.Config) error {
// First, convert config.Env array to []instruction.KeyValuePair
var kvps []instructions.KeyValuePair
for _, env := range config.Env {
pkg/commands/env_test.go
+3 -3
View File
@@ -19,12 +19,12 @@ import (
"testing"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
)
func TestUpdateEnvConfig(t *testing.T) {
cfg := &manifest.Schema2Config{
cfg := &v1.Config{
Env: []string{
"PATH=/path/to/dir",
"hey=hey",
@@ -55,7 +55,7 @@ func TestUpdateEnvConfig(t *testing.T) {
testutil.CheckErrorAndDeepEqual(t, false, nil, expectedEnvArray, cfg.Env)
}
func Test_EnvExecute(t *testing.T) {
cfg := &manifest.Schema2Config{
cfg := &v1.Config{
Env: []string{
"path=/usr/",
"home=/root",
pkg/commands/expose.go
+4 -5
View File
@@ -21,8 +21,8 @@ import (
"strings"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
@@ -30,12 +30,12 @@ type ExposeCommand struct {
cmd *instructions.ExposeCommand
}
func (r *ExposeCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (r *ExposeCommand) ExecuteCommand(config *v1.Config) error {
logrus.Info("cmd: EXPOSE")
// Grab the currently exposed ports
existingPorts := config.ExposedPorts
if existingPorts == nil {
existingPorts = make(map[manifest.Schema2Port]struct{})
existingPorts = make(map[string]struct{})
}
// Add any new ones in
for _, p := range r.cmd.Ports {
@@ -53,8 +53,7 @@ func (r *ExposeCommand) ExecuteCommand(config *manifest.Schema2Config) error {
return fmt.Errorf("Invalid protocol: %s", protocol)
}
logrus.Infof("Adding exposed port: %s", p)
var x struct{}
existingPorts[manifest.Schema2Port(p)] = x
existingPorts[p] = struct{}{}
}
config.ExposedPorts = existingPorts
return nil
pkg/commands/expose_test.go
+6 -6
View File
@@ -20,13 +20,13 @@ import (
"testing"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
)
func TestUpdateExposedPorts(t *testing.T) {
cfg := &manifest.Schema2Config{
ExposedPorts: manifest.Schema2PortSet{
cfg := &v1.Config{
ExposedPorts: map[string]struct{}{
"8080/tcp": {},
},
Env: []string{
@@ -51,7 +51,7 @@ func TestUpdateExposedPorts(t *testing.T) {
},
}
expectedPorts := manifest.Schema2PortSet{
expectedPorts := map[string]struct{}{
"8080/tcp": {},
"8081/tcp": {},
"8082/tcp": {},
@@ -66,8 +66,8 @@ func TestUpdateExposedPorts(t *testing.T) {
}
func TestInvalidProtocol(t *testing.T) {
cfg := &manifest.Schema2Config{
ExposedPorts: manifest.Schema2PortSet{},
cfg := &v1.Config{
ExposedPorts: map[string]struct{}{},
}
ports := []string{
pkg/commands/label.go
+3 -3
View File
@@ -20,8 +20,8 @@ import (
"strings"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
@@ -29,12 +29,12 @@ type LabelCommand struct {
cmd *instructions.LabelCommand
}
func (r *LabelCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (r *LabelCommand) ExecuteCommand(config *v1.Config) error {
logrus.Info("cmd: LABEL")
return updateLabels(r.cmd.Labels, config)
}
func updateLabels(labels []instructions.KeyValuePair, config *manifest.Schema2Config) error {
func updateLabels(labels []instructions.KeyValuePair, config *v1.Config) error {
existingLabels := config.Labels
if existingLabels == nil {
existingLabels = make(map[string]string)
pkg/commands/label_test.go
+2 -2
View File
@@ -20,12 +20,12 @@ import (
"testing"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
)
func TestUpdateLabels(t *testing.T) {
cfg := &manifest.Schema2Config{
cfg := &v1.Config{
Labels: map[string]string{
"foo": "bar",
},
pkg/commands/onbuild.go
+2 -2
View File
@@ -18,8 +18,8 @@ package commands
import (
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
@@ -28,7 +28,7 @@ type OnBuildCommand struct {
}
//ExecuteCommand adds the specified expression in Onbuild to the config
func (o *OnBuildCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (o *OnBuildCommand) ExecuteCommand(config *v1.Config) error {
logrus.Info("cmd: ONBUILD")
logrus.Infof("args: %s", o.cmd.Expression)
resolvedExpression, err := util.ResolveEnvironmentReplacement(o.cmd.Expression, config.Env, false)
pkg/commands/onbuild_test.go
+2 -2
View File
@@ -20,8 +20,8 @@ import (
"testing"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
)
var onbuildTests = []struct {
@@ -50,7 +50,7 @@ var onbuildTests = []struct {
func TestExecuteOnbuild(t *testing.T) {
for _, test := range onbuildTests {
cfg := &manifest.Schema2Config{
cfg := &v1.Config{
Env: []string{
"dir=/some/dir",
},
pkg/commands/run.go
+2 -2
View File
@@ -23,8 +23,8 @@ import (
"strings"
"syscall"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
@@ -32,7 +32,7 @@ type RunCommand struct {
cmd *instructions.RunCommand
}
func (r *RunCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (r *RunCommand) ExecuteCommand(config *v1.Config) error {
var newCommand []string
if r.cmd.PrependShell {
// This is the default shell on Linux
pkg/commands/user.go
+2 -2
View File
@@ -21,8 +21,8 @@ import (
"strings"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
@@ -30,7 +30,7 @@ type UserCommand struct {
cmd *instructions.UserCommand
}
func (r *UserCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (r *UserCommand) ExecuteCommand(config *v1.Config) error {
logrus.Info("cmd: USER")
u := r.cmd.User
userAndGroup := strings.Split(u, ":")
pkg/commands/user_test.go
+2 -2
View File
@@ -19,8 +19,8 @@ import (
"testing"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
)
var userTests = []struct {
@@ -82,7 +82,7 @@ var userTests = []struct {
func TestUpdateUser(t *testing.T) {
for _, test := range userTests {
cfg := &manifest.Schema2Config{
cfg := &v1.Config{
Env: []string{
"envuser=root",
"envgroup=root",
pkg/commands/volume.go
+2 -2
View File
@@ -21,8 +21,8 @@ import (
"strings"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
@@ -31,7 +31,7 @@ type VolumeCommand struct {
snapshotFiles []string
}
func (v *VolumeCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (v *VolumeCommand) ExecuteCommand(config *v1.Config) error {
logrus.Info("cmd: VOLUME")
volumes := v.cmd.Volumes
resolvedVolumes, err := util.ResolveEnvironmentReplacementList(volumes, config.Env, true)
pkg/commands/volume_test.go
+2 -2
View File
@@ -19,12 +19,12 @@ import (
"testing"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
)
func TestUpdateVolume(t *testing.T) {
cfg := &manifest.Schema2Config{
cfg := &v1.Config{
Env: []string{
"VOLUME=/etc",
},
pkg/commands/workdir.go
+2 -2
View File
@@ -21,8 +21,8 @@ import (
"path/filepath"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
"github.com/sirupsen/logrus"
)
@@ -31,7 +31,7 @@ type WorkdirCommand struct {
snapshotFiles []string
}
func (w *WorkdirCommand) ExecuteCommand(config *manifest.Schema2Config) error {
func (w *WorkdirCommand) ExecuteCommand(config *v1.Config) error {
logrus.Info("cmd: workdir")
workdirPath := w.cmd.Path
resolvedWorkingDir, err := util.ResolveEnvironmentReplacement(workdirPath, config.Env, true)
pkg/commands/workdir_test.go
+2 -2
View File
@@ -19,8 +19,8 @@ import (
"testing"
"github.com/GoogleContainerTools/kaniko/testutil"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/google/go-containerregistry/v1"
)
// Each test here changes the same WorkingDir field in the config
@@ -63,7 +63,7 @@ var workdirTests = []struct {
func TestWorkdirCommand(t *testing.T) {
cfg := &manifest.Schema2Config{
cfg := &v1.Config{
WorkingDir: "/",
Env: []string{
"path=usr/",
pkg/executor/executor.go
+73 -12
View File
@@ -17,17 +17,29 @@ limitations under the License.
package executor
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"github.com/google/go-containerregistry/v1/empty"
"github.com/google/go-containerregistry/v1/tarball"
"github.com/google/go-containerregistry/authn"
"github.com/google/go-containerregistry/name"
"github.com/google/go-containerregistry/v1"
"github.com/google/go-containerregistry/v1/mutate"
"github.com/google/go-containerregistry/v1/remote"
"github.com/GoogleContainerTools/kaniko/pkg/commands"
"github.com/GoogleContainerTools/kaniko/pkg/constants"
"github.com/GoogleContainerTools/kaniko/pkg/dockerfile"
"github.com/GoogleContainerTools/kaniko/pkg/image"
"github.com/GoogleContainerTools/kaniko/pkg/snapshot"
"github.com/GoogleContainerTools/kaniko/pkg/util"
"github.com/containers/image/manifest"
"github.com/docker/docker/builder/dockerfile/instructions"
"github.com/sirupsen/logrus"
)
@@ -46,8 +58,28 @@ func DoBuild(dockerfilePath, srcContext, destination, snapshotMode string, docke
baseImage := stages[0].BaseName
// Unpack file system to root
var sourceImage v1.Image
var ref name.Reference
logrus.Infof("Unpacking filesystem of %s...", baseImage)
if err := util.ExtractFileSystemFromImage(baseImage); err != nil {
if baseImage == constants.NoBaseImage {
logrus.Info("No base image, nothing to extract")
sourceImage = empty.Image
} else {
// Initialize source image
ref, err = name.ParseReference(baseImage, name.WeakValidation)
if err != nil {
return err
}
auth, err := authn.DefaultKeychain.Resolve(ref.Context().Registry)
if err != nil {
return err
}
sourceImage, err = remote.Image(ref, auth, http.DefaultTransport)
if err != nil {
return err
}
}
if err := util.GetFSFromImage(sourceImage); err != nil {
return err
}
@@ -63,21 +95,22 @@ func DoBuild(dockerfilePath, srcContext, destination, snapshotMode string, docke
return err
}
// Initialize source image
sourceImage, err := image.NewSourceImage(baseImage)
destRef, err := name.ParseReference(destination, name.WeakValidation)
if err != nil {
return err
}
// Set environment variables within the image
if err := image.SetEnvVariables(sourceImage); err != nil {
return err
}
imageConfig := sourceImage.Config()
imageConfig, err := sourceImage.ConfigFile()
if err != nil {
return err
}
// Currently only supports single stage builds
for _, stage := range stages {
if err := resolveOnBuild(&stage, imageConfig); err != nil {
if err := resolveOnBuild(&stage, &imageConfig.Config); err != nil {
return err
}
for _, cmd := range stage.Commands {
@@ -88,7 +121,7 @@ func DoBuild(dockerfilePath, srcContext, destination, snapshotMode string, docke
if dockerCommand == nil {
continue
}
if err := dockerCommand.ExecuteCommand(imageConfig); err != nil {
if err := dockerCommand.ExecuteCommand(&imageConfig.Config); err != nil {
return err
}
// Now, we get the files to snapshot from this command and take the snapshot
@@ -100,11 +133,25 @@ func DoBuild(dockerfilePath, srcContext, destination, snapshotMode string, docke
util.MoveVolumeWhitelistToWhitelist()
if contents == nil {
logrus.Info("No files were changed, appending empty layer to config.")
sourceImage.AppendConfigHistory(constants.Author, true)
continue
}
// Append the layer to the image
if err := sourceImage.AppendLayer(contents, constants.Author); err != nil {
opener := func() (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewReader(contents)), nil
}
layer, err := tarball.LayerFromOpener(opener)
if err != nil {
return err
}
sourceImage, err = mutate.Append(sourceImage,
mutate.Addendum{
Layer: layer,
History: v1.History{
Author: constants.Author,
},
},
)
if err != nil {
return err
}
}
@@ -113,7 +160,21 @@ func DoBuild(dockerfilePath, srcContext, destination, snapshotMode string, docke
if err := setDefaultEnv(); err != nil {
return err
}
return image.PushImage(sourceImage, destination, dockerInsecureSkipTLSVerify)
wo := remote.WriteOptions{}
if ref != nil {
wo.MountPaths = []name.Repository{ref.Context()}
}
pushAuth, err := authn.DefaultKeychain.Resolve(destRef.Context().Registry)
if err != nil {
return err
}
sourceImage, err = mutate.Config(sourceImage, imageConfig.Config)
if err != nil {
return err
}
return remote.Write(destRef, sourceImage, pushAuth, http.DefaultTransport, wo)
}
func getHasher(snapshotMode string) (func(string) (string, error), error) {
@@ -127,7 +188,7 @@ func getHasher(snapshotMode string) (func(string) (string, error), error) {
return nil, fmt.Errorf("%s is not a valid snapshot mode", snapshotMode)
}
func resolveOnBuild(stage *instructions.Stage, config *manifest.Schema2Config) error {
func resolveOnBuild(stage *instructions.Stage, config *v1.Config) error {
if config.OnBuild == nil {
return nil
}
pkg/image/image.go
+12 -65
View File
@@ -17,80 +17,27 @@ limitations under the License.
package image
import (
"fmt"
"os"
"strings"
"github.com/GoogleContainerTools/kaniko/pkg/version"
"github.com/containers/image/types"
"github.com/google/go-containerregistry/v1"
img "github.com/GoogleContainerTools/container-diff/pkg/image"
"github.com/GoogleContainerTools/kaniko/pkg/constants"
"github.com/containers/image/copy"
"github.com/containers/image/docker"
"github.com/containers/image/signature"
"github.com/containers/image/transports/alltransports"
"github.com/sirupsen/logrus"
)
// sourceImage is the image that will be modified by the executor
// NewSourceImage initializes the source image with the base image
func NewSourceImage(srcImg string) (*img.MutableSource, error) {
if srcImg == constants.NoBaseImage {
return img.NewMutableSource(nil)
}
logrus.Infof("Initializing source image %s", srcImg)
ref, err := docker.ParseReference("//" + srcImg)
if err != nil {
return nil, err
}
return img.NewMutableSource(ref)
}
// PushImage pushes the final image
func PushImage(ms *img.MutableSource, destImg string, dockerInsecureSkipTLSVerify bool) error {
srcRef := &img.ProxyReference{
ImageReference: nil,
Src: ms,
}
destRef, err := alltransports.ParseImageName("docker://" + destImg)
if err != nil {
return err
}
policyContext, err := getPolicyContext()
if err != nil {
return err
}
logrus.Infof("Pushing image to %s", destImg)
opts := &copy.Options{
DestinationCtx: &types.SystemContext{
DockerRegistryUserAgent: fmt.Sprintf("kaniko/executor-%s", version.Version()),
DockerInsecureSkipTLSVerify: dockerInsecureSkipTLSVerify,
},
}
return copy.Image(policyContext, destRef, srcRef, opts)
}
// SetEnvVariables sets environment variables as specified in the image
func SetEnvVariables(ms *img.MutableSource) error {
envVars := ms.Env()
for key, val := range envVars {
if err := os.Setenv(key, val); err != nil {
func SetEnvVariables(img v1.Image) error {
cfg, err := img.ConfigFile()
if err != nil {
return err
}
envVars := cfg.Config.Env
for _, envVar := range envVars {
split := strings.SplitN(envVar, "=", 2)
if err := os.Setenv(split[0], split[1]); err != nil {
return err
}
logrus.Debugf("Setting environment variable %s=%s", key, val)
logrus.Infof("Setting environment variable %s", envVar)
}
return nil
}
func getPolicyContext() (*signature.PolicyContext, error) {
policyContext, err := signature.NewPolicyContext(&signature.Policy{
Default: signature.PolicyRequirements{signature.NewPRInsecureAcceptAnything()},
})
if err != nil {
logrus.Debugf("Error retrieving policy context: %s", err)
return nil, err
}
return policyContext, nil
}
pkg/snapshot/snapshot.go
+1
View File
@@ -150,6 +150,7 @@ func (s *Snapshotter) snapShotFS(f io.Writer) (bool, error) {
return false, err
}
if maybeAdd {
logrus.Debugf("Adding %s to layer, because it was changed.", path)
filesAdded = true
if err := util.AddToTar(path, info, s.hardlinks, w); err != nil {
return false, err
pkg/util/fs_util.go
+185 -17
View File
@@ -17,6 +17,7 @@ limitations under the License.
package util
import (
"archive/tar"
"bufio"
"io"
"net/http"
@@ -25,38 +26,163 @@ import (
"strings"
"time"
pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
"github.com/google/go-containerregistry/v1"
"github.com/GoogleContainerTools/kaniko/pkg/constants"
"github.com/containers/image/docker"
"github.com/sirupsen/logrus"
)
var whitelist = []string{"/kaniko"}
var whitelist = []string{
"/kaniko",
// /var/run is a special case. It's common to mount in /var/run/docker.sock or something similar
// which leads to a special mount on the /var/run/docker.sock file itself, but the directory to exist
// in the image with no way to tell if it came from the base image or not.
"/var/run",
}
var volumeWhitelist = []string{}
// ExtractFileSystemFromImage pulls an image and unpacks it to a file system at root
func ExtractFileSystemFromImage(img string) error {
func GetFSFromImage(img v1.Image) error {
whitelist, err := fileSystemWhitelist(constants.WhitelistPath)
if err != nil {
return err
}
logrus.Infof("Whitelisted directories are %s", whitelist)
if img == constants.NoBaseImage {
logrus.Info("No base image, nothing to extract")
return nil
}
ref, err := docker.ParseReference("//" + img)
logrus.Infof("Mounted directories: %v", whitelist)
layers, err := img.Layers()
if err != nil {
return err
}
imgSrc, err := ref.NewImageSource(nil)
if err != nil {
return err
fs := map[string]struct{}{}
whiteouts := map[string]struct{}{}
for i := len(layers) - 1; i >= 0; i-- {
logrus.Infof("Unpacking layer: %d", i)
l := layers[i]
r, err := l.Uncompressed()
if err != nil {
return err
}
tr := tar.NewReader(r)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
path := filepath.Join("/", filepath.Clean(hdr.Name))
base := filepath.Base(path)
dir := filepath.Dir(path)
if strings.HasPrefix(base, ".wh.") {
logrus.Infof("Whiting out %s", path)
name := strings.TrimPrefix(base, ".wh.")
whiteouts[filepath.Join(dir, name)] = struct{}{}
continue
}
if checkWhiteouts(path, whiteouts) {
logrus.Infof("Not adding %s because it is whited out", path)
continue
}
if _, ok := fs[path]; ok {
logrus.Infof("Not adding %s because it was added by a prior layer", path)
continue
}
if checkWhitelist(path, whitelist) {
logrus.Infof("Not adding %s because it is whitelisted", path)
continue
}
fs[path] = struct{}{}
if err := extractFile("/", hdr, tr); err != nil {
return err
}
}
}
return pkgutil.GetFileSystemFromReference(ref, imgSrc, constants.RootDir, whitelist)
return nil
}
func unTar(r io.Reader, dest string) error {
tr := tar.NewReader(r)
for {
hdr, err := tr.Next()
if err == io.EOF {
break
}
if err != nil {
return err
}
if err := extractFile(dest, hdr, tr); err != nil {
return err
}
}
return nil
}
func extractFile(dest string, hdr *tar.Header, tr io.Reader) error {
path := filepath.Join(dest, filepath.Clean(hdr.Name))
base := filepath.Base(path)
dir := filepath.Dir(path)
mode := hdr.FileInfo().Mode()
switch hdr.Typeflag {
case tar.TypeReg:
logrus.Debugf("creating file %s", path)
// It's possible a file is in the tar before it's directory.
if _, err := os.Stat(dir); os.IsNotExist(err) {
logrus.Debugf("base %s for file %s does not exist. Creating.", base, path)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
}
currFile, err := os.Create(path)
if err != nil {
return err
}
// manually set permissions on file, since the default umask (022) will interfere
if err = os.Chmod(path, mode); err != nil {
return err
}
if _, err = io.Copy(currFile, tr); err != nil {
return err
}
currFile.Close()
case tar.TypeDir:
logrus.Debugf("creating dir %s", path)
if err := os.MkdirAll(path, mode); err != nil {
return err
}
// In some cases, MkdirAll doesn't change the permissions, so run Chmod
if err := os.Chmod(path, mode); err != nil {
return err
}
case tar.TypeLink:
logrus.Debugf("link from %s to %s", hdr.Linkname, path)
// The base directory for a link may not exist before it is created.
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
if err := os.Symlink(filepath.Clean(filepath.Join("/", hdr.Linkname)), path); err != nil {
return err
}
case tar.TypeSymlink:
logrus.Debugf("symlink from %s to %s", hdr.Linkname, path)
// The base directory for a symlink may not exist before it is created.
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
if err := os.Symlink(hdr.Linkname, path); err != nil {
return err
}
}
return nil
}
// PathInWhitelist returns true if the path is whitelisted
func PathInWhitelist(path, directory string) bool {
for _, c := range constants.KanikoBuildFiles {
if path == c {
@@ -65,7 +191,30 @@ func PathInWhitelist(path, directory string) bool {
}
for _, d := range whitelist {
dirPath := filepath.Join(directory, d)
if pkgutil.HasFilepathPrefix(path, dirPath) {
if HasFilepathPrefix(path, dirPath) {
return true
}
}
return false
}
func checkWhiteouts(path string, whiteouts map[string]struct{}) bool {
// Don't add the file if it or it's directory are whited out.
if _, ok := whiteouts[path]; ok {
return true
}
for wd := range whiteouts {
if HasFilepathPrefix(path, wd) {
logrus.Infof("Not adding %s because it's directory is whited out", path)
return true
}
}
return false
}
func checkWhitelist(path string, whitelist []string) bool {
for _, wl := range whitelist {
if HasFilepathPrefix(path, wl) {
return true
}
}
@@ -268,3 +417,22 @@ func CopyFile(src, dest string) error {
defer srcFile.Close()
return CreateFile(dest, srcFile, fi.Mode())
}
// HasFilepathPrefix checks if the given file path begins with prefix
func HasFilepathPrefix(path, prefix string) bool {
path = filepath.Clean(path)
prefix = filepath.Clean(prefix)
pathArray := strings.Split(path, "/")
prefixArray := strings.Split(prefix, "/")
if len(pathArray) < len(prefixArray) {
return false
}
for index := range prefixArray {
if prefixArray[index] == pathArray[index] {
continue
}
return false
}
return true
}
pkg/util/fs_util_test.go
+359 -1
View File
@@ -17,9 +17,12 @@ limitations under the License.
package util
import (
"archive/tar"
"bytes"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"sort"
"testing"
@@ -47,7 +50,7 @@ func Test_fileSystemWhitelist(t *testing.T) {
}
actualWhitelist, err := fileSystemWhitelist(path)
expectedWhitelist := []string{"/kaniko", "/proc", "/dev", "/dev/pts", "/sys"}
expectedWhitelist := []string{"/kaniko", "/proc", "/dev", "/dev/pts", "/sys", "/var/run"}
sort.Strings(actualWhitelist)
sort.Strings(expectedWhitelist)
testutil.CheckErrorAndDeepEqual(t, false, err, expectedWhitelist, actualWhitelist)
@@ -131,3 +134,358 @@ func Test_RelativeFiles(t *testing.T) {
testutil.CheckErrorAndDeepEqual(t, false, err, test.expectedFiles, actualFiles)
}
}
func Test_checkWhiteouts(t *testing.T) {
type args struct {
path string
whiteouts map[string]struct{}
}
tests := []struct {
name string
args args
want bool
}{
{
name: "file whited out",
args: args{
path: "/foo",
whiteouts: map[string]struct{}{"/foo": {}},
},
want: true,
},
{
name: "directory whited out",
args: args{
path: "/foo/bar",
whiteouts: map[string]struct{}{"/foo": {}},
},
want: true,
},
{
name: "grandparent whited out",
args: args{
path: "/foo/bar/baz",
whiteouts: map[string]struct{}{"/foo": {}},
},
want: true,
},
{
name: "sibling whited out",
args: args{
path: "/foo/bar/baz",
whiteouts: map[string]struct{}{"/foo/bat": {}},
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := checkWhiteouts(tt.args.path, tt.args.whiteouts); got != tt.want {
t.Errorf("checkWhiteouts() = %v, want %v", got, tt.want)
}
})
}
}
func Test_checkWhitelist(t *testing.T) {
type args struct {
path string
whitelist []string
}
tests := []struct {
name string
args args
want bool
}{
{
name: "file whitelisted",
args: args{
path: "/foo",
whitelist: []string{"/foo"},
},
want: true,
},
{
name: "directory whitelisted",
args: args{
path: "/foo/bar",
whitelist: []string{"/foo"},
},
want: true,
},
{
name: "grandparent whitelisted",
args: args{
path: "/foo/bar/baz",
whitelist: []string{"/foo"},
},
want: true,
},
{
name: "sibling whitelisted",
args: args{
path: "/foo/bar/baz",
whitelist: []string{"/foo/bat"},
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := checkWhitelist(tt.args.path, tt.args.whitelist); got != tt.want {
t.Errorf("checkWhitelist() = %v, want %v", got, tt.want)
}
})
}
}
func TestHasFilepathPrefix(t *testing.T) {
type args struct {
path string
prefix string
}
tests := []struct {
name string
args args
want bool
}{
{
name: "parent",
args: args{
path: "/foo/bar",
prefix: "/foo",
},
want: true,
},
{
name: "nested parent",
args: args{
path: "/foo/bar/baz",
prefix: "/foo/bar",
},
want: true,
},
{
name: "sibling",
args: args{
path: "/foo/bar",
prefix: "/bar",
},
want: false,
},
{
name: "nested sibling",
args: args{
path: "/foo/bar/baz",
prefix: "/foo/bar",
},
want: true,
},
{
name: "name prefix",
args: args{
path: "/foo2/bar",
prefix: "/foo",
},
want: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := HasFilepathPrefix(tt.args.path, tt.args.prefix); got != tt.want {
t.Errorf("HasFilepathPrefix() = %v, want %v", got, tt.want)
}
})
}
}
type checker func(root string, t *testing.T)
func fileExists(p string) checker {
return func(root string, t *testing.T) {
_, err := os.Stat(filepath.Join(root, p))
if err != nil {
t.Fatalf("File does not exist")
}
}
}
func fileMatches(p string, c []byte) checker {
return func(root string, t *testing.T) {
actual, err := ioutil.ReadFile(filepath.Join(root, p))
if err != nil {
t.Fatalf("error reading file: %s", p)
}
if !reflect.DeepEqual(actual, c) {
t.Errorf("file contents do not match. %v!=%v", actual, c)
}
}
}
func permissionsMatch(p string, perms os.FileMode) checker {
return func(root string, t *testing.T) {
fi, err := os.Stat(filepath.Join(root, p))
if err != nil {
t.Fatalf("error statting file %s", p)
}
if fi.Mode() != perms {
t.Errorf("Permissions do not match. %s != %s", fi.Mode(), perms)
}
}
}
func linkPointsTo(src, dst string) checker {
return func(root string, t *testing.T) {
link := filepath.Join(root, src)
got, err := os.Readlink(link)
if err != nil {
t.Fatalf("error reading link %s: %s", link, err)
}
if got != dst {
t.Errorf("link destination does not match: %s != %s", got, dst)
}
}
}
func fileHeader(name string, contents string, mode int64) *tar.Header {
return &tar.Header{
Name: name,
Size: int64(len(contents)),
Mode: mode,
Typeflag: tar.TypeReg,
}
}
func linkHeader(name, linkname string) *tar.Header {
return &tar.Header{
Name: name,
Size: 0,
Typeflag: tar.TypeSymlink,
Linkname: linkname,
}
}
func hardlinkHeader(name, linkname string) *tar.Header {
return &tar.Header{
Name: name,
Size: 0,
Typeflag: tar.TypeLink,
Linkname: linkname,
}
}
func dirHeader(name string, mode int64) *tar.Header {
return &tar.Header{
Name: name,
Size: 0,
Typeflag: tar.TypeDir,
Mode: mode,
}
}
func TestExtractFile(t *testing.T) {
type tc struct {
name string
hdrs []*tar.Header
contents []byte
checkers []checker
}
tcs := []tc{
{
name: "normal file",
contents: []byte("helloworld"),
hdrs: []*tar.Header{fileHeader("./bar", "helloworld", 0644)},
checkers: []checker{
fileExists("/bar"),
fileMatches("/bar", []byte("helloworld")),
permissionsMatch("/bar", 0644),
},
},
{
name: "normal file, directory does not exist",
contents: []byte("helloworld"),
hdrs: []*tar.Header{fileHeader("./foo/bar", "helloworld", 0644)},
checkers: []checker{
fileExists("/foo/bar"),
fileMatches("/foo/bar", []byte("helloworld")),
permissionsMatch("/foo/bar", 0644),
permissionsMatch("/foo", 0755|os.ModeDir),
},
},
{
name: "normal file, directory is created after",
contents: []byte("helloworld"),
hdrs: []*tar.Header{
fileHeader("./foo/bar", "helloworld", 0644),
dirHeader("./foo", 0722),
},
checkers: []checker{
fileExists("/foo/bar"),
fileMatches("/foo/bar", []byte("helloworld")),
permissionsMatch("/foo/bar", 0644),
permissionsMatch("/foo", 0722|os.ModeDir),
},
},
{
name: "symlink",
hdrs: []*tar.Header{linkHeader("./bar", "bar/bat")},
checkers: []checker{
linkPointsTo("/bar", "bar/bat"),
},
},
{
name: "symlink relative path",
hdrs: []*tar.Header{linkHeader("./bar", "./foo/bar/baz")},
checkers: []checker{
linkPointsTo("/bar", "./foo/bar/baz"),
},
},
{
name: "symlink parent does not exist",
hdrs: []*tar.Header{linkHeader("./foo/bar/baz", "../../bat")},
checkers: []checker{
linkPointsTo("/foo/bar/baz", "../../bat"),
},
},
{
name: "symlink parent does not exist",
hdrs: []*tar.Header{linkHeader("./foo/bar/baz", "../../bat")},
checkers: []checker{
linkPointsTo("/foo/bar/baz", "../../bat"),
permissionsMatch("/foo", 0755|os.ModeDir),
permissionsMatch("/foo/bar", 0755|os.ModeDir),
},
},
{
name: "hardlink",
hdrs: []*tar.Header{
fileHeader("/bin/gzip", "gzip-binary", 0751),
hardlinkHeader("/bin/uncompress", "/bin/gzip"),
},
checkers: []checker{
linkPointsTo("/bin/uncompress", "/bin/gzip"),
},
},
}
for _, tc := range tcs {
t.Run(tc.name, func(t *testing.T) {
tc := tc
t.Parallel()
r, err := ioutil.TempDir("", "")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(r)
for _, hdr := range tc.hdrs {
if err := extractFile(r, hdr, bytes.NewReader(tc.contents)); err != nil {
t.Fatal(err)
}
}
for _, checker := range tc.checkers {
checker(r, t)
}
})
}
}
pkg/util/tar_util.go
+3 -4
View File
@@ -26,7 +26,6 @@ import (
"path/filepath"
"syscall"
pkgutil "github.com/GoogleContainerTools/container-diff/pkg/util"
"github.com/docker/docker/pkg/archive"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
@@ -122,7 +121,7 @@ func UnpackLocalTarArchive(path, dest string) error {
return UnpackCompressedTar(path, dest)
} else if compressionLevel == archive.Bzip2 {
bzr := bzip2.NewReader(file)
return pkgutil.UnTar(bzr, dest, nil)
return unTar(bzr, dest)
}
}
if fileIsUncompressedTar(path) {
@@ -131,7 +130,7 @@ func UnpackLocalTarArchive(path, dest string) error {
return err
}
defer file.Close()
return pkgutil.UnTar(file, dest, nil)
return unTar(file, dest)
}
return errors.New("path does not lead to local tar archive")
}
@@ -195,5 +194,5 @@ func UnpackCompressedTar(path, dir string) error {
return err
}
defer gzr.Close()
return pkgutil.UnTar(gzr, dir, nil)
return unTar(gzr, dir)
}