Upgrade docker (#2440)

* somehow now the only thing that doesnt work is devices.Device

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>

* this gets rid of all the compiler errors in the vendored code

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>

* fixed some things but a bunch of tests and maybe some compiler steps are still failing

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>

* all the things i figured out how to fix so far

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>

* guess i had to redo go mods after rebasing again

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>

* update docker constants to be SHOUTY CASE now

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>

* include DestPath in resolveEnv

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>

* fix one mistake in Docker lib upgrade and some typos/deprecations in the file

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>

* last changes (hopefully) to update to new docker libs

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>

---------

Signed-off-by: Joe Kimmel <jkimmel@vmware.com>
This commit is contained in:
Joe Kimmel
2023-03-23 20:34:25 -04:00
committed by GitHub
parent 86a73c5439
commit 906cc96b59
1356 changed files with 96735 additions and 72375 deletions
+1 -1
View File
@@ -98,7 +98,7 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui
copyCmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: append(unresolvedSrcs, dest),
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: unresolvedSrcs, DestPath: dest},
Chown: a.cmd.Chown,
},
fileContext: a.fileContext,
+31 -33
View File
@@ -117,7 +117,7 @@ func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
buildArgs := &dockerfile.BuildArgs{}
type testCase struct {
desctiption string
description string
expectLayer bool
expectErr bool
count *int
@@ -128,7 +128,7 @@ func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
}
testCases := []testCase{
func() testCase {
err = ioutil.WriteFile(filepath.Join(tempDir, "foo.txt"), []byte("meow"), 0644)
err = os.WriteFile(filepath.Join(tempDir, "foo.txt"), []byte("meow"), 0644)
if err != nil {
t.Errorf("couldn't write tempfile %v", err)
t.FailNow()
@@ -142,14 +142,11 @@ func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
},
fileContext: util.FileContext{Root: tempDir},
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{
"foo.txt", "foo.txt",
},
},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{"foo.txt"}, DestPath: ""}},
}
count := 0
tc := testCase{
desctiption: "with valid image and valid layer",
description: "with valid image and valid layer",
count: &count,
expectedCount: 1,
expectLayer: true,
@@ -166,7 +163,7 @@ func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
func() testCase {
c := &CachingCopyCommand{}
tc := testCase{
desctiption: "with no image",
description: "with no image",
expectErr: true,
}
c.extractFn = func(_ string, _ *tar.Header, _ io.Reader) error {
@@ -183,7 +180,7 @@ func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
return nil
}
return testCase{
desctiption: "with image containing no layers",
description: "with image containing no layers",
expectErr: true,
command: c,
}
@@ -200,7 +197,7 @@ func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
return nil
}
tc := testCase{
desctiption: "with image one layer which has no tar content",
description: "with image one layer which has no tar content",
expectErr: false, // this one probably should fail but doesn't because of how ExecuteCommand and util.GetFSFromLayers are implemented - cvgw- 2019-11-25
expectLayer: true,
}
@@ -210,7 +207,7 @@ func Test_CachingCopyCommand_ExecuteCommand(t *testing.T) {
}
for _, tc := range testCases {
t.Run(tc.desctiption, func(t *testing.T) {
t.Run(tc.description, func(t *testing.T) {
c := tc.command
err := c.ExecuteCommand(config, buildArgs)
if !tc.expectErr && err != nil {
@@ -274,7 +271,8 @@ func TestCopyExecuteCmd(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: test.sourcesAndDest,
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: test.sourcesAndDest[0 : len(test.sourcesAndDest)-1],
DestPath: test.sourcesAndDest[len(test.sourcesAndDest)-1]},
},
fileContext: fileContext,
}
@@ -421,7 +419,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{srcDir, "dest"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{srcDir}, DestPath: "dest"},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -453,7 +451,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
defer os.RemoveAll(testDir)
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "bam.txt"), "dest/"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{filepath.Join(srcDir, "bam.txt")}, DestPath: "dest/"},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -480,7 +478,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
defer os.RemoveAll(testDir)
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "bam.txt"), "dest"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{filepath.Join(srcDir, "bam.txt")}, DestPath: "dest"},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -509,7 +507,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "bam.txt"), "dest"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{filepath.Join(srcDir, "bam.txt")}, DestPath: "dest"},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -538,7 +536,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "sym.link"), "dest/"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{filepath.Join(srcDir, "sym.link")}, DestPath: "dest/"},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -583,7 +581,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{filepath.Join(srcDir, "dead.link"), "dest/"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{filepath.Join(srcDir, "dead.link")}, DestPath: "dest/"},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -614,7 +612,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
t.Run("copy src symlink dir to a dir", func(t *testing.T) {
testDir, srcDir := setupDirs(t)
defer os.RemoveAll(testDir)
expected, err := ioutil.ReadDir(filepath.Join(testDir, srcDir))
expected, err := os.ReadDir(filepath.Join(testDir, srcDir))
if err != nil {
t.Fatal(err)
}
@@ -624,7 +622,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{"another", "dest"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{"another"}, DestPath: "dest"},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -638,13 +636,13 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
err = cmd.ExecuteCommand(cfg, dockerfile.NewBuildArgs([]string{}))
testutil.CheckNoError(t, err)
// Check if "dest" dir exists with contents of srcDir
actual, err := ioutil.ReadDir(filepath.Join(testDir, "dest"))
actual, err := os.ReadDir(filepath.Join(testDir, "dest"))
if err != nil {
t.Fatal(err)
}
for i, f := range actual {
testutil.CheckDeepEqual(t, expected[i].Name(), f.Name())
testutil.CheckDeepEqual(t, expected[i].Mode(), f.Mode())
testutil.CheckDeepEqual(t, expected[i].Type(), f.Type())
}
})
@@ -670,7 +668,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{srcDir, "dest"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{srcDir}, DestPath: "dest"},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -704,7 +702,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
t.Run("copy src symlink dir to a dir", func(t *testing.T) {
testDir, srcDir := setupDirs(t)
defer os.RemoveAll(testDir)
expected, err := ioutil.ReadDir(filepath.Join(testDir, srcDir))
expected, err := os.ReadDir(filepath.Join(testDir, srcDir))
if err != nil {
t.Fatal(err)
}
@@ -714,7 +712,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{"another", "dest"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{"another"}, DestPath: "dest"},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -728,13 +726,13 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
err = cmd.ExecuteCommand(cfg, dockerfile.NewBuildArgs([]string{}))
testutil.CheckNoError(t, err)
// Check if "dest" dir exists with bam.txt and "dest" dir is a symlink
actual, err := ioutil.ReadDir(filepath.Join(testDir, "dest"))
actual, err := os.ReadDir(filepath.Join(testDir, "dest"))
if err != nil {
t.Fatal(err)
}
for i, f := range actual {
testutil.CheckDeepEqual(t, expected[i].Name(), f.Name())
testutil.CheckDeepEqual(t, expected[i].Mode(), f.Mode())
testutil.CheckDeepEqual(t, expected[i].Type(), f.Type())
}
})
@@ -757,7 +755,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{srcDir, linkedDest},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{srcDir}, DestPath: linkedDest},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -802,7 +800,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{fmt.Sprintf("%s/bam.txt", srcDir), linkedDest},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{fmt.Sprintf("%s/bam.txt", srcDir)}, DestPath: linkedDest},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -850,7 +848,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{fmt.Sprintf("%s/bam.txt", srcDir), testDir},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{fmt.Sprintf("%s/bam.txt", srcDir)}, DestPath: testDir},
Chown: "alice:group",
},
fileContext: util.FileContext{Root: testDir},
@@ -895,7 +893,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{fmt.Sprintf("%s/bam.txt", srcDir), testDir},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{fmt.Sprintf("%s/bam.txt", srcDir)}, DestPath: testDir},
Chown: "missing:missing",
},
fileContext: util.FileContext{Root: testDir},
@@ -927,7 +925,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
dest := filepath.Join(testDir, "copy")
cmd := CopyCommand{
cmd: &instructions.CopyCommand{
SourcesAndDest: []string{srcDir, dest},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{srcDir}, DestPath: dest},
},
fileContext: util.FileContext{Root: testDir},
}
@@ -939,7 +937,7 @@ func TestCopyCommand_ExecuteCommand_Extended(t *testing.T) {
}
err := cmd.ExecuteCommand(cfg, dockerfile.NewBuildArgs([]string{}))
testutil.CheckNoError(t, err)
actual, err := ioutil.ReadDir(filepath.Join(dest, "another"))
actual, err := os.ReadDir(filepath.Join(dest, "another"))
if err != nil {
t.Fatal(err)
}
+2 -2
View File
@@ -49,8 +49,8 @@ const (
RootUser = "root"
// Docker command names
Cmd = "cmd"
Entrypoint = "entrypoint"
Cmd = "CMD"
Entrypoint = "ENTRYPOINT"
// Name of the .dockerignore file
Dockerignore = ".dockerignore"
+2 -2
View File
@@ -250,11 +250,11 @@ func Test_GetOnBuildInstructions(t *testing.T) {
stageToIdx: map[string]string{"builder": "0", "temp": "1"},
expCommands: []instructions.Command{
&instructions.CopyCommand{
SourcesAndDest: []string{"a.txt b.txt"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{"a.txt"}, DestPath: "b.txt"},
From: "0",
},
&instructions.CopyCommand{
SourcesAndDest: []string{"/app /app"},
SourcesAndDest: instructions.SourcesAndDest{SourcePaths: []string{"/app"}, DestPath: "/app"},
From: "1",
},
}},
+2 -2
View File
@@ -575,11 +575,11 @@ func CalculateDependencies(stages []config.KanikoStage, opts *config.KanikoOptio
if err != nil {
continue
}
resolved, err := util.ResolveEnvironmentReplacementList(cmd.SourcesAndDest, ba.ReplacementEnvs(cfg.Config.Env), true)
resolved, err := util.ResolveEnvironmentReplacementList(cmd.SourcesAndDest.SourcePaths, ba.ReplacementEnvs(cfg.Config.Env), true)
if err != nil {
return nil, err
}
depGraph[i] = append(depGraph[i], resolved[0:len(resolved)-1]...)
depGraph[i] = append(depGraph[i], resolved...)
}
case *instructions.EnvCommand:
if err := util.UpdateConfigEnv(cmd.Env, &cfg.Config, ba.ReplacementEnvs(cfg.Config.Env)); err != nil {
+13 -16
View File
@@ -369,8 +369,8 @@ COPY --from=second /bar /bat
initializeConfig = tt.args.mockInitConfig
}
f, _ := ioutil.TempFile("", "")
ioutil.WriteFile(f.Name(), []byte(tt.args.dockerfile), 0755)
f, _ := os.CreateTemp("", "")
os.WriteFile(f.Name(), []byte(tt.args.dockerfile), 0755)
opts := &config.KanikoOptions{
DockerfilePath: f.Name(),
CustomPlatform: platforms.Format(platforms.Normalize(platforms.DefaultSpec())),
@@ -898,17 +898,16 @@ func Test_stageBuilder_build(t *testing.T) {
}
copyCommandCacheKey := hash
dockerFile := fmt.Sprintf(`
FROM ubuntu:16.04
COPY %s foo.txt
`, filename)
f, _ := ioutil.TempFile("", "")
ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755)
FROM ubuntu:16.04
COPY %s foo.txt
`, filename)
f, _ := os.CreateTemp("", "")
os.WriteFile(f.Name(), []byte(dockerFile), 0755)
opts := &config.KanikoOptions{
DockerfilePath: f.Name(),
Cache: true,
CacheCopyLayers: true,
}
testStages, metaArgs, err := dockerfile.ParseStages(opts)
if err != nil {
t.Errorf("Failed to parse test dockerfile to stages: %s", err)
@@ -968,8 +967,8 @@ COPY %s foo.txt
FROM ubuntu:16.04
COPY %s foo.txt
`, filename)
f, _ := ioutil.TempFile("", "")
ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755)
f, _ := os.CreateTemp("", "")
os.WriteFile(f.Name(), []byte(dockerFile), 0755)
opts := &config.KanikoOptions{
DockerfilePath: f.Name(),
Cache: true,
@@ -1047,8 +1046,8 @@ FROM ubuntu:16.04
RUN foobar
COPY %s bar.txt
`, filename)
f, _ := ioutil.TempFile("", "")
ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755)
f, _ := os.CreateTemp("", "")
os.WriteFile(f.Name(), []byte(dockerFile), 0755)
opts := &config.KanikoOptions{
DockerfilePath: f.Name(),
}
@@ -1122,7 +1121,7 @@ COPY %s bar.txt
RUN foobar
`, filename)
f, _ := ioutil.TempFile("", "")
ioutil.WriteFile(f.Name(), []byte(dockerFile), 0755)
os.WriteFile(f.Name(), []byte(dockerFile), 0755)
opts := &config.KanikoOptions{
DockerfilePath: f.Name(),
}
@@ -1286,7 +1285,7 @@ RUN foobar
t.Run(tc.description, func(t *testing.T) {
var fileName string
if tc.commands == nil {
file, err := ioutil.TempFile("", "foo")
file, err := os.CreateTemp("", "foo")
if err != nil {
t.Error(err)
}
@@ -1348,7 +1347,6 @@ RUN foobar
if err != nil {
t.Errorf("Expected error to be nil but was %v", err)
}
fmt.Println(lc.receivedKeys)
assertCacheKeys(t, tc.expectedCacheKeys, lc.receivedKeys, "receive")
assertCacheKeys(t, tc.pushedCacheKeys, keys, "push")
@@ -1397,7 +1395,6 @@ func getCommands(fileContext util.FileContext, cmds []instructions.Command, cach
outCommands = append(outCommands, cmd)
}
return outCommands
}
func tempDirAndFile(t *testing.T) (string, []string) {
+9 -5
View File
@@ -88,16 +88,20 @@ func ResolveEnvironmentReplacement(value string, envs []string, isFilepath bool)
func ResolveEnvAndWildcards(sd instructions.SourcesAndDest, fileContext FileContext, envs []string) ([]string, string, error) {
// First, resolve any environment replacement
resolvedEnvs, err := ResolveEnvironmentReplacementList(sd, envs, true)
resolvedEnvs, err := ResolveEnvironmentReplacementList(sd.SourcePaths, envs, true)
if err != nil {
return nil, "", errors.Wrap(err, "failed to resolve environment")
}
if len(resolvedEnvs) == 0 {
return nil, "", errors.New("resolved envs is empty")
}
dest := resolvedEnvs[len(resolvedEnvs)-1]
dests, err := ResolveEnvironmentReplacementList([]string{sd.DestPath}, envs, true)
if err != nil {
return nil, "", errors.Wrap(err, "failed to resolve environment for dest path")
}
dest := dests[0]
// Resolve wildcards and get a list of resolved sources
srcs, err := ResolveSources(resolvedEnvs[0:len(resolvedEnvs)-1], fileContext.Root)
srcs, err := ResolveSources(resolvedEnvs, fileContext.Root)
if err != nil {
return nil, "", errors.Wrap(err, "failed to resolve sources")
}
@@ -226,8 +230,8 @@ func URLDestinationFilepath(rawurl, dest, cwd string, envs []string) (string, er
}
func IsSrcsValid(srcsAndDest instructions.SourcesAndDest, resolvedSources []string, fileContext FileContext) error {
srcs := srcsAndDest[:len(srcsAndDest)-1]
dest := srcsAndDest[len(srcsAndDest)-1]
srcs := srcsAndDest.SourcePaths
dest := srcsAndDest.DestPath
if !ContainsWildcards(srcs) {
totalSrcs := 0
+1 -1
View File
@@ -490,7 +490,7 @@ func Test_IsSrcsValid(t *testing.T) {
if err != nil {
t.Fatalf("error creating file context: %v", err)
}
err = IsSrcsValid(test.srcsAndDest, test.resolvedSources, fileContext)
err = IsSrcsValid(instructions.SourcesAndDest{SourcePaths: test.srcsAndDest[0 : len(test.srcsAndDest)-1], DestPath: test.srcsAndDest[len(test.srcsAndDest)-1]}, test.resolvedSources, fileContext)
testutil.CheckError(t, test.shouldErr, err)
})
}
+1 -1
View File
@@ -33,11 +33,11 @@ import (
"github.com/GoogleContainerTools/kaniko/pkg/config"
"github.com/GoogleContainerTools/kaniko/pkg/timing"
"github.com/docker/docker/builder/dockerignore"
"github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/fileutils"
v1 "github.com/google/go-containerregistry/pkg/v1"
"github.com/karrick/godirwalk"
"github.com/moby/buildkit/frontend/dockerfile/dockerignore"
otiai10Cpy "github.com/otiai10/copy"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
+3 -3
View File
@@ -1018,7 +1018,7 @@ func Test_GetFSFromLayers_with_whiteouts_include_whiteout_enabled(t *testing.T)
root := t.TempDir()
// Write a whiteout path
d1 := []byte("Hello World\n")
if err := ioutil.WriteFile(filepath.Join(root, "foobar"), d1, 0644); err != nil {
if err := os.WriteFile(filepath.Join(root, "foobar"), d1, 0644); err != nil {
t.Fatal(err)
}
@@ -1067,7 +1067,7 @@ func Test_GetFSFromLayers_with_whiteouts_include_whiteout_enabled(t *testing.T)
mockLayer := mockv1.NewMockLayer(ctrl)
mockLayer.EXPECT().MediaType().Return(types.OCILayer, nil)
rc := ioutil.NopCloser(buf)
rc := io.NopCloser(buf)
mockLayer.EXPECT().Uncompressed().Return(rc, nil)
secondLayerFiles := []string{
@@ -1082,7 +1082,7 @@ func Test_GetFSFromLayers_with_whiteouts_include_whiteout_enabled(t *testing.T)
mockLayer2 := mockv1.NewMockLayer(ctrl)
mockLayer2.EXPECT().MediaType().Return(types.OCILayer, nil)
rc = ioutil.NopCloser(buf)
rc = io.NopCloser(buf)
mockLayer2.EXPECT().Uncompressed().Return(rc, nil)
layers := []v1.Layer{