mirror of
https://github.com/GoogleContainerTools/kaniko
synced 2026-10-06 18:32:30 +02:00
Christie Wilson
607af5f7a6
Kaniko uses mtime (as well as file contents and other attributes) to determine if files have changed. COPY and ADD commands should _always_ update the mtime, because they actually overwrite the files. However it turns out that the mtime can lag, so kaniko would sometimes add a new layer when using COPY or ADD on a file, and sometimes would not. This leads to a non-deterministic number of layers. To fix this, we have updated the kaniko commands to be more authoritative in declaring when they have changed a file (e.g. WORKDIR will now only create the directory when it doesn't exist) and we will trust those files and _always_ add them, instead of only adding them if they haven't changed. It is possible for RUN commands to also change the filesystem, in which case kaniko has no choice but to look at the filesystem to determine what has changed. For this case we have added a call to `sync` however we still cannot guarantee that sometimes the mtime will not lag, causing the number of layers to be non-deterministic. However when I tried to cause this behaviour with the RUN command, I couldn't. This changes the snapshotting logic a bit; before this change, the last command of the last stage in a Dockerfile would always scan the whole file system and ignore the files returned by the kaniko command. Instead we will now trust those files and assume that the snapshotting performed by previous commands will be adequate. Docker itself seems to rely on the storage driver to determine when files have changed and so doesn't have to deal with these problems directly. An alternative implementation would use `inotify` to track which files have changed. However that would mean watching every file in the filesystem, and adding new watches as files are added. Not only is there a limit on the number of files that can be watched, but according to the man pages a) this can take a significant amount of time b) there is complication around when events arrive (e.g. by the time they arrive, the files may have changed) and lastly c) events can be lost, which would mean we'd run into this non-deterministic behaviour again anyway. Fixes #251
86 lines
2.2 KiB
Go
86 lines
2.2 KiB
Go
/*
|
|
Copyright 2018 Google LLC
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package util
|
|
|
|
import (
|
|
"crypto/md5"
|
|
"encoding/hex"
|
|
"io"
|
|
"os"
|
|
"strconv"
|
|
"syscall"
|
|
|
|
"github.com/pkg/errors"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
// SetLogLevel sets the logrus logging level
|
|
func SetLogLevel(logLevel string) error {
|
|
lvl, err := logrus.ParseLevel(logLevel)
|
|
if err != nil {
|
|
return errors.Wrap(err, "parsing log level")
|
|
}
|
|
logrus.SetLevel(lvl)
|
|
return nil
|
|
}
|
|
|
|
// Hasher returns a hash function, used in snapshotting to determine if a file has changed
|
|
func Hasher() func(string) (string, error) {
|
|
hasher := func(p string) (string, error) {
|
|
h := md5.New()
|
|
fi, err := os.Lstat(p)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
h.Write([]byte(fi.Mode().String()))
|
|
h.Write([]byte(fi.ModTime().String()))
|
|
|
|
h.Write([]byte(strconv.FormatUint(uint64(fi.Sys().(*syscall.Stat_t).Uid), 36)))
|
|
h.Write([]byte(","))
|
|
h.Write([]byte(strconv.FormatUint(uint64(fi.Sys().(*syscall.Stat_t).Gid), 36)))
|
|
|
|
if fi.Mode().IsRegular() {
|
|
f, err := os.Open(p)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer f.Close()
|
|
if _, err := io.Copy(h, f); err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
|
|
return hex.EncodeToString(h.Sum(nil)), nil
|
|
}
|
|
return hasher
|
|
}
|
|
|
|
// MtimeHasher returns a hash function, which only looks at mtime to determine if a file has changed.
|
|
// Note that the mtime can lag, so it's possible that a file will have changed but the mtime may look the same.
|
|
func MtimeHasher() func(string) (string, error) {
|
|
hasher := func(p string) (string, error) {
|
|
h := md5.New()
|
|
fi, err := os.Lstat(p)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
h.Write([]byte(fi.ModTime().String()))
|
|
return hex.EncodeToString(h.Sum(nil)), nil
|
|
}
|
|
return hasher
|
|
}
|