diff --git a/Gopkg.lock b/Gopkg.lock index b11e5ee73..8aa683042 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -561,6 +561,14 @@ revision = "ab8a2e0c74be9d3be70b3184d9acc634935ded82" version = "1.1.4" +[[projects]] + digest = "1:75b5172f534d05a5abff749f1cf002351f834a2a5592812210aca5e139f9ddad" + name = "github.com/karrick/godirwalk" + packages = ["."] + pruneopts = "NUT" + revision = "cceff240ca8af695e41738831646717e80d2f846" + version = "v1.7.7" + [[projects]] digest = "1:d0164259ed17929689df11205194d80288e8ae25351778f7a3421a24774c36f8" name = "github.com/mattn/go-shellwords" @@ -1174,7 +1182,9 @@ "github.com/aws/aws-sdk-go/service/s3", "github.com/aws/aws-sdk-go/service/s3/s3manager", "github.com/docker/docker/builder/dockerfile", + "github.com/docker/docker/builder/dockerignore", "github.com/docker/docker/pkg/archive", + "github.com/docker/docker/pkg/fileutils", "github.com/docker/docker/pkg/signal", "github.com/genuinetools/amicontained/container", "github.com/google/go-cmp/cmp", @@ -1189,14 +1199,17 @@ "github.com/google/go-containerregistry/pkg/v1/remote", "github.com/google/go-containerregistry/pkg/v1/tarball", "github.com/google/go-github/github", + "github.com/karrick/godirwalk", "github.com/moby/buildkit/frontend/dockerfile/instructions", "github.com/moby/buildkit/frontend/dockerfile/parser", "github.com/moby/buildkit/frontend/dockerfile/shell", "github.com/pkg/errors", "github.com/sirupsen/logrus", "github.com/spf13/cobra", + "github.com/spf13/pflag", "golang.org/x/net/context", "golang.org/x/oauth2", + "golang.org/x/sync/errgroup", "k8s.io/client-go/discovery", ] solver-name = "gps-cdcl" diff --git a/pkg/snapshot/snapshot.go b/pkg/snapshot/snapshot.go index 8fee12764..39dbb60c5 100644 --- a/pkg/snapshot/snapshot.go +++ b/pkg/snapshot/snapshot.go @@ -19,10 +19,11 @@ package snapshot import ( "fmt" "io/ioutil" - "os" "path/filepath" "syscall" + "github.com/karrick/godirwalk" + "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/util" @@ -141,22 +142,22 @@ func (s *Snapshotter) TakeSnapshotFS() (string, error) { defer t.Close() // Save the fs state in a map to iterate over later. - memFs := map[string]os.FileInfo{} - filepath.Walk(s.directory, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if util.IsInWhitelist(path) { - if util.IsDestDir(path) { - logrus.Infof("Skipping paths under %s, as it is a whitelisted directory", path) - return filepath.SkipDir + memFs := map[string]*godirwalk.Dirent{} + godirwalk.Walk(s.directory, &godirwalk.Options{ + Callback: func(path string, ent *godirwalk.Dirent) error { + if util.IsInWhitelist(path) { + if util.IsDestDir(path) { + logrus.Infof("Skipping paths under %s, as it is a whitelisted directory", path) + return filepath.SkipDir + } + return nil } + memFs[path] = ent return nil - } - - memFs[path] = info - return nil - }) + }, + Unsorted: true, + }, + ) // First handle whiteouts for p := range memFs { diff --git a/vendor/github.com/karrick/godirwalk/LICENSE b/vendor/github.com/karrick/godirwalk/LICENSE new file mode 100644 index 000000000..01ce194c8 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/LICENSE @@ -0,0 +1,25 @@ +BSD 2-Clause License + +Copyright (c) 2017, Karrick McDermott +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/karrick/godirwalk/dirent.go b/vendor/github.com/karrick/godirwalk/dirent.go new file mode 100644 index 000000000..5a2772248 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/dirent.go @@ -0,0 +1,74 @@ +package godirwalk + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +// Dirent stores the name and file system mode type of discovered file system +// entries. +type Dirent struct { + name string + modeType os.FileMode +} + +// NewDirent returns a newly initialized Dirent structure, or an error. This +// function does not follow symbolic links. +// +// This function is rarely used, as Dirent structures are provided by other +// functions in this library that read and walk directories. +func NewDirent(osPathname string) (*Dirent, error) { + fi, err := os.Lstat(osPathname) + if err != nil { + return nil, errors.Wrap(err, "cannot lstat") + } + return &Dirent{ + name: filepath.Base(osPathname), + modeType: fi.Mode() & os.ModeType, + }, nil +} + +// Name returns the basename of the file system entry. +func (de Dirent) Name() string { return de.name } + +// ModeType returns the mode bits that specify the file system node type. We +// could make our own enum-like data type for encoding the file type, but Go's +// runtime already gives us architecture independent file modes, as discussed in +// `os/types.go`: +// +// Go's runtime FileMode type has same definition on all systems, so that +// information about files can be moved from one system to another portably. +func (de Dirent) ModeType() os.FileMode { return de.modeType } + +// IsDir returns true if and only if the Dirent represents a file system +// directory. Note that on some operating systems, more than one file mode bit +// may be set for a node. For instance, on Windows, a symbolic link that points +// to a directory will have both the directory and the symbolic link bits set. +func (de Dirent) IsDir() bool { return de.modeType&os.ModeDir != 0 } + +// IsRegular returns true if and only if the Dirent represents a regular +// file. That is, it ensures that no mode type bits are set. +func (de Dirent) IsRegular() bool { return de.modeType&os.ModeType == 0 } + +// IsSymlink returns true if and only if the Dirent represents a file system +// symbolic link. Note that on some operating systems, more than one file mode +// bit may be set for a node. For instance, on Windows, a symbolic link that +// points to a directory will have both the directory and the symbolic link bits +// set. +func (de Dirent) IsSymlink() bool { return de.modeType&os.ModeSymlink != 0 } + +// Dirents represents a slice of Dirent pointers, which are sortable by +// name. This type satisfies the `sort.Interface` interface. +type Dirents []*Dirent + +// Len returns the count of Dirent structures in the slice. +func (l Dirents) Len() int { return len(l) } + +// Less returns true if and only if the Name of the element specified by the +// first index is lexicographically less than that of the second index. +func (l Dirents) Less(i, j int) bool { return l[i].name < l[j].name } + +// Swap exchanges the two Dirent entries specified by the two provided indexes. +func (l Dirents) Swap(i, j int) { l[i], l[j] = l[j], l[i] } diff --git a/vendor/github.com/karrick/godirwalk/doc.go b/vendor/github.com/karrick/godirwalk/doc.go new file mode 100644 index 000000000..0dfdabd48 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/doc.go @@ -0,0 +1,34 @@ +/* +Package godirwalk provides functions to read and traverse directory trees. + +In short, why do I use this library? + +* It's faster than `filepath.Walk`. + +* It's more correct on Windows than `filepath.Walk`. + +* It's more easy to use than `filepath.Walk`. + +* It's more flexible than `filepath.Walk`. + +USAGE + +This library will normalize the provided top level directory name based on the +os-specific path separator by calling `filepath.Clean` on its first +argument. However it always provides the pathname created by using the correct +os-specific path separator when invoking the provided callback function. + + dirname := "some/directory/root" + err := godirwalk.Walk(dirname, &godirwalk.Options{ + Callback: func(osPathname string, de *godirwalk.Dirent) error { + fmt.Printf("%s %s\n", de.ModeType(), osPathname) + return nil + }, + }) + +This library not only provides functions for traversing a file system directory +tree, but also for obtaining a list of immediate descendants of a particular +directory, typically much more quickly than using `os.ReadDir` or +`os.ReadDirnames`. +*/ +package godirwalk diff --git a/vendor/github.com/karrick/godirwalk/readdir.go b/vendor/github.com/karrick/godirwalk/readdir.go new file mode 100644 index 000000000..2bba68975 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir.go @@ -0,0 +1,47 @@ +package godirwalk + +// ReadDirents returns a sortable slice of pointers to Dirent structures, each +// representing the file system name and mode type for one of the immediate +// descendant of the specified directory. If the specified directory is a +// symbolic link, it will be resolved. +// +// If an optional scratch buffer is provided that is at least one page of +// memory, it will be used when reading directory entries from the file system. +// +// children, err := godirwalk.ReadDirents(osDirname, nil) +// if err != nil { +// return nil, errors.Wrap(err, "cannot get list of directory children") +// } +// sort.Sort(children) +// for _, child := range children { +// fmt.Printf("%s %s\n", child.ModeType, child.Name) +// } +func ReadDirents(osDirname string, scratchBuffer []byte) (Dirents, error) { + return readdirents(osDirname, scratchBuffer) +} + +// ReadDirnames returns a slice of strings, representing the immediate +// descendants of the specified directory. If the specified directory is a +// symbolic link, it will be resolved. +// +// If an optional scratch buffer is provided that is at least one page of +// memory, it will be used when reading directory entries from the file system. +// +// Note that this function, depending on operating system, may or may not invoke +// the ReadDirents function, in order to prepare the list of immediate +// descendants. Therefore, if your program needs both the names and the file +// system mode types of descendants, it will always be faster to invoke +// ReadDirents directly, rather than calling this function, then looping over +// the results and calling os.Stat for each child. +// +// children, err := godirwalk.ReadDirnames(osDirname, nil) +// if err != nil { +// return nil, errors.Wrap(err, "cannot get list of directory children") +// } +// sort.Strings(children) +// for _, child := range children { +// fmt.Printf("%s\n", child) +// } +func ReadDirnames(osDirname string, scratchBuffer []byte) ([]string, error) { + return readdirnames(osDirname, scratchBuffer) +} diff --git a/vendor/github.com/karrick/godirwalk/readdir_unix.go b/vendor/github.com/karrick/godirwalk/readdir_unix.go new file mode 100644 index 000000000..04a628f7f --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir_unix.go @@ -0,0 +1,109 @@ +// +build darwin freebsd linux netbsd openbsd + +package godirwalk + +import ( + "os" + "path/filepath" + "syscall" + "unsafe" + + "github.com/pkg/errors" +) + +func readdirents(osDirname string, scratchBuffer []byte) (Dirents, error) { + dh, err := os.Open(osDirname) + if err != nil { + return nil, errors.Wrap(err, "cannot Open") + } + + var entries Dirents + + fd := int(dh.Fd()) + + if len(scratchBuffer) < MinimumScratchBufferSize { + scratchBuffer = make([]byte, DefaultScratchBufferSize) + } + + var de *syscall.Dirent + + for { + n, err := syscall.ReadDirent(fd, scratchBuffer) + if err != nil { + _ = dh.Close() // ignore potential error returned by Close + return nil, errors.Wrap(err, "cannot ReadDirent") + } + if n <= 0 { + break // end of directory reached + } + // Loop over the bytes returned by reading the directory entries. + buf := scratchBuffer[:n] + for len(buf) > 0 { + de = (*syscall.Dirent)(unsafe.Pointer(&buf[0])) // point entry to first syscall.Dirent in buffer + buf = buf[de.Reclen:] // advance buffer + + if inoFromDirent(de) == 0 { + continue // this item has been deleted, but not yet removed from directory + } + + nameSlice := nameFromDirent(de) + namlen := len(nameSlice) + if (namlen == 0) || (namlen == 1 && nameSlice[0] == '.') || (namlen == 2 && nameSlice[0] == '.' && nameSlice[1] == '.') { + continue // skip unimportant entries + } + osChildname := string(nameSlice) + + // Convert syscall constant, which is in purview of OS, to a + // constant defined by Go, assumed by this project to be stable. + var mode os.FileMode + switch de.Type { + case syscall.DT_REG: + // regular file + case syscall.DT_DIR: + mode = os.ModeDir + case syscall.DT_LNK: + mode = os.ModeSymlink + case syscall.DT_CHR: + mode = os.ModeDevice | os.ModeCharDevice + case syscall.DT_BLK: + mode = os.ModeDevice + case syscall.DT_FIFO: + mode = os.ModeNamedPipe + case syscall.DT_SOCK: + mode = os.ModeSocket + default: + // If syscall returned unknown type (e.g., DT_UNKNOWN, DT_WHT), + // then resolve actual mode by getting stat. + fi, err := os.Lstat(filepath.Join(osDirname, osChildname)) + if err != nil { + _ = dh.Close() // ignore potential error returned by Close + return nil, errors.Wrap(err, "cannot Stat") + } + // We only care about the bits that identify the type of a file + // system node, and can ignore append, exclusive, temporary, + // setuid, setgid, permission bits, and sticky bits, which are + // coincident to the bits that declare type of the file system + // node. + mode = fi.Mode() & os.ModeType + } + + entries = append(entries, &Dirent{name: osChildname, modeType: mode}) + } + } + if err = dh.Close(); err != nil { + return nil, err + } + return entries, nil +} + +func readdirnames(osDirname string, scratchBuffer []byte) ([]string, error) { + des, err := readdirents(osDirname, scratchBuffer) + if err != nil { + return nil, err + } + names := make([]string, len(des)) + for i, v := range des { + names[i] = v.name + } + return names, nil +} diff --git a/vendor/github.com/karrick/godirwalk/readdir_windows.go b/vendor/github.com/karrick/godirwalk/readdir_windows.go new file mode 100644 index 000000000..885a067a4 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/readdir_windows.go @@ -0,0 +1,54 @@ +package godirwalk + +import ( + "os" + + "github.com/pkg/errors" +) + +// The functions in this file are mere wrappers of what is already provided by +// standard library, in order to provide the same API as this library provides. +// +// The scratch buffer argument is ignored by this architecture. +// +// Please send PR or link to article if you know of a more performant way of +// enumerating directory contents and mode types on Windows. + +func readdirents(osDirname string, _ []byte) (Dirents, error) { + dh, err := os.Open(osDirname) + if err != nil { + return nil, errors.Wrap(err, "cannot Open") + } + + fileinfos, err := dh.Readdir(0) + if er := dh.Close(); err == nil { + err = er + } + if err != nil { + return nil, errors.Wrap(err, "cannot Readdir") + } + + entries := make(Dirents, len(fileinfos)) + for i, info := range fileinfos { + entries[i] = &Dirent{name: info.Name(), modeType: info.Mode() & os.ModeType} + } + + return entries, nil +} + +func readdirnames(osDirname string, _ []byte) ([]string, error) { + dh, err := os.Open(osDirname) + if err != nil { + return nil, errors.Wrap(err, "cannot Open") + } + + entries, err := dh.Readdirnames(0) + if er := dh.Close(); err == nil { + err = er + } + if err != nil { + return nil, errors.Wrap(err, "cannot Readdirnames") + } + + return entries, nil +} diff --git a/vendor/github.com/karrick/godirwalk/walk.go b/vendor/github.com/karrick/godirwalk/walk.go new file mode 100644 index 000000000..4c184ab87 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/walk.go @@ -0,0 +1,367 @@ +package godirwalk + +import ( + "os" + "path/filepath" + "sort" + + "github.com/pkg/errors" +) + +// DefaultScratchBufferSize specifies the size of the scratch buffer that will +// be allocated by Walk, ReadDirents, or ReadDirnames when a scratch buffer is +// not provided or the scratch buffer that is provided is smaller than +// MinimumScratchBufferSize bytes. This may seem like a large value; however, +// when a program intends to enumerate large directories, having a larger +// scratch buffer results in fewer operating system calls. +const DefaultScratchBufferSize = 64 * 1024 + +// MinimumScratchBufferSize specifies the minimum size of the scratch buffer +// that Walk, ReadDirents, and ReadDirnames will use when reading file entries +// from the operating system. It is initialized to the result from calling +// `os.Getpagesize()` during program startup. +var MinimumScratchBufferSize int + +func init() { + MinimumScratchBufferSize = os.Getpagesize() +} + +// Options provide parameters for how the Walk function operates. +type Options struct { + // ErrorCallback specifies a function to be invoked in the case of an error + // that could potentially be ignored while walking a file system + // hierarchy. When set to nil or left as its zero-value, any error condition + // causes Walk to immediately return the error describing what took + // place. When non-nil, this user supplied function is invoked with the OS + // pathname of the file system object that caused the error along with the + // error that took place. The return value of the supplied ErrorCallback + // function determines whether the error will cause Walk to halt immediately + // as it would were no ErrorCallback value provided, or skip this file + // system node yet continue on with the remaining nodes in the file system + // hierarchy. + // + // ErrorCallback is invoked both for errors that are returned by the + // runtime, and for errors returned by other user supplied callback + // functions. + ErrorCallback func(string, error) ErrorAction + + // FollowSymbolicLinks specifies whether Walk will follow symbolic links + // that refer to directories. When set to false or left as its zero-value, + // Walk will still invoke the callback function with symbolic link nodes, + // but if the symbolic link refers to a directory, it will not recurse on + // that directory. When set to true, Walk will recurse on symbolic links + // that refer to a directory. + FollowSymbolicLinks bool + + // Unsorted controls whether or not Walk will sort the immediate descendants + // of a directory by their relative names prior to visiting each of those + // entries. + // + // When set to false or left at its zero-value, Walk will get the list of + // immediate descendants of a particular directory, sort that list by + // lexical order of their names, and then visit each node in the list in + // sorted order. This will cause Walk to always traverse the same directory + // tree in the same order, however may be inefficient for directories with + // many immediate descendants. + // + // When set to true, Walk skips sorting the list of immediate descendants + // for a directory, and simply visits each node in the order the operating + // system enumerated them. This will be more fast, but with the side effect + // that the traversal order may be different from one invocation to the + // next. + Unsorted bool + + // Callback is a required function that Walk will invoke for every file + // system node it encounters. + Callback WalkFunc + + // PostChildrenCallback is an option function that Walk will invoke for + // every file system directory it encounters after its children have been + // processed. + PostChildrenCallback WalkFunc + + // ScratchBuffer is an optional byte slice to use as a scratch buffer for + // Walk to use when reading directory entries, to reduce amount of garbage + // generation. Not all architectures take advantage of the scratch + // buffer. If omitted or the provided buffer has fewer bytes than + // MinimumScratchBufferSize, then a buffer with DefaultScratchBufferSize + // bytes will be created and used once per Walk invocation. + ScratchBuffer []byte +} + +// ErrorAction defines a set of actions the Walk function could take based on +// the occurrence of an error while walking the file system. See the +// documentation for the ErrorCallback field of the Options structure for more +// information. +type ErrorAction int + +const ( + // Halt is the ErrorAction return value when the upstream code wants to halt + // the walk process when a runtime error takes place. It matches the default + // action the Walk function would take were no ErrorCallback provided. + Halt ErrorAction = iota + + // SkipNode is the ErrorAction return value when the upstream code wants to + // ignore the runtime error for the current file system node, skip + // processing of the node that caused the error, and continue walking the + // file system hierarchy with the remaining nodes. + SkipNode +) + +// WalkFunc is the type of the function called for each file system node visited +// by Walk. The pathname argument will contain the argument to Walk as a prefix; +// that is, if Walk is called with "dir", which is a directory containing the +// file "a", the provided WalkFunc will be invoked with the argument "dir/a", +// using the correct os.PathSeparator for the Go Operating System architecture, +// GOOS. The directory entry argument is a pointer to a Dirent for the node, +// providing access to both the basename and the mode type of the file system +// node. +// +// If an error is returned by the Callback or PostChildrenCallback functions, +// and no ErrorCallback function is provided, processing stops. If an +// ErrorCallback function is provided, then it is invoked with the OS pathname +// of the node that caused the error along along with the error. The return +// value of the ErrorCallback function determines whether to halt processing, or +// skip this node and continue processing remaining file system nodes. +// +// The exception is when the function returns the special value +// filepath.SkipDir. If the function returns filepath.SkipDir when invoked on a +// directory, Walk skips the directory's contents entirely. If the function +// returns filepath.SkipDir when invoked on a non-directory file system node, +// Walk skips the remaining files in the containing directory. Note that any +// supplied ErrorCallback function is not invoked with filepath.SkipDir when the +// Callback or PostChildrenCallback functions return that special value. +type WalkFunc func(osPathname string, directoryEntry *Dirent) error + +// Walk walks the file tree rooted at the specified directory, calling the +// specified callback function for each file system node in the tree, including +// root, symbolic links, and other node types. The nodes are walked in lexical +// order, which makes the output deterministic but means that for very large +// directories this function can be inefficient. +// +// This function is often much faster than filepath.Walk because it does not +// invoke os.Stat for every node it encounters, but rather obtains the file +// system node type when it reads the parent directory. +// +// If a runtime error occurs, either from the operating system or from the +// upstream Callback or PostChildrenCallback functions, processing typically +// halts. However, when an ErrorCallback function is provided in the provided +// Options structure, that function is invoked with the error along with the OS +// pathname of the file system node that caused the error. The ErrorCallback +// function's return value determines the action that Walk will then take. +// +// func main() { +// dirname := "." +// if len(os.Args) > 1 { +// dirname = os.Args[1] +// } +// err := godirwalk.Walk(dirname, &godirwalk.Options{ +// Callback: func(osPathname string, de *godirwalk.Dirent) error { +// fmt.Printf("%s %s\n", de.ModeType(), osPathname) +// return nil +// }, +// ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { +// // Your program may want to log the error somehow. +// fmt.Fprintf(os.Stderr, "ERROR: %s\n", err) +// +// // For the purposes of this example, a simple SkipNode will suffice, +// // although in reality perhaps additional logic might be called for. +// return godirwalk.SkipNode +// }, +// }) +// if err != nil { +// fmt.Fprintf(os.Stderr, "%s\n", err) +// os.Exit(1) +// } +// } +func Walk(pathname string, options *Options) error { + pathname = filepath.Clean(pathname) + + var fi os.FileInfo + var err error + + if options.FollowSymbolicLinks { + fi, err = os.Stat(pathname) + if err != nil { + return errors.Wrap(err, "cannot Stat") + } + } else { + fi, err = os.Lstat(pathname) + if err != nil { + return errors.Wrap(err, "cannot Lstat") + } + } + + mode := fi.Mode() + if mode&os.ModeDir == 0 { + return errors.Errorf("cannot Walk non-directory: %s", pathname) + } + + dirent := &Dirent{ + name: filepath.Base(pathname), + modeType: mode & os.ModeType, + } + + // If ErrorCallback is nil, set to a default value that halts the walk + // process on all operating system errors. This is done to allow error + // handling to be more succinct in the walk code. + if options.ErrorCallback == nil { + options.ErrorCallback = defaultErrorCallback + } + + if len(options.ScratchBuffer) < MinimumScratchBufferSize { + options.ScratchBuffer = make([]byte, DefaultScratchBufferSize) + } + + err = walk(pathname, dirent, options) + if err == filepath.SkipDir { + return nil // silence SkipDir for top level + } + return err +} + +// defaultErrorCallback always returns Halt because if the upstream code did not +// provide an ErrorCallback function, walking the file system hierarchy ought to +// halt upon any operating system error. +func defaultErrorCallback(_ string, _ error) ErrorAction { return Halt } + +// walk recursively traverses the file system node specified by pathname and the +// Dirent. +func walk(osPathname string, dirent *Dirent, options *Options) error { + err := options.Callback(osPathname, dirent) + if err != nil { + if err == filepath.SkipDir { + return err + } + err = errors.Wrap(err, "Callback") // wrap potential errors returned by callback + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err + } + + // On some platforms, an entry can have more than one mode type bit set. + // For instance, it could have both the symlink bit and the directory bit + // set indicating it's a symlink to a directory. + if dirent.IsSymlink() { + if !options.FollowSymbolicLinks { + return nil + } + // Only need to Stat entry if platform did not already have os.ModeDir + // set, such as would be the case for unix like operating systems. (This + // guard eliminates extra os.Stat check on Windows.) + if !dirent.IsDir() { + referent, err := os.Readlink(osPathname) + if err != nil { + err = errors.Wrap(err, "cannot Readlink") + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err + } + + var osp string + if filepath.IsAbs(referent) { + osp = referent + } else { + osp = filepath.Join(filepath.Dir(osPathname), referent) + } + + fi, err := os.Stat(osp) + if err != nil { + err = errors.Wrap(err, "cannot Stat") + if action := options.ErrorCallback(osp, err); action == SkipNode { + return nil + } + return err + } + dirent.modeType = fi.Mode() & os.ModeType + } + } + + if !dirent.IsDir() { + return nil + } + + // If get here, then specified pathname refers to a directory. + deChildren, err := ReadDirents(osPathname, options.ScratchBuffer) + if err != nil { + err = errors.Wrap(err, "cannot ReadDirents") + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err + } + + if !options.Unsorted { + sort.Sort(deChildren) // sort children entries unless upstream says to leave unsorted + } + + for _, deChild := range deChildren { + osChildname := filepath.Join(osPathname, deChild.name) + err = walk(osChildname, deChild, options) + if err != nil { + if err != filepath.SkipDir { + return err + } + // If received skipdir on a directory, stop processing that + // directory, but continue to its siblings. If received skipdir on a + // non-directory, stop processing remaining siblings. + if deChild.IsSymlink() { + // Only need to Stat entry if platform did not already have + // os.ModeDir set, such as would be the case for unix like + // operating systems. (This guard eliminates extra os.Stat check + // on Windows.) + if !deChild.IsDir() { + // Resolve symbolic link referent to determine whether node + // is directory or not. + referent, err := os.Readlink(osChildname) + if err != nil { + err = errors.Wrap(err, "cannot Readlink") + if action := options.ErrorCallback(osChildname, err); action == SkipNode { + continue // with next child + } + return err + } + + var osp string + if filepath.IsAbs(referent) { + osp = referent + } else { + osp = filepath.Join(osPathname, referent) + } + + fi, err := os.Stat(osp) + if err != nil { + err = errors.Wrap(err, "cannot Stat") + if action := options.ErrorCallback(osp, err); action == SkipNode { + continue // with next child + } + return err + } + deChild.modeType = fi.Mode() & os.ModeType + } + } + if !deChild.IsDir() { + // If not directory, return immediately, thus skipping remainder + // of siblings. + return nil + } + } + } + + if options.PostChildrenCallback == nil { + return nil + } + + err = options.PostChildrenCallback(osPathname, dirent) + if err == nil || err == filepath.SkipDir { + return err + } + + err = errors.Wrap(err, "PostChildrenCallback") // wrap potential errors returned by callback + if action := options.ErrorCallback(osPathname, err); action == SkipNode { + return nil + } + return err +} diff --git a/vendor/github.com/karrick/godirwalk/withFileno.go b/vendor/github.com/karrick/godirwalk/withFileno.go new file mode 100644 index 000000000..1dc04a717 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withFileno.go @@ -0,0 +1,9 @@ +// +build dragonfly freebsd openbsd netbsd + +package godirwalk + +import "syscall" + +func inoFromDirent(de *syscall.Dirent) uint64 { + return uint64(de.Fileno) +} diff --git a/vendor/github.com/karrick/godirwalk/withIno.go b/vendor/github.com/karrick/godirwalk/withIno.go new file mode 100644 index 000000000..47fc12540 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withIno.go @@ -0,0 +1,9 @@ +// +build darwin linux + +package godirwalk + +import "syscall" + +func inoFromDirent(de *syscall.Dirent) uint64 { + return de.Ino +} diff --git a/vendor/github.com/karrick/godirwalk/withNamlen.go b/vendor/github.com/karrick/godirwalk/withNamlen.go new file mode 100644 index 000000000..46a4af500 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withNamlen.go @@ -0,0 +1,29 @@ +// +build darwin dragonfly freebsd netbsd openbsd + +package godirwalk + +import ( + "reflect" + "syscall" + "unsafe" +) + +func nameFromDirent(de *syscall.Dirent) []byte { + // Because this GOOS' syscall.Dirent provides a Namlen field that says how + // long the name is, this function does not need to search for the NULL + // byte. + ml := int(de.Namlen) + + // Convert syscall.Dirent.Name, which is array of int8, to []byte, by + // overwriting Cap, Len, and Data slice header fields to values from + // syscall.Dirent fields. Setting the Cap, Len, and Data field values for + // the slice header modifies what the slice header points to, and in this + // case, the name buffer. + var name []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) + sh.Cap = ml + sh.Len = ml + sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) + + return name +} diff --git a/vendor/github.com/karrick/godirwalk/withoutNamlen.go b/vendor/github.com/karrick/godirwalk/withoutNamlen.go new file mode 100644 index 000000000..dcf9f3a97 --- /dev/null +++ b/vendor/github.com/karrick/godirwalk/withoutNamlen.go @@ -0,0 +1,36 @@ +// +build nacl linux solaris + +package godirwalk + +import ( + "bytes" + "reflect" + "syscall" + "unsafe" +) + +func nameFromDirent(de *syscall.Dirent) []byte { + // Because this GOOS' syscall.Dirent does not provide a field that specifies + // the name length, this function must first calculate the max possible name + // length, and then search for the NULL byte. + ml := int(uint64(de.Reclen) - uint64(unsafe.Offsetof(syscall.Dirent{}.Name))) + + // Convert syscall.Dirent.Name, which is array of int8, to []byte, by + // overwriting Cap, Len, and Data slice header fields to values from + // syscall.Dirent fields. Setting the Cap, Len, and Data field values for + // the slice header modifies what the slice header points to, and in this + // case, the name buffer. + var name []byte + sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) + sh.Cap = ml + sh.Len = ml + sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) + + if index := bytes.IndexByte(name, 0); index >= 0 { + // Found NULL byte; set slice's cap and len accordingly. + sh.Cap = index + sh.Len = index + } + + return name +}