diff --git a/go.mod b/go.mod index 9bce58de3..a74f30813 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/karrick/godirwalk v1.16.1 github.com/minio/highwayhash v1.0.2 github.com/moby/buildkit v0.11.6 - github.com/otiai10/copy v1.12.0 + github.com/otiai10/copy v1.14.0 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.10.0 diff --git a/go.sum b/go.sum index c2671684b..3a234ba7a 100644 --- a/go.sum +++ b/go.sum @@ -532,8 +532,8 @@ github.com/opencontainers/runtime-spec v1.1.0-rc.1/go.mod h1:jwyrGlmzljRJv/Fgzds github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= -github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= -github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= diff --git a/vendor/github.com/otiai10/copy/README.md b/vendor/github.com/otiai10/copy/README.md index 1cc8fc8c8..7c8b4e05a 100644 --- a/vendor/github.com/otiai10/copy/README.md +++ b/vendor/github.com/otiai10/copy/README.md @@ -86,6 +86,19 @@ type Options struct { // If given, copy.Copy refers to this fs.FS instead of the OS filesystem. // e.g., You can use embed.FS to copy files from embedded filesystem. FS fs.FS + + // NumOfWorkers represents the number of workers used for + // concurrent copying contents of directories. + // If 0 or 1, it does not use goroutine for copying directories. + // Please refer to https://pkg.go.dev/golang.org/x/sync/semaphore for more details. + NumOfWorkers int64 + + // PreferConcurrent is a function to determine whether or not + // to use goroutine for copying contents of directories. + // If PreferConcurrent is nil, which is default, it does concurrent + // copying for all directories. + // If NumOfWorkers is 0 or 1, this function will be ignored. + PreferConcurrent func(srcdir, destdir string) (bool, error) } ``` @@ -105,4 +118,4 @@ err := Copy("your/directory", "your/directory.copy", opt) ## License -[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fotiai10%2Fcopy.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fotiai10%2Fcopy?ref=badge_large) \ No newline at end of file +[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fotiai10%2Fcopy.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fotiai10%2Fcopy?ref=badge_large) diff --git a/vendor/github.com/otiai10/copy/copy.go b/vendor/github.com/otiai10/copy/copy.go index 085db7824..2979af937 100644 --- a/vendor/github.com/otiai10/copy/copy.go +++ b/vendor/github.com/otiai10/copy/copy.go @@ -1,12 +1,16 @@ package copy import ( + "context" "io" "io/fs" "io/ioutil" "os" "path/filepath" "time" + + "golang.org/x/sync/errgroup" + "golang.org/x/sync/semaphore" ) type timespec struct { @@ -18,6 +22,10 @@ type timespec struct { // Copy copies src to dest, doesn't matter if src is a directory or a file. func Copy(src, dest string, opts ...Options) error { opt := assureOptions(src, dest, opts...) + if opt.NumOfWorkers > 1 { + opt.intent.sem = semaphore.NewWeighted(opt.NumOfWorkers) + opt.intent.ctx = context.Background() + } if opt.FS != nil { info, err := fs.Stat(opt.FS, src) if err != nil { @@ -183,12 +191,15 @@ func dcopy(srcdir, destdir string, info os.FileInfo, opt Options) (err error) { return } - for _, content := range contents { - cs, cd := filepath.Join(srcdir, content.Name()), filepath.Join(destdir, content.Name()) - - if err = copyNextOrSkip(cs, cd, content, opt); err != nil { - // If any error, exit immediately - return + if yes, err := shouldCopyDirectoryConcurrent(opt, srcdir, destdir); err != nil { + return err + } else if yes { + if err := dcopyConcurrent(srcdir, destdir, contents, opt); err != nil { + return err + } + } else { + if err := dcopySequential(srcdir, destdir, contents, opt); err != nil { + return err } } @@ -207,6 +218,42 @@ func dcopy(srcdir, destdir string, info os.FileInfo, opt Options) (err error) { return } +func dcopySequential(srcdir, destdir string, contents []os.FileInfo, opt Options) error { + for _, content := range contents { + cs, cd := filepath.Join(srcdir, content.Name()), filepath.Join(destdir, content.Name()) + + if err := copyNextOrSkip(cs, cd, content, opt); err != nil { + // If any error, exit immediately + return err + } + } + return nil +} + +// Copy this directory concurrently regarding semaphore of opt.intent +func dcopyConcurrent(srcdir, destdir string, contents []os.FileInfo, opt Options) error { + group, ctx := errgroup.WithContext(opt.intent.ctx) + getRoutine := func(cs, cd string, content os.FileInfo) func() error { + return func() error { + if content.IsDir() { + return copyNextOrSkip(cs, cd, content, opt) + } + if err := opt.intent.sem.Acquire(ctx, 1); err != nil { + return err + } + err := copyNextOrSkip(cs, cd, content, opt) + opt.intent.sem.Release(1) + return err + } + } + for _, content := range contents { + csd := filepath.Join(srcdir, content.Name()) + cdd := filepath.Join(destdir, content.Name()) + group.Go(getRoutine(csd, cdd, content)) + } + return group.Wait() +} + func onDirExists(opt Options, srcdir, destdir string) (bool, error) { _, err := os.Stat(destdir) if err == nil && opt.OnDirExists != nil && destdir != opt.intent.dest { diff --git a/vendor/github.com/otiai10/copy/options.go b/vendor/github.com/otiai10/copy/options.go index 1b4e50833..1fbfcb14a 100644 --- a/vendor/github.com/otiai10/copy/options.go +++ b/vendor/github.com/otiai10/copy/options.go @@ -1,9 +1,12 @@ package copy import ( + "context" "io" "io/fs" "os" + + "golang.org/x/sync/semaphore" ) // Options specifies optional actions on copying. @@ -65,10 +68,28 @@ type Options struct { // e.g., You can use embed.FS to copy files from embedded filesystem. FS fs.FS - intent struct { - src string - dest string - } + // NumOfWorkers represents the number of workers used for + // concurrent copying contents of directories. + // If 0 or 1, it does not use goroutine for copying directories. + // Please refer to https://pkg.go.dev/golang.org/x/sync/semaphore for more details. + NumOfWorkers int64 + + // PreferConcurrent is a function to determine whether or not + // to use goroutine for copying contents of directories. + // If PreferConcurrent is nil, which is default, it does concurrent + // copying for all directories. + // If NumOfWorkers is 0 or 1, this function will be ignored. + PreferConcurrent func(srcdir, destdir string) (bool, error) + + // Internal use only + intent intent +} + +type intent struct { + src string + dest string + sem *semaphore.Weighted + ctx context.Context } // SymlinkAction represents what to do on symlink. @@ -112,10 +133,7 @@ func getDefaultOptions(src, dest string) Options { PreserveTimes: false, // Do not preserve the modification time CopyBufferSize: 0, // Do not specify, use default bufsize (32*1024) WrapReader: nil, // Do not wrap src files, use them as they are. - intent: struct { - src string - dest string - }{src, dest}, + intent: intent{src, dest, nil, nil}, } } @@ -141,3 +159,13 @@ func assureOptions(src, dest string, opts ...Options) Options { opts[0].intent.dest = defopt.intent.dest return opts[0] } + +func shouldCopyDirectoryConcurrent(opt Options, srcdir, destdir string) (bool, error) { + if opt.NumOfWorkers <= 1 { + return false, nil + } + if opt.PreferConcurrent == nil { + return true, nil + } + return opt.PreferConcurrent(srcdir, destdir) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c7ab8b66b..d83cc026b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -752,7 +752,7 @@ github.com/opencontainers/runtime-spec/specs-go github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalkdir -# github.com/otiai10/copy v1.12.0 +# github.com/otiai10/copy v1.14.0 ## explicit; go 1.18 github.com/otiai10/copy # github.com/pelletier/go-toml v1.9.5