This commit is contained in:
Nick Kubala
2018-08-01 13:02:54 -07:00
parent e45d1f6aac
commit e59f05e32f
1298 changed files with 38524 additions and 226785 deletions
-4
View File
@@ -122,10 +122,6 @@ func (d *driver) Lstat(p string) (os.FileInfo, error) {
return os.Lstat(p)
}
func (d *driver) Readlink(p string) (string, error) {
return os.Readlink(p)
}
func (d *driver) Mkdir(p string, mode os.FileMode) error {
return os.Mkdir(p, mode)
}
+5 -13
View File
@@ -6,7 +6,6 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"sort"
"github.com/containerd/continuity/devices"
@@ -26,18 +25,6 @@ func (d *driver) Mkfifo(path string, mode os.FileMode) error {
return devices.Mknod(path, mode, 0, 0)
}
// Lchmod changes the mode of an file not following symlinks.
func (d *driver) Lchmod(path string, mode os.FileMode) (err error) {
if !filepath.IsAbs(path) {
path, err = filepath.Abs(path)
if err != nil {
return
}
}
return sysx.Fchmodat(0, path, uint32(mode), sysx.AtSymlinkNofollow)
}
// Getxattr returns all of the extended attributes for the file at path p.
func (d *driver) Getxattr(p string) (map[string][]byte, error) {
xattrs, err := sysx.Listxattr(p)
@@ -120,3 +107,8 @@ func (d *driver) LSetxattr(path string, attrMap map[string][]byte) error {
func (d *driver) DeviceInfo(fi os.FileInfo) (maj uint64, min uint64, err error) {
return devices.DeviceInfo(fi)
}
// Readlink was forked on Windows to fix a Golang bug, use the "os" package here
func (d *driver) Readlink(p string) (string, error) {
return os.Readlink(p)
}
+7
View File
@@ -3,6 +3,7 @@ package driver
import (
"os"
"github.com/containerd/continuity/sysx"
"github.com/pkg/errors"
)
@@ -19,3 +20,9 @@ func (d *driver) Lchmod(path string, mode os.FileMode) (err error) {
// TODO: Use Window's equivalent
return os.Chmod(path, mode)
}
// Readlink is forked in order to support Volume paths which are used
// in container layers.
func (d *driver) Readlink(p string) (string, error) {
return sysx.Readlink(p)
}
+19
View File
@@ -0,0 +1,19 @@
package driver
import (
"os"
"golang.org/x/sys/unix"
)
// Lchmod changes the mode of a file not following symlinks.
func (d *driver) Lchmod(path string, mode os.FileMode) error {
// On Linux, file mode is not supported for symlinks,
// and fchmodat() does not support AT_SYMLINK_NOFOLLOW,
// so symlinks need to be skipped entirely.
if st, err := os.Stat(path); err == nil && st.Mode()&os.ModeSymlink != 0 {
return nil
}
return unix.Fchmodat(unix.AT_FDCWD, path, uint32(mode), 0)
}
+14
View File
@@ -0,0 +1,14 @@
// +build darwin freebsd solaris
package driver
import (
"os"
"golang.org/x/sys/unix"
)
// Lchmod changes the mode of a file not following symlinks.
func (d *driver) Lchmod(path string, mode os.FileMode) error {
return unix.Fchmodat(unix.AT_FDCWD, path, uint32(mode), unix.AT_SYMLINK_NOFOLLOW)
}
+18 -12
View File
@@ -49,20 +49,26 @@ func copyFileContent(dst, src *os.File) error {
return errors.Wrap(err, "unable to stat source")
}
n, err := unix.CopyFileRange(int(src.Fd()), nil, int(dst.Fd()), nil, int(st.Size()), 0)
if err != nil {
if err != unix.ENOSYS && err != unix.EXDEV {
return errors.Wrap(err, "copy file range failed")
size := st.Size()
first := true
srcFd := int(src.Fd())
dstFd := int(dst.Fd())
for size > 0 {
n, err := unix.CopyFileRange(srcFd, nil, dstFd, nil, int(size), 0)
if err != nil {
if (err != unix.ENOSYS && err != unix.EXDEV) || !first {
return errors.Wrap(err, "copy file range failed")
}
buf := bufferPool.Get().(*[]byte)
_, err = io.CopyBuffer(dst, src, *buf)
bufferPool.Put(buf)
return errors.Wrap(err, "userspace copy failed")
}
buf := bufferPool.Get().(*[]byte)
_, err = io.CopyBuffer(dst, src, *buf)
bufferPool.Put(buf)
return err
}
if int64(n) != st.Size() {
return errors.Wrapf(err, "short copy: %d of %d", int64(n), st.Size())
first = false
size -= int64(n)
}
return nil
+2 -2
View File
@@ -10,8 +10,8 @@ type Usage struct {
// DiskUsage counts the number of inodes and disk usage for the resources under
// path.
func DiskUsage(roots ...string) (Usage, error) {
return diskUsage(roots...)
func DiskUsage(ctx context.Context, roots ...string) (Usage, error) {
return diskUsage(ctx, roots...)
}
// DiffUsage counts the numbers of inodes and disk usage in the
+7 -1
View File
@@ -24,7 +24,7 @@ func newInode(stat *syscall.Stat_t) inode {
}
}
func diskUsage(roots ...string) (Usage, error) {
func diskUsage(ctx context.Context, roots ...string) (Usage, error) {
var (
size int64
@@ -37,6 +37,12 @@ func diskUsage(roots ...string) (Usage, error) {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
inoKey := newInode(fi.Sys().(*syscall.Stat_t))
if _, ok := inodes[inoKey]; !ok {
inodes[inoKey] = struct{}{}
+7 -1
View File
@@ -8,7 +8,7 @@ import (
"path/filepath"
)
func diskUsage(roots ...string) (Usage, error) {
func diskUsage(ctx context.Context, roots ...string) (Usage, error) {
var (
size int64
)
@@ -21,6 +21,12 @@ func diskUsage(roots ...string) (Usage, error) {
return err
}
select {
case <-ctx.Done():
return ctx.Err()
default:
}
size += fi.Size()
return nil
}); err != nil {
+10
View File
@@ -0,0 +1,10 @@
// +build !windows
package syscallx
import "syscall"
// Readlink returns the destination of the named symbolic link.
func Readlink(path string, buf []byte) (n int, err error) {
return syscall.Readlink(path, buf)
}
+96
View File
@@ -0,0 +1,96 @@
package syscallx
import (
"syscall"
"unsafe"
)
type reparseDataBuffer struct {
ReparseTag uint32
ReparseDataLength uint16
Reserved uint16
// GenericReparseBuffer
reparseBuffer byte
}
type mountPointReparseBuffer struct {
SubstituteNameOffset uint16
SubstituteNameLength uint16
PrintNameOffset uint16
PrintNameLength uint16
PathBuffer [1]uint16
}
type symbolicLinkReparseBuffer struct {
SubstituteNameOffset uint16
SubstituteNameLength uint16
PrintNameOffset uint16
PrintNameLength uint16
Flags uint32
PathBuffer [1]uint16
}
const (
_IO_REPARSE_TAG_MOUNT_POINT = 0xA0000003
_SYMLINK_FLAG_RELATIVE = 1
)
// Readlink returns the destination of the named symbolic link.
func Readlink(path string, buf []byte) (n int, err error) {
fd, err := syscall.CreateFile(syscall.StringToUTF16Ptr(path), syscall.GENERIC_READ, 0, nil, syscall.OPEN_EXISTING,
syscall.FILE_FLAG_OPEN_REPARSE_POINT|syscall.FILE_FLAG_BACKUP_SEMANTICS, 0)
if err != nil {
return -1, err
}
defer syscall.CloseHandle(fd)
rdbbuf := make([]byte, syscall.MAXIMUM_REPARSE_DATA_BUFFER_SIZE)
var bytesReturned uint32
err = syscall.DeviceIoControl(fd, syscall.FSCTL_GET_REPARSE_POINT, nil, 0, &rdbbuf[0], uint32(len(rdbbuf)), &bytesReturned, nil)
if err != nil {
return -1, err
}
rdb := (*reparseDataBuffer)(unsafe.Pointer(&rdbbuf[0]))
var s string
switch rdb.ReparseTag {
case syscall.IO_REPARSE_TAG_SYMLINK:
data := (*symbolicLinkReparseBuffer)(unsafe.Pointer(&rdb.reparseBuffer))
p := (*[0xffff]uint16)(unsafe.Pointer(&data.PathBuffer[0]))
s = syscall.UTF16ToString(p[data.SubstituteNameOffset/2 : (data.SubstituteNameOffset+data.SubstituteNameLength)/2])
if data.Flags&_SYMLINK_FLAG_RELATIVE == 0 {
if len(s) >= 4 && s[:4] == `\??\` {
s = s[4:]
switch {
case len(s) >= 2 && s[1] == ':': // \??\C:\foo\bar
// do nothing
case len(s) >= 4 && s[:4] == `UNC\`: // \??\UNC\foo\bar
s = `\\` + s[4:]
default:
// unexpected; do nothing
}
} else {
// unexpected; do nothing
}
}
case _IO_REPARSE_TAG_MOUNT_POINT:
data := (*mountPointReparseBuffer)(unsafe.Pointer(&rdb.reparseBuffer))
p := (*[0xffff]uint16)(unsafe.Pointer(&data.PathBuffer[0]))
s = syscall.UTF16ToString(p[data.SubstituteNameOffset/2 : (data.SubstituteNameOffset+data.SubstituteNameLength)/2])
if len(s) >= 4 && s[:4] == `\??\` { // \??\C:\foo\bar
if len(s) < 48 || s[:11] != `\??\Volume{` {
s = s[4:]
}
} else {
// unexpected; do nothing
}
default:
// the path is not a symlink or junction but another type of reparse
// point
return -1, syscall.ENOENT
}
n = copy(buf, []byte(s))
return n, nil
}
-18
View File
@@ -1,18 +0,0 @@
package sysx
const (
// AtSymlinkNoFollow defined from AT_SYMLINK_NOFOLLOW in <sys/fcntl.h>
AtSymlinkNofollow = 0x20
)
const (
// SYS_FCHMODAT defined from golang.org/sys/unix
SYS_FCHMODAT = 467
)
// These functions will be generated by generate.sh
// $ GOOS=darwin GOARCH=386 ./generate.sh chmod
// $ GOOS=darwin GOARCH=amd64 ./generate.sh chmod
//sys Fchmodat(dirfd int, path string, mode uint32, flags int) (err error)
-25
View File
@@ -1,25 +0,0 @@
// mksyscall.pl -l32 chmod_darwin.go
// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
package sysx
import (
"syscall"
"unsafe"
)
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) {
var _p0 *byte
_p0, err = syscall.BytePtrFromString(path)
if err != nil {
return
}
_, _, e1 := syscall.Syscall6(SYS_FCHMODAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0)
use(unsafe.Pointer(_p0))
if e1 != 0 {
err = errnoErr(e1)
}
return
}
-25
View File
@@ -1,25 +0,0 @@
// mksyscall.pl chmod_darwin.go
// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
package sysx
import (
"syscall"
"unsafe"
)
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) {
var _p0 *byte
_p0, err = syscall.BytePtrFromString(path)
if err != nil {
return
}
_, _, e1 := syscall.Syscall6(SYS_FCHMODAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0)
use(unsafe.Pointer(_p0))
if e1 != 0 {
err = errnoErr(e1)
}
return
}
-17
View File
@@ -1,17 +0,0 @@
package sysx
const (
// AtSymlinkNoFollow defined from AT_SYMLINK_NOFOLLOW in <sys/fcntl.h>
AtSymlinkNofollow = 0x200
)
const (
// SYS_FCHMODAT defined from golang.org/sys/unix
SYS_FCHMODAT = 490
)
// These functions will be generated by generate.sh
// $ GOOS=freebsd GOARCH=amd64 ./generate.sh chmod
//sys Fchmodat(dirfd int, path string, mode uint32, flags int) (err error)
-25
View File
@@ -1,25 +0,0 @@
// mksyscall.pl chmod_freebsd.go
// MACHINE GENERATED BY THE COMMAND ABOVE; DO NOT EDIT
package sysx
import (
"syscall"
"unsafe"
)
// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT
func Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) {
var _p0 *byte
_p0, err = syscall.BytePtrFromString(path)
if err != nil {
return
}
_, _, e1 := syscall.Syscall6(SYS_FCHMODAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0)
use(unsafe.Pointer(_p0))
if e1 != 0 {
err = errnoErr(e1)
}
return
}
-12
View File
@@ -1,12 +0,0 @@
package sysx
import "syscall"
const (
// AtSymlinkNoFollow defined from AT_SYMLINK_NOFOLLOW in /usr/include/linux/fcntl.h
AtSymlinkNofollow = 0x100
)
func Fchmodat(dirfd int, path string, mode uint32, flags int) error {
return syscall.Fchmodat(dirfd, path, mode, flags)
}
-11
View File
@@ -1,11 +0,0 @@
package sysx
import "golang.org/x/sys/unix"
const (
AtSymlinkNofollow = unix.AT_SYMLINK_NOFOLLOW
)
func Fchmodat(dirfd int, path string, mode uint32, flags int) error {
return unix.Fchmodat(dirfd, path, mode, flags)
}
+112
View File
@@ -0,0 +1,112 @@
package sysx
import (
"os"
"path/filepath"
"github.com/containerd/continuity/syscallx"
)
// Readlink returns the destination of the named symbolic link.
// If there is an error, it will be of type *PathError.
func Readlink(name string) (string, error) {
for len := 128; ; len *= 2 {
b := make([]byte, len)
n, e := fixCount(syscallx.Readlink(fixLongPath(name), b))
if e != nil {
return "", &os.PathError{Op: "readlink", Path: name, Err: e}
}
if n < len {
return string(b[0:n]), nil
}
}
}
// Many functions in package syscall return a count of -1 instead of 0.
// Using fixCount(call()) instead of call() corrects the count.
func fixCount(n int, err error) (int, error) {
if n < 0 {
n = 0
}
return n, err
}
// fixLongPath returns the extended-length (\\?\-prefixed) form of
// path when needed, in order to avoid the default 260 character file
// path limit imposed by Windows. If path is not easily converted to
// the extended-length form (for example, if path is a relative path
// or contains .. elements), or is short enough, fixLongPath returns
// path unmodified.
//
// See https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath
func fixLongPath(path string) string {
// Do nothing (and don't allocate) if the path is "short".
// Empirically (at least on the Windows Server 2013 builder),
// the kernel is arbitrarily okay with < 248 bytes. That
// matches what the docs above say:
// "When using an API to create a directory, the specified
// path cannot be so long that you cannot append an 8.3 file
// name (that is, the directory name cannot exceed MAX_PATH
// minus 12)." Since MAX_PATH is 260, 260 - 12 = 248.
//
// The MSDN docs appear to say that a normal path that is 248 bytes long
// will work; empirically the path must be less then 248 bytes long.
if len(path) < 248 {
// Don't fix. (This is how Go 1.7 and earlier worked,
// not automatically generating the \\?\ form)
return path
}
// The extended form begins with \\?\, as in
// \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt.
// The extended form disables evaluation of . and .. path
// elements and disables the interpretation of / as equivalent
// to \. The conversion here rewrites / to \ and elides
// . elements as well as trailing or duplicate separators. For
// simplicity it avoids the conversion entirely for relative
// paths or paths containing .. elements. For now,
// \\server\share paths are not converted to
// \\?\UNC\server\share paths because the rules for doing so
// are less well-specified.
if len(path) >= 2 && path[:2] == `\\` {
// Don't canonicalize UNC paths.
return path
}
if !filepath.IsAbs(path) {
// Relative path
return path
}
const prefix = `\\?`
pathbuf := make([]byte, len(prefix)+len(path)+len(`\`))
copy(pathbuf, prefix)
n := len(path)
r, w := 0, len(prefix)
for r < n {
switch {
case os.IsPathSeparator(path[r]):
// empty block
r++
case path[r] == '.' && (r+1 == n || os.IsPathSeparator(path[r+1])):
// /./
r++
case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])):
// /../ is currently unhandled
return path
default:
pathbuf[w] = '\\'
w++
for ; r < n && !os.IsPathSeparator(path[r]); r++ {
pathbuf[w] = path[r]
w++
}
}
}
// A drive's root directory needs a trailing \
if w == len(`\\?\c:`) {
pathbuf[w] = '\\'
w++
}
return string(pathbuf[:w])
}
-1
View File
@@ -8,7 +8,6 @@ package sysx
//sys setxattr(path string, attr string, data []byte, flags int) (err error)
//sys removexattr(path string, attr string, options int) (err error)
//sys listxattr(path string, dest []byte, options int) (sz int, err error)
//sys Fchmodat(dirfd int, path string, mode uint32, flags int) (err error)
const (
xattrNoFollow = 0x01