Added to vendor

vendor/github.com/containers/storage/drivers/devmapper/device_setup.go

@@ -0,0 +1,236 @@
+package devmapper
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type directLVMConfig struct {
+	Device              string
+	ThinpPercent        uint64
+	ThinpMetaPercent    uint64
+	AutoExtendPercent   uint64
+	AutoExtendThreshold uint64
+}
+
+var (
+	errThinpPercentMissing = errors.New("must set both `dm.thinp_percent` and `dm.thinp_metapercent` if either is specified")
+	errThinpPercentTooBig  = errors.New("combined `dm.thinp_percent` and `dm.thinp_metapercent` must not be greater than 100")
+	errMissingSetupDevice  = errors.New("must provide device path in `dm.setup_device` in order to configure direct-lvm")
+)
+
+func validateLVMConfig(cfg directLVMConfig) error {
+	if reflect.DeepEqual(cfg, directLVMConfig{}) {
+		return nil
+	}
+	if cfg.Device == "" {
+		return errMissingSetupDevice
+	}
+	if (cfg.ThinpPercent > 0 && cfg.ThinpMetaPercent == 0) || cfg.ThinpMetaPercent > 0 && cfg.ThinpPercent == 0 {
+		return errThinpPercentMissing
+	}
+
+	if cfg.ThinpPercent+cfg.ThinpMetaPercent > 100 {
+		return errThinpPercentTooBig
+	}
+	return nil
+}
+
+func checkDevAvailable(dev string) error {
+	lvmScan, err := exec.LookPath("lvmdiskscan")
+	if err != nil {
+		logrus.Debug("could not find lvmdiskscan")
+		return nil
+	}
+
+	out, err := exec.Command(lvmScan).CombinedOutput()
+	if err != nil {
+		logrus.WithError(err).Error(string(out))
+		return nil
+	}
+
+	if !bytes.Contains(out, []byte(dev)) {
+		return errors.Errorf("%s is not available for use with devicemapper", dev)
+	}
+	return nil
+}
+
+func checkDevInVG(dev string) error {
+	pvDisplay, err := exec.LookPath("pvdisplay")
+	if err != nil {
+		logrus.Debug("could not find pvdisplay")
+		return nil
+	}
+
+	out, err := exec.Command(pvDisplay, dev).CombinedOutput()
+	if err != nil {
+		logrus.WithError(err).Error(string(out))
+		return nil
+	}
+
+	scanner := bufio.NewScanner(bytes.NewReader(bytes.TrimSpace(out)))
+	for scanner.Scan() {
+		fields := strings.SplitAfter(strings.TrimSpace(scanner.Text()), "VG Name")
+		if len(fields) > 1 {
+			// got "VG Name" line"
+			vg := strings.TrimSpace(fields[1])
+			if len(vg) > 0 {
+				return errors.Errorf("%s is already part of a volume group %q: must remove this device from any volume group or provide a different device", dev, vg)
+			}
+			logrus.Error(fields)
+			break
+		}
+	}
+	return nil
+}
+
+func checkDevHasFS(dev string) error {
+	blkid, err := exec.LookPath("blkid")
+	if err != nil {
+		logrus.Debug("could not find blkid")
+		return nil
+	}
+
+	out, err := exec.Command(blkid, dev).CombinedOutput()
+	if err != nil {
+		logrus.WithError(err).Error(string(out))
+		return nil
+	}
+
+	fields := bytes.Fields(out)
+	for _, f := range fields {
+		kv := bytes.Split(f, []byte{'='})
+		if bytes.Equal(kv[0], []byte("TYPE")) {
+			v := bytes.Trim(kv[1], "\"")
+			if len(v) > 0 {
+				return errors.Errorf("%s has a filesystem already, use dm.directlvm_device_force=true if you want to wipe the device", dev)
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+func verifyBlockDevice(dev string, force bool) error {
+	if err := checkDevAvailable(dev); err != nil {
+		return err
+	}
+	if err := checkDevInVG(dev); err != nil {
+		return err
+	}
+
+	if force {
+		return nil
+	}
+
+	if err := checkDevHasFS(dev); err != nil {
+		return err
+	}
+	return nil
+}
+
+func readLVMConfig(root string) (directLVMConfig, error) {
+	var cfg directLVMConfig
+
+	p := filepath.Join(root, "setup-config.json")
+	b, err := ioutil.ReadFile(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return cfg, nil
+		}
+		return cfg, errors.Wrap(err, "error reading existing setup config")
+	}
+
+	// check if this is just an empty file, no need to produce a json error later if so
+	if len(b) == 0 {
+		return cfg, nil
+	}
+
+	err = json.Unmarshal(b, &cfg)
+	return cfg, errors.Wrap(err, "error unmarshaling previous device setup config")
+}
+
+func writeLVMConfig(root string, cfg directLVMConfig) error {
+	p := filepath.Join(root, "setup-config.json")
+	b, err := json.Marshal(cfg)
+	if err != nil {
+		return errors.Wrap(err, "error marshalling direct lvm config")
+	}
+	err = ioutil.WriteFile(p, b, 0600)
+	return errors.Wrap(err, "error writing direct lvm config to file")
+}
+
+func setupDirectLVM(cfg directLVMConfig) error {
+	lvmProfileDir := "/etc/lvm/profile"
+	binaries := []string{"pvcreate", "vgcreate", "lvcreate", "lvconvert", "lvchange", "thin_check"}
+
+	for _, bin := range binaries {
+		if _, err := exec.LookPath(bin); err != nil {
+			return errors.Wrap(err, "error looking up command `"+bin+"` while setting up direct lvm")
+		}
+	}
+
+	err := os.MkdirAll(lvmProfileDir, 0755)
+	if err != nil {
+		return errors.Wrap(err, "error creating lvm profile directory")
+	}
+
+	if cfg.AutoExtendPercent == 0 {
+		cfg.AutoExtendPercent = 20
+	}
+
+	if cfg.AutoExtendThreshold == 0 {
+		cfg.AutoExtendThreshold = 80
+	}
+
+	if cfg.ThinpPercent == 0 {
+		cfg.ThinpPercent = 95
+	}
+	if cfg.ThinpMetaPercent == 0 {
+		cfg.ThinpMetaPercent = 1
+	}
+
+	out, err := exec.Command("pvcreate", "-f", cfg.Device).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	out, err = exec.Command("vgcreate", "storage", cfg.Device).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpool", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpPercent)).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+	out, err = exec.Command("lvcreate", "--wipesignatures", "y", "-n", "thinpoolmeta", "storage", "--extents", fmt.Sprintf("%d%%VG", cfg.ThinpMetaPercent)).CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	out, err = exec.Command("lvconvert", "-y", "--zero", "n", "-c", "512K", "--thinpool", "storage/thinpool", "--poolmetadata", "storage/thinpoolmeta").CombinedOutput()
+	if err != nil {
+		return errors.Wrap(err, string(out))
+	}
+
+	profile := fmt.Sprintf("activation{\nthin_pool_autoextend_threshold=%d\nthin_pool_autoextend_percent=%d\n}", cfg.AutoExtendThreshold, cfg.AutoExtendPercent)
+	err = ioutil.WriteFile(lvmProfileDir+"/storage-thinpool.profile", []byte(profile), 0600)
+	if err != nil {
+		return errors.Wrap(err, "error writing storage thinp autoextend profile")
+	}
+
+	out, err = exec.Command("lvchange", "--metadataprofile", "storage-thinpool", "storage/thinpool").CombinedOutput()
+	return errors.Wrap(err, string(out))
+}

vendor/github.com/containers/storage/drivers/devmapper/devmapper_doc.go

@@ -0,0 +1,106 @@
+package devmapper
+
+// Definition of struct dm_task and sub structures (from lvm2)
+//
+// struct dm_ioctl {
+// 	/*
+// 	 * The version number is made up of three parts:
+// 	 * major - no backward or forward compatibility,
+// 	 * minor - only backwards compatible,
+// 	 * patch - both backwards and forwards compatible.
+// 	 *
+// 	 * All clients of the ioctl interface should fill in the
+// 	 * version number of the interface that they were
+// 	 * compiled with.
+// 	 *
+// 	 * All recognized ioctl commands (ie. those that don't
+// 	 * return -ENOTTY) fill out this field, even if the
+// 	 * command failed.
+// 	 */
+// 	uint32_t version[3];	/* in/out */
+// 	uint32_t data_size;	/* total size of data passed in
+// 				 * including this struct */
+
+// 	uint32_t data_start;	/* offset to start of data
+// 				 * relative to start of this struct */
+
+// 	uint32_t target_count;	/* in/out */
+// 	int32_t open_count;	/* out */
+// 	uint32_t flags;		/* in/out */
+
+// 	/*
+// 	 * event_nr holds either the event number (input and output) or the
+// 	 * udev cookie value (input only).
+// 	 * The DM_DEV_WAIT ioctl takes an event number as input.
+// 	 * The DM_SUSPEND, DM_DEV_REMOVE and DM_DEV_RENAME ioctls
+// 	 * use the field as a cookie to return in the DM_COOKIE
+// 	 * variable with the uevents they issue.
+// 	 * For output, the ioctls return the event number, not the cookie.
+// 	 */
+// 	uint32_t event_nr;      	/* in/out */
+// 	uint32_t padding;
+
+// 	uint64_t dev;		/* in/out */
+
+// 	char name[DM_NAME_LEN];	/* device name */
+// 	char uuid[DM_UUID_LEN];	/* unique identifier for
+// 				 * the block device */
+// 	char data[7];		/* padding or data */
+// };
+
+// struct target {
+// 	uint64_t start;
+// 	uint64_t length;
+// 	char *type;
+// 	char *params;
+
+// 	struct target *next;
+// };
+
+// typedef enum {
+// 	DM_ADD_NODE_ON_RESUME, /* add /dev/mapper node with dmsetup resume */
+// 	DM_ADD_NODE_ON_CREATE  /* add /dev/mapper node with dmsetup create */
+// } dm_add_node_t;
+
+// struct dm_task {
+// 	int type;
+// 	char *dev_name;
+// 	char *mangled_dev_name;
+
+// 	struct target *head, *tail;
+
+// 	int read_only;
+// 	uint32_t event_nr;
+// 	int major;
+// 	int minor;
+// 	int allow_default_major_fallback;
+// 	uid_t uid;
+// 	gid_t gid;
+// 	mode_t mode;
+// 	uint32_t read_ahead;
+// 	uint32_t read_ahead_flags;
+// 	union {
+// 		struct dm_ioctl *v4;
+// 	} dmi;
+// 	char *newname;
+// 	char *message;
+// 	char *geometry;
+// 	uint64_t sector;
+// 	int no_flush;
+// 	int no_open_count;
+// 	int skip_lockfs;
+// 	int query_inactive_table;
+// 	int suppress_identical_reload;
+// 	dm_add_node_t add_node;
+// 	uint64_t existing_table_size;
+// 	int cookie_set;
+// 	int new_uuid;
+// 	int secure_data;
+// 	int retry_remove;
+// 	int enable_checks;
+// 	int expected_errno;
+
+// 	char *uuid;
+// 	char *mangled_uuid;
+// };
+//

vendor/github.com/containers/storage/drivers/devmapper/driver.go

@@ -0,0 +1,240 @@
+// +build linux
+
+package devmapper
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"strconv"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/containers/storage/drivers"
+	"github.com/containers/storage/pkg/devicemapper"
+	"github.com/containers/storage/pkg/idtools"
+	"github.com/containers/storage/pkg/locker"
+	"github.com/containers/storage/pkg/mount"
+	"github.com/containers/storage/pkg/system"
+	units "github.com/docker/go-units"
+)
+
+func init() {
+	graphdriver.Register("devicemapper", Init)
+}
+
+// Driver contains the device set mounted and the home directory
+type Driver struct {
+	*DeviceSet
+	home    string
+	uidMaps []idtools.IDMap
+	gidMaps []idtools.IDMap
+	ctr     *graphdriver.RefCounter
+	locker  *locker.Locker
+}
+
+// Init creates a driver with the given home and the set of options.
+func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (graphdriver.Driver, error) {
+	deviceSet, err := NewDeviceSet(home, true, options, uidMaps, gidMaps)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := mount.MakePrivate(home); err != nil {
+		return nil, err
+	}
+
+	d := &Driver{
+		DeviceSet: deviceSet,
+		home:      home,
+		uidMaps:   uidMaps,
+		gidMaps:   gidMaps,
+		ctr:       graphdriver.NewRefCounter(graphdriver.NewDefaultChecker()),
+		locker:    locker.New(),
+	}
+
+	return graphdriver.NewNaiveDiffDriver(d, uidMaps, gidMaps), nil
+}
+
+func (d *Driver) String() string {
+	return "devicemapper"
+}
+
+// Status returns the status about the driver in a printable format.
+// Information returned contains Pool Name, Data File, Metadata file, disk usage by
+// the data and metadata, etc.
+func (d *Driver) Status() [][2]string {
+	s := d.DeviceSet.Status()
+
+	status := [][2]string{
+		{"Pool Name", s.PoolName},
+		{"Pool Blocksize", units.HumanSize(float64(s.SectorSize))},
+		{"Base Device Size", units.HumanSize(float64(s.BaseDeviceSize))},
+		{"Backing Filesystem", s.BaseDeviceFS},
+		{"Data file", s.DataFile},
+		{"Metadata file", s.MetadataFile},
+		{"Data Space Used", units.HumanSize(float64(s.Data.Used))},
+		{"Data Space Total", units.HumanSize(float64(s.Data.Total))},
+		{"Data Space Available", units.HumanSize(float64(s.Data.Available))},
+		{"Metadata Space Used", units.HumanSize(float64(s.Metadata.Used))},
+		{"Metadata Space Total", units.HumanSize(float64(s.Metadata.Total))},
+		{"Metadata Space Available", units.HumanSize(float64(s.Metadata.Available))},
+		{"Thin Pool Minimum Free Space", units.HumanSize(float64(s.MinFreeSpace))},
+		{"Udev Sync Supported", fmt.Sprintf("%v", s.UdevSyncSupported)},
+		{"Deferred Removal Enabled", fmt.Sprintf("%v", s.DeferredRemoveEnabled)},
+		{"Deferred Deletion Enabled", fmt.Sprintf("%v", s.DeferredDeleteEnabled)},
+		{"Deferred Deleted Device Count", fmt.Sprintf("%v", s.DeferredDeletedDeviceCount)},
+	}
+	if len(s.DataLoopback) > 0 {
+		status = append(status, [2]string{"Data loop file", s.DataLoopback})
+	}
+	if len(s.MetadataLoopback) > 0 {
+		status = append(status, [2]string{"Metadata loop file", s.MetadataLoopback})
+	}
+	if vStr, err := devicemapper.GetLibraryVersion(); err == nil {
+		status = append(status, [2]string{"Library Version", vStr})
+	}
+	return status
+}
+
+// Metadata returns a map of information about the device.
+func (d *Driver) Metadata(id string) (map[string]string, error) {
+	m, err := d.DeviceSet.exportDeviceMetadata(id)
+
+	if err != nil {
+		return nil, err
+	}
+
+	metadata := make(map[string]string)
+	metadata["DeviceId"] = strconv.Itoa(m.deviceID)
+	metadata["DeviceSize"] = strconv.FormatUint(m.deviceSize, 10)
+	metadata["DeviceName"] = m.deviceName
+	return metadata, nil
+}
+
+// Cleanup unmounts a device.
+func (d *Driver) Cleanup() error {
+	err := d.DeviceSet.Shutdown(d.home)
+
+	if err2 := mount.Unmount(d.home); err == nil {
+		err = err2
+	}
+
+	return err
+}
+
+// CreateReadWrite creates a layer that is writable for use as a container
+// file system.
+func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error {
+	return d.Create(id, parent, opts)
+}
+
+// Create adds a device with a given id and the parent.
+func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
+	var storageOpt map[string]string
+	if opts != nil {
+		storageOpt = opts.StorageOpt
+	}
+
+	if err := d.DeviceSet.AddDevice(id, parent, storageOpt); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// Remove removes a device with a given id, unmounts the filesystem.
+func (d *Driver) Remove(id string) error {
+	d.locker.Lock(id)
+	defer d.locker.Unlock(id)
+	if !d.DeviceSet.HasDevice(id) {
+		// Consider removing a non-existing device a no-op
+		// This is useful to be able to progress on container removal
+		// if the underlying device has gone away due to earlier errors
+		return nil
+	}
+
+	// This assumes the device has been properly Get/Put:ed and thus is unmounted
+	if err := d.DeviceSet.DeleteDevice(id, false); err != nil {
+		return fmt.Errorf("failed to remove device %s: %v", id, err)
+	}
+	return system.EnsureRemoveAll(path.Join(d.home, "mnt", id))
+}
+
+// Get mounts a device with given id into the root filesystem
+func (d *Driver) Get(id, mountLabel string) (string, error) {
+	d.locker.Lock(id)
+	defer d.locker.Unlock(id)
+	mp := path.Join(d.home, "mnt", id)
+	rootFs := path.Join(mp, "rootfs")
+	if count := d.ctr.Increment(mp); count > 1 {
+		return rootFs, nil
+	}
+
+	uid, gid, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
+	if err != nil {
+		d.ctr.Decrement(mp)
+		return "", err
+	}
+
+	// Create the target directories if they don't exist
+	if err := idtools.MkdirAllAs(path.Join(d.home, "mnt"), 0755, uid, gid); err != nil && !os.IsExist(err) {
+		d.ctr.Decrement(mp)
+		return "", err
+	}
+	if err := idtools.MkdirAs(mp, 0755, uid, gid); err != nil && !os.IsExist(err) {
+		d.ctr.Decrement(mp)
+		return "", err
+	}
+
+	// Mount the device
+	if err := d.DeviceSet.MountDevice(id, mp, mountLabel); err != nil {
+		d.ctr.Decrement(mp)
+		return "", err
+	}
+
+	if err := idtools.MkdirAllAs(rootFs, 0755, uid, gid); err != nil && !os.IsExist(err) {
+		d.ctr.Decrement(mp)
+		d.DeviceSet.UnmountDevice(id, mp)
+		return "", err
+	}
+
+	idFile := path.Join(mp, "id")
+	if _, err := os.Stat(idFile); err != nil && os.IsNotExist(err) {
+		// Create an "id" file with the container/image id in it to help reconstruct this in case
+		// of later problems
+		if err := ioutil.WriteFile(idFile, []byte(id), 0600); err != nil {
+			d.ctr.Decrement(mp)
+			d.DeviceSet.UnmountDevice(id, mp)
+			return "", err
+		}
+	}
+
+	return rootFs, nil
+}
+
+// Put unmounts a device and removes it.
+func (d *Driver) Put(id string) error {
+	d.locker.Lock(id)
+	defer d.locker.Unlock(id)
+	mp := path.Join(d.home, "mnt", id)
+	if count := d.ctr.Decrement(mp); count > 0 {
+		return nil
+	}
+	err := d.DeviceSet.UnmountDevice(id, mp)
+	if err != nil {
+		logrus.Errorf("devmapper: Error unmounting device %s: %s", id, err)
+	}
+	return err
+}
+
+// Exists checks to see if the device exists.
+func (d *Driver) Exists(id string) bool {
+	return d.DeviceSet.HasDevice(id)
+}
+
+// AdditionalImageStores returns additional image stores supported by the driver
+func (d *Driver) AdditionalImageStores() []string {
+	return nil
+}

vendor/github.com/containers/storage/drivers/devmapper/mount.go

@@ -0,0 +1,88 @@
+// +build linux
+
+package devmapper
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sys/unix"
+)
+
+// FIXME: this is copy-pasted from the aufs driver.
+// It should be moved into the core.
+
+// Mounted returns true if a mount point exists.
+func Mounted(mountpoint string) (bool, error) {
+	var mntpointSt unix.Stat_t
+	if err := unix.Stat(mountpoint, &mntpointSt); err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	var parentSt unix.Stat_t
+	if err := unix.Stat(filepath.Join(mountpoint, ".."), &parentSt); err != nil {
+		return false, err
+	}
+	return mntpointSt.Dev != parentSt.Dev, nil
+}
+
+type probeData struct {
+	fsName string
+	magic  string
+	offset uint64
+}
+
+// ProbeFsType returns the filesystem name for the given device id.
+func ProbeFsType(device string) (string, error) {
+	probes := []probeData{
+		{"btrfs", "_BHRfS_M", 0x10040},
+		{"ext4", "\123\357", 0x438},
+		{"xfs", "XFSB", 0},
+	}
+
+	maxLen := uint64(0)
+	for _, p := range probes {
+		l := p.offset + uint64(len(p.magic))
+		if l > maxLen {
+			maxLen = l
+		}
+	}
+
+	file, err := os.Open(device)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	buffer := make([]byte, maxLen)
+	l, err := file.Read(buffer)
+	if err != nil {
+		return "", err
+	}
+
+	if uint64(l) != maxLen {
+		return "", fmt.Errorf("devmapper: unable to detect filesystem type of %s, short read", device)
+	}
+
+	for _, p := range probes {
+		if bytes.Equal([]byte(p.magic), buffer[p.offset:p.offset+uint64(len(p.magic))]) {
+			return p.fsName, nil
+		}
+	}
+
+	return "", fmt.Errorf("devmapper: Unknown filesystem type on %s", device)
+}
+
+func joinMountOptions(a, b string) string {
+	if a == "" {
+		return b
+	}
+	if b == "" {
+		return a
+	}
+	return a + "," + b
+}