Merge pull request #130 from democratic-csi/next

Next

CHANGELOG.md

@@ -1,3 +1,15 @@
+# v1.4.3
+
+Released 2021-12-01
+
+- more appropriate handling of `size_bytes` for snapshots
+- more robust handling of `NodePublishVolume` to ensure the staging path is
+  actually mounted
+- allow control of the `mount` / `umount` / `findmnt` command timeout via
+  `MOUNT_DEFAULT_TIMEOUT` env var
+- minor fix for `zfs-generic-iscsi` with `targetCli` to work-around Ubuntu
+  18:04 bug (see #127)
+
 # v1.4.2
 
 Released 2021-09-29

docs/Nomad/README.md

@@ -0,0 +1,14 @@
+# setup
+
+```
+cat <<EOF > /etc/nomad.d/csi.hcl
+plugin "docker" {
+  config {
+    allow_privileged = true
+    volumes {
+      # required for bind mounting host directories
+      enabled = true
+    }
+  }
+}
+```

docs/Nomad/examples/democratic-csi-iscsi-controller.hcl

@@ -0,0 +1,43 @@
+job "democratic-csi-iscsi-controller" {
+  datacenters = ["dc1"]
+
+  group "controller" {
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = "docker.io/democraticcsi/democratic-csi:latest"
+
+        args = [
+          "--csi-version=1.5.0",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.iscsi",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=info",
+          "--csi-mode=controller",
+          "--server-socket=/csi/csi.sock",
+        ]
+      }
+
+      template {
+        destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+        data = <<EOH
+config
+EOH
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.iscsi"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+}

docs/Nomad/examples/democratic-csi-iscsi-node.hcl

@@ -0,0 +1,64 @@
+job "democratic-csi-iscsi-node" {
+  datacenters = ["dc1"]
+
+  # you can run node plugins as service jobs as well, but this ensures
+  # that all nodes in the DC have a copy.
+  type = "system"
+
+  group "nodes" {
+    task "plugin" {
+      driver = "docker"
+
+      env {
+        CSI_NODE_ID = "${attr.unique.hostname}"
+      }
+
+      config {
+        image = "docker.io/democraticcsi/democratic-csi:latest"
+
+        args = [
+          "--csi-version=1.5.0",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.iscsi",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=info",
+          "--csi-mode=node",
+          "--server-socket=/csi/csi.sock",
+        ]
+
+        # node plugins must run as privileged jobs because they
+        # mount disks to the host
+        privileged = true
+        ipc_mode = "host"
+        network_mode = "host"
+
+        mount {
+          type = "bind"
+          target = "/host"
+          source = "/"
+          readonly=false
+        }
+      }
+
+      template {
+        destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+        data = <<EOH
+config
+EOH
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.iscsi"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+}

docs/Nomad/examples/democratic-csi-nfs-controller.hcl

@@ -0,0 +1,43 @@
+job "democratic-csi-nfs-controller" {
+  datacenters = ["dc1"]
+
+  group "controller" {
+    task "plugin" {
+      driver = "docker"
+
+      config {
+        image = "docker.io/democraticcsi/democratic-csi:latest"
+
+        args = [
+          "--csi-version=1.5.0",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.nfs",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=info",
+          "--csi-mode=controller",
+          "--server-socket=/csi/csi.sock",
+        ]
+      }
+
+      template {
+        destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+        data = <<EOH
+config
+EOH
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.nfs"
+        type      = "controller"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+}

docs/Nomad/examples/democratic-csi-nfs-node.hcl

@@ -0,0 +1,57 @@
+job "democratic-csi-nfs-node" {
+  datacenters = ["dc1"]
+
+  # you can run node plugins as service jobs as well, but this ensures
+  # that all nodes in the DC have a copy.
+  type = "system"
+
+  group "nodes" {
+    task "plugin" {
+      driver = "docker"
+
+      env {
+        CSI_NODE_ID = "${attr.unique.hostname}"
+      }
+
+      config {
+        image = "docker.io/democraticcsi/democratic-csi:latest"
+
+        args = [
+          "--csi-version=1.5.0",
+          # must match the csi_plugin.id attribute below
+          "--csi-name=org.democratic-csi.nfs",
+          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
+          "--log-level=info",
+          "--csi-mode=node",
+          "--server-socket=/csi/csi.sock",
+        ]
+
+        # node plugins must run as privileged jobs because they
+        # mount disks to the host
+        privileged = true
+        ipc_mode = "host"
+        network_mode = "host"
+      }
+
+      template {
+        destination = "${NOMAD_TASK_DIR}/driver-config-file.yaml"
+
+        data = <<EOH
+config
+EOH
+      }
+
+      csi_plugin {
+        # must match --csi-name arg
+        id        = "org.democratic-csi.nfs"
+        type      = "node"
+        mount_dir = "/csi"
+      }
+
+      resources {
+        cpu    = 500
+        memory = 256
+      }
+    }
+  }
+}

docs/Nomad/examples/job-mysql.hcl

@@ -0,0 +1,66 @@
+job "mysql-server" {
+  datacenters = ["dc1"]
+  type        = "service"
+
+  group "mysql-server" {
+    count = 1
+
+    volume "mysql" {
+      type      = "csi"
+      read_only = false
+      
+      # iscsi
+      source    = "csi-volume-iscsi"
+      access_mode = "single-node-writer"
+      attachment_mode = "file-system"
+
+      # nfs
+      #source    = "csi-volume-nfs"
+      #access_mode = "multi-node-multi-writer"
+      #attachment_mode = "file-system"
+    }
+
+    network {
+      port "db" {
+        static = 3306
+      }
+    }
+
+    restart {
+      attempts = 10
+      interval = "5m"
+      delay    = "25s"
+      mode     = "delay"
+    }
+
+    task "mysql-server" {
+      driver = "docker"
+
+      volume_mount {
+        volume      = "mysql"
+        destination = "/srv"
+        read_only   = false
+      }
+
+      env {
+        MYSQL_ROOT_PASSWORD = "password"
+      }
+
+      config {
+        image = "hashicorp/mysql-portworx-demo:latest"
+        args  = ["--datadir", "/srv/mysql"]
+        ports = ["db"]
+      }
+
+      resources {
+        cpu    = 500
+        memory = 1024
+      }
+
+      service {
+        name = "mysql-server"
+        port = "db"
+      }
+    }
+  }
+}

docs/Nomad/examples/volume-iscsi.hcl

@@ -0,0 +1,11 @@
+type = "csi"
+id = "csi-volume-iscsi"
+name = "csi-volume-iscsi"
+plugin_id = "org.democratic-csi.iscsi"
+capacity_min = "1GiB"
+capacity_max = "1GiB"
+
+capability {
+  access_mode     = "single-node-writer"
+  attachment_mode = "file-system"
+}

docs/Nomad/examples/volume-nfs.hcl

@@ -0,0 +1,11 @@
+type = "csi"
+id = "csi-volume-nfs"
+name = "csi-volume-nfs"
+plugin_id = "org.democratic-csi.nfs"
+capacity_min = "1GiB"
+capacity_max = "1GiB"
+
+capability {
+  access_mode     = "multi-node-multi-writer"
+  attachment_mode = "file-system"
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "democratic-csi",
-  "version": "1.4.2",
+  "version": "1.4.3",
   "description": "kubernetes csi driver framework",
   "main": "bin/democratic-csi",
   "scripts": {
@@ -20,7 +20,7 @@
   "dependencies": {
     "@grpc/grpc-js": "^1.3.6",
     "@grpc/proto-loader": "^0.6.0",
-    "@kubernetes/client-node": "^0.15.1",
+    "@kubernetes/client-node": "^0.16.1",
     "async-mutex": "^0.3.1",
     "bunyan": "^1.8.15",
     "grpc-uds": "^0.1.6",
@@ -38,6 +38,6 @@
     "yargs": "^17.0.1"
   },
   "devDependencies": {
-    "eslint": "^7.22.0"
+    "eslint": "^8.1.0"
   }
 }

src/driver/controller-client-common/index.js

@@ -228,6 +228,12 @@ class ControllerClientCommonDriver extends CsiBaseDriver {
     return this.getControllerSnapshotBasePath() + "/" + snapshot_id;
   }
 
+  async getDirectoryUsage(path) {
+    let result = await this.exec("du", ["-s", "--block-size=1", path]);
+    let size = result.stdout.split("\t", 1)[0];
+    return size;
+  }
+
   exec(command, args, options = {}) {
     args = args || [];
 
@@ -664,13 +670,14 @@ class ControllerClientCommonDriver extends CsiBaseDriver {
       await driver.cloneDir(volume_path, snapshot_path);
     }
 
+    let size_bytes = await driver.getDirectoryUsage(snapshot_path);
     return {
       snapshot: {
         /**
          * The purpose of this field is to give CO guidance on how much space
          * is needed to create a volume from this snapshot.
          */
-        size_bytes: 0,
+        size_bytes,
         snapshot_id,
         source_volume_id: source_volume_id,
         //https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto

src/driver/controller-synology/http/index.js

@@ -20,9 +20,10 @@ class SynologyHttpClient {
 
   async login() {
     if (!this.sid) {
+      // See https://global.download.synology.com/download/Document/Software/DeveloperGuide/Os/DSM/All/enu/DSM_Login_Web_API_Guide_enu.pdf
       const data = {
         api: "SYNO.API.Auth",
-        version: "2",
+        version: "6",
         method: "login",
         account: this.options.username,
         passwd: this.options.password,

src/driver/controller-synology/index.js

@@ -830,7 +830,7 @@ class ControllerSynologyDriver extends CsiBaseDriver {
         `invalid source_volume_id: ${source_volume_id}`
       );
     }
-    
+
     // check for other snapshopts with the same name on other volumes and fail as appropriate
     // TODO: technically this should only be checking lun/snapshots relevant to this specific install of the driver
     // but alas an isolation/namespacing mechanism does not exist in synology
@@ -845,47 +845,35 @@ class ControllerSynologyDriver extends CsiBaseDriver {
     }
 
     // check for already exists
-    let snapshot = await httpClient.GetSnapshotByLunIDAndName(lun.lun_id, name);
+    let snapshot;
-    if (snapshot) {
+    snapshot = await httpClient.GetSnapshotByLunIDAndName(lun.lun_id, name);
-      return {
+    if (!snapshot) {
-        snapshot: {
+      let data = Object.assign({}, driver.options.iscsi.lunSnapshotTemplate, {
-          /**
+        src_lun_uuid: lun.uuid,
-           * The purpose of this field is to give CO guidance on how much space
+        taken_by: "democratic-csi",
-           * is needed to create a volume from this snapshot.
+        description: name, //check
-           */
+      });
-          size_bytes: 0,
+
-          snapshot_id: `/lun/${lun.lun_id}/${snapshot.uuid}`, // add shanpshot_uuid //fixme
+      await httpClient.CreateSnapshot(data);
-          source_volume_id: source_volume_id,
+      snapshot = await httpClient.GetSnapshotByLunIDAndName(lun.lun_id, name);
-          //https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto
+
-          creation_time: {
+      if (!snapshot) {
-            seconds: snapshot.time,
+        throw new Error(`failed to create snapshot`);
-            nanos: 0,
+      }
-          },
-          ready_to_use: true,
-        },
-      };
     }
 
-    let data = Object.assign({}, driver.options.iscsi.lunSnapshotTemplate, {
-      src_lun_uuid: lun.uuid,
-      taken_by: "democratic-csi",
-      description: name, //check
-    });
-
-    let response = await httpClient.CreateSnapshot(data);
-
     return {
       snapshot: {
         /**
          * The purpose of this field is to give CO guidance on how much space
          * is needed to create a volume from this snapshot.
          */
-        size_bytes: 0,
+        size_bytes: snapshot.total_size,
-        snapshot_id: `/lun/${lun.lun_id}/${response.body.data.snapshot_uuid}`,
+        snapshot_id: `/lun/${lun.lun_id}/${snapshot.uuid}`, // add shanpshot_uuid //fixme
         source_volume_id: source_volume_id,
         //https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto
         creation_time: {
-          seconds: Math.round(new Date().getTime() / 1000),
+          seconds: snapshot.time,
           nanos: 0,
         },
         ready_to_use: true,

src/driver/controller-zfs-generic/index.js

@@ -353,7 +353,21 @@ delete ${iscsiName}
 
     driver.ctx.logger.verbose("TargetCLI command: " + logCommand);
 
-    let response = await sshClient.exec(sshClient.buildCommand(command, args));
+    // https://github.com/democratic-csi/democratic-csi/issues/127
+    // https://bugs.launchpad.net/ubuntu/+source/python-configshell-fb/+bug/1776761
+    // can apply the linked patch with some modifications to overcome the
+    // KeyErrors or we can simply start a fake tty which does not seem to have
+    // a detrimental effect, only affects Ubuntu 18.04 and older
+    let options = {
+      pty: true,
+    };
+    let response = await sshClient.exec(
+      sshClient.buildCommand(command, args),
+      options
+    );
+    if (response.code != 0) {
+      throw new Error(response.stderr);
+    }
     driver.ctx.logger.verbose(
       "TargetCLI response: " + JSON.stringify(response)
     );

src/driver/controller-zfs-ssh/index.js

@@ -2,6 +2,7 @@ const { CsiBaseDriver } = require("../index");
 const SshClient = require("../../utils/ssh").SshClient;
 const { GrpcError, grpc } = require("../../utils/grpc");
 const sleep = require("../../utils/general").sleep;
+const getLargestNumber = require("../../utils/general").getLargestNumber;
 
 const { Zetabyte, ZfsSshProcessManager } = require("../../utils/zfs");
 
@@ -1248,7 +1249,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
       let sleep_time = 3000;
       let current_try = 1;
       let success = false;
-      while(!success && current_try <= max_tries) {
+      while (!success && current_try <= max_tries) {
         try {
           await zb.zfs.destroy(datasetName, { recurse: true, force: true });
           success = true;
@@ -1374,9 +1375,10 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
         properties.volsize = capacity_bytes;
         setProps = true;
 
-        if (this.options.zfs.zvolEnableReservation) {
+        // managed automatically for zvols
-          properties.refreservation = capacity_bytes;
+        //if (this.options.zfs.zvolEnableReservation) {
-        }
+        //  properties.refreservation = capacity_bytes;
+        //}
         break;
     }
 
@@ -1800,6 +1802,9 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
             "refquota",
             "avail",
             "used",
+            "volsize",
+            "referenced",
+            "logicalreferenced",
             VOLUME_CSI_NAME_PROPERTY_NAME,
             SNAPSHOT_CSI_NAME_PROPERTY_NAME,
             MANAGED_PROPERTY_NAME,
@@ -1864,6 +1869,20 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
           return;
         }
 
+        // TODO: properly handle use-case where datasetEnableQuotas is not turned on
+        let size_bytes = 0;
+        if (driverZfsResourceType == "filesystem") {
+          // independent of detached snapshots when creating a volume from a 'snapshot'
+          // we could be using detached clones (ie: send/receive)
+          // so we must be cognizant and use the highest possible value here
+          // note that whatever value is returned here can/will essentially impact the refquota
+          // value of a derived volume
+          size_bytes = getLargestNumber(row.referenced, row.logicalreferenced);
+        } else {
+          // get the size of the parent volume
+          size_bytes = row.volsize;
+        }
+
         if (source_volume_id)
           entries.push({
             snapshot: {
@@ -1874,7 +1893,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
                * In that vein, I think it's best to return 0 here given the
                * unknowns of 'cow' implications.
                */
-              size_bytes: 0,
+              size_bytes,
 
               // remove parent dataset details
               snapshot_id: row["name"].replace(
@@ -1920,6 +1939,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
     const driverZfsResourceType = this.getDriverZfsResourceType();
     const zb = await this.getZetabyte();
 
+    let size_bytes = 0;
     let detachedSnapshot = false;
     try {
       let tmpDetachedSnapshot = JSON.parse(
@@ -1976,6 +1996,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
       );
     }
 
+    const volumeDatasetName = volumeParentDatasetName + "/" + source_volume_id;
     const datasetName = datasetParentName + "/" + source_volume_id;
     snapshotProperties[SNAPSHOT_CSI_NAME_PROPERTY_NAME] = name;
     snapshotProperties[SNAPSHOT_CSI_SOURCE_VOLUME_ID_PROPERTY_NAME] =
@@ -2062,12 +2083,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
 
     if (detachedSnapshot) {
       tmpSnapshotName =
-        volumeParentDatasetName +
+        volumeDatasetName + "@" + VOLUME_SOURCE_DETACHED_SNAPSHOT_PREFIX + name;
-        "/" +
-        source_volume_id +
-        "@" +
-        VOLUME_SOURCE_DETACHED_SNAPSHOT_PREFIX +
-        name;
       snapshotDatasetName = datasetName + "/" + name;
 
       await zb.zfs.create(datasetName, { parents: true });
@@ -2123,11 +2139,17 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
         force: true,
         defer: true,
       });
+
+      // let things settle down
+      //await sleep(3000);
     } else {
       try {
         await zb.zfs.snapshot(fullSnapshotName, {
           properties: snapshotProperties,
         });
+
+        // let things settle down
+        //await sleep(3000);
       } catch (err) {
         if (err.toString().includes("dataset does not exist")) {
           throw new GrpcError(
@@ -2140,6 +2162,8 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
       }
     }
 
+    // TODO: let things settle to ensure proper size_bytes is reported
+    // sysctl -d vfs.zfs.txg.timeout  # vfs.zfs.txg.timeout: Max seconds worth of delta per txg
     let properties;
     properties = await zb.zfs.get(
       fullSnapshotName,
@@ -2150,6 +2174,11 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
         "refquota",
         "avail",
         "used",
+        "volsize",
+        "referenced",
+        "refreservation",
+        "logicalused",
+        "logicalreferenced",
         VOLUME_CSI_NAME_PROPERTY_NAME,
         SNAPSHOT_CSI_NAME_PROPERTY_NAME,
         SNAPSHOT_CSI_SOURCE_VOLUME_ID_PROPERTY_NAME,
@@ -2160,6 +2189,22 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
     properties = properties[fullSnapshotName];
     driver.ctx.logger.verbose("snapshot properties: %j", properties);
 
+    // TODO: properly handle use-case where datasetEnableQuotas is not turned on
+    if (driverZfsResourceType == "filesystem") {
+      // independent of detached snapshots when creating a volume from a 'snapshot'
+      // we could be using detached clones (ie: send/receive)
+      // so we must be cognizant and use the highest possible value here
+      // note that whatever value is returned here can/will essentially impact the refquota
+      // value of a derived volume
+      size_bytes = getLargestNumber(
+        properties.referenced.value,
+        properties.logicalreferenced.value
+      );
+    } else {
+      // get the size of the parent volume
+      size_bytes = properties.volsize.value;
+    }
+
     // set this just before sending out response so we know if volume completed
     // this should give us a relatively sane way to clean up artifacts over time
     await zb.zfs.set(fullSnapshotName, { [SUCCESS_PROPERTY_NAME]: "true" });
@@ -2173,7 +2218,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver {
          * In that vein, I think it's best to return 0 here given the
          * unknowns of 'cow' implications.
          */
-        size_bytes: 0,
+        size_bytes,
 
         // remove parent dataset details
         snapshot_id: properties.name.value.replace(

src/driver/freenas/api.js

@@ -3,6 +3,7 @@ const { CsiBaseDriver } = require("../index");
 const HttpClient = require("./http").Client;
 const TrueNASApiClient = require("./http/api").Api;
 const { Zetabyte } = require("../../utils/zfs");
+const getLargestNumber = require("../../utils/general").getLargestNumber;
 const sleep = require("../../utils/general").sleep;
 
 const Handlebars = require("handlebars");
@@ -1843,6 +1844,14 @@ class FreeNASApiDriver extends CsiBaseDriver {
     return client;
   }
 
+  async getMinimumVolumeSize() {
+    const driverZfsResourceType = this.getDriverZfsResourceType();
+    switch (driverZfsResourceType) {
+      case "filesystem":
+        return 1073741824;
+    }
+  }
+
   async getTrueNASHttpApiClient() {
     const driver = this;
     const httpClient = await this.getHttpClient();
@@ -2003,6 +2012,8 @@ class FreeNASApiDriver extends CsiBaseDriver {
     let zvolBlocksize = this.options.zfs.zvolBlocksize || "16K";
     let name = call.request.name;
     let volume_content_source = call.request.volume_content_source;
+    let minimum_volume_size = await driver.getMinimumVolumeSize();
+    let default_required_bytes = 1073741824;
 
     if (!datasetParentName) {
       throw new GrpcError(
@@ -2038,7 +2049,7 @@ class FreeNASApiDriver extends CsiBaseDriver {
       Object.keys(call.request.capacity_range).length === 0
     ) {
       call.request.capacity_range = {
-        required_bytes: 1073741824,
+        required_bytes: default_required_bytes,
       };
     }
 
@@ -2079,6 +2090,14 @@ class FreeNASApiDriver extends CsiBaseDriver {
       );
     }
 
+    // ensure *actual* capacity is not too small
+    if (minimum_volume_size > 0 && capacity_bytes < minimum_volume_size) {
+      throw new GrpcError(
+        grpc.status.OUT_OF_RANGE,
+        `volume capacity is smaller than the minimum: ${minimum_volume_size}`
+      );
+    }
+
     // ensure *actual* capacity is not greater than limit
     if (
       call.request.capacity_range.limit_bytes &&
@@ -2100,10 +2119,10 @@ class FreeNASApiDriver extends CsiBaseDriver {
       let size;
       switch (driverZfsResourceType) {
         case "volume":
-          size = properties["volsize"].value;
+          size = properties["volsize"].rawvalue;
           break;
         case "filesystem":
-          size = properties["refquota"].value;
+          size = properties["refquota"].rawvalue;
           break;
         default:
           throw new Error(
@@ -2197,10 +2216,13 @@ class FreeNASApiDriver extends CsiBaseDriver {
 
     // zvol enables reservation by default
     // this implements 'sparse' zvols
+    let sparse;
     if (driverZfsResourceType == "volume") {
+      // this is managed by the `sparse` option in the api
       if (!this.options.zfs.zvolEnableReservation) {
         volumeProperties.refreservation = 0;
       }
+      sparse = Boolean(!this.options.zfs.zvolEnableReservation);
     }
 
     let detachedClone = false;
@@ -2539,6 +2561,7 @@ class FreeNASApiDriver extends CsiBaseDriver {
         ...httpApiClient.getSystemProperties(volumeProperties),
         type: driverZfsResourceType.toUpperCase(),
         volsize: driverZfsResourceType == "volume" ? capacity_bytes : undefined,
+        sparse: driverZfsResourceType == "volume" ? sparse : undefined,
         create_ancestors: true,
         user_properties: httpApiClient.getPropertiesKeyValueArray(
           httpApiClient.getUserProperties(volumeProperties)
@@ -2913,9 +2936,10 @@ class FreeNASApiDriver extends CsiBaseDriver {
         properties.volsize = capacity_bytes;
         setProps = true;
 
-        if (this.options.zfs.zvolEnableReservation) {
+        // managed automatically for zvols
-          properties.refreservation = capacity_bytes;
+        //if (this.options.zfs.zvolEnableReservation) {
-        }
+        //  properties.refreservation = capacity_bytes;
+        //}
         break;
     }
 
@@ -2970,8 +2994,17 @@ class FreeNASApiDriver extends CsiBaseDriver {
 
     let properties;
     properties = await httpApiClient.DatasetGet(datasetName, ["available"]);
+    let minimum_volume_size = await driver.getMinimumVolumeSize();
 
-    return { available_capacity: Number(properties.available.rawvalue) };
+    return {
+      available_capacity: Number(properties.available.rawvalue),
+      minimum_volume_size:
+        minimum_volume_size > 0
+          ? {
+              value: Number(minimum_volume_size),
+            }
+          : undefined,
+    };
   }
 
   /**
@@ -3326,6 +3359,9 @@ class FreeNASApiDriver extends CsiBaseDriver {
           "refquota",
           "available",
           "used",
+          "volsize",
+          "referenced",
+          "logicalreferenced",
           VOLUME_CSI_NAME_PROPERTY_NAME,
           SNAPSHOT_CSI_NAME_PROPERTY_NAME,
           MANAGED_PROPERTY_NAME,
@@ -3546,6 +3582,20 @@ class FreeNASApiDriver extends CsiBaseDriver {
           return;
         }
 
+        // TODO: properly handle use-case where datasetEnableQuotas is not turned on
+        let size_bytes = 0;
+        if (driverZfsResourceType == "filesystem") {
+          // independent of detached snapshots when creating a volume from a 'snapshot'
+          // we could be using detached clones (ie: send/receive)
+          // so we must be cognizant and use the highest possible value here
+          // note that whatever value is returned here can/will essentially impact the refquota
+          // value of a derived volume
+          size_bytes = getLargestNumber(row.referenced, row.logicalreferenced);
+        } else {
+          // get the size of the parent volume
+          size_bytes = row.volsize;
+        }
+
         if (source_volume_id)
           entries.push({
             snapshot: {
@@ -3556,7 +3606,7 @@ class FreeNASApiDriver extends CsiBaseDriver {
                * In that vein, I think it's best to return 0 here given the
                * unknowns of 'cow' implications.
                */
-              size_bytes: 0,
+              size_bytes,
 
               // remove parent dataset details
               snapshot_id: row["name"].replace(
@@ -3606,6 +3656,7 @@ class FreeNASApiDriver extends CsiBaseDriver {
     const httpApiClient = await this.getTrueNASHttpApiClient();
     const zb = await this.getZetabyte();
 
+    let size_bytes = 0;
     let detachedSnapshot = false;
     try {
       let tmpDetachedSnapshot = JSON.parse(
@@ -3904,12 +3955,19 @@ class FreeNASApiDriver extends CsiBaseDriver {
       "refquota",
       "available",
       "used",
+      "volsize",
+      "referenced",
+      "refreservation",
+      "logicalused",
+      "logicalreferenced",
       VOLUME_CSI_NAME_PROPERTY_NAME,
       SNAPSHOT_CSI_NAME_PROPERTY_NAME,
       SNAPSHOT_CSI_SOURCE_VOLUME_ID_PROPERTY_NAME,
       MANAGED_PROPERTY_NAME,
     ];
 
+    // TODO: let things settle to ensure proper size_bytes is reported
+    // sysctl -d vfs.zfs.txg.timeout  # vfs.zfs.txg.timeout: Max seconds worth of delta per txg
     if (detachedSnapshot) {
       properties = await httpApiClient.DatasetGet(
         fullSnapshotName,
@@ -3924,6 +3982,22 @@ class FreeNASApiDriver extends CsiBaseDriver {
 
     driver.ctx.logger.verbose("snapshot properties: %j", properties);
 
+    // TODO: properly handle use-case where datasetEnableQuotas is not turned on
+    if (driverZfsResourceType == "filesystem") {
+      // independent of detached snapshots when creating a volume from a 'snapshot'
+      // we could be using detached clones (ie: send/receive)
+      // so we must be cognizant and use the highest possible value here
+      // note that whatever value is returned here can/will essentially impact the refquota
+      // value of a derived volume
+      size_bytes = getLargestNumber(
+        properties.referenced.rawvalue,
+        properties.logicalreferenced.rawvalue
+      );
+    } else {
+      // get the size of the parent volume
+      size_bytes = properties.volsize.rawvalue;
+    }
+
     // set this just before sending out response so we know if volume completed
     // this should give us a relatively sane way to clean up artifacts over time
     //await zb.zfs.set(fullSnapshotName, { [SUCCESS_PROPERTY_NAME]: "true" });
@@ -3946,7 +4020,7 @@ class FreeNASApiDriver extends CsiBaseDriver {
          * In that vein, I think it's best to return 0 here given the
          * unknowns of 'cow' implications.
          */
-        size_bytes: 0,
+        size_bytes,
 
         // remove parent dataset details
         snapshot_id: properties.name.value.replace(

src/driver/freenas/http/index.js

@@ -14,6 +14,20 @@ class Client {
   }
   getBaseURL() {
     const server = this.options;
+    if (!server.protocol) {
+      if (server.port) {
+        if (String(server.port).includes("80")) {
+          server.protocol = "http";
+        }
+        if (String(server.port).includes("443")) {
+          server.protocol = "https";
+        }
+      }
+    }
+    if (!server.protocol) {
+      server.protocol = "http";
+    }
+
     const options = {
       scheme: server.protocol,
       host: server.host,

src/driver/index.js

@@ -1054,6 +1054,15 @@ class CsiBaseDriver {
             normalized_staging_path = staging_target_path;
           }
 
+          // sanity check to ensure the staged path is actually mounted
+          result = await mount.pathIsMounted(normalized_staging_path);
+          if (!result) {
+            throw new GrpcError(
+              grpc.status.FAILED_PRECONDITION,
+              `staging path is not mounted: ${normalized_staging_path}`
+            );
+          }
+
           result = await mount.pathIsMounted(target_path);
           // if not mounted, mount
           if (!result) {

src/utils/general.js

@@ -34,5 +34,23 @@ function lockKeysFromRequest(call, serviceMethodName) {
   }
 }
 
+function getLargestNumber() {
+  let number;
+  for (let i = 0; i < arguments.length; i++) {
+    value = Number(arguments[i]);
+    if (isNaN(value)) {
+      continue;
+    }
+    if (isNaN(number)) {
+      number = value;
+      continue;
+    }
+    number = value > number ? value : number;
+  }
+
+  return number;
+}
+
 module.exports.sleep = sleep;
 module.exports.lockKeysFromRequest = lockKeysFromRequest;
+module.exports.getLargestNumber = getLargestNumber;

src/utils/mount.js

@@ -10,7 +10,7 @@ FINDMNT_COMMON_OPTIONS = [
   "--nofsroot", // prevents unwanted behavior with cifs volumes
 ];
 
-DEFAUT_TIMEOUT = 30000;
+DEFAUT_TIMEOUT = process.env.MOUNT_DEFAULT_TIMEOUT || 30000;
 
 class Mount {
   constructor(options = {}) {

src/utils/ssh.js

@@ -33,25 +33,28 @@ class SshClient {
       var conn = new Client();
 
       if (client.options.connection.debug == true) {
-        client.options.connection.debug = function(msg) {
+        client.options.connection.debug = function (msg) {
           client.debug(msg);
         };
       }
-      
 
       conn
-        .on("error", function(err) {
+        .on("error", function (err) {
           client.debug("Client :: error");
           reject(err);
         })
-        .on("ready", function() {
+        .on("ready", function () {
           client.debug("Client :: ready");
-          conn.exec(command, options, function(err, stream) {
+          //options.pty = true;
+          //options.env = {
+          //  TERM: "",
+          //};
+          conn.exec(command, options, function (err, stream) {
             if (err) reject(err);
             let stderr;
             let stdout;
             stream
-              .on("close", function(code, signal) {
+              .on("close", function (code, signal) {
                 client.debug(
                   "Stream :: close :: code: " + code + ", signal: " + signal
                 );
@@ -61,7 +64,7 @@ class SshClient {
                 resolve({ stderr, stdout, code, signal });
                 conn.end();
               })
-              .on("data", function(data) {
+              .on("data", function (data) {
                 client.debug("STDOUT: " + data);
                 if (stream_proxy) {
                   stream_proxy.stdout.emit("data", ...arguments);
@@ -71,7 +74,7 @@ class SshClient {
                 }
                 stdout = stdout.concat(data);
               })
-              .stderr.on("data", function(data) {
+              .stderr.on("data", function (data) {
                 client.debug("STDERR: " + data);
                 if (stream_proxy) {
                   stream_proxy.stderr.emit("data", ...arguments);
@@ -86,7 +89,7 @@ class SshClient {
         .connect(client.options.connection);
 
       if (stream_proxy) {
-        stream_proxy.on("kill", signal => {
+        stream_proxy.on("kill", (signal) => {
           conn.end();
         });
       }