support ntfs, more robust detection of device formatting, more robust logic for iscsi device detection

Signed-off-by: Travis Glenn Hansen <travisghansen@yahoo.com>

src/driver/index.js

@@ -550,6 +550,7 @@ class CsiBaseDriver {
     const iscsi = driver.getDefaultISCSIInstance();
     let result;
     let device;
+    let block_device_info;
 
     const volume_id = call.request.volume_id;
     if (!volume_id) {
@@ -978,6 +979,27 @@ class CsiBaseDriver {
                 fs_type = "ext4";
               }
 
+              if (fs_type == "ntfs") {
+                block_device_info = await filesystem.getBlockDevice(device);
+                let partition_count =
+                  await filesystem.getBlockDevicePartitionCount(device);
+                if (partition_count > 0) {
+                  device = await filesystem.getBlockDeviceLargestPartition(
+                    device
+                  );
+                } else {
+                  // partion/gpt
+                  await filesystem.partitionDevice(
+                    device,
+                    "gpt",
+                    "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7"
+                  );
+                  device = await filesystem.getBlockDeviceLargestPartition(
+                    device
+                  );
+                }
+              }
+
               if (await filesystem.isBlockDevice(device)) {
                 // format
                 result = await filesystem.deviceIsFormatted(device);
@@ -1043,10 +1065,39 @@ class CsiBaseDriver {
               staging_target_path
             );
             if (!result) {
+              // expand fs if necessary
+              if (await filesystem.isBlockDevice(device)) {
+                // go ahead and expand fs (this covers cloned setups where expand is not explicitly invoked)
+                switch (fs_type) {
+                  case "exfat":
+                  case "ntfs":
+                  case "vfat":
+                    //await filesystem.checkFilesystem(device, fs_info.type);
+                    await filesystem.expandFilesystem(device, fs_type);
+                    break;
+                }
+              }
+
+              let mount_fs_type = fs_type;
+              if (mount_fs_type == "ntfs") {
+                mount_fs_type = "ntfs3";
+              }
+
+              // handle volume_mount_group where appropriate
+              if (volume_mount_group) {
+                switch (fs_type) {
+                  case "exfat":
+                  case "ntfs":
+                  case "vfat":
+                    mount_flags.push(`gid=${volume_mount_group}`);
+                    break;
+                }
+              }
+
               await mount.mount(
                 device,
                 staging_target_path,
-                ["-t", fs_type].concat(["-o", mount_flags.join(",")])
+                ["-t", mount_fs_type].concat(["-o", mount_flags.join(",")])
               );
             }
 
@@ -1054,8 +1105,8 @@ class CsiBaseDriver {
             if (await filesystem.isBlockDevice(device)) {
               // go ahead and expand fs (this covers cloned setups where expand is not explicitly invoked)
               switch (fs_type) {
-                case "ext4":
                 case "ext3":
+                case "ext4":
                 case "ext4dev":
                   //await filesystem.checkFilesystem(device, fs_info.type);
                   try {
@@ -1094,6 +1145,11 @@ class CsiBaseDriver {
                     fs_type
                   );
                   break;
+                case "exfat":
+                case "ntfs":
+                case "vfat":
+                  // noop
+                  break;
                 default:
                   // unsupported filesystem
                   throw new GrpcError(
@@ -1613,7 +1669,11 @@ class CsiBaseDriver {
 
           // TODO: this could be made async to detach all simultaneously
           for (const block_device_info_i of realBlockDeviceInfos) {
-            if (block_device_info_i.tran == "iscsi") {
+            if (await filesystem.deviceIsIscsi(block_device_info_i.path)) {
+              let parent_block_device = await filesystem.getBlockDeviceParent(
+                block_device_info_i.path
+              );
+
               // figure out which iscsi session this belongs to and logout
               // scan /dev/disk/by-path/ip-*?
               // device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
@@ -1632,7 +1692,7 @@ class CsiBaseDriver {
                     session.attached_scsi_devices.host.devices.some(
                       (device) => {
                         if (
-                          device.attached_scsi_disk == block_device_info_i.name
+                          device.attached_scsi_disk == parent_block_device.name
                         ) {
                           return true;
                         }
@@ -2439,8 +2499,8 @@ class CsiBaseDriver {
             fs_type = fs_info.type;
             if (fs_type) {
               switch (fs_type) {
-                case "ext4":
                 case "ext3":
+                case "ext4":
                 case "ext4dev":
                   //await filesystem.checkFilesystem(device, fs_info.type);
                   await filesystem.expandFilesystem(device, fs_type);
@@ -2453,6 +2513,13 @@ class CsiBaseDriver {
                     await filesystem.expandFilesystem(device_path, fs_type);
                   }
                   break;
+                case "exfat":
+                case "ntfs":
+                case "vfat":
+                  // TODO: return error here, cannot be expanded while online
+                  //await filesystem.checkFilesystem(device, fs_info.type);
+                  //await filesystem.expandFilesystem(device, fs_type);
+                  break;
                 default:
                   // unsupported filesystem
                   throw new GrpcError(

src/utils/filesystem.js

@@ -303,7 +303,7 @@ class Filesystem {
   async getBlockDevice(device) {
     const filesystem = this;
     device = await filesystem.realpath(device);
-    let args = ["-a", "-b", "-l", "-J", "-O"];
+    let args = ["-a", "-b", "-J", "-O"];
     args.push(device);
     let result;
 
@@ -317,30 +317,121 @@ class Filesystem {
   }
 
   /**
-   * blkid -p -o export <device>
+   *
+   * @param {*} device
+   * @returns
+   */
+  async getBlockDeviceLargestPartition(device) {
+    const filesystem = this;
+    let block_device_info = await filesystem.getBlockDevice(device);
+    if (block_device_info.children) {
+      let child;
+      for (const child_i of block_device_info.children) {
+        if (child_i.type == "part") {
+          if (!child) {
+            child = child_i;
+          } else {
+            if (child_i.size > child.size) {
+              child = child_i;
+            }
+          }
+        }
+      }
+      return `${child.path}`;
+    }
+  }
+
+  /**
+   *
+   * @param {*} device
+   * @returns
+   */
+  async getBlockDevicePartitionCount(device) {
+    const filesystem = this;
+    let count = 0;
+    let block_device_info = await filesystem.getBlockDevice(device);
+    if (block_device_info.children) {
+      for (const child_i of block_device_info.children) {
+        if (child_i.type == "part") {
+          count++;
+        }
+      }
+    }
+    return count;
+  }
+
+  /**
+   * type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 = linux
+   * type=EBD0A0A2-B9E5-4433-87C0-68B6B72699C7 = ntfs
+   * type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B = EFI
+   *
+   * @param {*} device
+   * @param {*} label
+   * @param {*} type
+   */
+  async partitionDevice(
+    device,
+    label = "gpt",
+    type = "0FC63DAF-8483-4772-8E79-3D69D8477DE4"
+  ) {
+    const filesystem = this;
+    let args = [device];
+    let result;
+
+    try {
+      result = await filesystem.exec("sfdisk", args, {
+        stdin: `label: ${label}\n`,
+      });
+      result = await filesystem.exec("sfdisk", args, {
+        stdin: `type=${type}\n`,
+      });
+    } catch (err) {
+      throw err;
+    }
+  }
+
+  /**
    *
    * @param {*} device
    */
   async deviceIsFormatted(device) {
     const filesystem = this;
-    let args = ["-p", "-o", "export", device];
     let result;
 
     try {
-      result = await filesystem.exec("blkid", args);
+      result = await filesystem.getBlockDevice(device);
+      return result.fstype ? true : false;
     } catch (err) {
-      if (err.code == 2 && err.stderr.includes("No such device or address")) {
       throw err;
     }
-
-      if (err.code == 2) {
-        return false;
   }
 
-      throw err;
+  async deviceIsIscsi(device) {
+    const filesystem = this;
+    let result;
+
+    do {
+      if (result) {
+        device = `/dev/${result.pkname}`;
+      }
+      result = await filesystem.getBlockDevice(device);
+    } while (result.pkname);
+
+    return result && result.tran == "iscsi";
   }
 
-    return true;
+  async getBlockDeviceParent(device) {
+    const filesystem = this;
+    let result;
+
+    do {
+      if (result) {
+        device = `/dev/${result.pkname}`;
+      }
+      result = await filesystem.getBlockDevice(device);
+    } while (result.pkname);
+
+    return result;
   }
 
   /**
@@ -463,6 +554,9 @@ class Filesystem {
         args = args.concat(["filesystem", "resize", "max"]);
         args.push(device); // in this case should be a mounted path
         break;
+      case "exfat":
+        // https://github.com/exfatprogs/exfatprogs/issues/134
+        return;
       case "ext4":
       case "ext3":
       case "ext4dev":
@@ -470,6 +564,13 @@ class Filesystem {
         args = args.concat(options);
         args.push(device);
         break;
+      case "ntfs":
+        // must be unmounted
+        command = "ntfsresize";
+        args = args.concat(options);
+        //args = args.concat(["-s", "max"]);
+        args.push(device);
+        break;
       case "xfs":
         command = "xfs_growfs";
         args = args.concat(options);
@@ -526,6 +627,14 @@ class Filesystem {
         args.push("-f");
         args.push("-p");
         break;
+      case "ntfs":
+        /**
+         * -b, --clear-bad-sectors Clear the bad sector list
+         * -d, --clear-dirty       Clear the volume dirty flag
+         */
+        command = "ntfsfix";
+        args.push(device);
+        break;
       case "xfs":
         command = "xfs_repair";
         args = args.concat(["-o", "force_geometry"]);
@@ -612,6 +721,12 @@ class Filesystem {
       //options.timeout = DEFAULT_TIMEOUT;
     }
 
+    let stdin;
+    if (options.stdin) {
+      stdin = options.stdin;
+      delete options.stdin;
+    }
+
     const filesystem = this;
     args = args || [];
 
@@ -619,13 +734,27 @@ class Filesystem {
       args.unshift(command);
       command = filesystem.options.paths.sudo;
     }
-    console.log("executing filesystem command: %s %s", command, args.join(" "));
+    let command_log = `${command} ${args.join(" ")}`.trim();
+    if (stdin) {
+      command_log = `echo '${stdin}' | ${command_log}`
+        .trim()
+        .replace(/\n/, "\\n");
+    }
+    console.log("executing filesystem command: %s", command_log);
 
     return new Promise((resolve, reject) => {
       const child = filesystem.options.executor.spawn(command, args, options);
       let stdout = "";
       let stderr = "";
 
+      child.on("spawn", function () {
+        if (stdin) {
+          child.stdin.setEncoding("utf-8");
+          child.stdin.write(stdin);
+          child.stdin.end();
+        }
+      });
+
       child.stdout.on("data", function (data) {
         stdout = stdout + data;
       });

src/utils/general.js

@@ -170,7 +170,7 @@ function stringify(value) {
 }
 
 function default_supported_block_filesystems() {
-  return ["btrfs", "ext3", "ext4", "ext4dev", "xfs", "ntfs"];
+  return ["btrfs", "exfat", "ext3", "ext4", "ext4dev", "ntfs", "vfat", "xfs"];
 }
 
 function default_supported_file_filesystems() {