minor bug fixes, iscsi multipath/device-mapper support

src/driver/index.js

@@ -254,6 +254,7 @@ class CsiBaseDriver {
    * @param {*} call
    */
   async NodeStageVolume(call) {
+    const driver = this;
     const mount = new Mount();
     const filesystem = new Filesystem();
     const iscsi = new ISCSI();
@@ -310,46 +311,141 @@ class CsiBaseDriver {
         device = `//${volume_context.server}/${volume_context.share}`;
         break;
       case "iscsi":
-        // create DB entry
+        let portals = [];
-        // https://library.netapp.com/ecmdocs/ECMP1654943/html/GUID-8EC685B4-8CB6-40D8-A8D5-031A3899BCDC.html
+        if (volume_context.portal) {
-        // put these options in place to force targets managed by csi to be explicitly attached (in the case of unclearn shutdown etc)
+          portals.push(volume_context.portal.trim());
-        let nodeDB = {
-          "node.startup": "manual",
-        };
-        const nodeDBKeyPrefix = "node-db.";
-        for (const key in normalizedSecrets) {
-          if (key.startsWith(nodeDBKeyPrefix)) {
-            nodeDB[key.substr(nodeDBKeyPrefix.length)] = normalizedSecrets[key];
-          }
         }
-        await iscsi.iscsiadm.createNodeDBEntry(
-          volume_context.iqn,
-          volume_context.portal,
-          nodeDB
-        );
-        // login
-        await iscsi.iscsiadm.login(volume_context.iqn, volume_context.portal);
 
-        // find device name
+        if (volume_context.portals) {
-        device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
+          volume_context.portals.split(",").forEach((portal) => {
+            portals.push(portal.trim());
+          });
+        }
 
-        // can take some time for device to show up, loop for some period
+        // ensure full portal value
-        result = await filesystem.pathExists(device);
+        portals = portals.map((value) => {
-        let timer_start = Math.round(new Date().getTime() / 1000);
+          if (!value.includes(":")) {
-        let timer_max = 30;
+            value += ":3260";
-        while (!result) {
+          }
-          await sleep(2000);
+
+          return value.trim();
+        });
+
+        // ensure unique entries only
+        portals = [...new Set(portals)];
+
+        let iscsiDevices = [];
+
+        for (let portal of portals) {
+          // create DB entry
+          // https://library.netapp.com/ecmdocs/ECMP1654943/html/GUID-8EC685B4-8CB6-40D8-A8D5-031A3899BCDC.html
+          // put these options in place to force targets managed by csi to be explicitly attached (in the case of unclearn shutdown etc)
+          let nodeDB = {
+            "node.startup": "manual",
+          };
+          const nodeDBKeyPrefix = "node-db.";
+          for (const key in normalizedSecrets) {
+            if (key.startsWith(nodeDBKeyPrefix)) {
+              nodeDB[key.substr(nodeDBKeyPrefix.length)] =
+                normalizedSecrets[key];
+            }
+          }
+          await iscsi.iscsiadm.createNodeDBEntry(
+            volume_context.iqn,
+            portal,
+            nodeDB
+          );
+          // login
+          await iscsi.iscsiadm.login(volume_context.iqn, portal);
+
+          // find device name
+          device = `/dev/disk/by-path/ip-${portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
+          let deviceByPath = device;
+
+          // can take some time for device to show up, loop for some period
           result = await filesystem.pathExists(device);
-          let current_time = Math.round(new Date().getTime() / 1000);
+          let timer_start = Math.round(new Date().getTime() / 1000);
-          if (!result && current_time - timer_start > timer_max) {
+          let timer_max = 30;
-            throw new GrpcError(
+          let deviceCreated = result;
-              grpc.status.UNKNOWN,
+          while (!result) {
-              `hit timeout waiting for device node to appear: ${device}`
+            await sleep(2000);
+            result = await filesystem.pathExists(device);
+
+            if (result) {
+              deviceCreated = true;
+              break;
+            }
+
+            let current_time = Math.round(new Date().getTime() / 1000);
+            if (!result && current_time - timer_start > timer_max) {
+              driver.ctx.logger.warn(
+                `hit timeout waiting for device node to appear: ${device}`
+              );
+              break;
+            }
+          }
+
+          if (deviceCreated) {
+            device = await filesystem.realpath(device);
+            iscsiDevices.push(device);
+
+            driver.ctx.logger.info(
+              `successfully logged into portal ${portal} and created device ${deviceByPath} with realpath ${device}`
             );
           }
         }
 
-        device = await filesystem.realpath(device);
+        // filter duplicates
+        iscsiDevices = iscsiDevices.filter((value, index, self) => {
+          return self.indexOf(value) === index;
+        });
+
+        // only throw an error if we were not able to attach to *any* devices
+        if (iscsiDevices.length < 1) {
+          throw new GrpcError(
+            grpc.status.UNKNOWN,
+            `unable to attach any iscsi devices`
+          );
+        }
+
+        if (iscsiDevices.length != portals.length) {
+          driver.ctx.logger.warn(
+            `failed to attach all iscsi devices/targets/portals`
+          );
+
+          // TODO: allow a parameter to control this behavior in some form
+          if (false) {
+            throw new GrpcError(
+              grpc.status.UNKNOWN,
+              `unable to attach all iscsi devices`
+            );
+          }
+        }
+
+        // compare all device-mapper slaves with the newly created devices
+        // if any of the new devices are device-mapper slaves treat this as a
+        // multipath scenario
+        let allDeviceMapperSlaves = await filesystem.getAllDeviceMapperSlaveDevices();
+        let commonDevices = allDeviceMapperSlaves.filter((value) =>
+          iscsiDevices.includes(value)
+        );
+
+        const useMultipath = portals.length > 1 || commonDevices.length > 0;
+
+        // discover multipath device to use
+        if (useMultipath) {
+          device = await filesystem.getDeviceMapperDeviceFromSlaves(
+            iscsiDevices,
+            false
+          );
+
+          if (!device) {
+            throw new GrpcError(
+              grpc.status.UNKNOWN,
+              `failed to discover multipath device`
+            );
+          }
+        }
         break;
       default:
         throw new GrpcError(
@@ -463,6 +559,7 @@ class CsiBaseDriver {
     const iscsi = new ISCSI();
     let result;
     let is_block = false;
+    let is_device_mapper = false;
     let block_device_info;
     let access_type = "mount";
 
@@ -505,78 +602,100 @@ class CsiBaseDriver {
     }
 
     if (is_block) {
-      if (block_device_info.tran == "iscsi") {
+      let realBlockDeviceInfos = [];
-        // figure out which iscsi session this belongs to and logout
+      // detect if is a multipath device
-        // scan /dev/disk/by-path/ip-*?
+      is_device_mapper = await filesystem.isDeviceMapperDevice(
-        // device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
+        block_device_info.path
-        // parse output from `iscsiadm -m session -P 3`
+      );
-        let sessions = await iscsi.iscsiadm.getSessionsDetails();
-        for (let i = 0; i < sessions.length; i++) {
-          let session = sessions[i];
-          let is_attached_to_session = false;
 
-          if (
+      if (is_device_mapper) {
-            session.attached_scsi_devices &&
+        let realBlockDevices = await filesystem.getDeviceMapperDeviceSlaves(
-            session.attached_scsi_devices.host &&
+          block_device_info.path
-            session.attached_scsi_devices.host.devices
+        );
-          ) {
+        for (const realBlockDevice of realBlockDevices) {
-            is_attached_to_session = session.attached_scsi_devices.host.devices.some(
+          realBlockDeviceInfos.push(
-              (device) => {
+            await filesystem.getBlockDevice(realBlockDevice)
-                if (device.attached_scsi_disk == block_device_info.name) {
+          );
-                  return true;
+        }
+      } else {
+        realBlockDeviceInfos = [block_device_info];
+      }
+
+      // TODO: this could be made async to detach all simultaneously
+      for (const block_device_info_i of realBlockDeviceInfos) {
+        if (block_device_info_i.tran == "iscsi") {
+          // figure out which iscsi session this belongs to and logout
+          // scan /dev/disk/by-path/ip-*?
+          // device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
+          // parse output from `iscsiadm -m session -P 3`
+          let sessions = await iscsi.iscsiadm.getSessionsDetails();
+          for (let i = 0; i < sessions.length; i++) {
+            let session = sessions[i];
+            let is_attached_to_session = false;
+
+            if (
+              session.attached_scsi_devices &&
+              session.attached_scsi_devices.host &&
+              session.attached_scsi_devices.host.devices
+            ) {
+              is_attached_to_session = session.attached_scsi_devices.host.devices.some(
+                (device) => {
+                  if (device.attached_scsi_disk == block_device_info_i.name) {
+                    return true;
+                  }
+                  return false;
                 }
-                return false;
+              );
-              }
-            );
-          }
-
-          if (is_attached_to_session) {
-            let timer_start;
-            let timer_max;
-
-            timer_start = Math.round(new Date().getTime() / 1000);
-            timer_max = 30;
-            let loggedOut = false;
-            while (!loggedOut) {
-              try {
-                await iscsi.iscsiadm.logout(session.target, [
-                  session.persistent_portal,
-                ]);
-                loggedOut = true;
-              } catch (err) {
-                await sleep(2000);
-                let current_time = Math.round(new Date().getTime() / 1000);
-                if (current_time - timer_start > timer_max) {
-                  // not throwing error for now as future invocations would not enter code path anyhow
-                  loggedOut = true;
-                  //throw new GrpcError(
-                  //  grpc.status.UNKNOWN,
-                  //  `hit timeout trying to logout of iscsi target: ${session.persistent_portal}`
-                  //);
-                }
-              }
             }
 
-            timer_start = Math.round(new Date().getTime() / 1000);
+            if (is_attached_to_session) {
-            timer_max = 30;
+              let timer_start;
-            let deletedEntry = false;
+              let timer_max;
-            while (!deletedEntry) {
+
-              try {
+              timer_start = Math.round(new Date().getTime() / 1000);
-                await iscsi.iscsiadm.deleteNodeDBEntry(
+              timer_max = 30;
-                  session.target,
+              let loggedOut = false;
-                  session.persistent_portal
+              while (!loggedOut) {
-                );
+                try {
-                deletedEntry = true;
+                  await iscsi.iscsiadm.logout(session.target, [
-              } catch (err) {
+                    session.persistent_portal,
-                await sleep(2000);
+                  ]);
-                let current_time = Math.round(new Date().getTime() / 1000);
+                  loggedOut = true;
-                if (current_time - timer_start > timer_max) {
+                } catch (err) {
-                  // not throwing error for now as future invocations would not enter code path anyhow
+                  await sleep(2000);
+                  let current_time = Math.round(new Date().getTime() / 1000);
+                  if (current_time - timer_start > timer_max) {
+                    // not throwing error for now as future invocations would not enter code path anyhow
+                    loggedOut = true;
+                    //throw new GrpcError(
+                    //  grpc.status.UNKNOWN,
+                    //  `hit timeout trying to logout of iscsi target: ${session.persistent_portal}`
+                    //);
+                  }
+                }
+              }
+
+              timer_start = Math.round(new Date().getTime() / 1000);
+              timer_max = 30;
+              let deletedEntry = false;
+              while (!deletedEntry) {
+                try {
+                  await iscsi.iscsiadm.deleteNodeDBEntry(
+                    session.target,
+                    session.persistent_portal
+                  );
                   deletedEntry = true;
-                  //throw new GrpcError(
+                } catch (err) {
-                  //  grpc.status.UNKNOWN,
+                  await sleep(2000);
-                  //  `hit timeout trying to delete iscsi node DB entry: ${session.target}, ${session.persistent_portal}`
+                  let current_time = Math.round(new Date().getTime() / 1000);
-                  //);
+                  if (current_time - timer_start > timer_max) {
+                    // not throwing error for now as future invocations would not enter code path anyhow
+                    deletedEntry = true;
+                    //throw new GrpcError(
+                    //  grpc.status.UNKNOWN,
+                    //  `hit timeout trying to delete iscsi node DB entry: ${session.target}, ${session.persistent_portal}`
+                    //);
+                  }
                 }
               }
             }

src/utils/filesystem.js

@@ -48,14 +48,156 @@ class Filesystem {
     const device_path = await filesystem.realpath(device);
     const blockdevices = await filesystem.getAllBlockDevices();
 
-    return blockdevices.some((i) => {
+    return blockdevices.some(async (i) => {
-      if (i.path == device_path) {
+      if ((await filesystem.realpath(i.path)) == device_path) {
         return true;
       }
       return false;
     });
   }
 
+  /**
+   * Attempt to discover if the device is a device-mapper device
+   *
+   * @param {*} device
+   */
+  async isDeviceMapperDevice(device) {
+    const filesystem = this;
+    const isBlock = await filesystem.isBlockDevice(device);
+
+    if (!isBlock) {
+      return false;
+    }
+
+    device = await filesystem.realpath(device);
+
+    return device.includes("dm-");
+  }
+
+  async isDeviceMapperSlaveDevice(device) {
+    const filesystem = this;
+    device = await filesystem.realpath(device);
+  }
+
+  /**
+   * Get all device-mapper devices (ie: dm-0, dm-1, dm-N...)
+   */
+  async getAllDeviceMapperDevices() {
+    const filesystem = this;
+    let result;
+    let devices = [];
+    let args = [
+      "-c",
+      'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;',
+    ];
+
+    try {
+      result = await filesystem.exec("sh", args);
+
+      for (const dm of result.stdout.trim().split("\n")) {
+        devices.push("/dev/" + dm.split(":")[0].trim());
+      }
+      return devices;
+    } catch (err) {
+      throw err;
+    }
+  }
+
+  async getAllDeviceMapperSlaveDevices() {
+    const filesystem = this;
+    let result;
+    let args = [
+      "-c",
+      'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;',
+    ];
+    let slaves = [];
+
+    try {
+      result = await filesystem.exec("sh", args);
+
+      for (const dm of result.stdout.trim().split("\n")) {
+        const realDevices = dm
+          .split(":")[1]
+          .split(" ")
+          .map((value) => {
+            return "/dev/" + value.trim();
+          });
+        slaves.push(...realDevices);
+      }
+      return slaves;
+    } catch (err) {
+      throw err;
+    }
+  }
+
+  /**
+   * Get all slave devices connected to a device-mapper device
+   *
+   * @param {*} device
+   */
+  async getDeviceMapperDeviceSlaves(device) {
+    const filesystem = this;
+    device = await filesystem.realpath(device);
+    let device_info = await filesystem.getBlockDevice(device);
+    const slaves = [];
+
+    let result;
+    let args = [`/sys/block/${device_info.kname}/slaves/`];
+
+    try {
+      result = await filesystem.exec("ls", args);
+
+      for (const entry of result.stdout.split("\n")) {
+        if (entry.trim().length < 1) {
+          continue;
+        }
+
+        slaves.push("/dev/" + entry.trim());
+      }
+      return slaves;
+    } catch (err) {
+      throw err;
+    }
+  }
+
+  async getDeviceMapperDeviceFromSlaves(slaves, matchAll = true) {
+    const filesystem = this;
+    let result;
+
+    // get mapping of dm devices to real devices
+    let args = [
+      "-c",
+      'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;',
+    ];
+
+    result = await filesystem.exec("sh", args);
+
+    for (const dm of result.stdout.trim().split("\n")) {
+      const dmDevice = "/dev/" + dm.split(":")[0].trim();
+      const realDevices = dm
+        .split(":")[1]
+        .split(" ")
+        .map((value) => {
+          return "/dev/" + value.trim();
+        });
+      const intersectDevices = slaves.filter((value) =>
+        realDevices.includes(value)
+      );
+
+      if (matchAll === false && intersectDevices.length > 0) {
+        return dmDevice;
+      }
+
+      // if all 3 have the same elements we have a winner
+      if (
+        intersectDevices.length == realDevices.length &&
+        realDevices.length == slaves.length
+      ) {
+        return dmDevice;
+      }
+    }
+  }
+
   /**
    * create symlink
    *

src/utils/iscsi.js

@@ -25,7 +25,7 @@ class ISCSI {
 
     if (!options.executor) {
       options.executor = {
-        spawn: cp.spawn
+        spawn: cp.spawn,
       };
     }
 
@@ -47,7 +47,7 @@ class ISCSI {
         const entries = result.stdout.trim().split("\n");
         const interfaces = [];
         let fields;
-        entries.forEach(entry => {
+        entries.forEach((entry) => {
           fields = entry.split(" ");
           interfaces.push({
             iface_name: fields[0],
@@ -55,7 +55,7 @@ class ISCSI {
             hwaddress: getIscsiValue(fields[1].split(",")[1]),
             ipaddress: getIscsiValue(fields[1].split(",")[2]),
             net_ifacename: getIscsiValue(fields[1].split(",")[3]),
-            initiatorname: getIscsiValue(fields[1].split(",")[4])
+            initiatorname: getIscsiValue(fields[1].split(",")[4]),
           });
         });
 
@@ -75,7 +75,7 @@ class ISCSI {
         const entries = result.stdout.trim().split("\n");
         const i = {};
         let fields, key, value;
-        entries.forEach(entry => {
+        entries.forEach((entry) => {
           if (entry.startsWith("#")) return;
           fields = entry.split("=");
           key = fields[0].trim();
@@ -103,7 +103,7 @@ class ISCSI {
           "-p",
           portal,
           "-o",
-          "new"
+          "new",
         ]);
         await iscsi.exec(options.paths.iscsiadm, args);
         for (let attribute in attributes) {
@@ -120,7 +120,7 @@ class ISCSI {
             "--name",
             attribute,
             "--value",
-            attributes[attribute]
+            attributes[attribute],
           ]);
           await iscsi.exec(options.paths.iscsiadm, args);
         }
@@ -142,7 +142,7 @@ class ISCSI {
           "-p",
           portal,
           "-o",
-          "delete"
+          "delete",
         ]);
         await iscsi.exec(options.paths.iscsiadm, args);
       },
@@ -174,7 +174,7 @@ class ISCSI {
         const entries = result.stdout.trim().split("\n");
         const sessions = [];
         let fields;
-        entries.forEach(entry => {
+        entries.forEach((entry) => {
           fields = entry.split(" ");
           sessions.push({
             protocol: entry.split(":")[0],
@@ -182,7 +182,7 @@ class ISCSI {
             portal: fields[2].split(",")[0],
             target_portal_group_tag: fields[2].split(",")[1],
             iqn: fields[3].split(":")[0],
-            target: fields[3].split(":")[1]
+            target: fields[3].split(":")[1],
           });
         });
 
@@ -212,6 +212,7 @@ class ISCSI {
           return [];
         }
 
+        let currentTarget;
         let sessionGroups = [];
         let currentSession = [];
 
@@ -221,13 +222,21 @@ class ISCSI {
         entries.shift();
         entries.shift();
 
+        // this should break up the lines into groups of lines
+        // where each group is the full details of a single session
+        // note that the output of the command bundles/groups all sessions
+        // by target so extra logic is needed to hanle that
+        // alternatively we could get all sessions using getSessions()
+        // and then invoke `iscsiadm -m session -P 3 -r <session id>` in a loop
         for (let i = 0; i < entries.length; i++) {
           let entry = entries[i];
           if (entry.startsWith("Target:")) {
+            currentTarget = entry;
+          } else if (entry.trim().startsWith("Current Portal:")) {
             if (currentSession.length > 0) {
               sessionGroups.push(currentSession);
             }
-            currentSession = [entry];
+            currentSession = [currentTarget, entry];
           } else {
             currentSession.push(entry);
           }
@@ -261,11 +270,7 @@ class ISCSI {
               .trim()
               .replace(/ /g, "_")
               .replace(/\W/g, "");
-            let value = line
+            let value = line.split(":").slice(1).join(":").trim();
-              .split(":")
-              .slice(1)
-              .join(":")
-              .trim();
 
             if (currentSection) {
               session[currentSection] = session[currentSection] || {};
@@ -282,7 +287,7 @@ class ISCSI {
                         .split(":")
                         .slice(1)
                         .join(":")
-                        .trim()
+                        .trim(),
                     };
                     while (
                       sessionLines[j + 1] &&
@@ -308,7 +313,7 @@ class ISCSI {
                             .split(":")
                             .slice(1)
                             .join(":")
-                            .trim()
+                            .trim(),
                         });
                       }
 
@@ -322,7 +327,7 @@ class ISCSI {
                   key = key.charAt(0).toLowerCase() + key.slice(1);
                   key = key.replace(
                     /[A-Z]/g,
-                    letter => `_${letter.toLowerCase()}`
+                    (letter) => `_${letter.toLowerCase()}`
                   );
                   break;
               }
@@ -367,12 +372,12 @@ class ISCSI {
 
         const entries = result.stdout.trim().split("\n");
         const targets = [];
-        entries.forEach(entry => {
+        entries.forEach((entry) => {
           targets.push({
             portal: entry.split(",")[0],
             target_portal_group_tag: entry.split(" ")[0].split(",")[1],
             iqn: entry.split(" ")[1].split(":")[0],
-            target: entry.split(" ")[1].split(":")[1]
+            target: entry.split(" ")[1].split(":")[1],
           });
         });
 
@@ -432,7 +437,7 @@ class ISCSI {
         }
 
         return true;
-      }
+      },
     };
   }
 
@@ -460,15 +465,15 @@ class ISCSI {
     }
 
     return new Promise((resolve, reject) => {
-      child.stdout.on("data", function(data) {
+      child.stdout.on("data", function (data) {
         stdout = stdout + data;
       });
 
-      child.stderr.on("data", function(data) {
+      child.stderr.on("data", function (data) {
         stderr = stderr + data;
       });
 
-      child.on("close", function(code) {
+      child.on("close", function (code) {
         const result = { code, stdout, stderr };
         if (timeout) {
           clearTimeout(timeout);