proper support for both native nvme multipath and DM multipath

Signed-off-by: Travis Glenn Hansen <travisghansen@yahoo.com>

contrib/scale-nvmet-start.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# simple script to 'start' nvmet on TrueNAS SCALE
+#
+# to reinstall nvmetcli simply rm /usr/sbin/nvmetcli
+
+# debug
+#set -x
+
+# exit non-zero
+set -e
+
+SCRIPTDIR="$(
+  cd -- "$(dirname "$0")" >/dev/null 2>&1
+  pwd -P
+)"
+cd "${SCRIPTDIR}"
+
+: "${NVMETCONFIG:="${SCRIPTDIR}/nvmet-config.json"}"
+
+export PATH=${HOME}/.local/bin:${PATH}
+
+modules=()
+modules+=("nvmet")
+modules+=("nvmet-fc")
+modules+=("nvmet-rdma")
+modules+=("nvmet-tcp")
+
+for module in "${modules[@]}"; do
+  modprobe "${module}"
+done
+
+which nvmetcli &>/dev/null || {
+  which pip &>/dev/null || {
+    wget -O get-pip.py https://bootstrap.pypa.io/get-pip.py
+    python get-pip.py --user
+    rm get-pip.py
+  }
+
+  if [[ ! -d nvmetcli ]]; then
+    git clone git://git.infradead.org/users/hch/nvmetcli.git
+  fi
+
+  cd nvmetcli
+
+  # install to root home dir
+  python3 setup.py install --user
+
+  # install to root home dir
+  pip install configshell_fb --user
+
+  # remove source
+  cd "${SCRIPTDIR}"
+  rm -rf nvmetcli
+}
+
+cd "${SCRIPTDIR}"
+nvmetcli restore "${NVMETCONFIG}"

src/driver/index.js

@@ -14,6 +14,7 @@ const registry = require("../utils/registry");
 const semver = require("semver");
 const GeneralUtils = require("../utils/general");
 const { Zetabyte } = require("../utils/zfs");
+const { transport } = require("winston");
 
 const __REGISTRY_NS__ = "CsiBaseDriver";
 
@@ -982,7 +983,8 @@ class CsiBaseDriver {
                 try {
                   await GeneralUtils.retry(15, 2000, async () => {
                     namespaceDevice =
-                      await nvmeof.namespaceDevicePathByNQNNamespace(
+                      await nvmeof.namespaceDevicePathByTransportNQNNamespace(
+                        nvmeofConnection.transport,
                         nvmeofConnection.nqn,
                         nvmeofConnection.nsid
                       );
@@ -1126,6 +1128,14 @@ class CsiBaseDriver {
                       `failed to discover multipath device`
                     );
                   }
+                } else {
+                  // only throw an error if we were not able to attach to *any* devices
+                  if (nvmeofNamespaceDevices.length > 1) {
+                    throw new GrpcError(
+                      grpc.status.UNKNOWN,
+                      `too many nvme namespace devices, neither DM nor native multipath enabled`
+                    );
+                  }
                 }
               }
             }
@@ -2348,6 +2358,7 @@ class CsiBaseDriver {
         }
 
         if (is_block) {
+          let breakdeviceloop = false;
           let realBlockDeviceInfos = [];
           // detect if is a multipath device
           is_device_mapper = await filesystem.isDeviceMapperDevice(
@@ -2369,6 +2380,9 @@ class CsiBaseDriver {
 
           // TODO: this could be made async to detach all simultaneously
           for (const block_device_info_i of realBlockDeviceInfos) {
+            if (breakdeviceloop) {
+              break;
+            }
             switch (block_device_info_i.tran) {
               case "iscsi":
                 {
@@ -2475,7 +2489,15 @@ class CsiBaseDriver {
                     let nqn = await nvmeof.nqnByNamespaceDeviceName(
                       block_device_info_i.name
                     );
-                    await nvmeof.disconnectByNQN(nqn);
+                    if (nqn) {
+                      await nvmeof.disconnectByNQN(nqn);
+                      /**
+                       * the above disconnects *all* devices with the nqn so we
+                       * do NOT want to keep iterating all the 'real' devices
+                       * in the case of DM multipath
+                       */
+                      breakdeviceloop = true;
+                    }
                   }
                 }
                 break;

src/utils/nvmeof.js

@@ -106,9 +106,9 @@ class NVMEoF {
 
   /**
    * Rescans the NVME namespaces
-   * 
+   *
-   * @param {*} device 
+   * @param {*} device
-   * @param {*} args 
+   * @param {*} args
    */
   async rescanNamespace(device, args = []) {
     const nvmeof = this;
@@ -170,20 +170,29 @@ class NVMEoF {
     return result.stdout.trim() == "Y";
   }
 
-  async namespaceDevicePathByNQNNamespace(nqn, namespace) {
+  async namespaceDevicePathByTransportNQNNamespace(transport, nqn, namespace) {
     const nvmeof = this;
-    let result = await nvmeof.list(["-v"]);
+    transport = nvmeof.parseTransport(transport);
-    for (let device of result.Devices) {
+    let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled();
-      for (let subsytem of device.Subsystems) {
+    if (nativeMultipathEnabled) {
-        if (subsytem.SubsystemNQN != nqn) {
+      let subsystem = await nvmeof.getSubsystemByNQN(nqn);
-          continue;
+      if (subsystem) {
-        } else {
+        for (let i_namespace of subsystem.Namespaces) {
-          for (let i_namespace of subsytem.Namespaces) {
+          if (i_namespace.NSID != namespace) {
-            if (i_namespace.NSID != namespace) {
+            continue;
-              continue;
+          } else {
-            } else {
+            return `/dev/${i_namespace.NameSpace}`;
-              return `/dev/${i_namespace.NameSpace}`;
+          }
-            }
+        }
+      }
+    } else {
+      let controller = await nvmeof.getControllerByTransportNQN(transport, nqn);
+      if (controller) {
+        for (let i_namespace of controller.Namespaces) {
+          if (i_namespace.NSID != namespace) {
+            continue;
+          } else {
+            return `/dev/${i_namespace.NameSpace}`;
           }
         }
       }
@@ -193,45 +202,60 @@ class NVMEoF {
   async controllerDevicePathByTransportNQN(transport, nqn) {
     const nvmeof = this;
     transport = nvmeof.parseTransport(transport);
+    let controller = await nvmeof.getControllerByTransportNQN(transport, nqn);
+    if (controller) {
+      return `/dev/${controller.Controller}`;
+    }
+  }
+
+  async getSubsystemByNQN(nqn) {
+    const nvmeof = this;
     let result = await nvmeof.list(["-v"]);
     for (let device of result.Devices) {
-      for (let subsytem of device.Subsystems) {
+      for (let subsystem of device.Subsystems) {
-        if (subsytem.SubsystemNQN != nqn) {
+        if (subsystem.SubsystemNQN == nqn) {
+          return subsystem;
+        }
+      }
+    }
+  }
+
+  async getControllerByTransportNQN(transport, nqn) {
+    const nvmeof = this;
+    transport = nvmeof.parseTransport(transport);
+    let subsystem = await nvmeof.getSubsystemByNQN(nqn);
+    if (subsystem) {
+      for (let controller of subsystem.Controllers) {
+        if (controller.Transport != transport.type) {
           continue;
-        } else {
+        }
-          for (let controller of subsytem.Controllers) {
-            if (controller.Transport != transport.type) {
-              continue;
-            }
 
-            let controllerAddress = controller.Address;
+        let controllerAddress = controller.Address;
-            let parts = controllerAddress.split(",");
+        let parts = controllerAddress.split(",");
 
-            let traddr;
+        let traddr;
-            let trsvcid;
+        let trsvcid;
-            for (let i_part of parts) {
+        for (let i_part of parts) {
-              let i_parts = i_part.split("=");
+          let i_parts = i_part.split("=");
-              switch (i_parts[0]) {
+          switch (i_parts[0]) {
-                case "traddr":
+            case "traddr":
-                  traddr = i_parts[1];
+              traddr = i_parts[1];
-                  break;
+              break;
-                case "trsvcid":
+            case "trsvcid":
-                  trsvcid = i_parts[1];
+              trsvcid = i_parts[1];
-                  break;
+              break;
-              }
-            }
-
-            if (traddr != transport.address) {
-              continue;
-            }
-
-            if (transport.service && trsvcid != transport.service) {
-              continue;
-            }
-
-            return `/dev/${controller.Controller}`;
           }
         }
+
+        if (traddr != transport.address) {
+          continue;
+        }
+
+        if (transport.service && trsvcid != transport.service) {
+          continue;
+        }
+
+        return controller;
       }
     }
   }
@@ -240,13 +264,27 @@ class NVMEoF {
     const nvmeof = this;
     name = name.replace("/dev/", "");
     let result = await nvmeof.list(["-v"]);
-    for (let device of result.Devices) {
+    let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled();
-      for (let subsytem of device.Subsystems) {
+
-        for (let namespace of subsytem.Namespaces) {
+    if (nativeMultipathEnabled) {
-          if (namespace.NameSpace != name) {
+      for (let device of result.Devices) {
-            continue;
+        for (let subsystem of device.Subsystems) {
-          } else {
+          for (let namespace of subsystem.Namespaces) {
-            return subsytem.SubsystemNQN;
+            if (namespace.NameSpace == name) {
+              return subsystem.SubsystemNQN;
+            }
+          }
+        }
+      }
+    } else {
+      for (let device of result.Devices) {
+        for (let subsystem of device.Subsystems) {
+          for (let controller of subsystem.Controllers) {
+            for (let namespace of controller.Namespaces) {
+              if (namespace.NameSpace == name) {
+                return subsystem.SubsystemNQN;
+              }
+            }
           }
         }
       }