From afba7d852745a7ad4735cf8c5de6e007fa9c35a9 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Thu, 22 Dec 2022 01:24:43 -0700 Subject: [PATCH] proper support for both native nvme multipath and DM multipath Signed-off-by: Travis Glenn Hansen --- contrib/scale-nvmet-start.sh | 58 ++++++++++++++ src/driver/index.js | 26 +++++- src/utils/nvmeof.js | 148 ++++++++++++++++++++++------------- 3 files changed, 175 insertions(+), 57 deletions(-) create mode 100755 contrib/scale-nvmet-start.sh diff --git a/contrib/scale-nvmet-start.sh b/contrib/scale-nvmet-start.sh new file mode 100755 index 0000000..42fbae8 --- /dev/null +++ b/contrib/scale-nvmet-start.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# simple script to 'start' nvmet on TrueNAS SCALE +# +# to reinstall nvmetcli simply rm /usr/sbin/nvmetcli + +# debug +#set -x + +# exit non-zero +set -e + +SCRIPTDIR="$( + cd -- "$(dirname "$0")" >/dev/null 2>&1 + pwd -P +)" +cd "${SCRIPTDIR}" + +: "${NVMETCONFIG:="${SCRIPTDIR}/nvmet-config.json"}" + +export PATH=${HOME}/.local/bin:${PATH} + +modules=() +modules+=("nvmet") +modules+=("nvmet-fc") +modules+=("nvmet-rdma") +modules+=("nvmet-tcp") + +for module in "${modules[@]}"; do + modprobe "${module}" +done + +which nvmetcli &>/dev/null || { + which pip &>/dev/null || { + wget -O get-pip.py https://bootstrap.pypa.io/get-pip.py + python get-pip.py --user + rm get-pip.py + } + + if [[ ! -d nvmetcli ]]; then + git clone git://git.infradead.org/users/hch/nvmetcli.git + fi + + cd nvmetcli + + # install to root home dir + python3 setup.py install --user + + # install to root home dir + pip install configshell_fb --user + + # remove source + cd "${SCRIPTDIR}" + rm -rf nvmetcli +} + +cd "${SCRIPTDIR}" +nvmetcli restore "${NVMETCONFIG}" diff --git a/src/driver/index.js b/src/driver/index.js index d3d1c37..b2201d3 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -14,6 +14,7 @@ const registry = require("../utils/registry"); const semver = require("semver"); const GeneralUtils = require("../utils/general"); const { Zetabyte } = require("../utils/zfs"); +const { transport } = require("winston"); const __REGISTRY_NS__ = "CsiBaseDriver"; @@ -982,7 +983,8 @@ class CsiBaseDriver { try { await GeneralUtils.retry(15, 2000, async () => { namespaceDevice = - await nvmeof.namespaceDevicePathByNQNNamespace( + await nvmeof.namespaceDevicePathByTransportNQNNamespace( + nvmeofConnection.transport, nvmeofConnection.nqn, nvmeofConnection.nsid ); @@ -1126,6 +1128,14 @@ class CsiBaseDriver { `failed to discover multipath device` ); } + } else { + // only throw an error if we were not able to attach to *any* devices + if (nvmeofNamespaceDevices.length > 1) { + throw new GrpcError( + grpc.status.UNKNOWN, + `too many nvme namespace devices, neither DM nor native multipath enabled` + ); + } } } } @@ -2348,6 +2358,7 @@ class CsiBaseDriver { } if (is_block) { + let breakdeviceloop = false; let realBlockDeviceInfos = []; // detect if is a multipath device is_device_mapper = await filesystem.isDeviceMapperDevice( @@ -2369,6 +2380,9 @@ class CsiBaseDriver { // TODO: this could be made async to detach all simultaneously for (const block_device_info_i of realBlockDeviceInfos) { + if (breakdeviceloop) { + break; + } switch (block_device_info_i.tran) { case "iscsi": { @@ -2475,7 +2489,15 @@ class CsiBaseDriver { let nqn = await nvmeof.nqnByNamespaceDeviceName( block_device_info_i.name ); - await nvmeof.disconnectByNQN(nqn); + if (nqn) { + await nvmeof.disconnectByNQN(nqn); + /** + * the above disconnects *all* devices with the nqn so we + * do NOT want to keep iterating all the 'real' devices + * in the case of DM multipath + */ + breakdeviceloop = true; + } } } break; diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index bd5898d..43b399f 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -106,9 +106,9 @@ class NVMEoF { /** * Rescans the NVME namespaces - * - * @param {*} device - * @param {*} args + * + * @param {*} device + * @param {*} args */ async rescanNamespace(device, args = []) { const nvmeof = this; @@ -170,20 +170,29 @@ class NVMEoF { return result.stdout.trim() == "Y"; } - async namespaceDevicePathByNQNNamespace(nqn, namespace) { + async namespaceDevicePathByTransportNQNNamespace(transport, nqn, namespace) { const nvmeof = this; - let result = await nvmeof.list(["-v"]); - for (let device of result.Devices) { - for (let subsytem of device.Subsystems) { - if (subsytem.SubsystemNQN != nqn) { - continue; - } else { - for (let i_namespace of subsytem.Namespaces) { - if (i_namespace.NSID != namespace) { - continue; - } else { - return `/dev/${i_namespace.NameSpace}`; - } + transport = nvmeof.parseTransport(transport); + let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled(); + if (nativeMultipathEnabled) { + let subsystem = await nvmeof.getSubsystemByNQN(nqn); + if (subsystem) { + for (let i_namespace of subsystem.Namespaces) { + if (i_namespace.NSID != namespace) { + continue; + } else { + return `/dev/${i_namespace.NameSpace}`; + } + } + } + } else { + let controller = await nvmeof.getControllerByTransportNQN(transport, nqn); + if (controller) { + for (let i_namespace of controller.Namespaces) { + if (i_namespace.NSID != namespace) { + continue; + } else { + return `/dev/${i_namespace.NameSpace}`; } } } @@ -193,45 +202,60 @@ class NVMEoF { async controllerDevicePathByTransportNQN(transport, nqn) { const nvmeof = this; transport = nvmeof.parseTransport(transport); + let controller = await nvmeof.getControllerByTransportNQN(transport, nqn); + if (controller) { + return `/dev/${controller.Controller}`; + } + } + + async getSubsystemByNQN(nqn) { + const nvmeof = this; let result = await nvmeof.list(["-v"]); for (let device of result.Devices) { - for (let subsytem of device.Subsystems) { - if (subsytem.SubsystemNQN != nqn) { + for (let subsystem of device.Subsystems) { + if (subsystem.SubsystemNQN == nqn) { + return subsystem; + } + } + } + } + + async getControllerByTransportNQN(transport, nqn) { + const nvmeof = this; + transport = nvmeof.parseTransport(transport); + let subsystem = await nvmeof.getSubsystemByNQN(nqn); + if (subsystem) { + for (let controller of subsystem.Controllers) { + if (controller.Transport != transport.type) { continue; - } else { - for (let controller of subsytem.Controllers) { - if (controller.Transport != transport.type) { - continue; - } + } - let controllerAddress = controller.Address; - let parts = controllerAddress.split(","); + let controllerAddress = controller.Address; + let parts = controllerAddress.split(","); - let traddr; - let trsvcid; - for (let i_part of parts) { - let i_parts = i_part.split("="); - switch (i_parts[0]) { - case "traddr": - traddr = i_parts[1]; - break; - case "trsvcid": - trsvcid = i_parts[1]; - break; - } - } - - if (traddr != transport.address) { - continue; - } - - if (transport.service && trsvcid != transport.service) { - continue; - } - - return `/dev/${controller.Controller}`; + let traddr; + let trsvcid; + for (let i_part of parts) { + let i_parts = i_part.split("="); + switch (i_parts[0]) { + case "traddr": + traddr = i_parts[1]; + break; + case "trsvcid": + trsvcid = i_parts[1]; + break; } } + + if (traddr != transport.address) { + continue; + } + + if (transport.service && trsvcid != transport.service) { + continue; + } + + return controller; } } } @@ -240,13 +264,27 @@ class NVMEoF { const nvmeof = this; name = name.replace("/dev/", ""); let result = await nvmeof.list(["-v"]); - for (let device of result.Devices) { - for (let subsytem of device.Subsystems) { - for (let namespace of subsytem.Namespaces) { - if (namespace.NameSpace != name) { - continue; - } else { - return subsytem.SubsystemNQN; + let nativeMultipathEnabled = await nvmeof.nativeMultipathEnabled(); + + if (nativeMultipathEnabled) { + for (let device of result.Devices) { + for (let subsystem of device.Subsystems) { + for (let namespace of subsystem.Namespaces) { + if (namespace.NameSpace == name) { + return subsystem.SubsystemNQN; + } + } + } + } + } else { + for (let device of result.Devices) { + for (let subsystem of device.Subsystems) { + for (let controller of subsystem.Controllers) { + for (let namespace of controller.Namespaces) { + if (namespace.NameSpace == name) { + return subsystem.SubsystemNQN; + } + } } } }