From 5076ca36649b5a44c744f84ee850acdd92bf0185 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Sat, 28 Nov 2020 22:56:28 -0700 Subject: [PATCH] minor bug fixes, iscsi multipath/device-mapper support --- src/driver/index.js | 317 +++++++++++++++++++++++++++------------- src/utils/filesystem.js | 146 +++++++++++++++++- src/utils/iscsi.js | 53 ++++--- 3 files changed, 391 insertions(+), 125 deletions(-) diff --git a/src/driver/index.js b/src/driver/index.js index 056b7c6..388bb57 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -254,6 +254,7 @@ class CsiBaseDriver { * @param {*} call */ async NodeStageVolume(call) { + const driver = this; const mount = new Mount(); const filesystem = new Filesystem(); const iscsi = new ISCSI(); @@ -310,46 +311,141 @@ class CsiBaseDriver { device = `//${volume_context.server}/${volume_context.share}`; break; case "iscsi": - // create DB entry - // https://library.netapp.com/ecmdocs/ECMP1654943/html/GUID-8EC685B4-8CB6-40D8-A8D5-031A3899BCDC.html - // put these options in place to force targets managed by csi to be explicitly attached (in the case of unclearn shutdown etc) - let nodeDB = { - "node.startup": "manual", - }; - const nodeDBKeyPrefix = "node-db."; - for (const key in normalizedSecrets) { - if (key.startsWith(nodeDBKeyPrefix)) { - nodeDB[key.substr(nodeDBKeyPrefix.length)] = normalizedSecrets[key]; - } + let portals = []; + if (volume_context.portal) { + portals.push(volume_context.portal.trim()); } - await iscsi.iscsiadm.createNodeDBEntry( - volume_context.iqn, - volume_context.portal, - nodeDB - ); - // login - await iscsi.iscsiadm.login(volume_context.iqn, volume_context.portal); - // find device name - device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`; + if (volume_context.portals) { + volume_context.portals.split(",").forEach((portal) => { + portals.push(portal.trim()); + }); + } - // can take some time for device to show up, loop for some period - result = await filesystem.pathExists(device); - let timer_start = Math.round(new Date().getTime() / 1000); - let timer_max = 30; - while (!result) { - await sleep(2000); + // ensure full portal value + portals = portals.map((value) => { + if (!value.includes(":")) { + value += ":3260"; + } + + return value.trim(); + }); + + // ensure unique entries only + portals = [...new Set(portals)]; + + let iscsiDevices = []; + + for (let portal of portals) { + // create DB entry + // https://library.netapp.com/ecmdocs/ECMP1654943/html/GUID-8EC685B4-8CB6-40D8-A8D5-031A3899BCDC.html + // put these options in place to force targets managed by csi to be explicitly attached (in the case of unclearn shutdown etc) + let nodeDB = { + "node.startup": "manual", + }; + const nodeDBKeyPrefix = "node-db."; + for (const key in normalizedSecrets) { + if (key.startsWith(nodeDBKeyPrefix)) { + nodeDB[key.substr(nodeDBKeyPrefix.length)] = + normalizedSecrets[key]; + } + } + await iscsi.iscsiadm.createNodeDBEntry( + volume_context.iqn, + portal, + nodeDB + ); + // login + await iscsi.iscsiadm.login(volume_context.iqn, portal); + + // find device name + device = `/dev/disk/by-path/ip-${portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`; + let deviceByPath = device; + + // can take some time for device to show up, loop for some period result = await filesystem.pathExists(device); - let current_time = Math.round(new Date().getTime() / 1000); - if (!result && current_time - timer_start > timer_max) { - throw new GrpcError( - grpc.status.UNKNOWN, - `hit timeout waiting for device node to appear: ${device}` + let timer_start = Math.round(new Date().getTime() / 1000); + let timer_max = 30; + let deviceCreated = result; + while (!result) { + await sleep(2000); + result = await filesystem.pathExists(device); + + if (result) { + deviceCreated = true; + break; + } + + let current_time = Math.round(new Date().getTime() / 1000); + if (!result && current_time - timer_start > timer_max) { + driver.ctx.logger.warn( + `hit timeout waiting for device node to appear: ${device}` + ); + break; + } + } + + if (deviceCreated) { + device = await filesystem.realpath(device); + iscsiDevices.push(device); + + driver.ctx.logger.info( + `successfully logged into portal ${portal} and created device ${deviceByPath} with realpath ${device}` ); } } - device = await filesystem.realpath(device); + // filter duplicates + iscsiDevices = iscsiDevices.filter((value, index, self) => { + return self.indexOf(value) === index; + }); + + // only throw an error if we were not able to attach to *any* devices + if (iscsiDevices.length < 1) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unable to attach any iscsi devices` + ); + } + + if (iscsiDevices.length != portals.length) { + driver.ctx.logger.warn( + `failed to attach all iscsi devices/targets/portals` + ); + + // TODO: allow a parameter to control this behavior in some form + if (false) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unable to attach all iscsi devices` + ); + } + } + + // compare all device-mapper slaves with the newly created devices + // if any of the new devices are device-mapper slaves treat this as a + // multipath scenario + let allDeviceMapperSlaves = await filesystem.getAllDeviceMapperSlaveDevices(); + let commonDevices = allDeviceMapperSlaves.filter((value) => + iscsiDevices.includes(value) + ); + + const useMultipath = portals.length > 1 || commonDevices.length > 0; + + // discover multipath device to use + if (useMultipath) { + device = await filesystem.getDeviceMapperDeviceFromSlaves( + iscsiDevices, + false + ); + + if (!device) { + throw new GrpcError( + grpc.status.UNKNOWN, + `failed to discover multipath device` + ); + } + } break; default: throw new GrpcError( @@ -463,6 +559,7 @@ class CsiBaseDriver { const iscsi = new ISCSI(); let result; let is_block = false; + let is_device_mapper = false; let block_device_info; let access_type = "mount"; @@ -505,78 +602,100 @@ class CsiBaseDriver { } if (is_block) { - if (block_device_info.tran == "iscsi") { - // figure out which iscsi session this belongs to and logout - // scan /dev/disk/by-path/ip-*? - // device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`; - // parse output from `iscsiadm -m session -P 3` - let sessions = await iscsi.iscsiadm.getSessionsDetails(); - for (let i = 0; i < sessions.length; i++) { - let session = sessions[i]; - let is_attached_to_session = false; + let realBlockDeviceInfos = []; + // detect if is a multipath device + is_device_mapper = await filesystem.isDeviceMapperDevice( + block_device_info.path + ); - if ( - session.attached_scsi_devices && - session.attached_scsi_devices.host && - session.attached_scsi_devices.host.devices - ) { - is_attached_to_session = session.attached_scsi_devices.host.devices.some( - (device) => { - if (device.attached_scsi_disk == block_device_info.name) { - return true; + if (is_device_mapper) { + let realBlockDevices = await filesystem.getDeviceMapperDeviceSlaves( + block_device_info.path + ); + for (const realBlockDevice of realBlockDevices) { + realBlockDeviceInfos.push( + await filesystem.getBlockDevice(realBlockDevice) + ); + } + } else { + realBlockDeviceInfos = [block_device_info]; + } + + // TODO: this could be made async to detach all simultaneously + for (const block_device_info_i of realBlockDeviceInfos) { + if (block_device_info_i.tran == "iscsi") { + // figure out which iscsi session this belongs to and logout + // scan /dev/disk/by-path/ip-*? + // device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`; + // parse output from `iscsiadm -m session -P 3` + let sessions = await iscsi.iscsiadm.getSessionsDetails(); + for (let i = 0; i < sessions.length; i++) { + let session = sessions[i]; + let is_attached_to_session = false; + + if ( + session.attached_scsi_devices && + session.attached_scsi_devices.host && + session.attached_scsi_devices.host.devices + ) { + is_attached_to_session = session.attached_scsi_devices.host.devices.some( + (device) => { + if (device.attached_scsi_disk == block_device_info_i.name) { + return true; + } + return false; } - return false; - } - ); - } - - if (is_attached_to_session) { - let timer_start; - let timer_max; - - timer_start = Math.round(new Date().getTime() / 1000); - timer_max = 30; - let loggedOut = false; - while (!loggedOut) { - try { - await iscsi.iscsiadm.logout(session.target, [ - session.persistent_portal, - ]); - loggedOut = true; - } catch (err) { - await sleep(2000); - let current_time = Math.round(new Date().getTime() / 1000); - if (current_time - timer_start > timer_max) { - // not throwing error for now as future invocations would not enter code path anyhow - loggedOut = true; - //throw new GrpcError( - // grpc.status.UNKNOWN, - // `hit timeout trying to logout of iscsi target: ${session.persistent_portal}` - //); - } - } + ); } - timer_start = Math.round(new Date().getTime() / 1000); - timer_max = 30; - let deletedEntry = false; - while (!deletedEntry) { - try { - await iscsi.iscsiadm.deleteNodeDBEntry( - session.target, - session.persistent_portal - ); - deletedEntry = true; - } catch (err) { - await sleep(2000); - let current_time = Math.round(new Date().getTime() / 1000); - if (current_time - timer_start > timer_max) { - // not throwing error for now as future invocations would not enter code path anyhow + if (is_attached_to_session) { + let timer_start; + let timer_max; + + timer_start = Math.round(new Date().getTime() / 1000); + timer_max = 30; + let loggedOut = false; + while (!loggedOut) { + try { + await iscsi.iscsiadm.logout(session.target, [ + session.persistent_portal, + ]); + loggedOut = true; + } catch (err) { + await sleep(2000); + let current_time = Math.round(new Date().getTime() / 1000); + if (current_time - timer_start > timer_max) { + // not throwing error for now as future invocations would not enter code path anyhow + loggedOut = true; + //throw new GrpcError( + // grpc.status.UNKNOWN, + // `hit timeout trying to logout of iscsi target: ${session.persistent_portal}` + //); + } + } + } + + timer_start = Math.round(new Date().getTime() / 1000); + timer_max = 30; + let deletedEntry = false; + while (!deletedEntry) { + try { + await iscsi.iscsiadm.deleteNodeDBEntry( + session.target, + session.persistent_portal + ); deletedEntry = true; - //throw new GrpcError( - // grpc.status.UNKNOWN, - // `hit timeout trying to delete iscsi node DB entry: ${session.target}, ${session.persistent_portal}` - //); + } catch (err) { + await sleep(2000); + let current_time = Math.round(new Date().getTime() / 1000); + if (current_time - timer_start > timer_max) { + // not throwing error for now as future invocations would not enter code path anyhow + deletedEntry = true; + //throw new GrpcError( + // grpc.status.UNKNOWN, + // `hit timeout trying to delete iscsi node DB entry: ${session.target}, ${session.persistent_portal}` + //); + } } } } diff --git a/src/utils/filesystem.js b/src/utils/filesystem.js index fb03909..4e59b28 100644 --- a/src/utils/filesystem.js +++ b/src/utils/filesystem.js @@ -48,14 +48,156 @@ class Filesystem { const device_path = await filesystem.realpath(device); const blockdevices = await filesystem.getAllBlockDevices(); - return blockdevices.some((i) => { - if (i.path == device_path) { + return blockdevices.some(async (i) => { + if ((await filesystem.realpath(i.path)) == device_path) { return true; } return false; }); } + /** + * Attempt to discover if the device is a device-mapper device + * + * @param {*} device + */ + async isDeviceMapperDevice(device) { + const filesystem = this; + const isBlock = await filesystem.isBlockDevice(device); + + if (!isBlock) { + return false; + } + + device = await filesystem.realpath(device); + + return device.includes("dm-"); + } + + async isDeviceMapperSlaveDevice(device) { + const filesystem = this; + device = await filesystem.realpath(device); + } + + /** + * Get all device-mapper devices (ie: dm-0, dm-1, dm-N...) + */ + async getAllDeviceMapperDevices() { + const filesystem = this; + let result; + let devices = []; + let args = [ + "-c", + 'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;', + ]; + + try { + result = await filesystem.exec("sh", args); + + for (const dm of result.stdout.trim().split("\n")) { + devices.push("/dev/" + dm.split(":")[0].trim()); + } + return devices; + } catch (err) { + throw err; + } + } + + async getAllDeviceMapperSlaveDevices() { + const filesystem = this; + let result; + let args = [ + "-c", + 'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;', + ]; + let slaves = []; + + try { + result = await filesystem.exec("sh", args); + + for (const dm of result.stdout.trim().split("\n")) { + const realDevices = dm + .split(":")[1] + .split(" ") + .map((value) => { + return "/dev/" + value.trim(); + }); + slaves.push(...realDevices); + } + return slaves; + } catch (err) { + throw err; + } + } + + /** + * Get all slave devices connected to a device-mapper device + * + * @param {*} device + */ + async getDeviceMapperDeviceSlaves(device) { + const filesystem = this; + device = await filesystem.realpath(device); + let device_info = await filesystem.getBlockDevice(device); + const slaves = []; + + let result; + let args = [`/sys/block/${device_info.kname}/slaves/`]; + + try { + result = await filesystem.exec("ls", args); + + for (const entry of result.stdout.split("\n")) { + if (entry.trim().length < 1) { + continue; + } + + slaves.push("/dev/" + entry.trim()); + } + return slaves; + } catch (err) { + throw err; + } + } + + async getDeviceMapperDeviceFromSlaves(slaves, matchAll = true) { + const filesystem = this; + let result; + + // get mapping of dm devices to real devices + let args = [ + "-c", + 'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;', + ]; + + result = await filesystem.exec("sh", args); + + for (const dm of result.stdout.trim().split("\n")) { + const dmDevice = "/dev/" + dm.split(":")[0].trim(); + const realDevices = dm + .split(":")[1] + .split(" ") + .map((value) => { + return "/dev/" + value.trim(); + }); + const intersectDevices = slaves.filter((value) => + realDevices.includes(value) + ); + + if (matchAll === false && intersectDevices.length > 0) { + return dmDevice; + } + + // if all 3 have the same elements we have a winner + if ( + intersectDevices.length == realDevices.length && + realDevices.length == slaves.length + ) { + return dmDevice; + } + } + } + /** * create symlink * diff --git a/src/utils/iscsi.js b/src/utils/iscsi.js index dbca409..36216f7 100644 --- a/src/utils/iscsi.js +++ b/src/utils/iscsi.js @@ -25,7 +25,7 @@ class ISCSI { if (!options.executor) { options.executor = { - spawn: cp.spawn + spawn: cp.spawn, }; } @@ -47,7 +47,7 @@ class ISCSI { const entries = result.stdout.trim().split("\n"); const interfaces = []; let fields; - entries.forEach(entry => { + entries.forEach((entry) => { fields = entry.split(" "); interfaces.push({ iface_name: fields[0], @@ -55,7 +55,7 @@ class ISCSI { hwaddress: getIscsiValue(fields[1].split(",")[1]), ipaddress: getIscsiValue(fields[1].split(",")[2]), net_ifacename: getIscsiValue(fields[1].split(",")[3]), - initiatorname: getIscsiValue(fields[1].split(",")[4]) + initiatorname: getIscsiValue(fields[1].split(",")[4]), }); }); @@ -75,7 +75,7 @@ class ISCSI { const entries = result.stdout.trim().split("\n"); const i = {}; let fields, key, value; - entries.forEach(entry => { + entries.forEach((entry) => { if (entry.startsWith("#")) return; fields = entry.split("="); key = fields[0].trim(); @@ -103,7 +103,7 @@ class ISCSI { "-p", portal, "-o", - "new" + "new", ]); await iscsi.exec(options.paths.iscsiadm, args); for (let attribute in attributes) { @@ -120,7 +120,7 @@ class ISCSI { "--name", attribute, "--value", - attributes[attribute] + attributes[attribute], ]); await iscsi.exec(options.paths.iscsiadm, args); } @@ -142,7 +142,7 @@ class ISCSI { "-p", portal, "-o", - "delete" + "delete", ]); await iscsi.exec(options.paths.iscsiadm, args); }, @@ -174,7 +174,7 @@ class ISCSI { const entries = result.stdout.trim().split("\n"); const sessions = []; let fields; - entries.forEach(entry => { + entries.forEach((entry) => { fields = entry.split(" "); sessions.push({ protocol: entry.split(":")[0], @@ -182,7 +182,7 @@ class ISCSI { portal: fields[2].split(",")[0], target_portal_group_tag: fields[2].split(",")[1], iqn: fields[3].split(":")[0], - target: fields[3].split(":")[1] + target: fields[3].split(":")[1], }); }); @@ -212,6 +212,7 @@ class ISCSI { return []; } + let currentTarget; let sessionGroups = []; let currentSession = []; @@ -221,13 +222,21 @@ class ISCSI { entries.shift(); entries.shift(); + // this should break up the lines into groups of lines + // where each group is the full details of a single session + // note that the output of the command bundles/groups all sessions + // by target so extra logic is needed to hanle that + // alternatively we could get all sessions using getSessions() + // and then invoke `iscsiadm -m session -P 3 -r ` in a loop for (let i = 0; i < entries.length; i++) { let entry = entries[i]; if (entry.startsWith("Target:")) { + currentTarget = entry; + } else if (entry.trim().startsWith("Current Portal:")) { if (currentSession.length > 0) { sessionGroups.push(currentSession); } - currentSession = [entry]; + currentSession = [currentTarget, entry]; } else { currentSession.push(entry); } @@ -261,11 +270,7 @@ class ISCSI { .trim() .replace(/ /g, "_") .replace(/\W/g, ""); - let value = line - .split(":") - .slice(1) - .join(":") - .trim(); + let value = line.split(":").slice(1).join(":").trim(); if (currentSection) { session[currentSection] = session[currentSection] || {}; @@ -282,7 +287,7 @@ class ISCSI { .split(":") .slice(1) .join(":") - .trim() + .trim(), }; while ( sessionLines[j + 1] && @@ -308,7 +313,7 @@ class ISCSI { .split(":") .slice(1) .join(":") - .trim() + .trim(), }); } @@ -322,7 +327,7 @@ class ISCSI { key = key.charAt(0).toLowerCase() + key.slice(1); key = key.replace( /[A-Z]/g, - letter => `_${letter.toLowerCase()}` + (letter) => `_${letter.toLowerCase()}` ); break; } @@ -367,12 +372,12 @@ class ISCSI { const entries = result.stdout.trim().split("\n"); const targets = []; - entries.forEach(entry => { + entries.forEach((entry) => { targets.push({ portal: entry.split(",")[0], target_portal_group_tag: entry.split(" ")[0].split(",")[1], iqn: entry.split(" ")[1].split(":")[0], - target: entry.split(" ")[1].split(":")[1] + target: entry.split(" ")[1].split(":")[1], }); }); @@ -432,7 +437,7 @@ class ISCSI { } return true; - } + }, }; } @@ -460,15 +465,15 @@ class ISCSI { } return new Promise((resolve, reject) => { - child.stdout.on("data", function(data) { + child.stdout.on("data", function (data) { stdout = stdout + data; }); - child.stderr.on("data", function(data) { + child.stderr.on("data", function (data) { stderr = stderr + data; }); - child.on("close", function(code) { + child.on("close", function (code) { const result = { code, stdout, stderr }; if (timeout) { clearTimeout(timeout);