minor bug fixes, iscsi multipath/device-mapper support

This commit is contained in:
Travis Glenn Hansen 2020-11-28 22:56:28 -07:00
parent c4a36750cd
commit 5076ca3664
3 changed files with 391 additions and 125 deletions

View File

@ -254,6 +254,7 @@ class CsiBaseDriver {
* @param {*} call
*/
async NodeStageVolume(call) {
const driver = this;
const mount = new Mount();
const filesystem = new Filesystem();
const iscsi = new ISCSI();
@ -310,46 +311,141 @@ class CsiBaseDriver {
device = `//${volume_context.server}/${volume_context.share}`;
break;
case "iscsi":
// create DB entry
// https://library.netapp.com/ecmdocs/ECMP1654943/html/GUID-8EC685B4-8CB6-40D8-A8D5-031A3899BCDC.html
// put these options in place to force targets managed by csi to be explicitly attached (in the case of unclearn shutdown etc)
let nodeDB = {
"node.startup": "manual",
};
const nodeDBKeyPrefix = "node-db.";
for (const key in normalizedSecrets) {
if (key.startsWith(nodeDBKeyPrefix)) {
nodeDB[key.substr(nodeDBKeyPrefix.length)] = normalizedSecrets[key];
}
let portals = [];
if (volume_context.portal) {
portals.push(volume_context.portal.trim());
}
await iscsi.iscsiadm.createNodeDBEntry(
volume_context.iqn,
volume_context.portal,
nodeDB
);
// login
await iscsi.iscsiadm.login(volume_context.iqn, volume_context.portal);
// find device name
device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
if (volume_context.portals) {
volume_context.portals.split(",").forEach((portal) => {
portals.push(portal.trim());
});
}
// can take some time for device to show up, loop for some period
result = await filesystem.pathExists(device);
let timer_start = Math.round(new Date().getTime() / 1000);
let timer_max = 30;
while (!result) {
await sleep(2000);
// ensure full portal value
portals = portals.map((value) => {
if (!value.includes(":")) {
value += ":3260";
}
return value.trim();
});
// ensure unique entries only
portals = [...new Set(portals)];
let iscsiDevices = [];
for (let portal of portals) {
// create DB entry
// https://library.netapp.com/ecmdocs/ECMP1654943/html/GUID-8EC685B4-8CB6-40D8-A8D5-031A3899BCDC.html
// put these options in place to force targets managed by csi to be explicitly attached (in the case of unclearn shutdown etc)
let nodeDB = {
"node.startup": "manual",
};
const nodeDBKeyPrefix = "node-db.";
for (const key in normalizedSecrets) {
if (key.startsWith(nodeDBKeyPrefix)) {
nodeDB[key.substr(nodeDBKeyPrefix.length)] =
normalizedSecrets[key];
}
}
await iscsi.iscsiadm.createNodeDBEntry(
volume_context.iqn,
portal,
nodeDB
);
// login
await iscsi.iscsiadm.login(volume_context.iqn, portal);
// find device name
device = `/dev/disk/by-path/ip-${portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
let deviceByPath = device;
// can take some time for device to show up, loop for some period
result = await filesystem.pathExists(device);
let current_time = Math.round(new Date().getTime() / 1000);
if (!result && current_time - timer_start > timer_max) {
throw new GrpcError(
grpc.status.UNKNOWN,
`hit timeout waiting for device node to appear: ${device}`
let timer_start = Math.round(new Date().getTime() / 1000);
let timer_max = 30;
let deviceCreated = result;
while (!result) {
await sleep(2000);
result = await filesystem.pathExists(device);
if (result) {
deviceCreated = true;
break;
}
let current_time = Math.round(new Date().getTime() / 1000);
if (!result && current_time - timer_start > timer_max) {
driver.ctx.logger.warn(
`hit timeout waiting for device node to appear: ${device}`
);
break;
}
}
if (deviceCreated) {
device = await filesystem.realpath(device);
iscsiDevices.push(device);
driver.ctx.logger.info(
`successfully logged into portal ${portal} and created device ${deviceByPath} with realpath ${device}`
);
}
}
device = await filesystem.realpath(device);
// filter duplicates
iscsiDevices = iscsiDevices.filter((value, index, self) => {
return self.indexOf(value) === index;
});
// only throw an error if we were not able to attach to *any* devices
if (iscsiDevices.length < 1) {
throw new GrpcError(
grpc.status.UNKNOWN,
`unable to attach any iscsi devices`
);
}
if (iscsiDevices.length != portals.length) {
driver.ctx.logger.warn(
`failed to attach all iscsi devices/targets/portals`
);
// TODO: allow a parameter to control this behavior in some form
if (false) {
throw new GrpcError(
grpc.status.UNKNOWN,
`unable to attach all iscsi devices`
);
}
}
// compare all device-mapper slaves with the newly created devices
// if any of the new devices are device-mapper slaves treat this as a
// multipath scenario
let allDeviceMapperSlaves = await filesystem.getAllDeviceMapperSlaveDevices();
let commonDevices = allDeviceMapperSlaves.filter((value) =>
iscsiDevices.includes(value)
);
const useMultipath = portals.length > 1 || commonDevices.length > 0;
// discover multipath device to use
if (useMultipath) {
device = await filesystem.getDeviceMapperDeviceFromSlaves(
iscsiDevices,
false
);
if (!device) {
throw new GrpcError(
grpc.status.UNKNOWN,
`failed to discover multipath device`
);
}
}
break;
default:
throw new GrpcError(
@ -463,6 +559,7 @@ class CsiBaseDriver {
const iscsi = new ISCSI();
let result;
let is_block = false;
let is_device_mapper = false;
let block_device_info;
let access_type = "mount";
@ -505,78 +602,100 @@ class CsiBaseDriver {
}
if (is_block) {
if (block_device_info.tran == "iscsi") {
// figure out which iscsi session this belongs to and logout
// scan /dev/disk/by-path/ip-*?
// device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
// parse output from `iscsiadm -m session -P 3`
let sessions = await iscsi.iscsiadm.getSessionsDetails();
for (let i = 0; i < sessions.length; i++) {
let session = sessions[i];
let is_attached_to_session = false;
let realBlockDeviceInfos = [];
// detect if is a multipath device
is_device_mapper = await filesystem.isDeviceMapperDevice(
block_device_info.path
);
if (
session.attached_scsi_devices &&
session.attached_scsi_devices.host &&
session.attached_scsi_devices.host.devices
) {
is_attached_to_session = session.attached_scsi_devices.host.devices.some(
(device) => {
if (device.attached_scsi_disk == block_device_info.name) {
return true;
if (is_device_mapper) {
let realBlockDevices = await filesystem.getDeviceMapperDeviceSlaves(
block_device_info.path
);
for (const realBlockDevice of realBlockDevices) {
realBlockDeviceInfos.push(
await filesystem.getBlockDevice(realBlockDevice)
);
}
} else {
realBlockDeviceInfos = [block_device_info];
}
// TODO: this could be made async to detach all simultaneously
for (const block_device_info_i of realBlockDeviceInfos) {
if (block_device_info_i.tran == "iscsi") {
// figure out which iscsi session this belongs to and logout
// scan /dev/disk/by-path/ip-*?
// device = `/dev/disk/by-path/ip-${volume_context.portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
// parse output from `iscsiadm -m session -P 3`
let sessions = await iscsi.iscsiadm.getSessionsDetails();
for (let i = 0; i < sessions.length; i++) {
let session = sessions[i];
let is_attached_to_session = false;
if (
session.attached_scsi_devices &&
session.attached_scsi_devices.host &&
session.attached_scsi_devices.host.devices
) {
is_attached_to_session = session.attached_scsi_devices.host.devices.some(
(device) => {
if (device.attached_scsi_disk == block_device_info_i.name) {
return true;
}
return false;
}
return false;
}
);
}
if (is_attached_to_session) {
let timer_start;
let timer_max;
timer_start = Math.round(new Date().getTime() / 1000);
timer_max = 30;
let loggedOut = false;
while (!loggedOut) {
try {
await iscsi.iscsiadm.logout(session.target, [
session.persistent_portal,
]);
loggedOut = true;
} catch (err) {
await sleep(2000);
let current_time = Math.round(new Date().getTime() / 1000);
if (current_time - timer_start > timer_max) {
// not throwing error for now as future invocations would not enter code path anyhow
loggedOut = true;
//throw new GrpcError(
// grpc.status.UNKNOWN,
// `hit timeout trying to logout of iscsi target: ${session.persistent_portal}`
//);
}
}
);
}
timer_start = Math.round(new Date().getTime() / 1000);
timer_max = 30;
let deletedEntry = false;
while (!deletedEntry) {
try {
await iscsi.iscsiadm.deleteNodeDBEntry(
session.target,
session.persistent_portal
);
deletedEntry = true;
} catch (err) {
await sleep(2000);
let current_time = Math.round(new Date().getTime() / 1000);
if (current_time - timer_start > timer_max) {
// not throwing error for now as future invocations would not enter code path anyhow
if (is_attached_to_session) {
let timer_start;
let timer_max;
timer_start = Math.round(new Date().getTime() / 1000);
timer_max = 30;
let loggedOut = false;
while (!loggedOut) {
try {
await iscsi.iscsiadm.logout(session.target, [
session.persistent_portal,
]);
loggedOut = true;
} catch (err) {
await sleep(2000);
let current_time = Math.round(new Date().getTime() / 1000);
if (current_time - timer_start > timer_max) {
// not throwing error for now as future invocations would not enter code path anyhow
loggedOut = true;
//throw new GrpcError(
// grpc.status.UNKNOWN,
// `hit timeout trying to logout of iscsi target: ${session.persistent_portal}`
//);
}
}
}
timer_start = Math.round(new Date().getTime() / 1000);
timer_max = 30;
let deletedEntry = false;
while (!deletedEntry) {
try {
await iscsi.iscsiadm.deleteNodeDBEntry(
session.target,
session.persistent_portal
);
deletedEntry = true;
//throw new GrpcError(
// grpc.status.UNKNOWN,
// `hit timeout trying to delete iscsi node DB entry: ${session.target}, ${session.persistent_portal}`
//);
} catch (err) {
await sleep(2000);
let current_time = Math.round(new Date().getTime() / 1000);
if (current_time - timer_start > timer_max) {
// not throwing error for now as future invocations would not enter code path anyhow
deletedEntry = true;
//throw new GrpcError(
// grpc.status.UNKNOWN,
// `hit timeout trying to delete iscsi node DB entry: ${session.target}, ${session.persistent_portal}`
//);
}
}
}
}

View File

@ -48,14 +48,156 @@ class Filesystem {
const device_path = await filesystem.realpath(device);
const blockdevices = await filesystem.getAllBlockDevices();
return blockdevices.some((i) => {
if (i.path == device_path) {
return blockdevices.some(async (i) => {
if ((await filesystem.realpath(i.path)) == device_path) {
return true;
}
return false;
});
}
/**
* Attempt to discover if the device is a device-mapper device
*
* @param {*} device
*/
async isDeviceMapperDevice(device) {
const filesystem = this;
const isBlock = await filesystem.isBlockDevice(device);
if (!isBlock) {
return false;
}
device = await filesystem.realpath(device);
return device.includes("dm-");
}
async isDeviceMapperSlaveDevice(device) {
const filesystem = this;
device = await filesystem.realpath(device);
}
/**
* Get all device-mapper devices (ie: dm-0, dm-1, dm-N...)
*/
async getAllDeviceMapperDevices() {
const filesystem = this;
let result;
let devices = [];
let args = [
"-c",
'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;',
];
try {
result = await filesystem.exec("sh", args);
for (const dm of result.stdout.trim().split("\n")) {
devices.push("/dev/" + dm.split(":")[0].trim());
}
return devices;
} catch (err) {
throw err;
}
}
async getAllDeviceMapperSlaveDevices() {
const filesystem = this;
let result;
let args = [
"-c",
'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;',
];
let slaves = [];
try {
result = await filesystem.exec("sh", args);
for (const dm of result.stdout.trim().split("\n")) {
const realDevices = dm
.split(":")[1]
.split(" ")
.map((value) => {
return "/dev/" + value.trim();
});
slaves.push(...realDevices);
}
return slaves;
} catch (err) {
throw err;
}
}
/**
* Get all slave devices connected to a device-mapper device
*
* @param {*} device
*/
async getDeviceMapperDeviceSlaves(device) {
const filesystem = this;
device = await filesystem.realpath(device);
let device_info = await filesystem.getBlockDevice(device);
const slaves = [];
let result;
let args = [`/sys/block/${device_info.kname}/slaves/`];
try {
result = await filesystem.exec("ls", args);
for (const entry of result.stdout.split("\n")) {
if (entry.trim().length < 1) {
continue;
}
slaves.push("/dev/" + entry.trim());
}
return slaves;
} catch (err) {
throw err;
}
}
async getDeviceMapperDeviceFromSlaves(slaves, matchAll = true) {
const filesystem = this;
let result;
// get mapping of dm devices to real devices
let args = [
"-c",
'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;',
];
result = await filesystem.exec("sh", args);
for (const dm of result.stdout.trim().split("\n")) {
const dmDevice = "/dev/" + dm.split(":")[0].trim();
const realDevices = dm
.split(":")[1]
.split(" ")
.map((value) => {
return "/dev/" + value.trim();
});
const intersectDevices = slaves.filter((value) =>
realDevices.includes(value)
);
if (matchAll === false && intersectDevices.length > 0) {
return dmDevice;
}
// if all 3 have the same elements we have a winner
if (
intersectDevices.length == realDevices.length &&
realDevices.length == slaves.length
) {
return dmDevice;
}
}
}
/**
* create symlink
*

View File

@ -25,7 +25,7 @@ class ISCSI {
if (!options.executor) {
options.executor = {
spawn: cp.spawn
spawn: cp.spawn,
};
}
@ -47,7 +47,7 @@ class ISCSI {
const entries = result.stdout.trim().split("\n");
const interfaces = [];
let fields;
entries.forEach(entry => {
entries.forEach((entry) => {
fields = entry.split(" ");
interfaces.push({
iface_name: fields[0],
@ -55,7 +55,7 @@ class ISCSI {
hwaddress: getIscsiValue(fields[1].split(",")[1]),
ipaddress: getIscsiValue(fields[1].split(",")[2]),
net_ifacename: getIscsiValue(fields[1].split(",")[3]),
initiatorname: getIscsiValue(fields[1].split(",")[4])
initiatorname: getIscsiValue(fields[1].split(",")[4]),
});
});
@ -75,7 +75,7 @@ class ISCSI {
const entries = result.stdout.trim().split("\n");
const i = {};
let fields, key, value;
entries.forEach(entry => {
entries.forEach((entry) => {
if (entry.startsWith("#")) return;
fields = entry.split("=");
key = fields[0].trim();
@ -103,7 +103,7 @@ class ISCSI {
"-p",
portal,
"-o",
"new"
"new",
]);
await iscsi.exec(options.paths.iscsiadm, args);
for (let attribute in attributes) {
@ -120,7 +120,7 @@ class ISCSI {
"--name",
attribute,
"--value",
attributes[attribute]
attributes[attribute],
]);
await iscsi.exec(options.paths.iscsiadm, args);
}
@ -142,7 +142,7 @@ class ISCSI {
"-p",
portal,
"-o",
"delete"
"delete",
]);
await iscsi.exec(options.paths.iscsiadm, args);
},
@ -174,7 +174,7 @@ class ISCSI {
const entries = result.stdout.trim().split("\n");
const sessions = [];
let fields;
entries.forEach(entry => {
entries.forEach((entry) => {
fields = entry.split(" ");
sessions.push({
protocol: entry.split(":")[0],
@ -182,7 +182,7 @@ class ISCSI {
portal: fields[2].split(",")[0],
target_portal_group_tag: fields[2].split(",")[1],
iqn: fields[3].split(":")[0],
target: fields[3].split(":")[1]
target: fields[3].split(":")[1],
});
});
@ -212,6 +212,7 @@ class ISCSI {
return [];
}
let currentTarget;
let sessionGroups = [];
let currentSession = [];
@ -221,13 +222,21 @@ class ISCSI {
entries.shift();
entries.shift();
// this should break up the lines into groups of lines
// where each group is the full details of a single session
// note that the output of the command bundles/groups all sessions
// by target so extra logic is needed to hanle that
// alternatively we could get all sessions using getSessions()
// and then invoke `iscsiadm -m session -P 3 -r <session id>` in a loop
for (let i = 0; i < entries.length; i++) {
let entry = entries[i];
if (entry.startsWith("Target:")) {
currentTarget = entry;
} else if (entry.trim().startsWith("Current Portal:")) {
if (currentSession.length > 0) {
sessionGroups.push(currentSession);
}
currentSession = [entry];
currentSession = [currentTarget, entry];
} else {
currentSession.push(entry);
}
@ -261,11 +270,7 @@ class ISCSI {
.trim()
.replace(/ /g, "_")
.replace(/\W/g, "");
let value = line
.split(":")
.slice(1)
.join(":")
.trim();
let value = line.split(":").slice(1).join(":").trim();
if (currentSection) {
session[currentSection] = session[currentSection] || {};
@ -282,7 +287,7 @@ class ISCSI {
.split(":")
.slice(1)
.join(":")
.trim()
.trim(),
};
while (
sessionLines[j + 1] &&
@ -308,7 +313,7 @@ class ISCSI {
.split(":")
.slice(1)
.join(":")
.trim()
.trim(),
});
}
@ -322,7 +327,7 @@ class ISCSI {
key = key.charAt(0).toLowerCase() + key.slice(1);
key = key.replace(
/[A-Z]/g,
letter => `_${letter.toLowerCase()}`
(letter) => `_${letter.toLowerCase()}`
);
break;
}
@ -367,12 +372,12 @@ class ISCSI {
const entries = result.stdout.trim().split("\n");
const targets = [];
entries.forEach(entry => {
entries.forEach((entry) => {
targets.push({
portal: entry.split(",")[0],
target_portal_group_tag: entry.split(" ")[0].split(",")[1],
iqn: entry.split(" ")[1].split(":")[0],
target: entry.split(" ")[1].split(":")[1]
target: entry.split(" ")[1].split(":")[1],
});
});
@ -432,7 +437,7 @@ class ISCSI {
}
return true;
}
},
};
}
@ -460,15 +465,15 @@ class ISCSI {
}
return new Promise((resolve, reject) => {
child.stdout.on("data", function(data) {
child.stdout.on("data", function (data) {
stdout = stdout + data;
});
child.stderr.on("data", function(data) {
child.stderr.on("data", function (data) {
stderr = stderr + data;
});
child.on("close", function(code) {
child.on("close", function (code) {
const result = { code, stdout, stderr };
if (timeout) {
clearTimeout(timeout);