From c4a36750cdb2febf6941e47d81b2a374c589a230 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Fri, 27 Nov 2020 14:19:13 -0700 Subject: [PATCH] many fixes, support auto-detection of truenas binary paths and apiVersion, fix config typo, better support for potential race conditions with deleting shares --- examples/freenas-iscsi.yaml | 3 + examples/freenas-nfs.yaml | 3 + examples/freenas-smb.yaml | 3 + examples/zfs-generic-iscsi.yaml | 2 +- src/driver/controller-zfs-generic/index.js | 24 +- src/driver/controller-zfs-ssh/index.js | 39 ++-- src/driver/freenas/http/index.js | 2 +- src/driver/freenas/index.js | 247 ++++++++++++++++++--- 8 files changed, 258 insertions(+), 65 deletions(-) diff --git a/examples/freenas-iscsi.yaml b/examples/freenas-iscsi.yaml index 99a83fd..979a9a0 100644 --- a/examples/freenas-iscsi.yaml +++ b/examples/freenas-iscsi.yaml @@ -12,6 +12,7 @@ httpConnection: password: allowInsecure: true # use apiVersion 2 for TrueNAS-12 and up (will work on 11.x in some scenarios as well) + # leave unset for auto-detection #apiVersion: 2 sshConnection: host: server address @@ -28,6 +29,8 @@ zfs: # the example below is useful for TrueNAS 12 #cli: # sudoEnabled: true + # + # leave paths unset for auto-detection # paths: # zfs: /usr/local/sbin/zfs # zpool: /usr/local/sbin/zpool diff --git a/examples/freenas-nfs.yaml b/examples/freenas-nfs.yaml index 141d305..0baa7b2 100644 --- a/examples/freenas-nfs.yaml +++ b/examples/freenas-nfs.yaml @@ -12,6 +12,7 @@ httpConnection: password: allowInsecure: true # use apiVersion 2 for TrueNAS-12 and up (will work on 11.x in some scenarios as well) + # leave unset for auto-detection #apiVersion: 2 sshConnection: host: server address @@ -28,6 +29,8 @@ zfs: # the example below is useful for TrueNAS 12 #cli: # sudoEnabled: true + # + # leave paths unset for auto-detection # paths: # zfs: /usr/local/sbin/zfs # zpool: /usr/local/sbin/zpool diff --git a/examples/freenas-smb.yaml b/examples/freenas-smb.yaml index 6704b11..0e20e89 100644 --- a/examples/freenas-smb.yaml +++ b/examples/freenas-smb.yaml @@ -12,6 +12,7 @@ httpConnection: password: allowInsecure: true # use apiVersion 2 for TrueNAS-12 and up (will work on 11.x in some scenarios as well) + # leave unset for auto-detection #apiVersion: 2 sshConnection: host: server address @@ -28,6 +29,8 @@ zfs: # the example below is useful for TrueNAS 12 #cli: # sudoEnabled: true + # + # leave paths unset for auto-detection # paths: # zfs: /usr/local/sbin/zfs # zpool: /usr/local/sbin/zpool diff --git a/examples/zfs-generic-iscsi.yaml b/examples/zfs-generic-iscsi.yaml index 5a74b55..3a647af 100644 --- a/examples/zfs-generic-iscsi.yaml +++ b/examples/zfs-generic-iscsi.yaml @@ -51,7 +51,7 @@ iscsi: # http://www.linux-iscsi.org/wiki/ISCSI # https://bugzilla.redhat.com/show_bug.cgi?id=1659195 # http://atodorov.org/blog/2015/04/07/how-to-configure-iscsi-target-on-red-hat-enterprise-linux-7/ - shareStragetyTargetCli: + shareStrategyTargetCli: #sudoEnabled: true basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664" tpg: diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index d91bb23..2cb0810 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -26,7 +26,7 @@ class ControllerZfsGenericDriver extends ControllerZfsSshBaseDriver { * @param {*} datasetName */ async createShare(call, datasetName) { - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); const sshClient = this.getSshClient(); let properties; @@ -105,25 +105,25 @@ class ControllerZfsGenericDriver extends ControllerZfsSshBaseDriver { switch (this.options.iscsi.shareStrategy) { case "targetCli": - basename = this.options.iscsi.shareStragetyTargetCli.basename; + basename = this.options.iscsi.shareStrategyTargetCli.basename; let setAttributesText = ""; let setAuthText = ""; - if (this.options.iscsi.shareStragetyTargetCli.tpg) { - if (this.options.iscsi.shareStragetyTargetCli.tpg.attributes) { + if (this.options.iscsi.shareStrategyTargetCli.tpg) { + if (this.options.iscsi.shareStrategyTargetCli.tpg.attributes) { for (const attributeName in this.options.iscsi - .shareStragetyTargetCli.tpg.attributes) { + .shareStrategyTargetCli.tpg.attributes) { const attributeValue = this.options.iscsi - .shareStragetyTargetCli.tpg.attributes[attributeName]; + .shareStrategyTargetCli.tpg.attributes[attributeName]; setAttributesText += "\n"; setAttributesText += `set attribute ${attributeName}=${attributeValue}`; } } - if (this.options.iscsi.shareStragetyTargetCli.tpg.auth) { + if (this.options.iscsi.shareStrategyTargetCli.tpg.auth) { for (const attributeName in this.options.iscsi - .shareStragetyTargetCli.tpg.auth) { + .shareStrategyTargetCli.tpg.auth) { const attributeValue = this.options.iscsi - .shareStragetyTargetCli.tpg.auth[attributeName]; + .shareStrategyTargetCli.tpg.auth[attributeName]; setAttributesText += "\n"; setAttributesText += `set auth ${attributeName}=${attributeValue}`; } @@ -178,7 +178,7 @@ create /backstores/block/${iscsiName} } async deleteShare(call, datasetName) { - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); const sshClient = this.getSshClient(); let response; @@ -210,7 +210,7 @@ create /backstores/block/${iscsiName} iscsiName = iscsiName.toLowerCase(); switch (this.options.iscsi.shareStrategy) { case "targetCli": - basename = this.options.iscsi.shareStragetyTargetCli.basename; + basename = this.options.iscsi.shareStrategyTargetCli.basename; response = await this.targetCliCommand( ` cd /iscsi @@ -267,7 +267,7 @@ delete ${iscsiName} taregetCliCommand.push("|"); taregetCliCommand.push("targetcli"); - if (this.options.iscsi.shareStragetyTargetCli.sudoEnabled) { + if (this.options.iscsi.shareStrategyTargetCli.sudoEnabled) { command = "sudo"; args.unshift("sh"); } diff --git a/src/driver/controller-zfs-ssh/index.js b/src/driver/controller-zfs-ssh/index.js index d0fe409..c32d41d 100644 --- a/src/driver/controller-zfs-ssh/index.js +++ b/src/driver/controller-zfs-ssh/index.js @@ -122,7 +122,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { }); } - getZetabyte() { + async getZetabyte() { const sshClient = this.getSshClient(); const options = {}; options.executor = new ZfsSshProcessManager(sshClient); @@ -130,6 +130,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { if ( this.options.zfs.hasOwnProperty("cli") && + this.options.zfs.cli && this.options.zfs.cli.hasOwnProperty("paths") ) { options.paths = this.options.zfs.cli.paths; @@ -137,20 +138,26 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { if ( this.options.zfs.hasOwnProperty("cli") && + this.options.zfs.cli && this.options.zfs.cli.hasOwnProperty("sudoEnabled") ) { options.sudo = this.getSudoEnabled(); } + if (typeof this.setZetabyteCustomOptions === "function") { + await this.setZetabyteCustomOptions(options); + } + return new Zetabyte(options); } getSudoEnabled() { - return this.options.zfs.cli.sudoEnabled === true; + return this.options.zfs.cli && this.options.zfs.cli.sudoEnabled === true; } - getSudoPath() { - return this.options.zfs.cli.paths.sudo || "/usr/bin/sudo"; + async getSudoPath() { + const zb = await this.getZetabyte(); + return zb.options.paths.sudo || "/usr/bin/sudo"; } getDatasetParentName() { @@ -175,7 +182,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { } async removeSnapshotsFromDatatset(datasetName, options = {}) { - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); await zb.zfs.destroy(datasetName + "@%", options); } @@ -265,7 +272,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { const driver = this; const driverZfsResourceType = this.getDriverZfsResourceType(); const sshClient = this.getSshClient(); - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); let datasetParentName = this.getVolumeParentDatasetName(); let snapshotParentDatasetName = this.getDetachedSnapshotParentDatasetName(); @@ -687,7 +694,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { properties.mountpoint.value, ]); if (this.getSudoEnabled()) { - command = this.getSudoPath() + " " + command; + command = (await this.getSudoPath()) + " " + command; } driver.ctx.logger.verbose("set permission command: %s", command); @@ -710,7 +717,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { properties.mountpoint.value, ]); if (this.getSudoEnabled()) { - command = this.getSudoPath() + " " + command; + command = (await this.getSudoPath()) + " " + command; } driver.ctx.logger.verbose("set ownership command: %s", command); @@ -727,7 +734,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { properties.mountpoint.value, ]); if (this.getSudoEnabled()) { - command = this.getSudoPath() + " " + command; + command = (await this.getSudoPath()) + " " + command; } driver.ctx.logger.verbose("set acl command: %s", command); @@ -799,7 +806,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { */ async DeleteVolume(call) { const driver = this; - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); let datasetParentName = this.getVolumeParentDatasetName(); let name = call.request.volume_id; @@ -904,7 +911,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { async ControllerExpandVolume(call) { const driver = this; const driverZfsResourceType = this.getDriverZfsResourceType(); - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); let datasetParentName = this.getVolumeParentDatasetName(); let name = call.request.volume_id; @@ -1017,7 +1024,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { */ async GetCapacity(call) { const driver = this; - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); let datasetParentName = this.getVolumeParentDatasetName(); @@ -1054,7 +1061,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { async ListVolumes(call) { const driver = this; const driverZfsResourceType = this.getDriverZfsResourceType(); - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); let datasetParentName = this.getVolumeParentDatasetName(); let entries = []; @@ -1239,7 +1246,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { async ListSnapshots(call) { const driver = this; const driverZfsResourceType = this.getDriverZfsResourceType(); - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); let entries = []; let entries_length = 0; @@ -1471,7 +1478,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { async CreateSnapshot(call) { const driver = this; const driverZfsResourceType = this.getDriverZfsResourceType(); - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); let detachedSnapshot = false; try { @@ -1705,7 +1712,7 @@ class ControllerZfsSshBaseDriver extends CsiBaseDriver { */ async DeleteSnapshot(call) { const driver = this; - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); const snapshot_id = call.request.snapshot_id; diff --git a/src/driver/freenas/http/index.js b/src/driver/freenas/http/index.js index 6823c4e..44ffa33 100644 --- a/src/driver/freenas/http/index.js +++ b/src/driver/freenas/http/index.js @@ -4,7 +4,7 @@ const USER_AGENT = "democratic-csi-driver"; class Client { constructor(options = {}) { - this.options = options; + this.options = JSON.parse(JSON.stringify(options)); this.logger = console; // default to v1.0 for now diff --git a/src/driver/freenas/index.js b/src/driver/freenas/index.js index d9a030c..c246a4f 100644 --- a/src/driver/freenas/index.js +++ b/src/driver/freenas/index.js @@ -15,7 +15,6 @@ const FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME = "democratic-csi:freenas_iscsi_targettoextent_id"; const FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME = "democratic-csi:freenas_iscsi_assets_name"; - class FreeNASDriver extends ControllerZfsSshBaseDriver { /** * cannot make this a storage class parameter as storage class/etc context is *not* sent @@ -36,9 +35,30 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { } } - getHttpClient() { + async setZetabyteCustomOptions(options) { + if (!options.hasOwnProperty("paths")) { + const majorMinor = await this.getSystemVersionMajorMinor(); + const isScale = await this.getIsScale(); + if (!isScale && Number(majorMinor) >= 12) { + options.paths = { + zfs: "/usr/local/sbin/zfs", + zpool: "/usr/local/sbin/zpool", + sudo: "/usr/local/bin/sudo", + chroot: "/usr/sbin/chroot", + }; + } + } + } + + async getHttpClient(autoDetectVersion = true) { const client = new HttpClient(this.options.httpConnection); client.logger = this.ctx.logger; + + if (autoDetectVersion && !!!this.options.httpConnection.apiVersion) { + const apiVersion = await this.getApiVersion(); + client.setApiVersion(apiVersion); + } + return client; } @@ -62,7 +82,7 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { if (!match || Object.keys(match).length < 1) { return; } - const httpClient = this.getHttpClient(); + const httpClient = await this.getHttpClient(); let target; let page = 0; @@ -126,9 +146,9 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { */ async createShare(call, datasetName) { const driverShareType = this.getDriverShareType(); - const httpClient = this.getHttpClient(); + const httpClient = await this.getHttpClient(); const apiVersion = httpClient.getApiVersion(); - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); let properties; let endpoint; @@ -1017,19 +1037,22 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { async deleteShare(call, datasetName) { const driverShareType = this.getDriverShareType(); - const httpClient = this.getHttpClient(); + const httpClient = await this.getHttpClient(); const apiVersion = httpClient.getApiVersion(); - const zb = this.getZetabyte(); + const zb = await this.getZetabyte(); let properties; let response; let endpoint; let shareId; + let deleteAsset; + let sharePaths; switch (driverShareType) { case "nfs": try { properties = await zb.zfs.get(datasetName, [ + "mountpoint", FREENAS_NFS_SHARE_PROPERTY_NAME, ]); } catch (err) { @@ -1063,18 +1086,33 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { // assume share is gone for now if ([404, 500].includes(response.statusCode)) { } else { - response = await httpClient.delete(endpoint); + switch (apiVersion) { + case 1: + sharePaths = response.body.nfs_paths; + break; + case 2: + sharePaths = response.body.paths; + break; + } - // returns a 500 if does not exist - // v1 = 204 - // v2 = 200 - if (![200, 204].includes(response.statusCode)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting nfs share - share: ${shareId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); + deleteAsset = sharePaths.some((value) => { + return value == properties.mountpoint.value; + }); + + if (deleteAsset) { + response = await httpClient.delete(endpoint); + + // returns a 500 if does not exist + // v1 = 204 + // v2 = 200 + if (![200, 204].includes(response.statusCode)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting nfs share - share: ${shareId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } } } break; @@ -1089,6 +1127,7 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { case "smb": try { properties = await zb.zfs.get(datasetName, [ + "mountpoint", FREENAS_SMB_SHARE_PROPERTY_NAME, ]); } catch (err) { @@ -1125,18 +1164,33 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { // assume share is gone for now if ([404, 500].includes(response.statusCode)) { } else { - response = await httpClient.delete(endpoint); + switch (apiVersion) { + case 1: + sharePaths = [response.body.cifs_path]; + break; + case 2: + sharePaths = [response.body.path]; + break; + } - // returns a 500 if does not exist - // v1 = 204 - // v2 = 200 - if (![200, 204].includes(response.statusCode)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting smb share - share: ${shareId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); + deleteAsset = sharePaths.some((value) => { + return value == properties.mountpoint.value; + }); + + if (deleteAsset) { + response = await httpClient.delete(endpoint); + + // returns a 500 if does not exist + // v1 = 204 + // v2 = 200 + if (![200, 204].includes(response.statusCode)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting smb share - share: ${shareId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } } } break; @@ -1175,7 +1229,6 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { let iscsiName = properties[FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME].value; let assetName; - let deleteAsset; switch (apiVersion) { case 1: @@ -1310,10 +1363,18 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { switch (driverShareType) { case "iscsi": - this.ctx.logger.verbose("FreeNAS reloading ctld"); - await sshClient.exec( - sshClient.buildCommand("/etc/rc.d/ctld", ["reload"]) - ); + const isScale = this.getIsScale(); + if (isScale) { + this.ctx.logger.verbose("FreeNAS reloading scst"); + await sshClient.exec( + sshClient.buildCommand("systemctl", ["reload", "scst"]) + ); + } else { + this.ctx.logger.verbose("FreeNAS reloading ctld"); + await sshClient.exec( + sshClient.buildCommand("/etc/rc.d/ctld", ["reload"]) + ); + } break; } } @@ -1321,11 +1382,120 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { async getApiVersion() { const systemVersion = await this.getSystemVersion(); + if (systemVersion.v2) { + return 2; + } + return 1; } + async getIsFreeNAS() { + const systemVersion = await this.getSystemVersion(); + let version; + + if (systemVersion.v2) { + version = systemVersion.v2; + } else { + version = systemVersion.v1.fullversion; + } + + if (version.toLowerCase().includes("freenas")) { + return true; + } + + return false; + } + + async getIsTrueNAS() { + const systemVersion = await this.getSystemVersion(); + let version; + + if (systemVersion.v2) { + version = systemVersion.v2; + } else { + version = systemVersion.v1.fullversion; + } + + if (version.toLowerCase().includes("truenas")) { + return true; + } + + return false; + } + + async getIsScale() { + const systemVersion = await this.getSystemVersion(); + + if (systemVersion.v2 && systemVersion.v2.toLowerCase().includes("scale")) { + return true; + } + + return false; + } + + async getSystemVersionMajorMinor() { + const systemVersion = await this.getSystemVersion(); + let parts; + let parts_i; + let version; + + /* + systemVersion.v2 = "FreeNAS-11.2-U5"; + systemVersion.v2 = "TrueNAS-SCALE-20.11-MASTER-20201127-092915"; + systemVersion.v1 = { + fullversion: "FreeNAS-9.3-STABLE-201503200528", + fullversion: "FreeNAS-11.2-U5 (c129415c52)", + }; + + systemVersion.v2 = null; + */ + + if (systemVersion.v2) { + version = systemVersion.v2; + } else { + version = systemVersion.v1.fullversion; + } + + if (version) { + parts = version.split("-"); + parts_i = []; + parts.forEach((value) => { + let i = value.replace(/[^\d.]/g, ""); + if (i.length > 0) { + parts_i.push(i); + } + }); + + // join and resplit to deal with single elements which contain a decimal + parts_i = parts_i.join(".").split("."); + parts_i.splice(2); + return parts_i.join("."); + } + } + + async getSystemVersionMajor() { + const majorMinor = await this.getSystemVersionMajorMinor(); + return majorMinor.split(".")[0]; + } + + async setVersionInfoCache(versionInfo) { + const driver = this; + this.cache = this.cache || {}; + this.cache.versionInfo = versionInfo; + + // crude timeout + setTimeout(function () { + driver.cache.versionInfo = null; + }, 60 * 1000); + } + async getSystemVersion() { - const httpClient = this.getHttpClient(); + this.cache = this.cache || {}; + if (this.cache.versionInfo) { + return this.cache.versionInfo; + } + + const httpClient = await this.getHttpClient(false); const endpoint = "/system/version/"; let response; const startApiVersion = httpClient.getApiVersion(); @@ -1334,12 +1504,18 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { httpClient.setApiVersion(2); /** * FreeNAS-11.2-U5 + * TrueNAS-12.0-RELEASE + * TrueNAS-SCALE-20.11-MASTER-20201127-092915 */ try { response = await httpClient.get(endpoint); if (response.statusCode == 200) { versionInfo.v2 = response.body; } + + // return immediately to save on resources and silly requests + await this.setVersionInfoCache(versionInfo); + return versionInfo; } catch (e) {} httpClient.setApiVersion(1); @@ -1357,6 +1533,7 @@ class FreeNASDriver extends ControllerZfsSshBaseDriver { // reset apiVersion httpClient.setApiVersion(startApiVersion); + await this.setVersionInfoCache(versionInfo); return versionInfo; } }