diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1a0d09d..16cdb78 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -125,6 +125,7 @@ jobs: config: - truenas/scale/25.10/scale-iscsi.yaml - truenas/scale/25.10/scale-nfs.yaml + - truenas/scale/25.10/scale-nvmeof.yaml # 80 char limit - truenas/scale/25.10/scale-smb.yaml runs-on: diff --git a/Dockerfile b/Dockerfile index 5d70bfa..28bab3d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ###################### # golang builder ###################### -FROM golang:1.25.3-bookworm as ctrbuilder +FROM golang:1.25.3-bookworm AS ctrbuilder # /go/containerd/ctr ADD docker/ctr-mount-labels.diff /tmp @@ -61,6 +61,7 @@ WORKDIR /home/csi/app USER csi # prevent need to build re2 module +# https://github.com/uhop/install-artifact-from-github/wiki/Making-local-mirror ENV RE2_DOWNLOAD_MIRROR="https://grpc-uds-binaries.s3-us-west-2.amazonaws.com/re2" ENV RE2_DOWNLOAD_SKIP_PATH=1 diff --git a/ci/configs/truenas/scale/25.10/scale-nvmeof.yaml b/ci/configs/truenas/scale/25.10/scale-nvmeof.yaml new file mode 100644 index 0000000..34c5828 --- /dev/null +++ b/ci/configs/truenas/scale/25.10/scale-nvmeof.yaml @@ -0,0 +1,32 @@ +driver: freenas-api-nvmeof + +httpConnection: + protocol: http + host: ${TRUENAS_HOST} + port: 80 + #apiKey: + username: ${TRUENAS_USERNAME} + password: ${TRUENAS_PASSWORD} + +zfs: + datasetParentName: tank/ci/${CI_BUILD_KEY}/v + detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s + + zvolCompression: + zvolDedup: + zvolEnableReservation: false + zvolBlocksize: + +nvmeof: + transports: + - tcp://${TRUENAS_HOST}:4420 + namePrefix: "csi-ci-${CI_BUILD_KEY}-" + ports: + - 1 + +# https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 +_private: + csi: + volume: + idHash: + strategy: crc16 diff --git a/src/driver/factory.js b/src/driver/factory.js index 4c8fd23..da6874e 100644 --- a/src/driver/factory.js +++ b/src/driver/factory.js @@ -24,13 +24,20 @@ function factory(ctx, options) { case "freenas-nfs": case "freenas-smb": case "freenas-iscsi": + case "freenas-nvmeof": case "truenas-nfs": case "truenas-smb": case "truenas-iscsi": + case "truenas-nvmeof": return new FreeNASSshDriver(ctx, options); - case "freenas-api-iscsi": case "freenas-api-nfs": case "freenas-api-smb": + case "freenas-api-iscsi": + case "freenas-api-nvmeof": + case "truenas-api-nfs": + case "truenas-api-smb": + case "truenas-api-iscsi": + case "truenas-api-nvmeof": return new FreeNASApiDriver(ctx, options); case "synology-nfs": case "synology-smb": diff --git a/src/driver/freenas/api.js b/src/driver/freenas/api.js index 1e83022..65cf970 100644 --- a/src/driver/freenas/api.js +++ b/src/driver/freenas/api.js @@ -13,6 +13,8 @@ const semver = require("semver"); // freenas properties const FREENAS_NFS_SHARE_PROPERTY_NAME = "democratic-csi:freenas_nfs_share_id"; const FREENAS_SMB_SHARE_PROPERTY_NAME = "democratic-csi:freenas_smb_share_id"; + +// iscsi const FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME = "democratic-csi:freenas_iscsi_target_id"; const FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME = @@ -22,6 +24,14 @@ const FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME = const FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME = "democratic-csi:freenas_iscsi_assets_name"; +// nvmeof +const FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME = + "democratic-csi:freenas_nvmeof_subsystem_id"; +const FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME = + "democratic-csi:freenas_nvmeof_namespace_id"; +const FREENAS_NVMEOF_ASSETS_NAME_PROPERTY_NAME = + "democratic-csi:freenas_nvmeof_assets_name"; + // zfs common properties const MANAGED_PROPERTY_NAME = "democratic-csi:managed_resource"; const SUCCESS_PROPERTY_NAME = "democratic-csi:provision_success"; @@ -200,22 +210,572 @@ class FreeNASApiDriver extends CsiBaseDriver { switch (driverShareType) { case "nfs": - properties = await httpApiClient.DatasetGet(datasetName, [ - "mountpoint", - FREENAS_NFS_SHARE_PROPERTY_NAME, - ]); - this.ctx.logger.debug("zfs props data: %j", properties); + { + properties = await httpApiClient.DatasetGet(datasetName, [ + "mountpoint", + FREENAS_NFS_SHARE_PROPERTY_NAME, + ]); + this.ctx.logger.debug("zfs props data: %j", properties); - // create nfs share - if ( - !zb.helpers.isPropertyValueSet( - properties[FREENAS_NFS_SHARE_PROPERTY_NAME].value - ) - ) { - let nfsShareComment; - if (this.options.nfs.shareCommentTemplate) { - nfsShareComment = Handlebars.compile( - this.options.nfs.shareCommentTemplate + // create nfs share + if ( + !zb.helpers.isPropertyValueSet( + properties[FREENAS_NFS_SHARE_PROPERTY_NAME].value + ) + ) { + let nfsShareComment; + if (this.options.nfs.shareCommentTemplate) { + nfsShareComment = Handlebars.compile( + this.options.nfs.shareCommentTemplate + )({ + name: call.request.name, + parameters: call.request.parameters, + csi: { + name: this.ctx.args.csiName, + version: this.ctx.args.csiVersion, + }, + zfs: { + datasetName: datasetName, + }, + }); + } else { + nfsShareComment = `democratic-csi (${this.ctx.args.csiName}): ${datasetName}`; + } + + switch (apiVersion) { + case 1: + case 2: + switch (apiVersion) { + case 1: + share = { + nfs_paths: [properties.mountpoint.value], + nfs_comment: nfsShareComment || "", + nfs_network: + this.options.nfs.shareAllowedNetworks.join(","), + nfs_hosts: this.options.nfs.shareAllowedHosts.join(","), + nfs_alldirs: this.options.nfs.shareAlldirs, + nfs_ro: false, + nfs_quiet: false, + nfs_maproot_user: this.options.nfs.shareMaprootUser, + nfs_maproot_group: this.options.nfs.shareMaprootGroup, + nfs_mapall_user: this.options.nfs.shareMapallUser, + nfs_mapall_group: this.options.nfs.shareMapallGroup, + nfs_security: [], + }; + break; + case 2: + share = { + paths: [properties.mountpoint.value], + comment: nfsShareComment || "", + networks: this.options.nfs.shareAllowedNetworks, + hosts: this.options.nfs.shareAllowedHosts, + alldirs: this.options.nfs.shareAlldirs, + ro: false, + quiet: false, + maproot_user: this.options.nfs.shareMaprootUser, + maproot_group: this.options.nfs.shareMaprootGroup, + mapall_user: this.options.nfs.shareMapallUser, + mapall_group: this.options.nfs.shareMapallGroup, + security: [], + }; + break; + } + + if (isScale && semver.satisfies(truenasVersion, ">=23.10")) { + delete share.quiet; + delete share.nfs_quiet; + } + + if (isScale && semver.satisfies(truenasVersion, ">=22.12")) { + share.path = share.paths[0]; + delete share.paths; + delete share.alldirs; + } + + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.post("/sharing/nfs", share); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); + + /** + * v1 = 201 + * v2 = 200 + */ + if ([200, 201].includes(response.statusCode)) { + let sharePaths; + switch (apiVersion) { + case 1: + sharePaths = response.body.nfs_paths; + break; + case 2: + if (response.body.path) { + sharePaths = [response.body.path]; + } else { + sharePaths = response.body.paths; + } + break; + } + + // FreeNAS responding with bad data + if (!sharePaths.includes(properties.mountpoint.value)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS responded with incorrect share data: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + //set zfs property + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_NFS_SHARE_PROPERTY_NAME]: response.body.id, + }); + } else { + /** + * v1 = 409 + * v2 = 422 + */ + if ( + [409, 422].includes(response.statusCode) && + (JSON.stringify(response.body).includes( + "You can't share same filesystem with all hosts twice." + ) || + JSON.stringify(response.body).includes( + "Another NFS share already exports this dataset for some network" + )) + ) { + let lookupShare = + await httpApiClient.findResourceByProperties( + "/sharing/nfs", + (item) => { + if ( + (item.nfs_paths && + item.nfs_paths.includes( + properties.mountpoint.value + )) || + (item.paths && + item.paths.includes( + properties.mountpoint.value + )) || + (item.path && + item.path == properties.mountpoint.value) + ) { + return true; + } + return false; + } + ); + + if (!lookupShare) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS failed to find matching share` + ); + } + + //set zfs property + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_NFS_SHARE_PROPERTY_NAME]: lookupShare.id, + }); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating nfs share - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } + } + + volume_context = { + node_attach_driver: "nfs", + server: this.options.nfs.shareHost, + share: properties.mountpoint.value, + }; + return volume_context; + } + break; + /** + * TODO: smb need to be more defensive like iscsi and nfs + * ensuring the path is valid and the shareName + */ + case "smb": + { + properties = await httpApiClient.DatasetGet(datasetName, [ + "mountpoint", + FREENAS_SMB_SHARE_PROPERTY_NAME, + ]); + this.ctx.logger.debug("zfs props data: %j", properties); + + let smbName; + + if (this.options.smb.nameTemplate) { + smbName = Handlebars.compile(this.options.smb.nameTemplate)({ + name: call.request.name, + parameters: call.request.parameters, + }); + } else { + smbName = zb.helpers.extractLeafName(datasetName); + } + + if (this.options.smb.namePrefix) { + smbName = this.options.smb.namePrefix + smbName; + } + + if (this.options.smb.nameSuffix) { + smbName += this.options.smb.nameSuffix; + } + + smbName = smbName.toLowerCase(); + + this.ctx.logger.info( + "FreeNAS creating smb share with name: " + smbName + ); + + // create smb share + if ( + !zb.helpers.isPropertyValueSet( + properties[FREENAS_SMB_SHARE_PROPERTY_NAME].value + ) + ) { + /** + * The only required parameters are: + * - path + * - name + * + * Note that over time it appears the list of available parameters has increased + * so in an effort to best support old versions of FreeNAS we should check the + * presense of each parameter in the config and set the corresponding parameter in + * the API request *only* if present in the config. + */ + switch (apiVersion) { + case 1: + case 2: + share = { + name: smbName, + path: properties.mountpoint.value, + }; + + let propertyMapping = { + shareAuxiliaryConfigurationTemplate: "auxsmbconf", + shareHome: "home", + shareAllowedHosts: "hostsallow", + shareDeniedHosts: "hostsdeny", + shareDefaultPermissions: "default_permissions", + shareGuestOk: "guestok", + shareGuestOnly: "guestonly", + shareShowHiddenFiles: "showhiddenfiles", + shareRecycleBin: "recyclebin", + shareBrowsable: "browsable", + shareAccessBasedEnumeration: "abe", + shareTimeMachine: "timemachine", + shareStorageTask: "storage_task", + }; + + for (const key in propertyMapping) { + if (this.options.smb.hasOwnProperty(key)) { + let value; + switch (key) { + case "shareAuxiliaryConfigurationTemplate": + value = Handlebars.compile( + this.options.smb.shareAuxiliaryConfigurationTemplate + )({ + name: call.request.name, + parameters: call.request.parameters, + }); + break; + default: + value = this.options.smb[key]; + break; + } + share[propertyMapping[key]] = value; + } + } + + if (isScale && semver.satisfies(truenasVersion, ">=25.10")) { + let topLevelProperties = [ + "purpose", + "name", + "path", + "enabled", + "comment", + "readonly", + "browsable", + "access_based_share_enumeration", + "audit", + ]; + let disallowedOptions = ["abe"]; + share.purpose = "LEGACY_SHARE"; + share.options = { + purpose: "LEGACY_SHARE", + }; + for (const key in share) { + switch (key) { + case "options": + // ignore + break; + default: + if (!topLevelProperties.includes(key)) { + if (!disallowedOptions.includes(key)) { + share.options[key] = share[key]; + } + delete share[key]; + } + break; + } + } + } + + switch (apiVersion) { + case 1: + endpoint = "/sharing/cifs"; + + // rename keys with cifs_ prefix + for (const key in share) { + share["cifs_" + key] = share[key]; + delete share[key]; + } + + // convert to comma-separated list + if (share.cifs_hostsallow) { + share.cifs_hostsallow = share.cifs_hostsallow.join(","); + } + + // convert to comma-separated list + if (share.cifs_hostsdeny) { + share.cifs_hostsdeny = share.cifs_hostsdeny.join(","); + } + break; + case 2: + endpoint = "/sharing/smb"; + break; + } + + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.post(endpoint, share); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); + + /** + * v1 = 201 + * v2 = 200 + */ + if ([200, 201].includes(response.statusCode)) { + share = response.body; + let sharePath; + let shareName; + switch (apiVersion) { + case 1: + sharePath = response.body.cifs_path; + shareName = response.body.cifs_name; + break; + case 2: + sharePath = response.body.path; + shareName = response.body.name; + break; + } + + if (shareName != smbName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS responded with incorrect share data: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + if (sharePath != properties.mountpoint.value) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS responded with incorrect share data: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + //set zfs property + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_SMB_SHARE_PROPERTY_NAME]: response.body.id, + }); + } else { + /** + * v1 = 409 + * v2 = 422 + */ + if ( + [409, 422].includes(response.statusCode) && + JSON.stringify(response.body).includes( + "A share with this name already exists." + ) + ) { + let lookupShare = + await httpApiClient.findResourceByProperties( + endpoint, + (item) => { + if ( + (item.cifs_path && + item.cifs_path == properties.mountpoint.value && + item.cifs_name && + item.cifs_name == smbName) || + (item.path && + item.path == properties.mountpoint.value && + item.name && + item.name == smbName) + ) { + return true; + } + return false; + } + ); + + if (!lookupShare) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS failed to find matching share` + ); + } + + //set zfs property + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_SMB_SHARE_PROPERTY_NAME]: lookupShare.id, + }); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating smb share - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } + } + + volume_context = { + node_attach_driver: "smb", + server: this.options.smb.shareHost, + share: smbName, + }; + return volume_context; + } + break; + case "iscsi": + { + properties = await httpApiClient.DatasetGet(datasetName, [ + FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME, + FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME, + FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME, + ]); + this.ctx.logger.debug("zfs props data: %j", properties); + + let basename; + let iscsiName; + + if (this.options.iscsi.nameTemplate) { + iscsiName = Handlebars.compile(this.options.iscsi.nameTemplate)({ + name: call.request.name, + parameters: call.request.parameters, + }); + } else { + iscsiName = zb.helpers.extractLeafName(datasetName); + } + + if (this.options.iscsi.namePrefix) { + iscsiName = this.options.iscsi.namePrefix + iscsiName; + } + + if (this.options.iscsi.nameSuffix) { + iscsiName += this.options.iscsi.nameSuffix; + } + + // According to RFC3270, 'Each iSCSI node, whether an initiator or target, MUST have an iSCSI name. Initiators and targets MUST support the receipt of iSCSI names of up to the maximum length of 223 bytes.' + // https://kb.netapp.com/Advice_and_Troubleshooting/Miscellaneous/What_is_the_maximum_length_of_a_iSCSI_iqn_name + // https://tools.ietf.org/html/rfc3720 + // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 + iscsiName = iscsiName.toLowerCase(); + + let extentDiskName = "zvol/" + datasetName; + let maxZvolNameLength = await driver.getMaxZvolNameLength(); + driver.ctx.logger.debug( + "max zvol name length: %s", + maxZvolNameLength + ); + + /** + * limit is a FreeBSD limitation + * https://www.ixsystems.com/documentation/freenas/11.2-U5/storage.html#zfs-zvol-config-opts-tab + */ + if (extentDiskName.length > maxZvolNameLength) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `extent disk name cannot exceed ${maxZvolNameLength} characters: ${extentDiskName}` + ); + } + + // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 + if (isScale && iscsiName.length > 64) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `extent name cannot exceed 64 characters: ${iscsiName}` + ); + } + + this.ctx.logger.info( + "FreeNAS creating iscsi assets with name: " + iscsiName + ); + + let extentComment; + if (this.options.iscsi.extentCommentTemplate) { + extentComment = Handlebars.compile( + this.options.iscsi.extentCommentTemplate )({ name: call.request.name, parameters: call.request.parameters, @@ -228,1178 +788,810 @@ class FreeNASApiDriver extends CsiBaseDriver { }, }); } else { - nfsShareComment = `democratic-csi (${this.ctx.args.csiName}): ${datasetName}`; + extentComment = ""; } - switch (apiVersion) { - case 1: - case 2: - switch (apiVersion) { - case 1: - share = { - nfs_paths: [properties.mountpoint.value], - nfs_comment: nfsShareComment || "", - nfs_network: - this.options.nfs.shareAllowedNetworks.join(","), - nfs_hosts: this.options.nfs.shareAllowedHosts.join(","), - nfs_alldirs: this.options.nfs.shareAlldirs, - nfs_ro: false, - nfs_quiet: false, - nfs_maproot_user: this.options.nfs.shareMaprootUser, - nfs_maproot_group: this.options.nfs.shareMaprootGroup, - nfs_mapall_user: this.options.nfs.shareMapallUser, - nfs_mapall_group: this.options.nfs.shareMapallGroup, - nfs_security: [], - }; - break; - case 2: - share = { - paths: [properties.mountpoint.value], - comment: nfsShareComment || "", - networks: this.options.nfs.shareAllowedNetworks, - hosts: this.options.nfs.shareAllowedHosts, - alldirs: this.options.nfs.shareAlldirs, - ro: false, - quiet: false, - maproot_user: this.options.nfs.shareMaprootUser, - maproot_group: this.options.nfs.shareMaprootGroup, - mapall_user: this.options.nfs.shareMapallUser, - mapall_group: this.options.nfs.shareMapallGroup, - security: [], - }; - break; - } - - if (isScale && semver.satisfies(truenasVersion, ">=23.10")) { - delete share.quiet; - delete share.nfs_quiet; - } - - if (isScale && semver.satisfies(truenasVersion, ">=22.12")) { - share.path = share.paths[0]; - delete share.paths; - delete share.alldirs; - } - - response = await GeneralUtils.retry( - 3, - 1000, - async () => { - return await httpClient.post("/sharing/nfs", share); - }, - { - retryCondition: (err) => { - if (err.code == "ECONNRESET") { - return true; - } - if (err.code == "ECONNABORTED") { - return true; - } - if (err.response && err.response.statusCode == 504) { - return true; - } - return false; - }, - } - ); - - /** - * v1 = 201 - * v2 = 200 - */ - if ([200, 201].includes(response.statusCode)) { - let sharePaths; - switch (apiVersion) { - case 1: - sharePaths = response.body.nfs_paths; - break; - case 2: - if (response.body.path) { - sharePaths = [response.body.path]; - } else { - sharePaths = response.body.paths; - } - break; - } - - // FreeNAS responding with bad data - if (!sharePaths.includes(properties.mountpoint.value)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS responded with incorrect share data: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - //set zfs property - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_NFS_SHARE_PROPERTY_NAME]: response.body.id, - }); - } else { - /** - * v1 = 409 - * v2 = 422 - */ - if ( - [409, 422].includes(response.statusCode) && - (JSON.stringify(response.body).includes( - "You can't share same filesystem with all hosts twice." - ) || - JSON.stringify(response.body).includes( - "Another NFS share already exports this dataset for some network" - )) - ) { - let lookupShare = - await httpApiClient.findResourceByProperties( - "/sharing/nfs", - (item) => { - if ( - (item.nfs_paths && - item.nfs_paths.includes( - properties.mountpoint.value - )) || - (item.paths && - item.paths.includes(properties.mountpoint.value)) || - (item.path && - item.path == properties.mountpoint.value) - ) { - return true; - } - return false; - } - ); - - if (!lookupShare) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS failed to find matching share` - ); - } - - //set zfs property - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_NFS_SHARE_PROPERTY_NAME]: lookupShare.id, - }); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating nfs share - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } - break; - default: - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` - ); - } - } - - volume_context = { - node_attach_driver: "nfs", - server: this.options.nfs.shareHost, - share: properties.mountpoint.value, - }; - return volume_context; - - break; - /** - * TODO: smb need to be more defensive like iscsi and nfs - * ensuring the path is valid and the shareName - */ - case "smb": - properties = await httpApiClient.DatasetGet(datasetName, [ - "mountpoint", - FREENAS_SMB_SHARE_PROPERTY_NAME, - ]); - this.ctx.logger.debug("zfs props data: %j", properties); - - let smbName; - - if (this.options.smb.nameTemplate) { - smbName = Handlebars.compile(this.options.smb.nameTemplate)({ - name: call.request.name, - parameters: call.request.parameters, - }); - } else { - smbName = zb.helpers.extractLeafName(datasetName); - } - - if (this.options.smb.namePrefix) { - smbName = this.options.smb.namePrefix + smbName; - } - - if (this.options.smb.nameSuffix) { - smbName += this.options.smb.nameSuffix; - } - - smbName = smbName.toLowerCase(); - - this.ctx.logger.info( - "FreeNAS creating smb share with name: " + smbName - ); - - // create smb share - if ( - !zb.helpers.isPropertyValueSet( - properties[FREENAS_SMB_SHARE_PROPERTY_NAME].value + const extentInsecureTpc = this.options.iscsi.hasOwnProperty( + "extentInsecureTpc" ) - ) { - /** - * The only required parameters are: - * - path - * - name - * - * Note that over time it appears the list of available parameters has increased - * so in an effort to best support old versions of FreeNAS we should check the - * presense of each parameter in the config and set the corresponding parameter in - * the API request *only* if present in the config. - */ - switch (apiVersion) { - case 1: - case 2: - share = { - name: smbName, - path: properties.mountpoint.value, - }; - - let propertyMapping = { - shareAuxiliaryConfigurationTemplate: "auxsmbconf", - shareHome: "home", - shareAllowedHosts: "hostsallow", - shareDeniedHosts: "hostsdeny", - shareDefaultPermissions: "default_permissions", - shareGuestOk: "guestok", - shareGuestOnly: "guestonly", - shareShowHiddenFiles: "showhiddenfiles", - shareRecycleBin: "recyclebin", - shareBrowsable: "browsable", - shareAccessBasedEnumeration: "abe", - shareTimeMachine: "timemachine", - shareStorageTask: "storage_task", - }; - - for (const key in propertyMapping) { - if (this.options.smb.hasOwnProperty(key)) { - let value; - switch (key) { - case "shareAuxiliaryConfigurationTemplate": - value = Handlebars.compile( - this.options.smb.shareAuxiliaryConfigurationTemplate - )({ - name: call.request.name, - parameters: call.request.parameters, - }); - break; - default: - value = this.options.smb[key]; - break; - } - share[propertyMapping[key]] = value; - } - } - - if (isScale && semver.satisfies(truenasVersion, ">=25.10")) { - let topLevelProperties = [ - "purpose", - "name", - "path", - "enabled", - "comment", - "readonly", - "browsable", - "access_based_share_enumeration", - "audit", - ]; - let disallowedOptions = ["abe"]; - share.purpose = "LEGACY_SHARE"; - share.options = { - purpose: "LEGACY_SHARE", - }; - for (const key in share) { - switch (key) { - case "options": - // ignore - break; - default: - if (!topLevelProperties.includes(key)) { - if (!disallowedOptions.includes(key)) { - share.options[key] = share[key]; - } - delete share[key]; - } - break; - } - } - } - - switch (apiVersion) { - case 1: - endpoint = "/sharing/cifs"; - - // rename keys with cifs_ prefix - for (const key in share) { - share["cifs_" + key] = share[key]; - delete share[key]; - } - - // convert to comma-separated list - if (share.cifs_hostsallow) { - share.cifs_hostsallow = share.cifs_hostsallow.join(","); - } - - // convert to comma-separated list - if (share.cifs_hostsdeny) { - share.cifs_hostsdeny = share.cifs_hostsdeny.join(","); - } - break; - case 2: - endpoint = "/sharing/smb"; - break; - } - - response = await GeneralUtils.retry( - 3, - 1000, - async () => { - return await httpClient.post(endpoint, share); - }, - { - retryCondition: (err) => { - if (err.code == "ECONNRESET") { - return true; - } - if (err.code == "ECONNABORTED") { - return true; - } - if (err.response && err.response.statusCode == 504) { - return true; - } - return false; - }, - } - ); - - /** - * v1 = 201 - * v2 = 200 - */ - if ([200, 201].includes(response.statusCode)) { - share = response.body; - let sharePath; - let shareName; - switch (apiVersion) { - case 1: - sharePath = response.body.cifs_path; - shareName = response.body.cifs_name; - break; - case 2: - sharePath = response.body.path; - shareName = response.body.name; - break; - } - - if (shareName != smbName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS responded with incorrect share data: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - if (sharePath != properties.mountpoint.value) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS responded with incorrect share data: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - //set zfs property - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_SMB_SHARE_PROPERTY_NAME]: response.body.id, - }); - } else { - /** - * v1 = 409 - * v2 = 422 - */ - if ( - [409, 422].includes(response.statusCode) && - JSON.stringify(response.body).includes( - "A share with this name already exists." - ) - ) { - let lookupShare = - await httpApiClient.findResourceByProperties( - endpoint, - (item) => { - if ( - (item.cifs_path && - item.cifs_path == properties.mountpoint.value && - item.cifs_name && - item.cifs_name == smbName) || - (item.path && - item.path == properties.mountpoint.value && - item.name && - item.name == smbName) - ) { - return true; - } - return false; - } - ); - - if (!lookupShare) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS failed to find matching share` - ); - } - - //set zfs property - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_SMB_SHARE_PROPERTY_NAME]: lookupShare.id, - }); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating smb share - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } - break; - default: - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` - ); - } - } - - volume_context = { - node_attach_driver: "smb", - server: this.options.smb.shareHost, - share: smbName, - }; - return volume_context; - - break; - case "iscsi": - properties = await httpApiClient.DatasetGet(datasetName, [ - FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME, - FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME, - FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME, - ]); - this.ctx.logger.debug("zfs props data: %j", properties); - - let basename; - let iscsiName; - - if (this.options.iscsi.nameTemplate) { - iscsiName = Handlebars.compile(this.options.iscsi.nameTemplate)({ - name: call.request.name, - parameters: call.request.parameters, - }); - } else { - iscsiName = zb.helpers.extractLeafName(datasetName); - } - - if (this.options.iscsi.namePrefix) { - iscsiName = this.options.iscsi.namePrefix + iscsiName; - } - - if (this.options.iscsi.nameSuffix) { - iscsiName += this.options.iscsi.nameSuffix; - } - - // According to RFC3270, 'Each iSCSI node, whether an initiator or target, MUST have an iSCSI name. Initiators and targets MUST support the receipt of iSCSI names of up to the maximum length of 223 bytes.' - // https://kb.netapp.com/Advice_and_Troubleshooting/Miscellaneous/What_is_the_maximum_length_of_a_iSCSI_iqn_name - // https://tools.ietf.org/html/rfc3720 - // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 - iscsiName = iscsiName.toLowerCase(); - - let extentDiskName = "zvol/" + datasetName; - let maxZvolNameLength = await driver.getMaxZvolNameLength(); - driver.ctx.logger.debug("max zvol name length: %s", maxZvolNameLength); - - /** - * limit is a FreeBSD limitation - * https://www.ixsystems.com/documentation/freenas/11.2-U5/storage.html#zfs-zvol-config-opts-tab - */ - if (extentDiskName.length > maxZvolNameLength) { - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `extent disk name cannot exceed ${maxZvolNameLength} characters: ${extentDiskName}` - ); - } - - // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 - if (isScale && iscsiName.length > 64) { - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `extent name cannot exceed 64 characters: ${iscsiName}` - ); - } - - this.ctx.logger.info( - "FreeNAS creating iscsi assets with name: " + iscsiName - ); - - let extentComment; - if (this.options.iscsi.extentCommentTemplate) { - extentComment = Handlebars.compile( - this.options.iscsi.extentCommentTemplate - )({ - name: call.request.name, - parameters: call.request.parameters, - csi: { - name: this.ctx.args.csiName, - version: this.ctx.args.csiVersion, - }, - zfs: { - datasetName: datasetName, - }, - }); - } else { - extentComment = ""; - } - - const extentInsecureTpc = this.options.iscsi.hasOwnProperty( - "extentInsecureTpc" - ) - ? this.options.iscsi.extentInsecureTpc - : true; - - const extentXenCompat = this.options.iscsi.hasOwnProperty( - "extentXenCompat" - ) - ? this.options.iscsi.extentXenCompat - : false; - - const extentBlocksize = this.options.iscsi.hasOwnProperty( - "extentBlocksize" - ) - ? this.options.iscsi.extentBlocksize - : 512; - - const extentDisablePhysicalBlocksize = - this.options.iscsi.hasOwnProperty("extentDisablePhysicalBlocksize") - ? this.options.iscsi.extentDisablePhysicalBlocksize + ? this.options.iscsi.extentInsecureTpc : true; - const extentRpm = this.options.iscsi.hasOwnProperty("extentRpm") - ? this.options.iscsi.extentRpm - : "SSD"; - - let extentAvailThreshold = this.options.iscsi.hasOwnProperty( - "extentAvailThreshold" - ) - ? Number(this.options.iscsi.extentAvailThreshold) - : null; - - if (!(extentAvailThreshold > 0 && extentAvailThreshold <= 100)) { - extentAvailThreshold = null; - } - - switch (apiVersion) { - case 1: - response = await httpClient.get( - "/services/iscsi/globalconfiguration" - ); - if (response.statusCode != 200) { - throw new GrpcError( - grpc.status.UNKNOWN, - `error getting iscsi configuration - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - basename = response.body.iscsi_basename; - this.ctx.logger.verbose("FreeNAS ISCSI BASENAME: " + basename); - break; - case 2: - response = await httpClient.get("/iscsi/global"); - if (response.statusCode != 200) { - throw new GrpcError( - grpc.status.UNKNOWN, - `error getting iscsi configuration - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - basename = response.body.basename; - this.ctx.logger.verbose("FreeNAS ISCSI BASENAME: " + basename); - break; - default: - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` - ); - } - - // if we got all the way to the TARGETTOEXTENT then we fully finished - // otherwise we must do all assets every time due to the interdependence of IDs etc - if ( - !zb.helpers.isPropertyValueSet( - properties[FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME].value + const extentXenCompat = this.options.iscsi.hasOwnProperty( + "extentXenCompat" ) - ) { + ? this.options.iscsi.extentXenCompat + : false; + + const extentBlocksize = this.options.iscsi.hasOwnProperty( + "extentBlocksize" + ) + ? this.options.iscsi.extentBlocksize + : 512; + + const extentDisablePhysicalBlocksize = + this.options.iscsi.hasOwnProperty("extentDisablePhysicalBlocksize") + ? this.options.iscsi.extentDisablePhysicalBlocksize + : true; + + const extentRpm = this.options.iscsi.hasOwnProperty("extentRpm") + ? this.options.iscsi.extentRpm + : "SSD"; + + let extentAvailThreshold = this.options.iscsi.hasOwnProperty( + "extentAvailThreshold" + ) + ? Number(this.options.iscsi.extentAvailThreshold) + : null; + + if (!(extentAvailThreshold > 0 && extentAvailThreshold <= 100)) { + extentAvailThreshold = null; + } + switch (apiVersion) { - case 1: { - // create target - let target = { - iscsi_target_name: iscsiName, - iscsi_target_alias: "", // TODO: allow template for this - }; - - response = await httpClient.post( - "/services/iscsi/target", - target + case 1: + response = await httpClient.get( + "/services/iscsi/globalconfiguration" ); - - // 409 if invalid - if (response.statusCode != 201) { - target = null; - if ( - response.statusCode == 409 && - JSON.stringify(response.body).includes( - "Target name already exists" - ) - ) { - target = await httpApiClient.findResourceByProperties( - "/services/iscsi/target", - { - iscsi_target_name: iscsiName, - } - ); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating iscsi target - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } else { - target = response.body; - } - - if (!target) { + if (response.statusCode != 200) { throw new GrpcError( grpc.status.UNKNOWN, - `unknown error creating iscsi target` + `error getting iscsi configuration - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` ); } - - if (target.iscsi_target_name != iscsiName) { + basename = response.body.iscsi_basename; + this.ctx.logger.verbose("FreeNAS ISCSI BASENAME: " + basename); + break; + case 2: + response = await httpClient.get("/iscsi/global"); + if (response.statusCode != 200) { throw new GrpcError( grpc.status.UNKNOWN, - `mismatch name error creating iscsi target` + `error getting iscsi configuration - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` ); } + basename = response.body.basename; + this.ctx.logger.verbose("FreeNAS ISCSI BASENAME: " + basename); + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } - this.ctx.logger.verbose("FreeNAS ISCSI TARGET: %j", target); - - // set target.id on zvol - await zb.zfs.set(datasetName, { - [FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME]: target.id, - }); - - // create targetgroup(s) - // targetgroups do have IDs - for (let targetGroupConfig of this.options.iscsi.targetGroups) { - let targetGroup = { - iscsi_target: target.id, - iscsi_target_authgroup: - targetGroupConfig.targetGroupAuthGroup, - iscsi_target_authtype: targetGroupConfig.targetGroupAuthType - ? targetGroupConfig.targetGroupAuthType - : "None", - iscsi_target_portalgroup: - targetGroupConfig.targetGroupPortalGroup, - iscsi_target_initiatorgroup: - targetGroupConfig.targetGroupInitiatorGroup, - iscsi_target_initialdigest: "Auto", + // if we got all the way to the TARGETTOEXTENT then we fully finished + // otherwise we must do all assets every time due to the interdependence of IDs etc + if ( + !zb.helpers.isPropertyValueSet( + properties[FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME].value + ) + ) { + switch (apiVersion) { + case 1: { + // create target + let target = { + iscsi_target_name: iscsiName, + iscsi_target_alias: "", // TODO: allow template for this }; + response = await httpClient.post( - "/services/iscsi/targetgroup", - targetGroup + "/services/iscsi/target", + target ); // 409 if invalid if (response.statusCode != 201) { - targetGroup = null; - /** - * 404 gets returned with an unable to process response when the DB is corrupted (has invalid entries in essense) - * - * To resolve properly the DB should be cleaned up - * /usr/local/etc/rc.d/django stop - * /usr/local/etc/rc.d/nginx stop - * sqlite3 /data/freenas-v1.db - * - * // this deletes everything, probably not what you want - * // should have a better query to only find entries where associated assets no longer exist - * DELETE from services_iscsitargetgroups; - * - * /usr/local/etc/rc.d/django restart - * /usr/local/etc/rc.d/nginx restart - */ + target = null; if ( - response.statusCode == 404 || - (response.statusCode == 409 && - JSON.stringify(response.body).includes( - "cannot be duplicated on a target" - )) + response.statusCode == 409 && + JSON.stringify(response.body).includes( + "Target name already exists" + ) ) { - targetGroup = await httpApiClient.findResourceByProperties( - "/services/iscsi/targetgroup", + target = await httpApiClient.findResourceByProperties( + "/services/iscsi/target", { - iscsi_target: target.id, - iscsi_target_portalgroup: - targetGroupConfig.targetGroupPortalGroup, - iscsi_target_initiatorgroup: - targetGroupConfig.targetGroupInitiatorGroup, + iscsi_target_name: iscsiName, } ); } else { throw new GrpcError( grpc.status.UNKNOWN, - `received error creating iscsi targetgroup - code: ${ + `received error creating iscsi target - code: ${ response.statusCode } body: ${JSON.stringify(response.body)}` ); } } else { - targetGroup = response.body; + target = response.body; } - if (!targetGroup) { + if (!target) { throw new GrpcError( grpc.status.UNKNOWN, - `unknown error creating iscsi targetgroup` + `unknown error creating iscsi target` ); } - this.ctx.logger.verbose( - "FreeNAS ISCSI TARGET_GROUP: %j", - targetGroup - ); - } - - let extent = { - iscsi_target_extent_comment: extentComment, - iscsi_target_extent_type: "Disk", // Disk/File, after save Disk becomes "ZVOL" - iscsi_target_extent_name: iscsiName, - iscsi_target_extent_insecure_tpc: extentInsecureTpc, - //iscsi_target_extent_naa: "0x3822690834aae6c5", - iscsi_target_extent_disk: extentDiskName, - iscsi_target_extent_xen: extentXenCompat, - iscsi_target_extent_avail_threshold: extentAvailThreshold, - iscsi_target_extent_blocksize: Number(extentBlocksize), - iscsi_target_extent_pblocksize: extentDisablePhysicalBlocksize, - iscsi_target_extent_rpm: isNaN(Number(extentRpm)) - ? "SSD" - : Number(extentRpm), - iscsi_target_extent_ro: false, - }; - response = await httpClient.post( - "/services/iscsi/extent", - extent - ); - - // 409 if invalid - if (response.statusCode != 201) { - extent = null; - if ( - response.statusCode == 409 && - JSON.stringify(response.body).includes( - "Extent name must be unique" - ) - ) { - extent = await httpApiClient.findResourceByProperties( - "/services/iscsi/extent", - { iscsi_target_extent_name: iscsiName } - ); - } else { + if (target.iscsi_target_name != iscsiName) { throw new GrpcError( grpc.status.UNKNOWN, - `received error creating iscsi extent - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` + `mismatch name error creating iscsi target` ); } - } else { - extent = response.body; - } - if (!extent) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi extent` - ); - } + this.ctx.logger.verbose("FreeNAS ISCSI TARGET: %j", target); - if (extent.iscsi_target_extent_name != iscsiName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `mismatch name error creating iscsi extent` - ); - } - - this.ctx.logger.verbose("FreeNAS ISCSI EXTENT: %j", extent); - - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME]: extent.id, - }); - - // create targettoextent - let targetToExtent = { - iscsi_target: target.id, - iscsi_extent: extent.id, - iscsi_lunid: 0, - }; - response = await httpClient.post( - "/services/iscsi/targettoextent", - targetToExtent - ); - - // 409 if invalid - if (response.statusCode != 201) { - targetToExtent = null; - - // LUN ID is already being used for this target. - // Extent is already in this target. - if ( - response.statusCode == 409 && - (JSON.stringify(response.body).includes( - "Extent is already in this target." - ) || - JSON.stringify(response.body).includes( - "LUN ID is already being used for this target." - )) - ) { - targetToExtent = await httpApiClient.findResourceByProperties( - "/services/iscsi/targettoextent", - { - iscsi_target: target.id, - iscsi_extent: extent.id, - iscsi_lunid: 0, - } - ); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating iscsi targettoextent - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } else { - targetToExtent = response.body; - } - - if (!targetToExtent) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi targettoextent` - ); - } - this.ctx.logger.verbose( - "FreeNAS ISCSI TARGET_TO_EXTENT: %j", - targetToExtent - ); - - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME]: - targetToExtent.id, - }); - - break; - } - case 2: - // create target and targetgroup - //let targetId; - let targetGroups = []; - for (let targetGroupConfig of this.options.iscsi.targetGroups) { - targetGroups.push({ - portal: targetGroupConfig.targetGroupPortalGroup, - initiator: targetGroupConfig.targetGroupInitiatorGroup, - auth: - targetGroupConfig.targetGroupAuthGroup > 0 - ? targetGroupConfig.targetGroupAuthGroup - : null, - authmethod: - targetGroupConfig.targetGroupAuthType.length > 0 - ? targetGroupConfig.targetGroupAuthType - .toUpperCase() - .replace(" ", "_") - : "NONE", + // set target.id on zvol + await zb.zfs.set(datasetName, { + [FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME]: target.id, }); - } - let target = { - name: iscsiName, - alias: null, // cannot send "" error: handler error - driver: FreeNASDriver method: CreateVolume error: {"name":"GrpcError","code":2,"message":"received error creating iscsi target - code: 422 body: {\"iscsi_target_create.alias\":[{\"message\":\"Alias already exists\",\"errno\":22}]}"} - mode: "ISCSI", - groups: targetGroups, - }; - response = await httpClient.post("/iscsi/target", target); - - // 409 if invalid - if (response.statusCode != 200) { - target = null; - if ( - response.statusCode == 422 && - JSON.stringify(response.body).includes( - "Target name already exists" - ) - ) { - target = await httpApiClient.findResourceByProperties( - "/iscsi/target", - { - name: iscsiName, - } + // create targetgroup(s) + // targetgroups do have IDs + for (let targetGroupConfig of this.options.iscsi.targetGroups) { + let targetGroup = { + iscsi_target: target.id, + iscsi_target_authgroup: + targetGroupConfig.targetGroupAuthGroup, + iscsi_target_authtype: targetGroupConfig.targetGroupAuthType + ? targetGroupConfig.targetGroupAuthType + : "None", + iscsi_target_portalgroup: + targetGroupConfig.targetGroupPortalGroup, + iscsi_target_initiatorgroup: + targetGroupConfig.targetGroupInitiatorGroup, + iscsi_target_initialdigest: "Auto", + }; + response = await httpClient.post( + "/services/iscsi/targetgroup", + targetGroup ); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating iscsi target - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` + + // 409 if invalid + if (response.statusCode != 201) { + targetGroup = null; + /** + * 404 gets returned with an unable to process response when the DB is corrupted (has invalid entries in essense) + * + * To resolve properly the DB should be cleaned up + * /usr/local/etc/rc.d/django stop + * /usr/local/etc/rc.d/nginx stop + * sqlite3 /data/freenas-v1.db + * + * // this deletes everything, probably not what you want + * // should have a better query to only find entries where associated assets no longer exist + * DELETE from services_iscsitargetgroups; + * + * /usr/local/etc/rc.d/django restart + * /usr/local/etc/rc.d/nginx restart + */ + if ( + response.statusCode == 404 || + (response.statusCode == 409 && + JSON.stringify(response.body).includes( + "cannot be duplicated on a target" + )) + ) { + targetGroup = + await httpApiClient.findResourceByProperties( + "/services/iscsi/targetgroup", + { + iscsi_target: target.id, + iscsi_target_portalgroup: + targetGroupConfig.targetGroupPortalGroup, + iscsi_target_initiatorgroup: + targetGroupConfig.targetGroupInitiatorGroup, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi targetgroup - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } else { + targetGroup = response.body; + } + + if (!targetGroup) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi targetgroup` + ); + } + + this.ctx.logger.verbose( + "FreeNAS ISCSI TARGET_GROUP: %j", + targetGroup ); } - } else { - target = response.body; - } - if (!target) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi target` + let extent = { + iscsi_target_extent_comment: extentComment, + iscsi_target_extent_type: "Disk", // Disk/File, after save Disk becomes "ZVOL" + iscsi_target_extent_name: iscsiName, + iscsi_target_extent_insecure_tpc: extentInsecureTpc, + //iscsi_target_extent_naa: "0x3822690834aae6c5", + iscsi_target_extent_disk: extentDiskName, + iscsi_target_extent_xen: extentXenCompat, + iscsi_target_extent_avail_threshold: extentAvailThreshold, + iscsi_target_extent_blocksize: Number(extentBlocksize), + iscsi_target_extent_pblocksize: + extentDisablePhysicalBlocksize, + iscsi_target_extent_rpm: isNaN(Number(extentRpm)) + ? "SSD" + : Number(extentRpm), + iscsi_target_extent_ro: false, + }; + response = await httpClient.post( + "/services/iscsi/extent", + extent ); - } - if (target.name != iscsiName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `mismatch name error creating iscsi target` - ); - } - - // handle situations/race conditions where groups failed to be added/created on the target - // groups":[{"portal":1,"initiator":1,"auth":null,"authmethod":"NONE"},{"portal":2,"initiator":1,"auth":null,"authmethod":"NONE"}] - // TODO: this logic could be more intelligent but this should do for now as it appears in the failure scenario no groups are added - // in other words, I have never seen them invalid, only omitted so this should be enough - if (target.groups.length != targetGroups.length) { - response = await httpClient.put( - `/iscsi/target/id/${target.id}`, - { - groups: targetGroups, + // 409 if invalid + if (response.statusCode != 201) { + extent = null; + if ( + response.statusCode == 409 && + JSON.stringify(response.body).includes( + "Extent name must be unique" + ) + ) { + extent = await httpApiClient.findResourceByProperties( + "/services/iscsi/extent", + { iscsi_target_extent_name: iscsiName } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi extent - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); } - ); + } else { + extent = response.body; + } - if (response.statusCode != 200) { + if (!extent) { throw new GrpcError( grpc.status.UNKNOWN, - `failed setting target groups` + `unknown error creating iscsi extent` ); + } + + if (extent.iscsi_target_extent_name != iscsiName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `mismatch name error creating iscsi extent` + ); + } + + this.ctx.logger.verbose("FreeNAS ISCSI EXTENT: %j", extent); + + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME]: extent.id, + }); + + // create targettoextent + let targetToExtent = { + iscsi_target: target.id, + iscsi_extent: extent.id, + iscsi_lunid: 0, + }; + response = await httpClient.post( + "/services/iscsi/targettoextent", + targetToExtent + ); + + // 409 if invalid + if (response.statusCode != 201) { + targetToExtent = null; + + // LUN ID is already being used for this target. + // Extent is already in this target. + if ( + response.statusCode == 409 && + (JSON.stringify(response.body).includes( + "Extent is already in this target." + ) || + JSON.stringify(response.body).includes( + "LUN ID is already being used for this target." + )) + ) { + targetToExtent = + await httpApiClient.findResourceByProperties( + "/services/iscsi/targettoextent", + { + iscsi_target: target.id, + iscsi_extent: extent.id, + iscsi_lunid: 0, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi targettoextent - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } else { + targetToExtent = response.body; + } + + if (!targetToExtent) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi targettoextent` + ); + } + this.ctx.logger.verbose( + "FreeNAS ISCSI TARGET_TO_EXTENT: %j", + targetToExtent + ); + + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME]: + targetToExtent.id, + }); + + break; + } + case 2: + // create target and targetgroup + //let targetId; + let targetGroups = []; + for (let targetGroupConfig of this.options.iscsi.targetGroups) { + targetGroups.push({ + portal: targetGroupConfig.targetGroupPortalGroup, + initiator: targetGroupConfig.targetGroupInitiatorGroup, + auth: + targetGroupConfig.targetGroupAuthGroup > 0 + ? targetGroupConfig.targetGroupAuthGroup + : null, + authmethod: + targetGroupConfig.targetGroupAuthType.length > 0 + ? targetGroupConfig.targetGroupAuthType + .toUpperCase() + .replace(" ", "_") + : "NONE", + }); + } + let target = { + name: iscsiName, + alias: null, // cannot send "" error: handler error - driver: FreeNASDriver method: CreateVolume error: {"name":"GrpcError","code":2,"message":"received error creating iscsi target - code: 422 body: {\"iscsi_target_create.alias\":[{\"message\":\"Alias already exists\",\"errno\":22}]}"} + mode: "ISCSI", + groups: targetGroups, + }; + + response = await httpClient.post("/iscsi/target", target); + + // 409 if invalid + if (response.statusCode != 200) { + target = null; + if ( + response.statusCode == 422 && + JSON.stringify(response.body).includes( + "Target name already exists" + ) + ) { + target = await httpApiClient.findResourceByProperties( + "/iscsi/target", + { + name: iscsiName, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi target - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } } else { target = response.body; + } - // re-run sanity checks - if (!target) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi target` - ); - } + if (!target) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi target` + ); + } - if (target.name != iscsiName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `mismatch name error creating iscsi target` - ); - } + if (target.name != iscsiName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `mismatch name error creating iscsi target` + ); + } - if (target.groups.length != targetGroups.length) { + // handle situations/race conditions where groups failed to be added/created on the target + // groups":[{"portal":1,"initiator":1,"auth":null,"authmethod":"NONE"},{"portal":2,"initiator":1,"auth":null,"authmethod":"NONE"}] + // TODO: this logic could be more intelligent but this should do for now as it appears in the failure scenario no groups are added + // in other words, I have never seen them invalid, only omitted so this should be enough + if (target.groups.length != targetGroups.length) { + response = await httpClient.put( + `/iscsi/target/id/${target.id}`, + { + groups: targetGroups, + } + ); + + if (response.statusCode != 200) { throw new GrpcError( grpc.status.UNKNOWN, `failed setting target groups` ); + } else { + target = response.body; + + // re-run sanity checks + if (!target) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi target` + ); + } + + if (target.name != iscsiName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `mismatch name error creating iscsi target` + ); + } + + if (target.groups.length != targetGroups.length) { + throw new GrpcError( + grpc.status.UNKNOWN, + `failed setting target groups` + ); + } } } - } - this.ctx.logger.verbose("FreeNAS ISCSI TARGET: %j", target); + this.ctx.logger.verbose("FreeNAS ISCSI TARGET: %j", target); - // set target.id on zvol - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME]: target.id, - }); + // set target.id on zvol + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME]: target.id, + }); - let extent = { - comment: extentComment, - type: "DISK", // Disk/File, after save Disk becomes "ZVOL" - name: iscsiName, - //iscsi_target_extent_naa: "0x3822690834aae6c5", - disk: extentDiskName, - insecure_tpc: extentInsecureTpc, - xen: extentXenCompat, - avail_threshold: extentAvailThreshold, - blocksize: Number(extentBlocksize), - pblocksize: extentDisablePhysicalBlocksize, - rpm: "" + extentRpm, // should be a string - ro: false, - }; + let extent = { + comment: extentComment, + type: "DISK", // Disk/File, after save Disk becomes "ZVOL" + name: iscsiName, + //iscsi_target_extent_naa: "0x3822690834aae6c5", + disk: extentDiskName, + insecure_tpc: extentInsecureTpc, + xen: extentXenCompat, + avail_threshold: extentAvailThreshold, + blocksize: Number(extentBlocksize), + pblocksize: extentDisablePhysicalBlocksize, + rpm: "" + extentRpm, // should be a string + ro: false, + }; - response = await httpClient.post("/iscsi/extent", extent); + response = await httpClient.post("/iscsi/extent", extent); - // 409 if invalid - if (response.statusCode != 200) { - extent = null; - if ( - response.statusCode == 422 && - JSON.stringify(response.body).includes( - "Extent name must be unique" - ) - ) { - extent = await httpApiClient.findResourceByProperties( - "/iscsi/extent", - { - name: iscsiName, - } - ); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating iscsi extent - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } else { - extent = response.body; - } - - if (!extent) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi extent` - ); - } - - if (extent.name != iscsiName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `mismatch name error creating iscsi extent` - ); - } - - this.ctx.logger.verbose("FreeNAS ISCSI EXTENT: %j", extent); - - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME]: extent.id, - }); - - // create targettoextent - let targetToExtent = { - target: target.id, - extent: extent.id, - lunid: 0, - }; - response = await httpClient.post( - "/iscsi/targetextent", - targetToExtent - ); - - if (response.statusCode != 200) { - targetToExtent = null; - - // LUN ID is already being used for this target. - // Extent is already in this target. - if ( - response.statusCode == 422 && - (JSON.stringify(response.body).includes( - "Extent is already in this target." - ) || + // 409 if invalid + if (response.statusCode != 200) { + extent = null; + if ( + response.statusCode == 422 && JSON.stringify(response.body).includes( - "LUN ID is already being used for this target." - )) - ) { - targetToExtent = await httpApiClient.findResourceByProperties( - "/iscsi/targetextent", - { - target: target.id, - extent: extent.id, - lunid: 0, - } - ); + "Extent name must be unique" + ) + ) { + extent = await httpApiClient.findResourceByProperties( + "/iscsi/extent", + { + name: iscsiName, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi extent - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } } else { + extent = response.body; + } + + if (!extent) { throw new GrpcError( grpc.status.UNKNOWN, - `received error creating iscsi targetextent - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` + `unknown error creating iscsi extent` ); } - } else { - targetToExtent = response.body; - } - if (!targetToExtent) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi targetextent` + if (extent.name != iscsiName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `mismatch name error creating iscsi extent` + ); + } + + this.ctx.logger.verbose("FreeNAS ISCSI EXTENT: %j", extent); + + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME]: extent.id, + }); + + // create targettoextent + let targetToExtent = { + target: target.id, + extent: extent.id, + lunid: 0, + }; + response = await httpClient.post( + "/iscsi/targetextent", + targetToExtent ); - } - this.ctx.logger.verbose( - "FreeNAS ISCSI TARGET_TO_EXTENT: %j", - targetToExtent - ); - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME]: - targetToExtent.id, - }); + if (response.statusCode != 200) { + targetToExtent = null; - break; - default: + // LUN ID is already being used for this target. + // Extent is already in this target. + if ( + response.statusCode == 422 && + (JSON.stringify(response.body).includes( + "Extent is already in this target." + ) || + JSON.stringify(response.body).includes( + "LUN ID is already being used for this target." + )) + ) { + targetToExtent = + await httpApiClient.findResourceByProperties( + "/iscsi/targetextent", + { + target: target.id, + extent: extent.id, + lunid: 0, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi targetextent - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } else { + targetToExtent = response.body; + } + + if (!targetToExtent) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi targetextent` + ); + } + this.ctx.logger.verbose( + "FreeNAS ISCSI TARGET_TO_EXTENT: %j", + targetToExtent + ); + + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME]: + targetToExtent.id, + }); + + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } + } + + // iqn = target + let iqn = basename + ":" + iscsiName; + this.ctx.logger.info("FreeNAS iqn: " + iqn); + + // store this off to make delete process more bullet proof + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME]: iscsiName, + }); + + volume_context = { + node_attach_driver: "iscsi", + portal: this.options.iscsi.targetPortal || "", + portals: this.options.iscsi.targetPortals + ? this.options.iscsi.targetPortals.join(",") + : "", + interface: this.options.iscsi.interface || "", + iqn: iqn, + lun: 0, + }; + return volume_context; + } + break; + + case "nvmeof": + { + switch (apiVersion) { + case 1: throw new GrpcError( grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` + `nvmeof feature is only available with version 2 of the api` ); + break; } + + if (!isScale || semver.satisfies(truenasVersion, "<25.10")) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `nvmeof feature is only available with TrueNAS version 25.10 and above` + ); + } + + properties = await httpApiClient.DatasetGet(datasetName, [ + FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME, + FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME, + FREENAS_NVMEOF_ASSETS_NAME_PROPERTY_NAME, + ]); + this.ctx.logger.debug("zfs props data: %j", properties); + + let nvmeofName; + + if (this.options.nvmeof.nameTemplate) { + nvmeofName = Handlebars.compile(this.options.nvmeof.nameTemplate)({ + name: call.request.name, + parameters: call.request.parameters, + }); + } else { + nvmeofName = zb.helpers.extractLeafName(datasetName); + } + + if (this.options.nvmeof.namePrefix) { + nvmeofName = this.options.nvmeof.namePrefix + nvmeofName; + } + + if (this.options.nvmeof.nameSuffix) { + nvmeofName += this.options.nvmeof.nameSuffix; + } + + // According to RFC3270, 'Each iSCSI node, whether an initiator or target, MUST have an iSCSI name. Initiators and targets MUST support the receipt of iSCSI names of up to the maximum length of 223 bytes.' + // https://kb.netapp.com/Advice_and_Troubleshooting/Miscellaneous/What_is_the_maximum_length_of_a_iSCSI_iqn_name + // https://tools.ietf.org/html/rfc3720 + // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 + nvmeofName = nvmeofName.toLowerCase(); + + let namespaceDiskName = "zvol/" + datasetName; + let maxZvolNameLength = await driver.getMaxZvolNameLength(); + driver.ctx.logger.debug( + "max zvol name length: %s", + maxZvolNameLength + ); + + if (namespaceDiskName.length > maxZvolNameLength) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `namespace disk name cannot exceed ${maxZvolNameLength} characters: ${namespaceDiskName}` + ); + } + + // TODO: get basenqn from global config, add nvemofName to it and ensure full nqn is <= 223 + // // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 + // if (isScale && nvmeofName.length > 64) { + // throw new GrpcError( + // grpc.status.FAILED_PRECONDITION, + // `extent name cannot exceed 64 characters: ${nvmeofName}` + // ); + // } + + this.ctx.logger.info( + "FreeNAS creating nvmeof assets with name: " + nvmeofName + ); + + // http:///api/docs/current/api_methods_nvmet.subsys.create.html + let subsystemTemplate = _.get( + this.options, + "nvmeof.subsystemTemplate", + {} + ); + subsystemTemplate = subsystemTemplate || {}; + + // http:///api/docs/current/api_methods_nvmet.namespace.create.html + let namespaceTemplate = _.get( + this.options, + "nvmeof.namespaceTemplate", + {} + ); + namespaceTemplate = namespaceTemplate || {}; + + // create subsystem + let subsystem; + switch (apiVersion) { + case 2: + subsystem = await httpApiClient.NvmetSubsysCreate( + nvmeofName, + subsystemTemplate + ); + + break; + } + if (!subsystem) { + throw new GrpcError( + grpc.status.NOT_FOUND, + `unable to find nvmeof subsystem: ${nvmeofName}` + ); + } + this.ctx.logger.verbose("FreeNAS NVMEOF SUBSYSTEM: %j", subsystem); + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME]: subsystem.id, + }); + + // create subsystem + let namespace; + switch (apiVersion) { + case 2: + namespace = await httpApiClient.NvmetNamespaceCreate( + namespaceDiskName, + subsystem.id, + namespaceTemplate + ); + + break; + } + if (!namespace) { + throw new GrpcError( + grpc.status.NOT_FOUND, + `unable to find nvmeof namespace: ${namespaceDiskName}` + ); + } + this.ctx.logger.verbose("FreeNAS NVMEOF NAMESPACE: %j", namespace); + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME]: namespace.id, + }); + + // assign ports to subsystem + let ports = _.get(this.options, "nvmeof.ports", []); + for (const port_i of ports) { + const port = await httpApiClient.NvmetPortSubsysCreate( + port_i, + subsystem.id + ); + this.ctx.logger.verbose("FreeNAS NVMEOF PORT: %j", port); + } + + // TODO: assign hosts + + // store this off to make delete process more bullet proof + await httpApiClient.DatasetSet(datasetName, { + [FREENAS_NVMEOF_ASSETS_NAME_PROPERTY_NAME]: nvmeofName, + }); + + volume_context = { + node_attach_driver: "nvmeof", + transport: this.options.nvmeof.transport || "", + transports: this.options.nvmeof.transports + ? this.options.nvmeof.transports.join(",") + : "", + nqn: subsystem.subnqn, + nsid: namespace.nsid, + }; + return volume_context; } - - // iqn = target - let iqn = basename + ":" + iscsiName; - this.ctx.logger.info("FreeNAS iqn: " + iqn); - - // store this off to make delete process more bullet proof - await httpApiClient.DatasetSet(datasetName, { - [FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME]: iscsiName, - }); - - volume_context = { - node_attach_driver: "iscsi", - portal: this.options.iscsi.targetPortal || "", - portals: this.options.iscsi.targetPortals - ? this.options.iscsi.targetPortals.join(",") - : "", - interface: this.options.iscsi.interface || "", - iqn: iqn, - lun: 0, - }; - return volume_context; + break; default: throw new GrpcError( @@ -1415,6 +1607,16 @@ class FreeNASApiDriver extends CsiBaseDriver { const httpApiClient = await this.getTrueNASHttpApiClient(); const apiVersion = httpClient.getApiVersion(); const zb = await this.getZetabyte(); + const truenasVersion = await httpApiClient.getSystemVersionSemver(); + + if (!truenasVersion) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unable to detect TrueNAS version` + ); + } + + const isScale = await httpApiClient.getIsScale(); let properties; let response; @@ -1425,200 +1627,397 @@ class FreeNASApiDriver extends CsiBaseDriver { switch (driverShareType) { case "nfs": - try { - properties = await httpApiClient.DatasetGet(datasetName, [ - "mountpoint", - FREENAS_NFS_SHARE_PROPERTY_NAME, - ]); - } catch (err) { - if (err.toString().includes("dataset does not exist")) { - return; + { + try { + properties = await httpApiClient.DatasetGet(datasetName, [ + "mountpoint", + FREENAS_NFS_SHARE_PROPERTY_NAME, + ]); + } catch (err) { + if (err.toString().includes("dataset does not exist")) { + return; + } + throw err; } - throw err; - } - this.ctx.logger.debug("zfs props data: %j", properties); + this.ctx.logger.debug("zfs props data: %j", properties); - shareId = properties[FREENAS_NFS_SHARE_PROPERTY_NAME].value; + shareId = properties[FREENAS_NFS_SHARE_PROPERTY_NAME].value; - // only remove if the process has not succeeded already - if (zb.helpers.isPropertyValueSet(shareId)) { - // remove nfs share - switch (apiVersion) { - case 1: - case 2: - endpoint = "/sharing/nfs/"; - if (apiVersion == 2) { - endpoint += "id/"; - } - endpoint += shareId; - - response = await httpClient.get(endpoint); - - // assume share is gone for now - if ([404, 500].includes(response.statusCode)) { - } else { - switch (apiVersion) { - case 1: - sharePaths = response.body.nfs_paths; - break; - case 2: - if (response.body.path) { - sharePaths = [response.body.path]; - } else { - sharePaths = response.body.paths; - } - break; + // only remove if the process has not succeeded already + if (zb.helpers.isPropertyValueSet(shareId)) { + // remove nfs share + switch (apiVersion) { + case 1: + case 2: + endpoint = "/sharing/nfs/"; + if (apiVersion == 2) { + endpoint += "id/"; } + endpoint += shareId; - deleteAsset = sharePaths.some((value) => { - return value == properties.mountpoint.value; - }); + response = await httpClient.get(endpoint); - if (deleteAsset) { - response = await GeneralUtils.retry( - 3, - 1000, - async () => { - return await httpClient.delete(endpoint); - }, - { - retryCondition: (err) => { - if (err.code == "ECONNRESET") { - return true; - } - if (err.code == "ECONNABORTED") { - return true; - } - if (err.response && err.response.statusCode == 504) { - return true; - } - return false; - }, - } - ); - - // returns a 500 if does not exist - // v1 = 204 - // v2 = 200 - if (![200, 204].includes(response.statusCode)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting nfs share - share: ${shareId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); + // assume share is gone for now + if ([404, 500].includes(response.statusCode)) { + } else { + switch (apiVersion) { + case 1: + sharePaths = response.body.nfs_paths; + break; + case 2: + if (response.body.path) { + sharePaths = [response.body.path]; + } else { + sharePaths = response.body.paths; + } + break; } - // remove property to prevent delete race conditions - // due to id re-use by FreeNAS/TrueNAS - await httpApiClient.DatasetInherit( - datasetName, - FREENAS_NFS_SHARE_PROPERTY_NAME - ); + deleteAsset = sharePaths.some((value) => { + return value == properties.mountpoint.value; + }); + + if (deleteAsset) { + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.delete(endpoint); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); + + // returns a 500 if does not exist + // v1 = 204 + // v2 = 200 + if (![200, 204].includes(response.statusCode)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting nfs share - share: ${shareId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + // remove property to prevent delete race conditions + // due to id re-use by FreeNAS/TrueNAS + await httpApiClient.DatasetInherit( + datasetName, + FREENAS_NFS_SHARE_PROPERTY_NAME + ); + } } - } - break; - default: - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` - ); + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } } } break; case "smb": - try { - properties = await httpApiClient.DatasetGet(datasetName, [ - "mountpoint", - FREENAS_SMB_SHARE_PROPERTY_NAME, - ]); - } catch (err) { - if (err.toString().includes("dataset does not exist")) { - return; + { + try { + properties = await httpApiClient.DatasetGet(datasetName, [ + "mountpoint", + FREENAS_SMB_SHARE_PROPERTY_NAME, + ]); + } catch (err) { + if (err.toString().includes("dataset does not exist")) { + return; + } + throw err; } - throw err; - } - this.ctx.logger.debug("zfs props data: %j", properties); + this.ctx.logger.debug("zfs props data: %j", properties); - shareId = properties[FREENAS_SMB_SHARE_PROPERTY_NAME].value; + shareId = properties[FREENAS_SMB_SHARE_PROPERTY_NAME].value; - // only remove if the process has not succeeded already - if (zb.helpers.isPropertyValueSet(shareId)) { - // remove smb share - switch (apiVersion) { - case 1: - case 2: - switch (apiVersion) { - case 1: - endpoint = `/sharing/cifs/${shareId}`; - break; - case 2: - endpoint = `/sharing/smb/id/${shareId}`; - break; - } - - response = await httpClient.get(endpoint); - - // assume share is gone for now - if ([404, 500].includes(response.statusCode)) { - } else { + // only remove if the process has not succeeded already + if (zb.helpers.isPropertyValueSet(shareId)) { + // remove smb share + switch (apiVersion) { + case 1: + case 2: switch (apiVersion) { case 1: - sharePaths = [response.body.cifs_path]; + endpoint = `/sharing/cifs/${shareId}`; break; case 2: - sharePaths = [response.body.path]; + endpoint = `/sharing/smb/id/${shareId}`; break; } - deleteAsset = sharePaths.some((value) => { - return value == properties.mountpoint.value; - }); + response = await httpClient.get(endpoint); - if (deleteAsset) { - response = await GeneralUtils.retry( - 3, - 1000, - async () => { - return await httpClient.delete(endpoint); - }, - { - retryCondition: (err) => { - if (err.code == "ECONNRESET") { - return true; - } - if (err.code == "ECONNABORTED") { - return true; - } - if (err.response && err.response.statusCode == 504) { - return true; - } - return false; - }, - } - ); - - // returns a 500 if does not exist - // v1 = 204 - // v2 = 200 - if ( - ![200, 204].includes(response.statusCode) && - !JSON.stringify(response.body).includes("does not exist") - ) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting smb share - share: ${shareId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); + // assume share is gone for now + if ([404, 500].includes(response.statusCode)) { + } else { + switch (apiVersion) { + case 1: + sharePaths = [response.body.cifs_path]; + break; + case 2: + sharePaths = [response.body.path]; + break; } - // remove property to prevent delete race conditions - // due to id re-use by FreeNAS/TrueNAS - await httpApiClient.DatasetInherit( - datasetName, - FREENAS_SMB_SHARE_PROPERTY_NAME - ); + deleteAsset = sharePaths.some((value) => { + return value == properties.mountpoint.value; + }); + + if (deleteAsset) { + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.delete(endpoint); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); + + // returns a 500 if does not exist + // v1 = 204 + // v2 = 200 + if ( + ![200, 204].includes(response.statusCode) && + !JSON.stringify(response.body).includes("does not exist") + ) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting smb share - share: ${shareId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + // remove property to prevent delete race conditions + // due to id re-use by FreeNAS/TrueNAS + await httpApiClient.DatasetInherit( + datasetName, + FREENAS_SMB_SHARE_PROPERTY_NAME + ); + } + } + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } + } + } + break; + case "iscsi": + { + // Delete target + // NOTE: deleting a target inherently deletes associated targetgroup(s) and targettoextent(s) + + // Delete extent + try { + properties = await httpApiClient.DatasetGet(datasetName, [ + FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME, + FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME, + FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME, + FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME, + ]); + } catch (err) { + if (err.toString().includes("dataset does not exist")) { + return; + } + throw err; + } + + this.ctx.logger.debug("zfs props data: %j", properties); + + let targetId = + properties[FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME].value; + let extentId = + properties[FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME].value; + let iscsiName = + properties[FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME].value; + let assetName; + + switch (apiVersion) { + case 1: + case 2: + // only remove if the process has not succeeded already + if (zb.helpers.isPropertyValueSet(targetId)) { + // https://jira.ixsystems.com/browse/NAS-103952 + + // v1 - /services/iscsi/target/{id}/ + // v2 - /iscsi/target/id/{id} + endpoint = ""; + if (apiVersion == 1) { + endpoint += "/services"; + } + endpoint += "/iscsi/target/"; + if (apiVersion == 2) { + endpoint += "id/"; + } + endpoint += targetId; + response = await httpClient.get(endpoint); + + // assume is gone for now + if ([404, 500].includes(response.statusCode)) { + } else { + deleteAsset = true; + assetName = null; + + // checking if set for backwards compatibility + if (zb.helpers.isPropertyValueSet(iscsiName)) { + switch (apiVersion) { + case 1: + assetName = response.body.iscsi_target_name; + break; + case 2: + assetName = response.body.name; + break; + } + + if (assetName != iscsiName) { + deleteAsset = false; + } + } + + if (deleteAsset) { + let retries = 0; + let maxRetries = 5; + let retryWait = 1000; + response = await httpClient.delete(endpoint); + + // sometimes after an initiator has detached it takes a moment for TrueNAS to settle + // code: 422 body: {\"message\":\"Target csi-ci-55877e95sanity-node-expand-volume-e54f81fa-cd38e798 is in use.\",\"errno\":14} + while ( + response.statusCode == 422 && + retries < maxRetries && + _.get(response, "body.message").includes("Target") && + _.get(response, "body.message").includes("is in use") && + _.get(response, "body.errno") == 14 + ) { + retries++; + this.ctx.logger.debug( + "target: %s is in use, retry %s shortly", + targetId, + retries + ); + await GeneralUtils.sleep(retryWait); + response = await httpClient.delete(endpoint); + } + + if (![200, 204, 404].includes(response.statusCode)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting iscsi target - target: ${targetId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + // remove property to prevent delete race conditions + // due to id re-use by FreeNAS/TrueNAS + await httpApiClient.DatasetInherit( + datasetName, + FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME + ); + } else { + this.ctx.logger.debug( + "not deleting iscsitarget asset as it appears ID %s has been re-used: zfs name - %s, iscsitarget name - %s", + targetId, + iscsiName, + assetName + ); + } + } + } + + // only remove if the process has not succeeded already + if (zb.helpers.isPropertyValueSet(extentId)) { + // v1 - /services/iscsi/targettoextent/{id}/ + // v2 - /iscsi/targetextent/id/{id} + if (apiVersion == 1) { + endpoint = "/services/iscsi/extent/"; + } else { + endpoint = "/iscsi/extent/id/"; + } + endpoint += extentId; + response = await httpClient.get(endpoint); + + // assume is gone for now + if ([404, 500].includes(response.statusCode)) { + } else { + deleteAsset = true; + assetName = null; + + // checking if set for backwards compatibility + if (zb.helpers.isPropertyValueSet(iscsiName)) { + switch (apiVersion) { + case 1: + assetName = response.body.iscsi_target_extent_name; + break; + case 2: + assetName = response.body.name; + break; + } + + if (assetName != iscsiName) { + deleteAsset = false; + } + } + + if (deleteAsset) { + response = await httpClient.delete(endpoint); + if (![200, 204, 404].includes(response.statusCode)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting iscsi extent - extent: ${extentId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + // remove property to prevent delete race conditions + // due to id re-use by FreeNAS/TrueNAS + await httpApiClient.DatasetInherit( + datasetName, + FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME + ); + } else { + this.ctx.logger.debug( + "not deleting iscsiextent asset as it appears ID %s has been re-used: zfs name - %s, iscsiextent name - %s", + extentId, + iscsiName, + assetName + ); + } } } break; @@ -1630,195 +2029,90 @@ class FreeNASApiDriver extends CsiBaseDriver { } } break; - case "iscsi": - // Delete target - // NOTE: deleting a target inherently deletes associated targetgroup(s) and targettoextent(s) - // Delete extent - try { - properties = await httpApiClient.DatasetGet(datasetName, [ - FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME, - FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME, - FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME, - FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME, - ]); - } catch (err) { - if (err.toString().includes("dataset does not exist")) { - return; + case "nvmeof": + { + switch (apiVersion) { + case 1: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `nvmeof feature is only available with version 2 of the api` + ); + break; } - throw err; - } - this.ctx.logger.debug("zfs props data: %j", properties); - - let targetId = properties[FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME].value; - let extentId = properties[FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME].value; - let iscsiName = - properties[FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME].value; - let assetName; - - switch (apiVersion) { - case 1: - case 2: - // only remove if the process has not succeeded already - if (zb.helpers.isPropertyValueSet(targetId)) { - // https://jira.ixsystems.com/browse/NAS-103952 - - // v1 - /services/iscsi/target/{id}/ - // v2 - /iscsi/target/id/{id} - endpoint = ""; - if (apiVersion == 1) { - endpoint += "/services"; - } - endpoint += "/iscsi/target/"; - if (apiVersion == 2) { - endpoint += "id/"; - } - endpoint += targetId; - response = await httpClient.get(endpoint); - - // assume is gone for now - if ([404, 500].includes(response.statusCode)) { - } else { - deleteAsset = true; - assetName = null; - - // checking if set for backwards compatibility - if (zb.helpers.isPropertyValueSet(iscsiName)) { - switch (apiVersion) { - case 1: - assetName = response.body.iscsi_target_name; - break; - case 2: - assetName = response.body.name; - break; - } - - if (assetName != iscsiName) { - deleteAsset = false; - } - } - - if (deleteAsset) { - let retries = 0; - let maxRetries = 5; - let retryWait = 1000; - response = await httpClient.delete(endpoint); - - // sometimes after an initiator has detached it takes a moment for TrueNAS to settle - // code: 422 body: {\"message\":\"Target csi-ci-55877e95sanity-node-expand-volume-e54f81fa-cd38e798 is in use.\",\"errno\":14} - while ( - response.statusCode == 422 && - retries < maxRetries && - _.get(response, "body.message").includes("Target") && - _.get(response, "body.message").includes("is in use") && - _.get(response, "body.errno") == 14 - ) { - retries++; - this.ctx.logger.debug( - "target: %s is in use, retry %s shortly", - targetId, - retries - ); - await GeneralUtils.sleep(retryWait); - response = await httpClient.delete(endpoint); - } - - if (![200, 204, 404].includes(response.statusCode)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting iscsi target - target: ${targetId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - // remove property to prevent delete race conditions - // due to id re-use by FreeNAS/TrueNAS - await httpApiClient.DatasetInherit( - datasetName, - FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME - ); - } else { - this.ctx.logger.debug( - "not deleting iscsitarget asset as it appears ID %s has been re-used: zfs name - %s, iscsitarget name - %s", - targetId, - iscsiName, - assetName - ); - } - } - } - - // only remove if the process has not succeeded already - if (zb.helpers.isPropertyValueSet(extentId)) { - // v1 - /services/iscsi/targettoextent/{id}/ - // v2 - /iscsi/targetextent/id/{id} - if (apiVersion == 1) { - endpoint = "/services/iscsi/extent/"; - } else { - endpoint = "/iscsi/extent/id/"; - } - endpoint += extentId; - response = await httpClient.get(endpoint); - - // assume is gone for now - if ([404, 500].includes(response.statusCode)) { - } else { - deleteAsset = true; - assetName = null; - - // checking if set for backwards compatibility - if (zb.helpers.isPropertyValueSet(iscsiName)) { - switch (apiVersion) { - case 1: - assetName = response.body.iscsi_target_extent_name; - break; - case 2: - assetName = response.body.name; - break; - } - - if (assetName != iscsiName) { - deleteAsset = false; - } - } - - if (deleteAsset) { - response = await httpClient.delete(endpoint); - if (![200, 204, 404].includes(response.statusCode)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting iscsi extent - extent: ${extentId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - // remove property to prevent delete race conditions - // due to id re-use by FreeNAS/TrueNAS - await httpApiClient.DatasetInherit( - datasetName, - FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME - ); - } else { - this.ctx.logger.debug( - "not deleting iscsiextent asset as it appears ID %s has been re-used: zfs name - %s, iscsiextent name - %s", - extentId, - iscsiName, - assetName - ); - } - } - } - break; - default: + if (!isScale || semver.satisfies(truenasVersion, "<25.10")) { throw new GrpcError( grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` + `nvmeof feature is only available with TrueNAS version 25.10 and above` ); + } + + try { + properties = await httpApiClient.DatasetGet(datasetName, [ + FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME, + FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME, + FREENAS_NVMEOF_ASSETS_NAME_PROPERTY_NAME, + ]); + } catch (err) { + if (err.toString().includes("dataset does not exist")) { + return; + } + throw err; + } + this.ctx.logger.debug("zfs props data: %j", properties); + + let subsystemId = + properties[FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME].value; + let namespaceId = + properties[FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME].value; + + // remove namespace + if (zb.helpers.isPropertyValueSet(namespaceId)) { + await GeneralUtils.retry( + 15, + 2000, + async () => { + await httpApiClient.NvmetNamespaceDeleteById(namespaceId); + }, + { + retryCondition: (err) => { + return true; + }, + } + ); + + await httpApiClient.DatasetInherit( + datasetName, + FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME + ); + } + + // remove subsystem + if (zb.helpers.isPropertyValueSet(subsystemId)) { + await GeneralUtils.retry( + 15, + 2000, + async () => { + await httpApiClient.NvmetSubsysDeleteById(subsystemId, { + force: true, + }); + }, + { + retryCondition: (err) => { + return true; + }, + } + ); + + await httpApiClient.DatasetInherit( + datasetName, + FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME + ); + } } break; + default: throw new GrpcError( grpc.status.FAILED_PRECONDITION, @@ -2008,6 +2302,8 @@ class FreeNASApiDriver extends CsiBaseDriver { return "filesystem"; case "freenas-api-iscsi": case "truenas-api-iscsi": + case "freenas-api-nvmeof": + case "truenas-api-nvmeof": return "volume"; default: throw new Error("unknown driver: " + this.ctx.args.driver); @@ -2025,6 +2321,9 @@ class FreeNASApiDriver extends CsiBaseDriver { case "freenas-api-iscsi": case "truenas-api-iscsi": return "iscsi"; + case "freenas-api-nvmeof": + case "truenas-api-nvmeof": + return "nvmeof"; default: throw new Error("unknown driver: " + this.ctx.args.driver); } diff --git a/src/driver/freenas/http/api.js b/src/driver/freenas/http/api.js index a5bb236..4687ce1 100644 --- a/src/driver/freenas/http/api.js +++ b/src/driver/freenas/http/api.js @@ -645,6 +645,356 @@ class Api { throw new Error(JSON.stringify(response.body)); } + async NvmetSubsysList(data = {}) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = "/nvmet/subsys"; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.get(endpoint, data); + + if (response.statusCode == 200) { + return response.body; + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetSubsysCreate(subsysName, data = {}) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + data.name = subsysName; + data.allow_any_host = true; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = "/nvmet/subsys"; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.post(endpoint, data); + + if (response.statusCode == 200) { + return response.body; + } + + if ( + response.statusCode == 422 && + JSON.stringify(response.body).includes("already exists") + ) { + return this.NvmetSubsysGetByName(subsysName); + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetSubsysGetByName(subsysName, data = {}) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + data.name = subsysName; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = "/nvmet/subsys"; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.get(endpoint, data); + + if (response.statusCode == 200) { + for (const subsys of response.body) { + if (subsys.name == subsysName) { + return subsys; + } + } + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetSubsysGetById(id, data = {}) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = `/nvmet/subsys/id/${id}`; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.get(endpoint, data); + + if (response.statusCode == 200) { + return response.body; + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetSubsysDeleteById(id, data = {}) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = `/nvmet/subsys/id/${id}`; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.delete(endpoint, data); + + if (response.statusCode == 200) { + return; + } + + if ( + response.statusCode == 422 && + JSON.stringify(response.body).includes("does not exist") + ) { + return; + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetPortList(data = {}) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = "/nvmet/port"; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.get(endpoint, data); + + if (response.statusCode == 200) { + return response.body; + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetPortSubsysList(data = {}) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = "/nvmet/port_subsys"; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.get(endpoint, data); + + if (response.statusCode == 200) { + return response.body; + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetPortSubsysCreate(port_id, subsys_id) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + let data = { + port_id, + subsys_id, + }; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = "/nvmet/port_subsys"; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.post(endpoint, data); + + if (response.statusCode == 200) { + return response.body; + } + + //already exists + if ( + response.statusCode == 422 && + JSON.stringify(response.body).includes("already exists") + ) { + response = await this.NvmetPortSubsysList({ port_id, subsys_id }); + if (Array.isArray(response) && response.length == 1) { + return response[0]; + } + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetNamespaceCreate(zvol, subsysId, data = {}) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + zvol = String(zvol); + if (zvol.startsWith("/dev/")) { + zvol = zvol.substring(5); + } + + if (zvol.startsWith("/")) { + zvol = zvol.substring(1); + } + + if (!zvol.startsWith("zvol/")) { + zvol = `zvol/${zvol}`; + } + + data.device_type = "ZVOL"; + data.device_path = zvol; + data.subsys_id = subsysId; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = "/nvmet/namespace"; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.post(endpoint, data); + + if (response.statusCode == 200) { + return response.body; + } + + if ( + response.statusCode == 422 && + JSON.stringify(response.body).includes("already exists") + ) { + return this.NvmetSubsysGetByName(subsysName); + } + + if ( + response.statusCode == 422 && + JSON.stringify(response.body).includes("already used by subsystem") + ) { + //This device_path already used by subsystem: csi-pvc-111-clustera + return this.NvmetNamespaceGetByDeivcePath(zvol); + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetNamespaceGetByDeivcePath(zvol) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + zvol = String(zvol); + if (zvol.startsWith("/dev/")) { + zvol = zvol.substring(5); + } + + if (zvol.startsWith("/")) { + zvol = zvol.substring(1); + } + + if (!zvol.startsWith("zvol/")) { + zvol = `zvol/${zvol}`; + } + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = "/nvmet/namespace"; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + let data = { + device_path: zvol, + }; + + response = await httpClient.get(endpoint, data); + + if (response.statusCode == 200) { + if (Array.isArray(response.body) && response.body.length == 1) { + return response.body[0]; + } + } + + throw new Error(JSON.stringify(response.body)); + } + + async NvmetNamespaceDeleteById(id) { + const httpClient = await this.getHttpClient(false); + const zb = await this.getZetabyte(); + const systemVersionSemver = await this.getSystemVersionSemver(); + + let response; + let endpoint; + + if (semver.satisfies(systemVersionSemver, ">=25.10")) { + endpoint = `/nvmet/namespace/id/${id}`; + } else { + throw new Error("nvmet is unavailable with TrueNAS versions <25.10"); + } + + response = await httpClient.delete(endpoint); + + if (response.statusCode == 200) { + return; + } + + if ( + response.statusCode == 422 && + JSON.stringify(response.body).includes("does not exist") + ) { + return; + } + + throw new Error(JSON.stringify(response.body)); + } + async CloneCreate(snapshotName, datasetName, data = {}) { const httpClient = await this.getHttpClient(false); const zb = await this.getZetabyte(); diff --git a/src/driver/freenas/ssh.js b/src/driver/freenas/ssh.js index f18bb09..ab26768 100644 --- a/src/driver/freenas/ssh.js +++ b/src/driver/freenas/ssh.js @@ -13,6 +13,8 @@ const semver = require("semver"); // freenas properties const FREENAS_NFS_SHARE_PROPERTY_NAME = "democratic-csi:freenas_nfs_share_id"; const FREENAS_SMB_SHARE_PROPERTY_NAME = "democratic-csi:freenas_smb_share_id"; + +// iscsi const FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME = "democratic-csi:freenas_iscsi_target_id"; const FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME = @@ -22,6 +24,14 @@ const FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME = const FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME = "democratic-csi:freenas_iscsi_assets_name"; +// nvmeof +const FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME = + "democratic-csi:freenas_nvmeof_subsystem_id"; +const FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME = + "democratic-csi:freenas_nvmeof_namespace_id"; +const FREENAS_NVMEOF_ASSETS_NAME_PROPERTY_NAME = + "democratic-csi:freenas_nvmeof_assets_name"; + // used for in-memory cache of the version info const FREENAS_SYSTEM_VERSION_CACHE_KEY = "freenas:system_version"; const __REGISTRY_NS__ = "FreeNASSshDriver"; @@ -176,6 +186,9 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver { case "freenas-iscsi": case "truenas-iscsi": return "iscsi"; + case "freenas-nvmeof": + case "truenas-nvmeof": + return "nvmeof"; default: throw new Error("unknown driver: " + this.ctx.args.driver); } @@ -293,23 +306,574 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver { switch (driverShareType) { case "nfs": - properties = await zb.zfs.get(datasetName, [ - "mountpoint", - FREENAS_NFS_SHARE_PROPERTY_NAME, - ]); - properties = properties[datasetName]; - this.ctx.logger.debug("zfs props data: %j", properties); + { + properties = await zb.zfs.get(datasetName, [ + "mountpoint", + FREENAS_NFS_SHARE_PROPERTY_NAME, + ]); + properties = properties[datasetName]; + this.ctx.logger.debug("zfs props data: %j", properties); - // create nfs share - if ( - !zb.helpers.isPropertyValueSet( - properties[FREENAS_NFS_SHARE_PROPERTY_NAME].value - ) - ) { - let nfsShareComment; - if (this.options.nfs.shareCommentTemplate) { - nfsShareComment = Handlebars.compile( - this.options.nfs.shareCommentTemplate + // create nfs share + if ( + !zb.helpers.isPropertyValueSet( + properties[FREENAS_NFS_SHARE_PROPERTY_NAME].value + ) + ) { + let nfsShareComment; + if (this.options.nfs.shareCommentTemplate) { + nfsShareComment = Handlebars.compile( + this.options.nfs.shareCommentTemplate + )({ + name: call.request.name, + parameters: call.request.parameters, + csi: { + name: this.ctx.args.csiName, + version: this.ctx.args.csiVersion, + }, + zfs: { + datasetName: datasetName, + }, + }); + } else { + nfsShareComment = `democratic-csi (${this.ctx.args.csiName}): ${datasetName}`; + } + + switch (apiVersion) { + case 1: + case 2: + switch (apiVersion) { + case 1: + share = { + nfs_paths: [properties.mountpoint.value], + nfs_comment: nfsShareComment || "", + nfs_network: + this.options.nfs.shareAllowedNetworks.join(","), + nfs_hosts: this.options.nfs.shareAllowedHosts.join(","), + nfs_alldirs: this.options.nfs.shareAlldirs, + nfs_ro: false, + nfs_quiet: false, + nfs_maproot_user: this.options.nfs.shareMaprootUser, + nfs_maproot_group: this.options.nfs.shareMaprootGroup, + nfs_mapall_user: this.options.nfs.shareMapallUser, + nfs_mapall_group: this.options.nfs.shareMapallGroup, + nfs_security: [], + }; + break; + case 2: + share = { + paths: [properties.mountpoint.value], + comment: nfsShareComment || "", + networks: this.options.nfs.shareAllowedNetworks, + hosts: this.options.nfs.shareAllowedHosts, + alldirs: this.options.nfs.shareAlldirs, + ro: false, + quiet: false, + maproot_user: this.options.nfs.shareMaprootUser, + maproot_group: this.options.nfs.shareMaprootGroup, + mapall_user: this.options.nfs.shareMapallUser, + mapall_group: this.options.nfs.shareMapallGroup, + security: [], + }; + break; + } + + if (isScale && semver.satisfies(truenasVersion, ">=23.10")) { + delete share.quiet; + delete share.nfs_quiet; + } + + if (isScale && semver.satisfies(truenasVersion, ">=22.12")) { + share.path = share.paths[0]; + delete share.paths; + delete share.alldirs; + } + + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.post("/sharing/nfs", share); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); + + /** + * v1 = 201 + * v2 = 200 + */ + if ([200, 201].includes(response.statusCode)) { + let sharePaths; + switch (apiVersion) { + case 1: + sharePaths = response.body.nfs_paths; + break; + case 2: + if (response.body.path) { + sharePaths = [response.body.path]; + } else { + sharePaths = response.body.paths; + } + break; + } + + // FreeNAS responding with bad data + if (!sharePaths.includes(properties.mountpoint.value)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS responded with incorrect share data: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + //set zfs property + await zb.zfs.set(datasetName, { + [FREENAS_NFS_SHARE_PROPERTY_NAME]: response.body.id, + }); + } else { + /** + * v1 = 409 + * v2 = 422 + */ + if ( + [409, 422].includes(response.statusCode) && + (JSON.stringify(response.body).includes( + "You can't share same filesystem with all hosts twice." + ) || + JSON.stringify(response.body).includes( + "Another NFS share already exports this dataset for some network" + )) + ) { + let lookupShare = await this.findResourceByProperties( + "/sharing/nfs", + (item) => { + if ( + (item.nfs_paths && + item.nfs_paths.includes( + properties.mountpoint.value + )) || + (item.paths && + item.paths.includes(properties.mountpoint.value)) || + (item.path && + item.path == properties.mountpoint.value) + ) { + return true; + } + return false; + } + ); + + if (!lookupShare) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS failed to find matching share` + ); + } + + //set zfs property + await zb.zfs.set(datasetName, { + [FREENAS_NFS_SHARE_PROPERTY_NAME]: lookupShare.id, + }); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating nfs share - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } + } + + volume_context = { + node_attach_driver: "nfs", + server: this.options.nfs.shareHost, + share: properties.mountpoint.value, + }; + return volume_context; + } + + break; + /** + * TODO: smb need to be more defensive like iscsi and nfs + * ensuring the path is valid and the shareName + */ + case "smb": + { + properties = await zb.zfs.get(datasetName, [ + "mountpoint", + FREENAS_SMB_SHARE_PROPERTY_NAME, + ]); + properties = properties[datasetName]; + this.ctx.logger.debug("zfs props data: %j", properties); + + let smbName; + + if (this.options.smb.nameTemplate) { + smbName = Handlebars.compile(this.options.smb.nameTemplate)({ + name: call.request.name, + parameters: call.request.parameters, + }); + } else { + smbName = zb.helpers.extractLeafName(datasetName); + } + + if (this.options.smb.namePrefix) { + smbName = this.options.smb.namePrefix + smbName; + } + + if (this.options.smb.nameSuffix) { + smbName += this.options.smb.nameSuffix; + } + + smbName = smbName.toLowerCase(); + + this.ctx.logger.info( + "FreeNAS creating smb share with name: " + smbName + ); + + // create smb share + if ( + !zb.helpers.isPropertyValueSet( + properties[FREENAS_SMB_SHARE_PROPERTY_NAME].value + ) + ) { + /** + * The only required parameters are: + * - path + * - name + * + * Note that over time it appears the list of available parameters has increased + * so in an effort to best support old versions of FreeNAS we should check the + * presense of each parameter in the config and set the corresponding parameter in + * the API request *only* if present in the config. + */ + switch (apiVersion) { + case 1: + case 2: + share = { + name: smbName, + path: properties.mountpoint.value, + }; + + let propertyMapping = { + shareAuxiliaryConfigurationTemplate: "auxsmbconf", + shareHome: "home", + shareAllowedHosts: "hostsallow", + shareDeniedHosts: "hostsdeny", + shareDefaultPermissions: "default_permissions", + shareGuestOk: "guestok", + shareGuestOnly: "guestonly", + shareShowHiddenFiles: "showhiddenfiles", + shareRecycleBin: "recyclebin", + shareBrowsable: "browsable", + shareAccessBasedEnumeration: "abe", + shareTimeMachine: "timemachine", + shareStorageTask: "storage_task", + }; + + for (const key in propertyMapping) { + if (this.options.smb.hasOwnProperty(key)) { + let value; + switch (key) { + case "shareAuxiliaryConfigurationTemplate": + value = Handlebars.compile( + this.options.smb.shareAuxiliaryConfigurationTemplate + )({ + name: call.request.name, + parameters: call.request.parameters, + }); + break; + default: + value = this.options.smb[key]; + break; + } + share[propertyMapping[key]] = value; + } + } + + if (isScale && semver.satisfies(truenasVersion, ">=25.10")) { + let topLevelProperties = [ + "purpose", + "name", + "path", + "enabled", + "comment", + "readonly", + "browsable", + "access_based_share_enumeration", + "audit", + ]; + let disallowedOptions = ["abe"]; + share.purpose = "LEGACY_SHARE"; + share.options = { + purpose: "LEGACY_SHARE", + }; + for (const key in share) { + switch (key) { + case "options": + // ignore + break; + default: + if (!topLevelProperties.includes(key)) { + if (!disallowedOptions.includes(key)) { + share.options[key] = share[key]; + } + delete share[key]; + } + break; + } + } + } + + switch (apiVersion) { + case 1: + endpoint = "/sharing/cifs"; + + // rename keys with cifs_ prefix + for (const key in share) { + share["cifs_" + key] = share[key]; + delete share[key]; + } + + // convert to comma-separated list + if (share.cifs_hostsallow) { + share.cifs_hostsallow = share.cifs_hostsallow.join(","); + } + + // convert to comma-separated list + if (share.cifs_hostsdeny) { + share.cifs_hostsdeny = share.cifs_hostsdeny.join(","); + } + break; + case 2: + endpoint = "/sharing/smb"; + break; + } + + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.post(endpoint, share); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); + + /** + * v1 = 201 + * v2 = 200 + */ + if ([200, 201].includes(response.statusCode)) { + share = response.body; + let sharePath; + let shareName; + switch (apiVersion) { + case 1: + sharePath = response.body.cifs_path; + shareName = response.body.cifs_name; + break; + case 2: + sharePath = response.body.path; + shareName = response.body.name; + break; + } + + if (shareName != smbName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS responded with incorrect share data: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + if (sharePath != properties.mountpoint.value) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS responded with incorrect share data: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + //set zfs property + await zb.zfs.set(datasetName, { + [FREENAS_SMB_SHARE_PROPERTY_NAME]: response.body.id, + }); + } else { + /** + * v1 = 409 + * v2 = 422 + */ + if ( + [409, 422].includes(response.statusCode) && + JSON.stringify(response.body).includes( + "A share with this name already exists." + ) + ) { + let lookupShare = await this.findResourceByProperties( + endpoint, + (item) => { + if ( + (item.cifs_path && + item.cifs_path == properties.mountpoint.value && + item.cifs_name && + item.cifs_name == smbName) || + (item.path && + item.path == properties.mountpoint.value && + item.name && + item.name == smbName) + ) { + return true; + } + return false; + } + ); + + if (!lookupShare) { + throw new GrpcError( + grpc.status.UNKNOWN, + `FreeNAS failed to find matching share` + ); + } + + //set zfs property + await zb.zfs.set(datasetName, { + [FREENAS_SMB_SHARE_PROPERTY_NAME]: lookupShare.id, + }); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating smb share - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } + } + + volume_context = { + node_attach_driver: "smb", + server: this.options.smb.shareHost, + share: smbName, + }; + return volume_context; + } + + break; + case "iscsi": + { + properties = await zb.zfs.get(datasetName, [ + FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME, + FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME, + FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME, + ]); + properties = properties[datasetName]; + this.ctx.logger.debug("zfs props data: %j", properties); + + let basename; + let iscsiName; + + if (this.options.iscsi.nameTemplate) { + iscsiName = Handlebars.compile(this.options.iscsi.nameTemplate)({ + name: call.request.name, + parameters: call.request.parameters, + }); + } else { + iscsiName = zb.helpers.extractLeafName(datasetName); + } + + if (this.options.iscsi.namePrefix) { + iscsiName = this.options.iscsi.namePrefix + iscsiName; + } + + if (this.options.iscsi.nameSuffix) { + iscsiName += this.options.iscsi.nameSuffix; + } + + // According to RFC3270, 'Each iSCSI node, whether an initiator or target, MUST have an iSCSI name. Initiators and targets MUST support the receipt of iSCSI names of up to the maximum length of 223 bytes.' + // https://kb.netapp.com/Advice_and_Troubleshooting/Miscellaneous/What_is_the_maximum_length_of_a_iSCSI_iqn_name + // https://tools.ietf.org/html/rfc3720 + // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 + iscsiName = iscsiName.toLowerCase(); + + let extentDiskName = "zvol/" + datasetName; + let maxZvolNameLength = await driver.getMaxZvolNameLength(); + driver.ctx.logger.debug( + "max zvol name length: %s", + maxZvolNameLength + ); + + /** + * limit is a FreeBSD limitation + * https://www.ixsystems.com/documentation/freenas/11.2-U5/storage.html#zfs-zvol-config-opts-tab + */ + + if (extentDiskName.length > maxZvolNameLength) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `extent disk name cannot exceed ${maxZvolNameLength} characters: ${extentDiskName}` + ); + } + + // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 + if (isScale && iscsiName.length > 64) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `extent name cannot exceed 64 characters: ${iscsiName}` + ); + } + + this.ctx.logger.info( + "FreeNAS creating iscsi assets with name: " + iscsiName + ); + + let extentComment; + if (this.options.iscsi.extentCommentTemplate) { + extentComment = Handlebars.compile( + this.options.iscsi.extentCommentTemplate )({ name: call.request.name, parameters: call.request.parameters, @@ -322,1178 +886,808 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver { }, }); } else { - nfsShareComment = `democratic-csi (${this.ctx.args.csiName}): ${datasetName}`; + extentComment = ""; } - switch (apiVersion) { - case 1: - case 2: - switch (apiVersion) { - case 1: - share = { - nfs_paths: [properties.mountpoint.value], - nfs_comment: nfsShareComment || "", - nfs_network: - this.options.nfs.shareAllowedNetworks.join(","), - nfs_hosts: this.options.nfs.shareAllowedHosts.join(","), - nfs_alldirs: this.options.nfs.shareAlldirs, - nfs_ro: false, - nfs_quiet: false, - nfs_maproot_user: this.options.nfs.shareMaprootUser, - nfs_maproot_group: this.options.nfs.shareMaprootGroup, - nfs_mapall_user: this.options.nfs.shareMapallUser, - nfs_mapall_group: this.options.nfs.shareMapallGroup, - nfs_security: [], - }; - break; - case 2: - share = { - paths: [properties.mountpoint.value], - comment: nfsShareComment || "", - networks: this.options.nfs.shareAllowedNetworks, - hosts: this.options.nfs.shareAllowedHosts, - alldirs: this.options.nfs.shareAlldirs, - ro: false, - quiet: false, - maproot_user: this.options.nfs.shareMaprootUser, - maproot_group: this.options.nfs.shareMaprootGroup, - mapall_user: this.options.nfs.shareMapallUser, - mapall_group: this.options.nfs.shareMapallGroup, - security: [], - }; - break; - } - - if (isScale && semver.satisfies(truenasVersion, ">=23.10")) { - delete share.quiet; - delete share.nfs_quiet; - } - - if (isScale && semver.satisfies(truenasVersion, ">=22.12")) { - share.path = share.paths[0]; - delete share.paths; - delete share.alldirs; - } - - response = await GeneralUtils.retry( - 3, - 1000, - async () => { - return await httpClient.post("/sharing/nfs", share); - }, - { - retryCondition: (err) => { - if (err.code == "ECONNRESET") { - return true; - } - if (err.code == "ECONNABORTED") { - return true; - } - if (err.response && err.response.statusCode == 504) { - return true; - } - return false; - }, - } - ); - - /** - * v1 = 201 - * v2 = 200 - */ - if ([200, 201].includes(response.statusCode)) { - let sharePaths; - switch (apiVersion) { - case 1: - sharePaths = response.body.nfs_paths; - break; - case 2: - if (response.body.path) { - sharePaths = [response.body.path]; - } else { - sharePaths = response.body.paths; - } - break; - } - - // FreeNAS responding with bad data - if (!sharePaths.includes(properties.mountpoint.value)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS responded with incorrect share data: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - //set zfs property - await zb.zfs.set(datasetName, { - [FREENAS_NFS_SHARE_PROPERTY_NAME]: response.body.id, - }); - } else { - /** - * v1 = 409 - * v2 = 422 - */ - if ( - [409, 422].includes(response.statusCode) && - (JSON.stringify(response.body).includes( - "You can't share same filesystem with all hosts twice." - ) || - JSON.stringify(response.body).includes( - "Another NFS share already exports this dataset for some network" - )) - ) { - let lookupShare = await this.findResourceByProperties( - "/sharing/nfs", - (item) => { - if ( - (item.nfs_paths && - item.nfs_paths.includes( - properties.mountpoint.value - )) || - (item.paths && - item.paths.includes(properties.mountpoint.value)) || - (item.path && item.path == properties.mountpoint.value) - ) { - return true; - } - return false; - } - ); - - if (!lookupShare) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS failed to find matching share` - ); - } - - //set zfs property - await zb.zfs.set(datasetName, { - [FREENAS_NFS_SHARE_PROPERTY_NAME]: lookupShare.id, - }); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating nfs share - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } - break; - default: - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` - ); - } - } - - volume_context = { - node_attach_driver: "nfs", - server: this.options.nfs.shareHost, - share: properties.mountpoint.value, - }; - return volume_context; - - break; - /** - * TODO: smb need to be more defensive like iscsi and nfs - * ensuring the path is valid and the shareName - */ - case "smb": - properties = await zb.zfs.get(datasetName, [ - "mountpoint", - FREENAS_SMB_SHARE_PROPERTY_NAME, - ]); - properties = properties[datasetName]; - this.ctx.logger.debug("zfs props data: %j", properties); - - let smbName; - - if (this.options.smb.nameTemplate) { - smbName = Handlebars.compile(this.options.smb.nameTemplate)({ - name: call.request.name, - parameters: call.request.parameters, - }); - } else { - smbName = zb.helpers.extractLeafName(datasetName); - } - - if (this.options.smb.namePrefix) { - smbName = this.options.smb.namePrefix + smbName; - } - - if (this.options.smb.nameSuffix) { - smbName += this.options.smb.nameSuffix; - } - - smbName = smbName.toLowerCase(); - - this.ctx.logger.info( - "FreeNAS creating smb share with name: " + smbName - ); - - // create smb share - if ( - !zb.helpers.isPropertyValueSet( - properties[FREENAS_SMB_SHARE_PROPERTY_NAME].value + const extentInsecureTpc = this.options.iscsi.hasOwnProperty( + "extentInsecureTpc" ) - ) { - /** - * The only required parameters are: - * - path - * - name - * - * Note that over time it appears the list of available parameters has increased - * so in an effort to best support old versions of FreeNAS we should check the - * presense of each parameter in the config and set the corresponding parameter in - * the API request *only* if present in the config. - */ - switch (apiVersion) { - case 1: - case 2: - share = { - name: smbName, - path: properties.mountpoint.value, - }; - - let propertyMapping = { - shareAuxiliaryConfigurationTemplate: "auxsmbconf", - shareHome: "home", - shareAllowedHosts: "hostsallow", - shareDeniedHosts: "hostsdeny", - shareDefaultPermissions: "default_permissions", - shareGuestOk: "guestok", - shareGuestOnly: "guestonly", - shareShowHiddenFiles: "showhiddenfiles", - shareRecycleBin: "recyclebin", - shareBrowsable: "browsable", - shareAccessBasedEnumeration: "abe", - shareTimeMachine: "timemachine", - shareStorageTask: "storage_task", - }; - - for (const key in propertyMapping) { - if (this.options.smb.hasOwnProperty(key)) { - let value; - switch (key) { - case "shareAuxiliaryConfigurationTemplate": - value = Handlebars.compile( - this.options.smb.shareAuxiliaryConfigurationTemplate - )({ - name: call.request.name, - parameters: call.request.parameters, - }); - break; - default: - value = this.options.smb[key]; - break; - } - share[propertyMapping[key]] = value; - } - } - - if (isScale && semver.satisfies(truenasVersion, ">=25.10")) { - let topLevelProperties = [ - "purpose", - "name", - "path", - "enabled", - "comment", - "readonly", - "browsable", - "access_based_share_enumeration", - "audit", - ]; - let disallowedOptions = ["abe"]; - share.purpose = "LEGACY_SHARE"; - share.options = { - purpose: "LEGACY_SHARE", - }; - for (const key in share) { - switch (key) { - case "options": - // ignore - break; - default: - if (!topLevelProperties.includes(key)) { - if (!disallowedOptions.includes(key)) { - share.options[key] = share[key]; - } - delete share[key]; - } - break; - } - } - } - - switch (apiVersion) { - case 1: - endpoint = "/sharing/cifs"; - - // rename keys with cifs_ prefix - for (const key in share) { - share["cifs_" + key] = share[key]; - delete share[key]; - } - - // convert to comma-separated list - if (share.cifs_hostsallow) { - share.cifs_hostsallow = share.cifs_hostsallow.join(","); - } - - // convert to comma-separated list - if (share.cifs_hostsdeny) { - share.cifs_hostsdeny = share.cifs_hostsdeny.join(","); - } - break; - case 2: - endpoint = "/sharing/smb"; - break; - } - - response = await GeneralUtils.retry( - 3, - 1000, - async () => { - return await httpClient.post(endpoint, share); - }, - { - retryCondition: (err) => { - if (err.code == "ECONNRESET") { - return true; - } - if (err.code == "ECONNABORTED") { - return true; - } - if (err.response && err.response.statusCode == 504) { - return true; - } - return false; - }, - } - ); - - /** - * v1 = 201 - * v2 = 200 - */ - if ([200, 201].includes(response.statusCode)) { - share = response.body; - let sharePath; - let shareName; - switch (apiVersion) { - case 1: - sharePath = response.body.cifs_path; - shareName = response.body.cifs_name; - break; - case 2: - sharePath = response.body.path; - shareName = response.body.name; - break; - } - - if (shareName != smbName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS responded with incorrect share data: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - if (sharePath != properties.mountpoint.value) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS responded with incorrect share data: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - //set zfs property - await zb.zfs.set(datasetName, { - [FREENAS_SMB_SHARE_PROPERTY_NAME]: response.body.id, - }); - } else { - /** - * v1 = 409 - * v2 = 422 - */ - if ( - [409, 422].includes(response.statusCode) && - JSON.stringify(response.body).includes( - "A share with this name already exists." - ) - ) { - let lookupShare = await this.findResourceByProperties( - endpoint, - (item) => { - if ( - (item.cifs_path && - item.cifs_path == properties.mountpoint.value && - item.cifs_name && - item.cifs_name == smbName) || - (item.path && - item.path == properties.mountpoint.value && - item.name && - item.name == smbName) - ) { - return true; - } - return false; - } - ); - - if (!lookupShare) { - throw new GrpcError( - grpc.status.UNKNOWN, - `FreeNAS failed to find matching share` - ); - } - - //set zfs property - await zb.zfs.set(datasetName, { - [FREENAS_SMB_SHARE_PROPERTY_NAME]: lookupShare.id, - }); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating smb share - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } - break; - default: - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` - ); - } - } - - volume_context = { - node_attach_driver: "smb", - server: this.options.smb.shareHost, - share: smbName, - }; - return volume_context; - - break; - case "iscsi": - properties = await zb.zfs.get(datasetName, [ - FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME, - FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME, - FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME, - ]); - properties = properties[datasetName]; - this.ctx.logger.debug("zfs props data: %j", properties); - - let basename; - let iscsiName; - - if (this.options.iscsi.nameTemplate) { - iscsiName = Handlebars.compile(this.options.iscsi.nameTemplate)({ - name: call.request.name, - parameters: call.request.parameters, - }); - } else { - iscsiName = zb.helpers.extractLeafName(datasetName); - } - - if (this.options.iscsi.namePrefix) { - iscsiName = this.options.iscsi.namePrefix + iscsiName; - } - - if (this.options.iscsi.nameSuffix) { - iscsiName += this.options.iscsi.nameSuffix; - } - - // According to RFC3270, 'Each iSCSI node, whether an initiator or target, MUST have an iSCSI name. Initiators and targets MUST support the receipt of iSCSI names of up to the maximum length of 223 bytes.' - // https://kb.netapp.com/Advice_and_Troubleshooting/Miscellaneous/What_is_the_maximum_length_of_a_iSCSI_iqn_name - // https://tools.ietf.org/html/rfc3720 - // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 - iscsiName = iscsiName.toLowerCase(); - - let extentDiskName = "zvol/" + datasetName; - let maxZvolNameLength = await driver.getMaxZvolNameLength(); - driver.ctx.logger.debug("max zvol name length: %s", maxZvolNameLength); - - /** - * limit is a FreeBSD limitation - * https://www.ixsystems.com/documentation/freenas/11.2-U5/storage.html#zfs-zvol-config-opts-tab - */ - - if (extentDiskName.length > maxZvolNameLength) { - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `extent disk name cannot exceed ${maxZvolNameLength} characters: ${extentDiskName}` - ); - } - - // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 - if (isScale && iscsiName.length > 64) { - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `extent name cannot exceed 64 characters: ${iscsiName}` - ); - } - - this.ctx.logger.info( - "FreeNAS creating iscsi assets with name: " + iscsiName - ); - - let extentComment; - if (this.options.iscsi.extentCommentTemplate) { - extentComment = Handlebars.compile( - this.options.iscsi.extentCommentTemplate - )({ - name: call.request.name, - parameters: call.request.parameters, - csi: { - name: this.ctx.args.csiName, - version: this.ctx.args.csiVersion, - }, - zfs: { - datasetName: datasetName, - }, - }); - } else { - extentComment = ""; - } - - const extentInsecureTpc = this.options.iscsi.hasOwnProperty( - "extentInsecureTpc" - ) - ? this.options.iscsi.extentInsecureTpc - : true; - - const extentXenCompat = this.options.iscsi.hasOwnProperty( - "extentXenCompat" - ) - ? this.options.iscsi.extentXenCompat - : false; - - const extentBlocksize = this.options.iscsi.hasOwnProperty( - "extentBlocksize" - ) - ? this.options.iscsi.extentBlocksize - : 512; - - const extentDisablePhysicalBlocksize = - this.options.iscsi.hasOwnProperty("extentDisablePhysicalBlocksize") - ? this.options.iscsi.extentDisablePhysicalBlocksize + ? this.options.iscsi.extentInsecureTpc : true; - const extentRpm = this.options.iscsi.hasOwnProperty("extentRpm") - ? this.options.iscsi.extentRpm - : "SSD"; - - let extentAvailThreshold = this.options.iscsi.hasOwnProperty( - "extentAvailThreshold" - ) - ? Number(this.options.iscsi.extentAvailThreshold) - : null; - - if (!(extentAvailThreshold > 0 && extentAvailThreshold <= 100)) { - extentAvailThreshold = null; - } - - switch (apiVersion) { - case 1: - response = await httpClient.get( - "/services/iscsi/globalconfiguration" - ); - if (response.statusCode != 200) { - throw new GrpcError( - grpc.status.UNKNOWN, - `error getting iscsi configuration - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - basename = response.body.iscsi_basename; - this.ctx.logger.verbose("FreeNAS ISCSI BASENAME: " + basename); - break; - case 2: - response = await httpClient.get("/iscsi/global"); - if (response.statusCode != 200) { - throw new GrpcError( - grpc.status.UNKNOWN, - `error getting iscsi configuration - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - basename = response.body.basename; - this.ctx.logger.verbose("FreeNAS ISCSI BASENAME: " + basename); - break; - default: - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` - ); - } - - // if we got all the way to the TARGETTOEXTENT then we fully finished - // otherwise we must do all assets every time due to the interdependence of IDs etc - if ( - !zb.helpers.isPropertyValueSet( - properties[FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME].value + const extentXenCompat = this.options.iscsi.hasOwnProperty( + "extentXenCompat" ) - ) { + ? this.options.iscsi.extentXenCompat + : false; + + const extentBlocksize = this.options.iscsi.hasOwnProperty( + "extentBlocksize" + ) + ? this.options.iscsi.extentBlocksize + : 512; + + const extentDisablePhysicalBlocksize = + this.options.iscsi.hasOwnProperty("extentDisablePhysicalBlocksize") + ? this.options.iscsi.extentDisablePhysicalBlocksize + : true; + + const extentRpm = this.options.iscsi.hasOwnProperty("extentRpm") + ? this.options.iscsi.extentRpm + : "SSD"; + + let extentAvailThreshold = this.options.iscsi.hasOwnProperty( + "extentAvailThreshold" + ) + ? Number(this.options.iscsi.extentAvailThreshold) + : null; + + if (!(extentAvailThreshold > 0 && extentAvailThreshold <= 100)) { + extentAvailThreshold = null; + } + switch (apiVersion) { - case 1: { - // create target - let target = { - iscsi_target_name: iscsiName, - iscsi_target_alias: "", // TODO: allow template for this - }; - - response = await httpClient.post( - "/services/iscsi/target", - target + case 1: + response = await httpClient.get( + "/services/iscsi/globalconfiguration" ); - - // 409 if invalid - if (response.statusCode != 201) { - target = null; - if ( - response.statusCode == 409 && - JSON.stringify(response.body).includes( - "Target name already exists" - ) - ) { - target = await this.findResourceByProperties( - "/services/iscsi/target", - { - iscsi_target_name: iscsiName, - } - ); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating iscsi target - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } else { - target = response.body; - } - - if (!target) { + if (response.statusCode != 200) { throw new GrpcError( grpc.status.UNKNOWN, - `unknown error creating iscsi target` + `error getting iscsi configuration - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` ); } - - if (target.iscsi_target_name != iscsiName) { + basename = response.body.iscsi_basename; + this.ctx.logger.verbose("FreeNAS ISCSI BASENAME: " + basename); + break; + case 2: + response = await httpClient.get("/iscsi/global"); + if (response.statusCode != 200) { throw new GrpcError( grpc.status.UNKNOWN, - `mismatch name error creating iscsi target` + `error getting iscsi configuration - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` ); } + basename = response.body.basename; + this.ctx.logger.verbose("FreeNAS ISCSI BASENAME: " + basename); + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } - this.ctx.logger.verbose("FreeNAS ISCSI TARGET: %j", target); - - // set target.id on zvol - await zb.zfs.set(datasetName, { - [FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME]: target.id, - }); - - // create targetgroup(s) - // targetgroups do have IDs - for (let targetGroupConfig of this.options.iscsi.targetGroups) { - let targetGroup = { - iscsi_target: target.id, - iscsi_target_authgroup: - targetGroupConfig.targetGroupAuthGroup, - iscsi_target_authtype: targetGroupConfig.targetGroupAuthType - ? targetGroupConfig.targetGroupAuthType - : "None", - iscsi_target_portalgroup: - targetGroupConfig.targetGroupPortalGroup, - iscsi_target_initiatorgroup: - targetGroupConfig.targetGroupInitiatorGroup, - iscsi_target_initialdigest: "Auto", + // if we got all the way to the TARGETTOEXTENT then we fully finished + // otherwise we must do all assets every time due to the interdependence of IDs etc + if ( + !zb.helpers.isPropertyValueSet( + properties[FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME].value + ) + ) { + switch (apiVersion) { + case 1: { + // create target + let target = { + iscsi_target_name: iscsiName, + iscsi_target_alias: "", // TODO: allow template for this }; + response = await httpClient.post( - "/services/iscsi/targetgroup", - targetGroup + "/services/iscsi/target", + target ); // 409 if invalid if (response.statusCode != 201) { - targetGroup = null; - /** - * 404 gets returned with an unable to process response when the DB is corrupted (has invalid entries in essense) - * - * To resolve properly the DB should be cleaned up - * /usr/local/etc/rc.d/django stop - * /usr/local/etc/rc.d/nginx stop - * sqlite3 /data/freenas-v1.db - * - * // this deletes everything, probably not what you want - * // should have a better query to only find entries where associated assets no longer exist - * DELETE from services_iscsitargetgroups; - * - * /usr/local/etc/rc.d/django restart - * /usr/local/etc/rc.d/nginx restart - */ + target = null; if ( - response.statusCode == 404 || - (response.statusCode == 409 && - JSON.stringify(response.body).includes( - "cannot be duplicated on a target" - )) + response.statusCode == 409 && + JSON.stringify(response.body).includes( + "Target name already exists" + ) ) { - targetGroup = await this.findResourceByProperties( - "/services/iscsi/targetgroup", + target = await this.findResourceByProperties( + "/services/iscsi/target", { - iscsi_target: target.id, - iscsi_target_portalgroup: - targetGroupConfig.targetGroupPortalGroup, - iscsi_target_initiatorgroup: - targetGroupConfig.targetGroupInitiatorGroup, + iscsi_target_name: iscsiName, } ); } else { throw new GrpcError( grpc.status.UNKNOWN, - `received error creating iscsi targetgroup - code: ${ + `received error creating iscsi target - code: ${ response.statusCode } body: ${JSON.stringify(response.body)}` ); } } else { - targetGroup = response.body; + target = response.body; } - if (!targetGroup) { + if (!target) { throw new GrpcError( grpc.status.UNKNOWN, - `unknown error creating iscsi targetgroup` + `unknown error creating iscsi target` ); } - this.ctx.logger.verbose( - "FreeNAS ISCSI TARGET_GROUP: %j", - targetGroup - ); - } - - let extent = { - iscsi_target_extent_comment: extentComment, - iscsi_target_extent_type: "Disk", // Disk/File, after save Disk becomes "ZVOL" - iscsi_target_extent_name: iscsiName, - iscsi_target_extent_insecure_tpc: extentInsecureTpc, - //iscsi_target_extent_naa: "0x3822690834aae6c5", - iscsi_target_extent_disk: extentDiskName, - iscsi_target_extent_xen: extentXenCompat, - iscsi_target_extent_avail_threshold: extentAvailThreshold, - iscsi_target_extent_blocksize: Number(extentBlocksize), - iscsi_target_extent_pblocksize: extentDisablePhysicalBlocksize, - iscsi_target_extent_rpm: isNaN(Number(extentRpm)) - ? "SSD" - : Number(extentRpm), - iscsi_target_extent_ro: false, - }; - response = await httpClient.post( - "/services/iscsi/extent", - extent - ); - - // 409 if invalid - if (response.statusCode != 201) { - extent = null; - if ( - response.statusCode == 409 && - JSON.stringify(response.body).includes( - "Extent name must be unique" - ) - ) { - extent = await this.findResourceByProperties( - "/services/iscsi/extent", - { iscsi_target_extent_name: iscsiName } - ); - } else { + if (target.iscsi_target_name != iscsiName) { throw new GrpcError( grpc.status.UNKNOWN, - `received error creating iscsi extent - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` + `mismatch name error creating iscsi target` ); } - } else { - extent = response.body; - } - if (!extent) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi extent` - ); - } + this.ctx.logger.verbose("FreeNAS ISCSI TARGET: %j", target); - if (extent.iscsi_target_extent_name != iscsiName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `mismatch name error creating iscsi extent` - ); - } - - this.ctx.logger.verbose("FreeNAS ISCSI EXTENT: %j", extent); - - await zb.zfs.set(datasetName, { - [FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME]: extent.id, - }); - - // create targettoextent - let targetToExtent = { - iscsi_target: target.id, - iscsi_extent: extent.id, - iscsi_lunid: 0, - }; - response = await httpClient.post( - "/services/iscsi/targettoextent", - targetToExtent - ); - - // 409 if invalid - if (response.statusCode != 201) { - targetToExtent = null; - - // LUN ID is already being used for this target. - // Extent is already in this target. - if ( - response.statusCode == 409 && - (JSON.stringify(response.body).includes( - "Extent is already in this target." - ) || - JSON.stringify(response.body).includes( - "LUN ID is already being used for this target." - )) - ) { - targetToExtent = await this.findResourceByProperties( - "/services/iscsi/targettoextent", - { - iscsi_target: target.id, - iscsi_extent: extent.id, - iscsi_lunid: 0, - } - ); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating iscsi targettoextent - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } else { - targetToExtent = response.body; - } - - if (!targetToExtent) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi targettoextent` - ); - } - this.ctx.logger.verbose( - "FreeNAS ISCSI TARGET_TO_EXTENT: %j", - targetToExtent - ); - - await zb.zfs.set(datasetName, { - [FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME]: - targetToExtent.id, - }); - - break; - } - case 2: - // create target and targetgroup - //let targetId; - let targetGroups = []; - for (let targetGroupConfig of this.options.iscsi.targetGroups) { - targetGroups.push({ - portal: targetGroupConfig.targetGroupPortalGroup, - initiator: targetGroupConfig.targetGroupInitiatorGroup, - auth: - targetGroupConfig.targetGroupAuthGroup > 0 - ? targetGroupConfig.targetGroupAuthGroup - : null, - authmethod: - targetGroupConfig.targetGroupAuthType.length > 0 - ? targetGroupConfig.targetGroupAuthType - .toUpperCase() - .replace(" ", "_") - : "NONE", + // set target.id on zvol + await zb.zfs.set(datasetName, { + [FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME]: target.id, }); - } - let target = { - name: iscsiName, - alias: null, // cannot send "" error: handler error - driver: FreeNASDriver method: CreateVolume error: {"name":"GrpcError","code":2,"message":"received error creating iscsi target - code: 422 body: {\"iscsi_target_create.alias\":[{\"message\":\"Alias already exists\",\"errno\":22}]}"} - mode: "ISCSI", - groups: targetGroups, - }; - response = await httpClient.post("/iscsi/target", target); - - // 409 if invalid - if (response.statusCode != 200) { - target = null; - if ( - response.statusCode == 422 && - JSON.stringify(response.body).includes( - "Target name already exists" - ) - ) { - target = await this.findResourceByProperties( - "/iscsi/target", - { - name: iscsiName, - } + // create targetgroup(s) + // targetgroups do have IDs + for (let targetGroupConfig of this.options.iscsi.targetGroups) { + let targetGroup = { + iscsi_target: target.id, + iscsi_target_authgroup: + targetGroupConfig.targetGroupAuthGroup, + iscsi_target_authtype: targetGroupConfig.targetGroupAuthType + ? targetGroupConfig.targetGroupAuthType + : "None", + iscsi_target_portalgroup: + targetGroupConfig.targetGroupPortalGroup, + iscsi_target_initiatorgroup: + targetGroupConfig.targetGroupInitiatorGroup, + iscsi_target_initialdigest: "Auto", + }; + response = await httpClient.post( + "/services/iscsi/targetgroup", + targetGroup ); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating iscsi target - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` + + // 409 if invalid + if (response.statusCode != 201) { + targetGroup = null; + /** + * 404 gets returned with an unable to process response when the DB is corrupted (has invalid entries in essense) + * + * To resolve properly the DB should be cleaned up + * /usr/local/etc/rc.d/django stop + * /usr/local/etc/rc.d/nginx stop + * sqlite3 /data/freenas-v1.db + * + * // this deletes everything, probably not what you want + * // should have a better query to only find entries where associated assets no longer exist + * DELETE from services_iscsitargetgroups; + * + * /usr/local/etc/rc.d/django restart + * /usr/local/etc/rc.d/nginx restart + */ + if ( + response.statusCode == 404 || + (response.statusCode == 409 && + JSON.stringify(response.body).includes( + "cannot be duplicated on a target" + )) + ) { + targetGroup = await this.findResourceByProperties( + "/services/iscsi/targetgroup", + { + iscsi_target: target.id, + iscsi_target_portalgroup: + targetGroupConfig.targetGroupPortalGroup, + iscsi_target_initiatorgroup: + targetGroupConfig.targetGroupInitiatorGroup, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi targetgroup - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } else { + targetGroup = response.body; + } + + if (!targetGroup) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi targetgroup` + ); + } + + this.ctx.logger.verbose( + "FreeNAS ISCSI TARGET_GROUP: %j", + targetGroup ); } - } else { - target = response.body; - } - if (!target) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi target` + let extent = { + iscsi_target_extent_comment: extentComment, + iscsi_target_extent_type: "Disk", // Disk/File, after save Disk becomes "ZVOL" + iscsi_target_extent_name: iscsiName, + iscsi_target_extent_insecure_tpc: extentInsecureTpc, + //iscsi_target_extent_naa: "0x3822690834aae6c5", + iscsi_target_extent_disk: extentDiskName, + iscsi_target_extent_xen: extentXenCompat, + iscsi_target_extent_avail_threshold: extentAvailThreshold, + iscsi_target_extent_blocksize: Number(extentBlocksize), + iscsi_target_extent_pblocksize: + extentDisablePhysicalBlocksize, + iscsi_target_extent_rpm: isNaN(Number(extentRpm)) + ? "SSD" + : Number(extentRpm), + iscsi_target_extent_ro: false, + }; + response = await httpClient.post( + "/services/iscsi/extent", + extent ); - } - if (target.name != iscsiName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `mismatch name error creating iscsi target` - ); - } - - // handle situations/race conditions where groups failed to be added/created on the target - // groups":[{"portal":1,"initiator":1,"auth":null,"authmethod":"NONE"},{"portal":2,"initiator":1,"auth":null,"authmethod":"NONE"}] - // TODO: this logic could be more intelligent but this should do for now as it appears in the failure scenario no groups are added - // in other words, I have never seen them invalid, only omitted so this should be enough - if (target.groups.length != targetGroups.length) { - response = await httpClient.put( - `/iscsi/target/id/${target.id}`, - { - groups: targetGroups, + // 409 if invalid + if (response.statusCode != 201) { + extent = null; + if ( + response.statusCode == 409 && + JSON.stringify(response.body).includes( + "Extent name must be unique" + ) + ) { + extent = await this.findResourceByProperties( + "/services/iscsi/extent", + { iscsi_target_extent_name: iscsiName } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi extent - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); } - ); + } else { + extent = response.body; + } - if (response.statusCode != 200) { + if (!extent) { throw new GrpcError( grpc.status.UNKNOWN, - `failed setting target groups` + `unknown error creating iscsi extent` ); + } + + if (extent.iscsi_target_extent_name != iscsiName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `mismatch name error creating iscsi extent` + ); + } + + this.ctx.logger.verbose("FreeNAS ISCSI EXTENT: %j", extent); + + await zb.zfs.set(datasetName, { + [FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME]: extent.id, + }); + + // create targettoextent + let targetToExtent = { + iscsi_target: target.id, + iscsi_extent: extent.id, + iscsi_lunid: 0, + }; + response = await httpClient.post( + "/services/iscsi/targettoextent", + targetToExtent + ); + + // 409 if invalid + if (response.statusCode != 201) { + targetToExtent = null; + + // LUN ID is already being used for this target. + // Extent is already in this target. + if ( + response.statusCode == 409 && + (JSON.stringify(response.body).includes( + "Extent is already in this target." + ) || + JSON.stringify(response.body).includes( + "LUN ID is already being used for this target." + )) + ) { + targetToExtent = await this.findResourceByProperties( + "/services/iscsi/targettoextent", + { + iscsi_target: target.id, + iscsi_extent: extent.id, + iscsi_lunid: 0, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi targettoextent - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } else { + targetToExtent = response.body; + } + + if (!targetToExtent) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi targettoextent` + ); + } + this.ctx.logger.verbose( + "FreeNAS ISCSI TARGET_TO_EXTENT: %j", + targetToExtent + ); + + await zb.zfs.set(datasetName, { + [FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME]: + targetToExtent.id, + }); + + break; + } + case 2: + // create target and targetgroup + //let targetId; + let targetGroups = []; + for (let targetGroupConfig of this.options.iscsi.targetGroups) { + targetGroups.push({ + portal: targetGroupConfig.targetGroupPortalGroup, + initiator: targetGroupConfig.targetGroupInitiatorGroup, + auth: + targetGroupConfig.targetGroupAuthGroup > 0 + ? targetGroupConfig.targetGroupAuthGroup + : null, + authmethod: + targetGroupConfig.targetGroupAuthType.length > 0 + ? targetGroupConfig.targetGroupAuthType + .toUpperCase() + .replace(" ", "_") + : "NONE", + }); + } + let target = { + name: iscsiName, + alias: null, // cannot send "" error: handler error - driver: FreeNASDriver method: CreateVolume error: {"name":"GrpcError","code":2,"message":"received error creating iscsi target - code: 422 body: {\"iscsi_target_create.alias\":[{\"message\":\"Alias already exists\",\"errno\":22}]}"} + mode: "ISCSI", + groups: targetGroups, + }; + + response = await httpClient.post("/iscsi/target", target); + + // 409 if invalid + if (response.statusCode != 200) { + target = null; + if ( + response.statusCode == 422 && + JSON.stringify(response.body).includes( + "Target name already exists" + ) + ) { + target = await this.findResourceByProperties( + "/iscsi/target", + { + name: iscsiName, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi target - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } } else { target = response.body; + } - // re-run sanity checks - if (!target) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi target` - ); - } + if (!target) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi target` + ); + } - if (target.name != iscsiName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `mismatch name error creating iscsi target` - ); - } + if (target.name != iscsiName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `mismatch name error creating iscsi target` + ); + } - if (target.groups.length != targetGroups.length) { + // handle situations/race conditions where groups failed to be added/created on the target + // groups":[{"portal":1,"initiator":1,"auth":null,"authmethod":"NONE"},{"portal":2,"initiator":1,"auth":null,"authmethod":"NONE"}] + // TODO: this logic could be more intelligent but this should do for now as it appears in the failure scenario no groups are added + // in other words, I have never seen them invalid, only omitted so this should be enough + if (target.groups.length != targetGroups.length) { + response = await httpClient.put( + `/iscsi/target/id/${target.id}`, + { + groups: targetGroups, + } + ); + + if (response.statusCode != 200) { throw new GrpcError( grpc.status.UNKNOWN, `failed setting target groups` ); + } else { + target = response.body; + + // re-run sanity checks + if (!target) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi target` + ); + } + + if (target.name != iscsiName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `mismatch name error creating iscsi target` + ); + } + + if (target.groups.length != targetGroups.length) { + throw new GrpcError( + grpc.status.UNKNOWN, + `failed setting target groups` + ); + } } } - } - this.ctx.logger.verbose("FreeNAS ISCSI TARGET: %j", target); + this.ctx.logger.verbose("FreeNAS ISCSI TARGET: %j", target); - // set target.id on zvol - await zb.zfs.set(datasetName, { - [FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME]: target.id, - }); + // set target.id on zvol + await zb.zfs.set(datasetName, { + [FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME]: target.id, + }); - let extent = { - comment: extentComment, - type: "DISK", // Disk/File, after save Disk becomes "ZVOL" - name: iscsiName, - //iscsi_target_extent_naa: "0x3822690834aae6c5", - disk: extentDiskName, - insecure_tpc: extentInsecureTpc, - xen: extentXenCompat, - avail_threshold: extentAvailThreshold, - blocksize: Number(extentBlocksize), - pblocksize: extentDisablePhysicalBlocksize, - rpm: "" + extentRpm, // should be a string - ro: false, - }; + let extent = { + comment: extentComment, + type: "DISK", // Disk/File, after save Disk becomes "ZVOL" + name: iscsiName, + //iscsi_target_extent_naa: "0x3822690834aae6c5", + disk: extentDiskName, + insecure_tpc: extentInsecureTpc, + xen: extentXenCompat, + avail_threshold: extentAvailThreshold, + blocksize: Number(extentBlocksize), + pblocksize: extentDisablePhysicalBlocksize, + rpm: "" + extentRpm, // should be a string + ro: false, + }; - response = await httpClient.post("/iscsi/extent", extent); + response = await httpClient.post("/iscsi/extent", extent); - // 409 if invalid - if (response.statusCode != 200) { - extent = null; - if ( - response.statusCode == 422 && - JSON.stringify(response.body).includes( - "Extent name must be unique" - ) - ) { - extent = await this.findResourceByProperties( - "/iscsi/extent", - { - name: iscsiName, - } - ); - } else { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error creating iscsi extent - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - } else { - extent = response.body; - } - - if (!extent) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi extent` - ); - } - - if (extent.name != iscsiName) { - throw new GrpcError( - grpc.status.UNKNOWN, - `mismatch name error creating iscsi extent` - ); - } - - this.ctx.logger.verbose("FreeNAS ISCSI EXTENT: %j", extent); - - await zb.zfs.set(datasetName, { - [FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME]: extent.id, - }); - - // create targettoextent - let targetToExtent = { - target: target.id, - extent: extent.id, - lunid: 0, - }; - response = await httpClient.post( - "/iscsi/targetextent", - targetToExtent - ); - - if (response.statusCode != 200) { - targetToExtent = null; - - // LUN ID is already being used for this target. - // Extent is already in this target. - if ( - response.statusCode == 422 && - (JSON.stringify(response.body).includes( - "Extent is already in this target." - ) || + // 409 if invalid + if (response.statusCode != 200) { + extent = null; + if ( + response.statusCode == 422 && JSON.stringify(response.body).includes( - "LUN ID is already being used for this target." - )) - ) { - targetToExtent = await this.findResourceByProperties( - "/iscsi/targetextent", - { - target: target.id, - extent: extent.id, - lunid: 0, - } - ); + "Extent name must be unique" + ) + ) { + extent = await this.findResourceByProperties( + "/iscsi/extent", + { + name: iscsiName, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi extent - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } } else { + extent = response.body; + } + + if (!extent) { throw new GrpcError( grpc.status.UNKNOWN, - `received error creating iscsi targetextent - code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` + `unknown error creating iscsi extent` ); } - } else { - targetToExtent = response.body; - } - if (!targetToExtent) { - throw new GrpcError( - grpc.status.UNKNOWN, - `unknown error creating iscsi targetextent` + if (extent.name != iscsiName) { + throw new GrpcError( + grpc.status.UNKNOWN, + `mismatch name error creating iscsi extent` + ); + } + + this.ctx.logger.verbose("FreeNAS ISCSI EXTENT: %j", extent); + + await zb.zfs.set(datasetName, { + [FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME]: extent.id, + }); + + // create targettoextent + let targetToExtent = { + target: target.id, + extent: extent.id, + lunid: 0, + }; + response = await httpClient.post( + "/iscsi/targetextent", + targetToExtent ); - } - this.ctx.logger.verbose( - "FreeNAS ISCSI TARGET_TO_EXTENT: %j", - targetToExtent - ); - await zb.zfs.set(datasetName, { - [FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME]: - targetToExtent.id, - }); + if (response.statusCode != 200) { + targetToExtent = null; - break; - default: + // LUN ID is already being used for this target. + // Extent is already in this target. + if ( + response.statusCode == 422 && + (JSON.stringify(response.body).includes( + "Extent is already in this target." + ) || + JSON.stringify(response.body).includes( + "LUN ID is already being used for this target." + )) + ) { + targetToExtent = await this.findResourceByProperties( + "/iscsi/targetextent", + { + target: target.id, + extent: extent.id, + lunid: 0, + } + ); + } else { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error creating iscsi targetextent - code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + } else { + targetToExtent = response.body; + } + + if (!targetToExtent) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unknown error creating iscsi targetextent` + ); + } + this.ctx.logger.verbose( + "FreeNAS ISCSI TARGET_TO_EXTENT: %j", + targetToExtent + ); + + await zb.zfs.set(datasetName, { + [FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME]: + targetToExtent.id, + }); + + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } + } + + // iqn = target + let iqn = basename + ":" + iscsiName; + this.ctx.logger.info("FreeNAS iqn: " + iqn); + + // store this off to make delete process more bullet proof + await zb.zfs.set(datasetName, { + [FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME]: iscsiName, + }); + + volume_context = { + node_attach_driver: "iscsi", + portal: this.options.iscsi.targetPortal || "", + portals: this.options.iscsi.targetPortals + ? this.options.iscsi.targetPortals.join(",") + : "", + interface: this.options.iscsi.interface || "", + iqn: iqn, + lun: 0, + }; + return volume_context; + } + break; + + case "nvmeof": + { + switch (apiVersion) { + case 1: throw new GrpcError( grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` + `nvmeof feature is only available with version 2 of the api` ); + break; } + + if (!isScale || semver.satisfies(truenasVersion, "<25.10")) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `nvmeof feature is only available with TrueNAS version 25.10 and above` + ); + } + + properties = await zb.zfs.get(datasetName, [ + FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME, + FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME, + FREENAS_NVMEOF_ASSETS_NAME_PROPERTY_NAME, + ]); + properties = properties[datasetName]; + this.ctx.logger.debug("zfs props data: %j", properties); + + let nvmeofName; + + if (this.options.nvmeof.nameTemplate) { + nvmeofName = Handlebars.compile(this.options.nvmeof.nameTemplate)({ + name: call.request.name, + parameters: call.request.parameters, + }); + } else { + nvmeofName = zb.helpers.extractLeafName(datasetName); + } + + if (this.options.nvmeof.namePrefix) { + nvmeofName = this.options.nvmeof.namePrefix + nvmeofName; + } + + if (this.options.nvmeof.nameSuffix) { + nvmeofName += this.options.nvmeof.nameSuffix; + } + + // According to RFC3270, 'Each iSCSI node, whether an initiator or target, MUST have an iSCSI name. Initiators and targets MUST support the receipt of iSCSI names of up to the maximum length of 223 bytes.' + // https://kb.netapp.com/Advice_and_Troubleshooting/Miscellaneous/What_is_the_maximum_length_of_a_iSCSI_iqn_name + // https://tools.ietf.org/html/rfc3720 + // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 + nvmeofName = nvmeofName.toLowerCase(); + + let namespaceDiskName = "zvol/" + datasetName; + let maxZvolNameLength = await driver.getMaxZvolNameLength(); + driver.ctx.logger.debug( + "max zvol name length: %s", + maxZvolNameLength + ); + + if (namespaceDiskName.length > maxZvolNameLength) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `namespace disk name cannot exceed ${maxZvolNameLength} characters: ${namespaceDiskName}` + ); + } + + // TODO: get basenqn from global config, add nvemofName to it and ensure full nqn is <= 223 + // // https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203 + // if (isScale && nvmeofName.length > 64) { + // throw new GrpcError( + // grpc.status.FAILED_PRECONDITION, + // `extent name cannot exceed 64 characters: ${nvmeofName}` + // ); + // } + + this.ctx.logger.info( + "FreeNAS creating nvmeof assets with name: " + nvmeofName + ); + + // http:///api/docs/current/api_methods_nvmet.subsys.create.html + let subsystemTemplate = _.get( + this.options, + "nvmeof.subsystemTemplate", + {} + ); + subsystemTemplate = subsystemTemplate || {}; + + // http:///api/docs/current/api_methods_nvmet.namespace.create.html + let namespaceTemplate = _.get( + this.options, + "nvmeof.namespaceTemplate", + {} + ); + namespaceTemplate = namespaceTemplate || {}; + + // create subsystem + let subsystem; + switch (apiVersion) { + case 2: + subsystem = await httpApiClient.NvmetSubsysCreate( + nvmeofName, + subsystemTemplate + ); + + break; + } + if (!subsystem) { + throw new GrpcError( + grpc.status.NOT_FOUND, + `unable to find nvmeof subsystem: ${nvmeofName}` + ); + } + this.ctx.logger.verbose("FreeNAS NVMEOF SUBSYSTEM: %j", subsystem); + await zb.zfs.set(datasetName, { + [FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME]: subsystem.id, + }); + + // create subsystem + let namespace; + switch (apiVersion) { + case 2: + namespace = await httpApiClient.NvmetNamespaceCreate( + namespaceDiskName, + subsystem.id, + namespaceTemplate + ); + + break; + } + if (!namespace) { + throw new GrpcError( + grpc.status.NOT_FOUND, + `unable to find nvmeof namespace: ${namespaceDiskName}` + ); + } + this.ctx.logger.verbose("FreeNAS NVMEOF NAMESPACE: %j", namespace); + await zb.zfs.set(datasetName, { + [FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME]: namespace.id, + }); + + // assign ports to subsystem + let ports = _.get(this.options, "nvmeof.ports", []); + for (const port_i of ports) { + const port = await httpApiClient.NvmetPortSubsysCreate( + port_i, + subsystem.id + ); + this.ctx.logger.verbose("FreeNAS NVMEOF PORT: %j", port); + } + + // TODO: assign hosts + + // store this off to make delete process more bullet proof + await zb.zfs.set(datasetName, { + [FREENAS_NVMEOF_ASSETS_NAME_PROPERTY_NAME]: nvmeofName, + }); + + volume_context = { + node_attach_driver: "nvmeof", + transport: this.options.nvmeof.transport || "", + transports: this.options.nvmeof.transports + ? this.options.nvmeof.transports.join(",") + : "", + nqn: subsystem.subnqn, + nsid: namespace.nsid, + }; + return volume_context; } - - // iqn = target - let iqn = basename + ":" + iscsiName; - this.ctx.logger.info("FreeNAS iqn: " + iqn); - - // store this off to make delete process more bullet proof - await zb.zfs.set(datasetName, { - [FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME]: iscsiName, - }); - - volume_context = { - node_attach_driver: "iscsi", - portal: this.options.iscsi.targetPortal || "", - portals: this.options.iscsi.targetPortals - ? this.options.iscsi.targetPortals.join(",") - : "", - interface: this.options.iscsi.interface || "", - iqn: iqn, - lun: 0, - }; - return volume_context; + break; default: throw new GrpcError( @@ -1506,8 +1700,19 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver { async deleteShare(call, datasetName) { const driverShareType = this.getDriverShareType(); const httpClient = await this.getHttpClient(); + const httpApiClient = await this.getTrueNASHttpApiClient(); const apiVersion = httpClient.getApiVersion(); const zb = await this.getZetabyte(); + const truenasVersion = await httpApiClient.getSystemVersionSemver(); + + if (!truenasVersion) { + throw new GrpcError( + grpc.status.UNKNOWN, + `unable to detect TrueNAS version` + ); + } + + const isScale = await httpApiClient.getIsScale(); let properties; let response; @@ -1518,202 +1723,400 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver { switch (driverShareType) { case "nfs": - try { - properties = await zb.zfs.get(datasetName, [ - "mountpoint", - FREENAS_NFS_SHARE_PROPERTY_NAME, - ]); - } catch (err) { - if (err.toString().includes("dataset does not exist")) { - return; + { + try { + properties = await zb.zfs.get(datasetName, [ + "mountpoint", + FREENAS_NFS_SHARE_PROPERTY_NAME, + ]); + } catch (err) { + if (err.toString().includes("dataset does not exist")) { + return; + } + throw err; } - throw err; - } - properties = properties[datasetName]; - this.ctx.logger.debug("zfs props data: %j", properties); + properties = properties[datasetName]; + this.ctx.logger.debug("zfs props data: %j", properties); - shareId = properties[FREENAS_NFS_SHARE_PROPERTY_NAME].value; + shareId = properties[FREENAS_NFS_SHARE_PROPERTY_NAME].value; - // only remove if the process has not succeeded already - if (zb.helpers.isPropertyValueSet(shareId)) { - // remove nfs share - switch (apiVersion) { - case 1: - case 2: - endpoint = "/sharing/nfs/"; - if (apiVersion == 2) { - endpoint += "id/"; - } - endpoint += shareId; - - response = await httpClient.get(endpoint); - - // assume share is gone for now - if ([404, 500].includes(response.statusCode)) { - } else { - switch (apiVersion) { - case 1: - sharePaths = response.body.nfs_paths; - break; - case 2: - if (response.body.path) { - sharePaths = [response.body.path]; - } else { - sharePaths = response.body.paths; - } - break; + // only remove if the process has not succeeded already + if (zb.helpers.isPropertyValueSet(shareId)) { + // remove nfs share + switch (apiVersion) { + case 1: + case 2: + endpoint = "/sharing/nfs/"; + if (apiVersion == 2) { + endpoint += "id/"; } + endpoint += shareId; - deleteAsset = sharePaths.some((value) => { - return value == properties.mountpoint.value; - }); + response = await httpClient.get(endpoint); - if (deleteAsset) { - response = await GeneralUtils.retry( - 3, - 1000, - async () => { - return await httpClient.delete(endpoint); - }, - { - retryCondition: (err) => { - if (err.code == "ECONNRESET") { - return true; - } - if (err.code == "ECONNABORTED") { - return true; - } - if (err.response && err.response.statusCode == 504) { - return true; - } - return false; - }, - } - ); - - // returns a 500 if does not exist - // v1 = 204 - // v2 = 200 - if (![200, 204].includes(response.statusCode)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting nfs share - share: ${shareId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); + // assume share is gone for now + if ([404, 500].includes(response.statusCode)) { + } else { + switch (apiVersion) { + case 1: + sharePaths = response.body.nfs_paths; + break; + case 2: + if (response.body.path) { + sharePaths = [response.body.path]; + } else { + sharePaths = response.body.paths; + } + break; } - // remove property to prevent delete race conditions - // due to id re-use by FreeNAS/TrueNAS - await zb.zfs.inherit( - datasetName, - FREENAS_NFS_SHARE_PROPERTY_NAME - ); + deleteAsset = sharePaths.some((value) => { + return value == properties.mountpoint.value; + }); + + if (deleteAsset) { + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.delete(endpoint); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); + + // returns a 500 if does not exist + // v1 = 204 + // v2 = 200 + if (![200, 204].includes(response.statusCode)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting nfs share - share: ${shareId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + // remove property to prevent delete race conditions + // due to id re-use by FreeNAS/TrueNAS + await zb.zfs.inherit( + datasetName, + FREENAS_NFS_SHARE_PROPERTY_NAME + ); + } } - } - break; - default: - throw new GrpcError( - grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` - ); + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } } } break; case "smb": - try { - properties = await zb.zfs.get(datasetName, [ - "mountpoint", - FREENAS_SMB_SHARE_PROPERTY_NAME, - ]); - } catch (err) { - if (err.toString().includes("dataset does not exist")) { - return; + { + try { + properties = await zb.zfs.get(datasetName, [ + "mountpoint", + FREENAS_SMB_SHARE_PROPERTY_NAME, + ]); + } catch (err) { + if (err.toString().includes("dataset does not exist")) { + return; + } + throw err; } - throw err; - } - properties = properties[datasetName]; - this.ctx.logger.debug("zfs props data: %j", properties); + properties = properties[datasetName]; + this.ctx.logger.debug("zfs props data: %j", properties); - shareId = properties[FREENAS_SMB_SHARE_PROPERTY_NAME].value; + shareId = properties[FREENAS_SMB_SHARE_PROPERTY_NAME].value; - // only remove if the process has not succeeded already - if (zb.helpers.isPropertyValueSet(shareId)) { - // remove smb share - switch (apiVersion) { - case 1: - case 2: - switch (apiVersion) { - case 1: - endpoint = `/sharing/cifs/${shareId}`; - break; - case 2: - endpoint = `/sharing/smb/id/${shareId}`; - break; - } - - response = await httpClient.get(endpoint); - - // assume share is gone for now - if ([404, 500].includes(response.statusCode)) { - } else { + // only remove if the process has not succeeded already + if (zb.helpers.isPropertyValueSet(shareId)) { + // remove smb share + switch (apiVersion) { + case 1: + case 2: switch (apiVersion) { case 1: - sharePaths = [response.body.cifs_path]; + endpoint = `/sharing/cifs/${shareId}`; break; case 2: - sharePaths = [response.body.path]; + endpoint = `/sharing/smb/id/${shareId}`; break; } - deleteAsset = sharePaths.some((value) => { - return value == properties.mountpoint.value; - }); + response = await httpClient.get(endpoint); - if (deleteAsset) { - response = await GeneralUtils.retry( - 3, - 1000, - async () => { - return await httpClient.delete(endpoint); - }, - { - retryCondition: (err) => { - if (err.code == "ECONNRESET") { - return true; - } - if (err.code == "ECONNABORTED") { - return true; - } - if (err.response && err.response.statusCode == 504) { - return true; - } - return false; - }, - } - ); - - // returns a 500 if does not exist - // v1 = 204 - // v2 = 200 - if ( - ![200, 204].includes(response.statusCode) && - !JSON.stringify(response.body).includes("does not exist") - ) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting smb share - share: ${shareId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); + // assume share is gone for now + if ([404, 500].includes(response.statusCode)) { + } else { + switch (apiVersion) { + case 1: + sharePaths = [response.body.cifs_path]; + break; + case 2: + sharePaths = [response.body.path]; + break; } - // remove property to prevent delete race conditions - // due to id re-use by FreeNAS/TrueNAS - await zb.zfs.inherit( - datasetName, - FREENAS_SMB_SHARE_PROPERTY_NAME - ); + deleteAsset = sharePaths.some((value) => { + return value == properties.mountpoint.value; + }); + + if (deleteAsset) { + response = await GeneralUtils.retry( + 3, + 1000, + async () => { + return await httpClient.delete(endpoint); + }, + { + retryCondition: (err) => { + if (err.code == "ECONNRESET") { + return true; + } + if (err.code == "ECONNABORTED") { + return true; + } + if (err.response && err.response.statusCode == 504) { + return true; + } + return false; + }, + } + ); + + // returns a 500 if does not exist + // v1 = 204 + // v2 = 200 + if ( + ![200, 204].includes(response.statusCode) && + !JSON.stringify(response.body).includes("does not exist") + ) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting smb share - share: ${shareId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + // remove property to prevent delete race conditions + // due to id re-use by FreeNAS/TrueNAS + await zb.zfs.inherit( + datasetName, + FREENAS_SMB_SHARE_PROPERTY_NAME + ); + } + } + break; + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown apiVersion ${apiVersion}` + ); + } + } + } + break; + case "iscsi": + { + // Delete target + // NOTE: deletting a target inherently deletes associated targetgroup(s) and targettoextent(s) + + // Delete extent + try { + properties = await zb.zfs.get(datasetName, [ + FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME, + FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME, + FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME, + FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME, + ]); + } catch (err) { + if (err.toString().includes("dataset does not exist")) { + return; + } + throw err; + } + + properties = properties[datasetName]; + this.ctx.logger.debug("zfs props data: %j", properties); + + let targetId = + properties[FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME].value; + let extentId = + properties[FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME].value; + let iscsiName = + properties[FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME].value; + let assetName; + + switch (apiVersion) { + case 1: + case 2: + // only remove if the process has not succeeded already + if (zb.helpers.isPropertyValueSet(targetId)) { + // https://jira.ixsystems.com/browse/NAS-103952 + + // v1 - /services/iscsi/target/{id}/ + // v2 - /iscsi/target/id/{id} + endpoint = ""; + if (apiVersion == 1) { + endpoint += "/services"; + } + endpoint += "/iscsi/target/"; + if (apiVersion == 2) { + endpoint += "id/"; + } + endpoint += targetId; + response = await httpClient.get(endpoint); + + // assume is gone for now + if ([404, 500].includes(response.statusCode)) { + } else { + deleteAsset = true; + assetName = null; + + // checking if set for backwards compatibility + if (zb.helpers.isPropertyValueSet(iscsiName)) { + switch (apiVersion) { + case 1: + assetName = response.body.iscsi_target_name; + break; + case 2: + assetName = response.body.name; + break; + } + + if (assetName != iscsiName) { + deleteAsset = false; + } + } + + if (deleteAsset) { + let retries = 0; + let maxRetries = 5; + let retryWait = 1000; + response = await httpClient.delete(endpoint); + + // sometimes after an initiator has detached it takes a moment for TrueNAS to settle + // code: 422 body: {\"message\":\"Target csi-ci-55877e95sanity-node-expand-volume-e54f81fa-cd38e798 is in use.\",\"errno\":14} + while ( + response.statusCode == 422 && + retries < maxRetries && + _.get(response, "body.message").includes("Target") && + _.get(response, "body.message").includes("is in use") && + _.get(response, "body.errno") == 14 + ) { + retries++; + this.ctx.logger.debug( + "target: %s is in use, retry %s shortly", + targetId, + retries + ); + await GeneralUtils.sleep(retryWait); + response = await httpClient.delete(endpoint); + } + + if (![200, 204].includes(response.statusCode)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting iscsi target - target: ${targetId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + // remove property to prevent delete race conditions + // due to id re-use by FreeNAS/TrueNAS + await zb.zfs.inherit( + datasetName, + FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME + ); + } else { + this.ctx.logger.debug( + "not deleting iscsitarget asset as it appears ID %s has been re-used: zfs name - %s, iscsitarget name - %s", + targetId, + iscsiName, + assetName + ); + } + } + } + + // only remove if the process has not succeeded already + if (zb.helpers.isPropertyValueSet(extentId)) { + // v1 - /services/iscsi/targettoextent/{id}/ + // v2 - /iscsi/targetextent/id/{id} + if (apiVersion == 1) { + endpoint = "/services/iscsi/extent/"; + } else { + endpoint = "/iscsi/extent/id/"; + } + endpoint += extentId; + response = await httpClient.get(endpoint); + + // assume is gone for now + if ([404, 500].includes(response.statusCode)) { + } else { + deleteAsset = true; + assetName = null; + + // checking if set for backwards compatibility + if (zb.helpers.isPropertyValueSet(iscsiName)) { + switch (apiVersion) { + case 1: + assetName = response.body.iscsi_target_extent_name; + break; + case 2: + assetName = response.body.name; + break; + } + + if (assetName != iscsiName) { + deleteAsset = false; + } + } + + if (deleteAsset) { + response = await httpClient.delete(endpoint); + if (![200, 204].includes(response.statusCode)) { + throw new GrpcError( + grpc.status.UNKNOWN, + `received error deleting iscsi extent - extent: ${extentId} code: ${ + response.statusCode + } body: ${JSON.stringify(response.body)}` + ); + } + + // remove property to prevent delete race conditions + // due to id re-use by FreeNAS/TrueNAS + await zb.zfs.inherit( + datasetName, + FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME + ); + } else { + this.ctx.logger.debug( + "not deleting iscsiextent asset as it appears ID %s has been re-used: zfs name - %s, iscsiextent name - %s", + extentId, + iscsiName, + assetName + ); + } } } break; @@ -1725,196 +2128,92 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver { } } break; - case "iscsi": - // Delete target - // NOTE: deletting a target inherently deletes associated targetgroup(s) and targettoextent(s) - // Delete extent - try { - properties = await zb.zfs.get(datasetName, [ - FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME, - FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME, - FREENAS_ISCSI_TARGETTOEXTENT_ID_PROPERTY_NAME, - FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME, - ]); - } catch (err) { - if (err.toString().includes("dataset does not exist")) { - return; + case "nvmeof": + { + switch (apiVersion) { + case 1: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `nvmeof feature is only available with version 2 of the api` + ); + break; } - throw err; - } - properties = properties[datasetName]; - this.ctx.logger.debug("zfs props data: %j", properties); - - let targetId = properties[FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME].value; - let extentId = properties[FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME].value; - let iscsiName = - properties[FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME].value; - let assetName; - - switch (apiVersion) { - case 1: - case 2: - // only remove if the process has not succeeded already - if (zb.helpers.isPropertyValueSet(targetId)) { - // https://jira.ixsystems.com/browse/NAS-103952 - - // v1 - /services/iscsi/target/{id}/ - // v2 - /iscsi/target/id/{id} - endpoint = ""; - if (apiVersion == 1) { - endpoint += "/services"; - } - endpoint += "/iscsi/target/"; - if (apiVersion == 2) { - endpoint += "id/"; - } - endpoint += targetId; - response = await httpClient.get(endpoint); - - // assume is gone for now - if ([404, 500].includes(response.statusCode)) { - } else { - deleteAsset = true; - assetName = null; - - // checking if set for backwards compatibility - if (zb.helpers.isPropertyValueSet(iscsiName)) { - switch (apiVersion) { - case 1: - assetName = response.body.iscsi_target_name; - break; - case 2: - assetName = response.body.name; - break; - } - - if (assetName != iscsiName) { - deleteAsset = false; - } - } - - if (deleteAsset) { - let retries = 0; - let maxRetries = 5; - let retryWait = 1000; - response = await httpClient.delete(endpoint); - - // sometimes after an initiator has detached it takes a moment for TrueNAS to settle - // code: 422 body: {\"message\":\"Target csi-ci-55877e95sanity-node-expand-volume-e54f81fa-cd38e798 is in use.\",\"errno\":14} - while ( - response.statusCode == 422 && - retries < maxRetries && - _.get(response, "body.message").includes("Target") && - _.get(response, "body.message").includes("is in use") && - _.get(response, "body.errno") == 14 - ) { - retries++; - this.ctx.logger.debug( - "target: %s is in use, retry %s shortly", - targetId, - retries - ); - await GeneralUtils.sleep(retryWait); - response = await httpClient.delete(endpoint); - } - - if (![200, 204].includes(response.statusCode)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting iscsi target - target: ${targetId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - // remove property to prevent delete race conditions - // due to id re-use by FreeNAS/TrueNAS - await zb.zfs.inherit( - datasetName, - FREENAS_ISCSI_TARGET_ID_PROPERTY_NAME - ); - } else { - this.ctx.logger.debug( - "not deleting iscsitarget asset as it appears ID %s has been re-used: zfs name - %s, iscsitarget name - %s", - targetId, - iscsiName, - assetName - ); - } - } - } - - // only remove if the process has not succeeded already - if (zb.helpers.isPropertyValueSet(extentId)) { - // v1 - /services/iscsi/targettoextent/{id}/ - // v2 - /iscsi/targetextent/id/{id} - if (apiVersion == 1) { - endpoint = "/services/iscsi/extent/"; - } else { - endpoint = "/iscsi/extent/id/"; - } - endpoint += extentId; - response = await httpClient.get(endpoint); - - // assume is gone for now - if ([404, 500].includes(response.statusCode)) { - } else { - deleteAsset = true; - assetName = null; - - // checking if set for backwards compatibility - if (zb.helpers.isPropertyValueSet(iscsiName)) { - switch (apiVersion) { - case 1: - assetName = response.body.iscsi_target_extent_name; - break; - case 2: - assetName = response.body.name; - break; - } - - if (assetName != iscsiName) { - deleteAsset = false; - } - } - - if (deleteAsset) { - response = await httpClient.delete(endpoint); - if (![200, 204].includes(response.statusCode)) { - throw new GrpcError( - grpc.status.UNKNOWN, - `received error deleting iscsi extent - extent: ${extentId} code: ${ - response.statusCode - } body: ${JSON.stringify(response.body)}` - ); - } - - // remove property to prevent delete race conditions - // due to id re-use by FreeNAS/TrueNAS - await zb.zfs.inherit( - datasetName, - FREENAS_ISCSI_EXTENT_ID_PROPERTY_NAME - ); - } else { - this.ctx.logger.debug( - "not deleting iscsiextent asset as it appears ID %s has been re-used: zfs name - %s, iscsiextent name - %s", - extentId, - iscsiName, - assetName - ); - } - } - } - break; - default: + if (!isScale || semver.satisfies(truenasVersion, "<25.10")) { throw new GrpcError( grpc.status.FAILED_PRECONDITION, - `invalid configuration: unknown apiVersion ${apiVersion}` + `nvmeof feature is only available with TrueNAS version 25.10 and above` ); + } + + try { + properties = await zb.zfs.get(datasetName, [ + FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME, + FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME, + FREENAS_NVMEOF_ASSETS_NAME_PROPERTY_NAME, + ]); + } catch (err) { + if (err.toString().includes("dataset does not exist")) { + return; + } + throw err; + } + + properties = properties[datasetName]; + this.ctx.logger.debug("zfs props data: %j", properties); + + let subsystemId = + properties[FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME].value; + let namespaceId = + properties[FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME].value; + + // remove namespace + if (zb.helpers.isPropertyValueSet(namespaceId)) { + await GeneralUtils.retry( + 15, + 2000, + async () => { + await httpApiClient.NvmetNamespaceDeleteById(namespaceId); + }, + { + retryCondition: (err) => { + return true; + }, + } + ); + + await zb.zfs.inherit( + datasetName, + FREENAS_NVMEOF_NAMESPACE_ID_PROPERTY_NAME + ); + } + + // remove subsystem + if (zb.helpers.isPropertyValueSet(subsystemId)) { + await GeneralUtils.retry( + 15, + 2000, + async () => { + await httpApiClient.NvmetSubsysDeleteById(subsystemId, { + force: true, + }); + }, + { + retryCondition: (err) => { + return true; + }, + } + ); + + await zb.zfs.inherit( + datasetName, + FREENAS_NVMEOF_SUBSYSTEM_ID_PROPERTY_NAME + ); + } } break; + default: throw new GrpcError( grpc.status.FAILED_PRECONDITION, diff --git a/src/driver/index.js b/src/driver/index.js index f0fe630..406111c 100644 --- a/src/driver/index.js +++ b/src/driver/index.js @@ -124,10 +124,13 @@ class CsiBaseDriver { * @returns Mount */ getDefaultMountInstance() { - return this.ctx.registry.get(`${__REGISTRY_NS__}:default_mount_instance`, () => { - const filesystem = this.getDefaultFilesystemInstance(); - return new Mount({ filesystem }); - }); + return this.ctx.registry.get( + `${__REGISTRY_NS__}:default_mount_instance`, + () => { + const filesystem = this.getDefaultFilesystemInstance(); + return new Mount({ filesystem }); + } + ); } /** @@ -136,9 +139,12 @@ class CsiBaseDriver { * @returns ISCSI */ getDefaultISCSIInstance() { - return this.ctx.registry.get(`${__REGISTRY_NS__}:default_iscsi_instance`, () => { - return new ISCSI(); - }); + return this.ctx.registry.get( + `${__REGISTRY_NS__}:default_iscsi_instance`, + () => { + return new ISCSI(); + } + ); } /** @@ -148,37 +154,46 @@ class CsiBaseDriver { */ getDefaultNVMEoFInstance() { const driver = this; - return this.ctx.registry.get(`${__REGISTRY_NS__}:default_nvmeof_instance`, () => { - return new NVMEoF({ logger: driver.ctx.logger }); - }); + return this.ctx.registry.get( + `${__REGISTRY_NS__}:default_nvmeof_instance`, + () => { + return new NVMEoF({ logger: driver.ctx.logger }); + } + ); } getDefaultZetabyteInstance() { - return this.ctx.registry.get(`${__REGISTRY_NS__}:default_zb_instance`, () => { - return new Zetabyte({ - idempotent: true, - paths: { - zfs: "zfs", - zpool: "zpool", - sudo: "sudo", - chroot: "chroot", - }, - //logger: driver.ctx.logger, - executor: { - spawn: function () { - const command = `${arguments[0]} ${arguments[1].join(" ")}`; - return cp.exec(command); + return this.ctx.registry.get( + `${__REGISTRY_NS__}:default_zb_instance`, + () => { + return new Zetabyte({ + idempotent: true, + paths: { + zfs: "zfs", + zpool: "zpool", + sudo: "sudo", + chroot: "chroot", }, - }, - log_commands: true, - }); - }); + //logger: driver.ctx.logger, + executor: { + spawn: function () { + const command = `${arguments[0]} ${arguments[1].join(" ")}`; + return cp.exec(command); + }, + }, + log_commands: true, + }); + } + ); } getDefaultOneClientInstance() { - return this.ctx.registry.get(`${__REGISTRY_NS__}:default_oneclient_instance`, () => { - return new OneClient(); - }); + return this.ctx.registry.get( + `${__REGISTRY_NS__}:default_oneclient_instance`, + () => { + return new OneClient(); + } + ); } getDefaultObjectiveFSInstance() { @@ -198,11 +213,14 @@ class CsiBaseDriver { * @returns CsiProxyClient */ getDefaultCsiProxyClientInstance() { - return this.ctx.registry.get(`${__REGISTRY_NS__}:default_csi_proxy_instance`, () => { - const options = {}; - options.services = _.get(this.options, "node.csiProxy.services", {}); - return new CsiProxyClient(options); - }); + return this.ctx.registry.get( + `${__REGISTRY_NS__}:default_csi_proxy_instance`, + () => { + const options = {}; + options.services = _.get(this.options, "node.csiProxy.services", {}); + return new CsiProxyClient(options); + } + ); } getDefaultKubernetsConfigInstance() { @@ -1054,7 +1072,7 @@ class CsiBaseDriver { for (let nvmeofConnection of nvmeofConnections) { // connect try { - await GeneralUtils.retry(15, 2000, async () => { + await GeneralUtils.retry(30, 2000, async () => { await nvmeof.connectByNQNTransport( nvmeofConnection.nqn, nvmeofConnection.transport @@ -1069,15 +1087,36 @@ class CsiBaseDriver { continue; } + // wait for connection to actually be connected + try { + await GeneralUtils.retry(30, 2000, async () => { + let state = await nvmeof.getSubsystemStateByNQNTransport( + nvmeofConnection.nqn, + nvmeofConnection.transport + ); + if (state != "live") { + throw new Error("nvmeof connection is not live"); + } + }); + } catch (err) { + driver.ctx.logger.warn( + `error: ${JSON.stringify( + err + )} transport never became live: ${ + nvmeofConnection.transport + }` + ); + continue; + } + // find controller device let controllerDevice; try { - await GeneralUtils.retry(15, 2000, async () => { + await GeneralUtils.retry(30, 2000, async () => { controllerDevice = await nvmeof.controllerDevicePathByTransportNQN( nvmeofConnection.transport, - nvmeofConnection.nqn, - nvmeofConnection.nsid + nvmeofConnection.nqn ); if (!controllerDevice) { @@ -1488,11 +1527,13 @@ class CsiBaseDriver { // format result = await filesystem.deviceIsFormatted(device); if (!result) { - let formatOptions = _.get( - driver.options.node.format, - [fs_type, "customOptions"], - [] - ); + let formatOptions = [ + ..._.get( + driver.options.node.format, + [fs_type, "customOptions"], + [] + ), + ]; if (!Array.isArray(formatOptions)) { formatOptions = []; } diff --git a/src/utils/nvmeof.js b/src/utils/nvmeof.js index e2f47e1..82c7b63 100644 --- a/src/utils/nvmeof.js +++ b/src/utils/nvmeof.js @@ -218,6 +218,50 @@ class NVMEoF { return false; } + async parseTransportFromPath(path) { + let address; + let service; + switch (path.Transport) { + case "fc": + case "rdma": + case "tcp": + let controllerAddress = path.Address; + /** + * For backwards compatibility with older nvme-cli versions (at least < 2.2.1) + * old: "Address":"traddr=127.0.0.1 trsvcid=4420" + * new: "Address":"traddr=127.0.0.1,trsvcid=4420" + */ + controllerAddress = controllerAddress.replace( + new RegExp(/ ([a-z_]*=)/, "g"), + ",$1" + ); + let parts = controllerAddress.split(","); + + for (let i_part of parts) { + let i_parts = i_part.split("="); + switch (i_parts[0].trim()) { + case "traddr": + address = i_parts[1].trim(); + break; + case "trsvcid": + service = i_parts[1].trim(); + break; + } + } + + break; + case "pcie": + address = path.Address; + break; + } + + return { + type: path.Transport, + address, + service, + }; + } + async parseTransport(transport) { if (typeof transport === "object") { return transport; @@ -338,11 +382,17 @@ class NVMEoF { async controllerDevicePathByTransportNQN(transport, nqn) { const nvmeof = this; + transport = await nvmeof.parseTransport(transport); - let controller = await nvmeof.getControllerByTransportNQN(transport, nqn); - if (controller) { - return `/dev/${controller.Controller}`; + let path = await nvmeof.getSubsystemPathByNQNTransport(nqn, transport); + if (path) { + return `/dev/${path.Name}`; } + + // let controller = await nvmeof.getControllerByTransportNQN(transport, nqn); + // if (controller) { + // return `/dev/${controller.Controller}`; + // } } async getSubsystems() { @@ -396,7 +446,7 @@ class NVMEoF { for (let subsystem of subsystems) { if (subsystem.Namespaces) { for (let namespace of subsystem.Namespaces) { - if (namespace.NameSpace == name) { + if (namespace.NameSpace == name && subsystem.Controllers) { return subsystem.Controllers; } } @@ -433,37 +483,18 @@ class NVMEoF { continue; } - let controllerAddress = controller.Address; - /** - * For backwards compatibility with older nvme-cli versions (at least < 2.2.1) - * old: "Address":"traddr=127.0.0.1 trsvcid=4420" - * new: "Address":"traddr=127.0.0.1,trsvcid=4420" - */ - controllerAddress = controllerAddress.replace( - new RegExp(/ ([a-z_]*=)/, "g"), - ",$1" + let controller_transport = await nvmeof.parseTransportFromPath( + controller ); - let parts = controllerAddress.split(","); - let traddr; - let trsvcid; - for (let i_part of parts) { - let i_parts = i_part.split("="); - switch (i_parts[0].trim()) { - case "traddr": - traddr = i_parts[1].trim(); - break; - case "trsvcid": - trsvcid = i_parts[1].trim(); - break; - } - } - - if (traddr != transport.address) { + if (controller_transport.address != transport.address) { continue; } - if (transport.service && trsvcid != transport.service) { + if ( + transport.service && + controller_transport.service != transport.service + ) { continue; } @@ -515,6 +546,39 @@ class NVMEoF { nvmeof.logger.warn(`failed to find nqn for device: ${name}`); } + async getSubsystemStateByNQNTransport(nqn, transport) { + const nvmeof = this; + transport = await nvmeof.parseTransport(transport); + const path = await nvmeof.getSubsystemPathByNQNTransport(nqn, transport); + return path?.State; + } + + async getSubsystemPathByNQNTransport(nqn, transport) { + const nvmeof = this; + transport = await nvmeof.parseTransport(transport); + const subsysList = await nvmeof.listSubsys(["-v"]); + host_label: for (const host of subsysList) { + subsys_label: for (const subsys of host.Subsystems) { + if (subsys.NQN != nqn) { + continue; + } + path_label: for (const path of subsys.Paths) { + let parsed_path_transport = await nvmeof.parseTransportFromPath(path); + for (const key of Object.keys(transport)) { + if ( + ["type", "address", "service"].includes(key) && + transport[key] != parsed_path_transport[key] + ) { + break path_label; + } + } + + return path; + } + } + } + } + devicePathByModelNumberSerialNumber(modelNumber, serialNumber) { modelNumber = modelNumber.replaceAll(" ", "_"); serialNumber = serialNumber.replaceAll(" ", "_");