From 94f6df240b9d8d3f3a2d12a4d86e88463eb13519 Mon Sep 17 00:00:00 2001 From: Travis Glenn Hansen Date: Tue, 1 Feb 2022 11:35:58 -0700 Subject: [PATCH] further abstraction to support zfs-local Signed-off-by: Travis Glenn Hansen --- src/driver/controller-zfs-generic/index.js | 52 ++++++++++++-- src/driver/controller-zfs/index.js | 83 +++++++++++++++------- src/driver/freenas/ssh.js | 54 +++++++++++--- 3 files changed, 147 insertions(+), 42 deletions(-) diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index 075e20b..42b690f 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -1,12 +1,50 @@ -const { ControllerZfsSshBaseDriver } = require("../controller-zfs-ssh"); +const { ControllerZfsBaseDriver } = require("../controller-zfs"); const { GrpcError, grpc } = require("../../utils/grpc"); +const SshClient = require("../../utils/ssh").SshClient; const sleep = require("../../utils/general").sleep; +const { Zetabyte, ZfsSshProcessManager } = require("../../utils/zfs"); const Handlebars = require("handlebars"); const ISCSI_ASSETS_NAME_PROPERTY_NAME = "democratic-csi:iscsi_assets_name"; -class ControllerZfsGenericDriver extends ControllerZfsSshBaseDriver { +class ControllerZfsGenericDriver extends ControllerZfsBaseDriver { + getExecClient() { + return new SshClient({ + logger: this.ctx.logger, + connection: this.options.sshConnection, + }); + } + + async getZetabyte() { + const execClient = this.getExecClient(); + const options = {}; + options.executor = new ZfsSshProcessManager(execClient); + options.idempotent = true; + + if ( + this.options.zfs.hasOwnProperty("cli") && + this.options.zfs.cli && + this.options.zfs.cli.hasOwnProperty("paths") + ) { + options.paths = this.options.zfs.cli.paths; + } + + if ( + this.options.zfs.hasOwnProperty("cli") && + this.options.zfs.cli && + this.options.zfs.cli.hasOwnProperty("sudoEnabled") + ) { + options.sudo = this.getSudoEnabled(); + } + + if (typeof this.setZetabyteCustomOptions === "function") { + await this.setZetabyteCustomOptions(options); + } + + return new Zetabyte(options); + } + /** * cannot make this a storage class parameter as storage class/etc context is *not* sent * into various calls such as GetControllerCapabilities etc @@ -30,7 +68,7 @@ class ControllerZfsGenericDriver extends ControllerZfsSshBaseDriver { */ async createShare(call, datasetName) { const zb = await this.getZetabyte(); - const sshClient = this.getSshClient(); + const execClient = this.getExecClient(); let properties; let response; @@ -194,7 +232,7 @@ create /backstores/block/${iscsiName} async deleteShare(call, datasetName) { const zb = await this.getZetabyte(); - const sshClient = this.getSshClient(); + const execClient = this.getExecClient(); let response; let properties; @@ -317,7 +355,7 @@ delete ${iscsiName} } async targetCliCommand(data) { - const sshClient = this.getSshClient(); + const execClient = this.getExecClient(); const driver = this; data = data.trim(); @@ -361,8 +399,8 @@ delete ${iscsiName} let options = { pty: true, }; - let response = await sshClient.exec( - sshClient.buildCommand(command, args), + let response = await execClient.exec( + execClient.buildCommand(command, args), options ); if (response.code != 0) { diff --git a/src/driver/controller-zfs/index.js b/src/driver/controller-zfs/index.js index d596cc9..4936573 100644 --- a/src/driver/controller-zfs/index.js +++ b/src/driver/controller-zfs/index.js @@ -35,6 +35,9 @@ const VOLUME_CONTEXT_PROVISIONER_INSTANCE_ID_PROPERTY_NAME = * - async getZetabyte() * - async setZetabyteCustomOptions(options) // optional * - getDriverZfsResourceType() // return "filesystem" or "volume" + * - getFSTypes() // optional + * - getAccessModes() // optional + * - async getAccessibleTopology() // optional * - async createShare(call, datasetName) // return appropriate volume_context for Node operations * - async deleteShare(call, datasetName) // no return expected * - async expandVolume(call, datasetName) // no return expected, used for restarting services etc if needed @@ -182,6 +185,43 @@ class ControllerZfsBaseDriver extends CsiBaseDriver { await zb.zfs.destroy(datasetName + "@%", options); } + getFSTypes() { + const driverZfsResourceType = this.getDriverZfsResourceType(); + switch (driverZfsResourceType) { + case "filesystem": + return ["nfs", "cifs"]; + case "volume": + return ["ext3", "ext4", "ext4dev", "xfs"]; + } + } + + getAccessModes() { + const driverZfsResourceType = this.getDriverZfsResourceType(); + switch (driverZfsResourceType) { + case "filesystem": + return [ + "UNKNOWN", + "SINGLE_NODE_WRITER", + "SINGLE_NODE_SINGLE_WRITER", // added in v1.5.0 + "SINGLE_NODE_MULTI_WRITER", // added in v1.5.0 + "SINGLE_NODE_READER_ONLY", + "MULTI_NODE_READER_ONLY", + "MULTI_NODE_SINGLE_WRITER", + "MULTI_NODE_MULTI_WRITER", + ]; + case "volume": + return [ + "UNKNOWN", + "SINGLE_NODE_WRITER", + "SINGLE_NODE_SINGLE_WRITER", // added in v1.5.0 + "SINGLE_NODE_MULTI_WRITER", // added in v1.5.0 + "SINGLE_NODE_READER_ONLY", + "MULTI_NODE_READER_ONLY", + "MULTI_NODE_SINGLE_WRITER", + ]; + } + } + assertCapabilities(capabilities) { const driverZfsResourceType = this.getDriverZfsResourceType(); this.ctx.logger.verbose("validating capabilities: %j", capabilities); @@ -198,24 +238,13 @@ class ControllerZfsBaseDriver extends CsiBaseDriver { if ( capability.mount.fs_type && - !["nfs", "cifs"].includes(capability.mount.fs_type) + !this.getFSTypes().includes(capability.mount.fs_type) ) { message = `invalid fs_type ${capability.mount.fs_type}`; return false; } - if ( - ![ - "UNKNOWN", - "SINGLE_NODE_WRITER", - "SINGLE_NODE_SINGLE_WRITER", // added in v1.5.0 - "SINGLE_NODE_MULTI_WRITER", // added in v1.5.0 - "SINGLE_NODE_READER_ONLY", - "MULTI_NODE_READER_ONLY", - "MULTI_NODE_SINGLE_WRITER", - "MULTI_NODE_MULTI_WRITER", - ].includes(capability.access_mode.mode) - ) { + if (!this.getAccessModes().includes(capability.access_mode.mode)) { message = `invalid access_mode, ${capability.access_mode.mode}`; return false; } @@ -225,26 +254,14 @@ class ControllerZfsBaseDriver extends CsiBaseDriver { if (capability.access_type == "mount") { if ( capability.mount.fs_type && - !["ext3", "ext4", "ext4dev", "xfs"].includes( - capability.mount.fs_type - ) + !this.getFSTypes().includes(capability.mount.fs_type) ) { message = `invalid fs_type ${capability.mount.fs_type}`; return false; } } - if ( - ![ - "UNKNOWN", - "SINGLE_NODE_WRITER", - "SINGLE_NODE_SINGLE_WRITER", // added in v1.5.0 - "SINGLE_NODE_MULTI_WRITER", // added in v1.5.0 - "SINGLE_NODE_READER_ONLY", - "MULTI_NODE_READER_ONLY", - "MULTI_NODE_SINGLE_WRITER", - ].includes(capability.access_mode.mode) - ) { + if (!this.getAccessModes().includes(capability.access_mode.mode)) { message = `invalid access_mode, ${capability.access_mode.mode}`; return false; } @@ -344,6 +361,11 @@ class ControllerZfsBaseDriver extends CsiBaseDriver { } } + let accessible_topology; + if (typeof this.getAccessibleTopology === "function") { + accessible_topology = await this.getAccessibleTopology(); + } + let volume = { // remove parent dataset info volume_id: row["name"].replace(new RegExp("^" + datasetName + "/"), ""), @@ -353,6 +375,7 @@ class ControllerZfsBaseDriver extends CsiBaseDriver { : row["volsize"], content_source: volume_content_source, volume_context, + accessible_topology, }; return volume; @@ -1101,6 +1124,11 @@ class ControllerZfsBaseDriver extends CsiBaseDriver { // this should give us a relatively sane way to clean up artifacts over time await zb.zfs.set(datasetName, { [SUCCESS_PROPERTY_NAME]: "true" }); + let accessible_topology; + if (typeof this.getAccessibleTopology === "function") { + accessible_topology = await this.getAccessibleTopology(); + } + const res = { volume: { volume_id: name, @@ -1112,6 +1140,7 @@ class ControllerZfsBaseDriver extends CsiBaseDriver { : 0, content_source: volume_content_source, volume_context, + accessible_topology, }, }; diff --git a/src/driver/freenas/ssh.js b/src/driver/freenas/ssh.js index c688895..579f92c 100644 --- a/src/driver/freenas/ssh.js +++ b/src/driver/freenas/ssh.js @@ -1,6 +1,8 @@ -const { ControllerZfsSshBaseDriver } = require("../controller-zfs-ssh"); +const { ControllerZfsBaseDriver } = require("../controller-zfs"); const { GrpcError, grpc } = require("../../utils/grpc"); +const SshClient = require("../../utils/ssh").SshClient; const HttpClient = require("./http").Client; +const { Zetabyte, ZfsSshProcessManager } = require("../../utils/zfs"); const Handlebars = require("handlebars"); @@ -18,7 +20,43 @@ const FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME = // used for in-memory cache of the version info const FREENAS_SYSTEM_VERSION_CACHE_KEY = "freenas:system_version"; -class FreeNASSshDriver extends ControllerZfsSshBaseDriver { +class FreeNASSshDriver extends ControllerZfsBaseDriver { + getExecClient() { + return new SshClient({ + logger: this.ctx.logger, + connection: this.options.sshConnection, + }); + } + + async getZetabyte() { + const sshClient = this.getExecClient(); + const options = {}; + options.executor = new ZfsSshProcessManager(sshClient); + options.idempotent = true; + + if ( + this.options.zfs.hasOwnProperty("cli") && + this.options.zfs.cli && + this.options.zfs.cli.hasOwnProperty("paths") + ) { + options.paths = this.options.zfs.cli.paths; + } + + if ( + this.options.zfs.hasOwnProperty("cli") && + this.options.zfs.cli && + this.options.zfs.cli.hasOwnProperty("sudoEnabled") + ) { + options.sudo = this.getSudoEnabled(); + } + + if (typeof this.setZetabyteCustomOptions === "function") { + await this.setZetabyteCustomOptions(options); + } + + return new Zetabyte(options); + } + /** * cannot make this a storage class parameter as storage class/etc context is *not* sent * into various calls such as GetControllerCapabilities etc @@ -169,7 +207,7 @@ class FreeNASSshDriver extends ControllerZfsSshBaseDriver { async createShare(call, datasetName) { const driver = this; const driverShareType = this.getDriverShareType(); - const sshClient = this.getSshClient(); + const execClient = this.getExecClient(); const httpClient = await this.getHttpClient(); const apiVersion = httpClient.getApiVersion(); const zb = await this.getZetabyte(); @@ -1626,7 +1664,7 @@ class FreeNASSshDriver extends ControllerZfsSshBaseDriver { async expandVolume(call, datasetName) { const driverShareType = this.getDriverShareType(); - const sshClient = this.getSshClient(); + const execClient = this.getExecClient(); const zb = await this.getZetabyte(); switch (driverShareType) { @@ -1645,7 +1683,7 @@ class FreeNASSshDriver extends ControllerZfsSshBaseDriver { properties[FREENAS_ISCSI_ASSETS_NAME_PROPERTY_NAME].value; /** - * command = sshClient.buildCommand("systemctl", ["reload", "scst"]); + * command = execClient.buildCommand("systemctl", ["reload", "scst"]); * does not help ^ * * echo 1 > /sys/kernel/scst_tgt/devices/${iscsiName}/resync_size @@ -1657,13 +1695,13 @@ class FreeNASSshDriver extends ControllerZfsSshBaseDriver { * midclt resync_lun_size_for_zvol tank/foo/bar * works on SCALE only ^ */ - command = sshClient.buildCommand("sh", [ + command = execClient.buildCommand("sh", [ "-c", `echo 1 > /sys/kernel/scst_tgt/devices/${iscsiName}/resync_size`, ]); reload = true; } else { - command = sshClient.buildCommand("/etc/rc.d/ctld", ["reload"]); + command = execClient.buildCommand("/etc/rc.d/ctld", ["reload"]); reload = true; } @@ -1677,7 +1715,7 @@ class FreeNASSshDriver extends ControllerZfsSshBaseDriver { command ); - let response = await sshClient.exec(command); + let response = await execClient.exec(command); if (response.code != 0) { throw new GrpcError( grpc.status.UNKNOWN,