diff --git a/README.md b/README.md index 405e121..17d727d 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,7 @@ have access to resizing, snapshots, clones, etc functionality. - `zfs-generic-nfs` (works with any ZoL installation...ie: Ubuntu) - `zfs-generic-iscsi` (works with any ZoL installation...ie: Ubuntu) - `zfs-local-ephemeral-inline` (provisions node-local zfs datasets) + - `nfs-client` (crudely provisions storage using a shared nfs share/directory for all volumes) - framework for developing `csi` drivers If you have any interest in providing a `csi` driver, simply open an issue to @@ -137,3 +138,9 @@ Install `democratic-csi` as usual with `volumeSnapshotClasses` defined as approp - https://kubernetes.io/docs/concepts/storage/volume-snapshots/ - https://github.com/kubernetes-csi/external-snapshotter#usage + +# Related + +- https://github.com/nmaupu/freenas-provisioner +- https://github.com/travisghansen/freenas-iscsi-provisioner +- https://datamattsson.tumblr.com/post/624751011659202560/welcome-truenas-core-container-storage-provider diff --git a/examples/nfs-client.yaml b/examples/nfs-client.yaml new file mode 100644 index 0000000..f0bd490 --- /dev/null +++ b/examples/nfs-client.yaml @@ -0,0 +1,9 @@ +driver: nfs-client +instance_id: +nfs: + shareHost: server address + shareBasePath: "/some/path" + controllerBasePath: "/storage" + dirPermissionsMode: "0777" + dirPermissionsUser: root + dirPermissionsGroup: wheel diff --git a/src/driver/controller-nfs-client/index.js b/src/driver/controller-nfs-client/index.js new file mode 100644 index 0000000..ebe68b8 --- /dev/null +++ b/src/driver/controller-nfs-client/index.js @@ -0,0 +1,664 @@ +const { CsiBaseDriver } = require("../index"); +const { GrpcError, grpc } = require("../../utils/grpc"); +const cp = require("child_process"); +const { Mount } = require("../../utils/mount"); + +/** + * Crude nfs-client driver which simply creates directories to be mounted + * and uses rsync for cloning/snapshots + */ +class ControllerNfsClientDriver extends CsiBaseDriver { + constructor(ctx, options) { + super(...arguments); + + options = options || {}; + options.service = options.service || {}; + options.service.identity = options.service.identity || {}; + options.service.controller = options.service.controller || {}; + options.service.node = options.service.node || {}; + + options.service.identity.capabilities = + options.service.identity.capabilities || {}; + + options.service.controller.capabilities = + options.service.controller.capabilities || {}; + + options.service.node.capabilities = options.service.node.capabilities || {}; + + if (!("service" in options.service.identity.capabilities)) { + this.ctx.logger.debug("setting default identity service caps"); + + options.service.identity.capabilities.service = [ + //"UNKNOWN", + "CONTROLLER_SERVICE", + //"VOLUME_ACCESSIBILITY_CONSTRAINTS" + ]; + } + + if (!("volume_expansion" in options.service.identity.capabilities)) { + this.ctx.logger.debug("setting default identity volume_expansion caps"); + + options.service.identity.capabilities.volume_expansion = [ + //"UNKNOWN", + "ONLINE", + //"OFFLINE" + ]; + } + + if (!("rpc" in options.service.controller.capabilities)) { + this.ctx.logger.debug("setting default controller caps"); + + options.service.controller.capabilities.rpc = [ + //"UNKNOWN", + "CREATE_DELETE_VOLUME", + //"PUBLISH_UNPUBLISH_VOLUME", + //"LIST_VOLUMES", + //"GET_CAPACITY", + "CREATE_DELETE_SNAPSHOT", + //"LIST_SNAPSHOTS", + "CLONE_VOLUME", + //"PUBLISH_READONLY", + //"EXPAND_VOLUME", + ]; + } + + if (!("rpc" in options.service.node.capabilities)) { + this.ctx.logger.debug("setting default node caps"); + + options.service.node.capabilities.rpc = [ + //"UNKNOWN", + "STAGE_UNSTAGE_VOLUME", + "GET_VOLUME_STATS", + //"EXPAND_VOLUME" + ]; + } + } + + assertCapabilities(capabilities) { + this.ctx.logger.verbose("validating capabilities: %j", capabilities); + + let message = null; + //[{"access_mode":{"mode":"SINGLE_NODE_WRITER"},"mount":{"mount_flags":["noatime","_netdev"],"fs_type":"nfs"},"access_type":"mount"}] + const valid = capabilities.every((capability) => { + if (capability.access_type != "mount") { + message = `invalid access_type ${capability.access_type}`; + return false; + } + + if ( + capability.mount.fs_type && + !["nfs"].includes(capability.mount.fs_type) + ) { + message = `invalid fs_type ${capability.mount.fs_type}`; + return false; + } + + if ( + ![ + "UNKNOWN", + "SINGLE_NODE_WRITER", + "SINGLE_NODE_READER_ONLY", + "MULTI_NODE_READER_ONLY", + "MULTI_NODE_SINGLE_WRITER", + "MULTI_NODE_MULTI_WRITER", + ].includes(capability.access_mode.mode) + ) { + message = `invalid access_mode, ${capability.access_mode.mode}`; + return false; + } + + return true; + }); + + return { valid, message }; + } + + // path helpers + getVolumeExtraPath() { + return "/v"; + } + + getSnapshotExtraPath() { + return "/s"; + } + + // share paths + getShareBasePath() { + let path = this.options.nfs.shareBasePath; + if (!path) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: missing shareBasePath` + ); + } + + path = path.replace(/\/$/, ""); + if (!path) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: missing shareBasePath` + ); + } + + return path; + } + + getShareVolumeBasePath() { + return this.getShareBasePath() + this.getVolumeExtraPath(); + } + + getShareSnapshotBasePath() { + return this.getShareBasePath() + this.getSnapshotExtraPath(); + } + + getShareVolumePath(volume_id) { + return this.getShareVolumeBasePath() + "/" + volume_id; + } + + getShareSnapshotPath(snapshot_id) { + return this.getShareSnapshotBasePath() + "/" + snapshot_id; + } + + // controller paths + getControllerBasePath() { + let path = this.options.nfs.controllerBasePath; + if (!path) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: missing controllerBasePath` + ); + } + + path = path.replace(/\/$/, ""); + if (!path) { + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: missing controllerBasePath` + ); + } + + return path; + } + + getControllerVolumeBasePath() { + return this.getControllerBasePath() + this.getVolumeExtraPath(); + } + + getControllerSnapshotBasePath() { + return this.getControllerBasePath() + this.getSnapshotExtraPath(); + } + + getControllerVolumePath(volume_id) { + return this.getControllerVolumeBasePath() + "/" + volume_id; + } + + getControllerSnapshotPath(snapshot_id) { + return this.getControllerSnapshotBasePath() + "/" + snapshot_id; + } + + exec(command, args, options = {}) { + args = args || []; + + let timeout; + let stdout = ""; + let stderr = ""; + + if (options.sudo) { + args.unshift(command); + command = "sudo"; + } + console.log("executing command: %s %s", command, args.join(" ")); + const child = cp.spawn(command, args, options); + + let didTimeout = false; + if (options && options.timeout) { + timeout = setTimeout(() => { + didTimeout = true; + child.kill(options.killSignal || "SIGTERM"); + }, options.timeout); + } + + return new Promise((resolve, reject) => { + child.stdout.on("data", function (data) { + stdout = stdout + data; + }); + + child.stderr.on("data", function (data) { + stderr = stderr + data; + }); + + child.on("close", function (code) { + const result = { code, stdout, stderr }; + if (timeout) { + clearTimeout(timeout); + } + if (code) { + reject(result); + } else { + resolve(result); + } + }); + }); + } + + stripTrailingSlash(s) { + if (s.length > 1) { + return s.replace(/\/$/, ""); + } + + return s; + } + + async cloneDir(source_path, target_path) { + await this.exec("mkdir", ["-p", target_path]); + + /** + * trailing / is important + * rsync -a /mnt/storage/s/foo/ /mnt/storage/v/PVC-111/ + */ + await this.exec("rsync", [ + "-a", + this.stripTrailingSlash(source_path) + "/", + this.stripTrailingSlash(target_path) + "/", + ]); + } + + async getAvailableSpaceAtPath(path) { + //df --output=avail /mnt/storage/ + // Avail + //1481334328 + + const response = await this.exec("df", ["--output=avail", path]); + + return response.stdout.split("\n")[1].trim(); + } + + async deleteDir(path) { + await this.exec("rm", ["-rf", path]); + + return; + + /** + * trailing / is important + * rsync -a /mnt/storage/s/foo/ /mnt/storage/v/PVC-111/ + */ + await this.exec("rsync", [ + "-a", + "--delete", + this.stripTrailingSlash(empty_path) + "/", + this.stripTrailingSlash(path) + "/", + ]); + } + + /** + * Create a volume doing in essence the following: + * 1. create directory + * + * Should return 2 parameters + * 1. `server` - host/ip of the nfs server + * 2. `share` - path of the mount shared + * + * @param {*} call + */ + async CreateVolume(call) { + const driver = this; + + let name = call.request.name; + let volume_content_source = call.request.volume_content_source; + + if (!name) { + throw new GrpcError( + grpc.status.INVALID_ARGUMENT, + `volume name is required` + ); + } + + if (call.request.volume_capabilities) { + const result = this.assertCapabilities(call.request.volume_capabilities); + if (result.valid !== true) { + throw new GrpcError(grpc.status.INVALID_ARGUMENT, result.message); + } + } + + if ( + call.request.capacity_range.required_bytes > 0 && + call.request.capacity_range.limit_bytes > 0 && + call.request.capacity_range.required_bytes > + call.request.capacity_range.limit_bytes + ) { + throw new GrpcError( + grpc.status.OUT_OF_RANGE, + `required_bytes is greather than limit_bytes` + ); + } + + let capacity_bytes = + call.request.capacity_range.required_bytes || + call.request.capacity_range.limit_bytes; + + if (!capacity_bytes) { + //should never happen, value must be set + throw new GrpcError( + grpc.status.INVALID_ARGUMENT, + `volume capacity is required (either required_bytes or limit_bytes)` + ); + } + + // ensure *actual* capacity is not greater than limit + if ( + call.request.capacity_range.limit_bytes && + call.request.capacity_range.limit_bytes > 0 && + capacity_bytes > call.request.capacity_range.limit_bytes + ) { + throw new GrpcError( + grpc.status.OUT_OF_RANGE, + `required volume capacity is greater than limit` + ); + } + + const volume_path = driver.getControllerVolumePath(name); + + let response; + let source_path; + //let volume_content_source_snapshot_id; + //let volume_content_source_volume_id; + + // create target dir + response = await driver.exec("mkdir", ["-p", volume_path]); + + // create dataset + if (volume_content_source) { + switch (volume_content_source.type) { + // must be available when adverstising CREATE_DELETE_SNAPSHOT + // simply clone + case "snapshot": + source_path = driver.getControllerSnapshotPath( + volume_content_source.snapshot.snapshot_id + ); + break; + // must be available when adverstising CLONE_VOLUME + // create snapshot first, then clone + case "volume": + source_path = driver.getControllerVolumePath( + volume_content_source.volume.volume_id + ); + break; + default: + throw new GrpcError( + grpc.status.INVALID_ARGUMENT, + `invalid volume_content_source type: ${volume_content_source.type}` + ); + break; + } + + driver.ctx.logger.debug("controller source path: %s", source_path); + response = await driver.cloneDir(source_path, volume_path); + } + + // set mode + if (this.options.nfs.dirPermissionsMode) { + driver.ctx.logger.verbose( + "setting dir mode to: %s on dir: %s", + this.options.nfs.dirPermissionsMode, + volume_path + ); + response = await driver.exec("chmod", [ + this.options.nfs.dirPermissionsMode, + volume_path, + ]); + } + + // set ownership + if ( + this.options.nfs.dirPermissionsUser || + this.options.nfs.dirPermissionsGroup + ) { + driver.ctx.logger.verbose( + "setting ownership to: %s:%s on dir: %s", + this.options.nfs.dirPermissionsUser, + this.options.nfs.dirPermissionsGroup, + volume_path + ); + response = await driver.exec("chown", [ + (this.options.nfs.dirPermissionsUser + ? this.options.nfs.dirPermissionsUser + : "") + + ":" + + (this.options.nfs.dirPermissionsGroup + ? this.options.nfs.dirPermissionsGroup + : ""), + volume_path, + ]); + } + + let volume_context = { + node_attach_driver: "nfs", + server: this.options.nfs.shareHost, + share: driver.getShareVolumePath(name), + }; + + volume_context["provisioner_driver"] = driver.options.driver; + if (driver.options.instance_id) { + volume_context["provisioner_driver_instance_id"] = + driver.options.instance_id; + } + + const res = { + volume: { + volume_id: name, + //capacity_bytes: capacity_bytes, // kubernetes currently pukes if capacity is returned as 0 + capacity_bytes: 0, + content_source: volume_content_source, + volume_context, + }, + }; + + return res; + } + + /** + * Delete a volume + * + * Deleting a volume consists of the following steps: + * 1. delete directory + * + * @param {*} call + */ + async DeleteVolume(call) { + const driver = this; + + let name = call.request.volume_id; + + if (!name) { + throw new GrpcError( + grpc.status.INVALID_ARGUMENT, + `volume_id is required` + ); + } + + const volume_path = driver.getControllerVolumePath(name); + await driver.deleteDir(volume_path); + + return {}; + } + + /** + * + * @param {*} call + */ + async ControllerExpandVolume(call) { + throw new GrpcError( + grpc.status.UNIMPLEMENTED, + `operation not supported by driver` + ); + } + + /** + * TODO: consider volume_capabilities? + * + * @param {*} call + */ + async GetCapacity(call) { + // really capacity is not used at all with nfs in this fashion, so no reason to enable + // here even though it is technically feasible. + throw new GrpcError( + grpc.status.UNIMPLEMENTED, + `operation not supported by driver` + ); + + const driver = this; + + if (call.request.volume_capabilities) { + const result = this.assertCapabilities(call.request.volume_capabilities); + + if (result.valid !== true) { + return { available_capacity: 0 }; + } + } + + const available_capacity = await driver.getAvailableSpaceAtPath( + driver.getControllerBasePath() + ); + return { available_capacity }; + } + + /** + * + * TODO: check capability to ensure not asking about block volumes + * + * @param {*} call + */ + async ListVolumes(call) { + throw new GrpcError( + grpc.status.UNIMPLEMENTED, + `operation not supported by driver` + ); + } + + /** + * + * @param {*} call + */ + async ListSnapshots(call) { + throw new GrpcError( + grpc.status.UNIMPLEMENTED, + `operation not supported by driver` + ); + } + + /** + * + * @param {*} call + */ + async CreateSnapshot(call) { + const driver = this; + + // both these are required + let source_volume_id = call.request.source_volume_id; + let name = call.request.name; + + if (!source_volume_id) { + throw new GrpcError( + grpc.status.INVALID_ARGUMENT, + `snapshot source_volume_id is required` + ); + } + + if (!name) { + throw new GrpcError( + grpc.status.INVALID_ARGUMENT, + `snapshot name is required` + ); + } + + driver.ctx.logger.verbose("requested snapshot name: %s", name); + + let invalid_chars; + invalid_chars = name.match(/[^a-z0-9_\-:.+]+/gi); + if (invalid_chars) { + invalid_chars = String.prototype.concat( + ...new Set(invalid_chars.join("")) + ); + throw new GrpcError( + grpc.status.INVALID_ARGUMENT, + `snapshot name contains invalid characters: ${invalid_chars}` + ); + } + + // https://stackoverflow.com/questions/32106243/regex-to-remove-all-non-alpha-numeric-and-replace-spaces-with/32106277 + name = name.replace(/[^a-z0-9_\-:.+]+/gi, ""); + + driver.ctx.logger.verbose("cleansed snapshot name: %s", name); + + const snapshot_id = `${source_volume_id}-${name}`; + const volume_path = driver.getControllerVolumePath(source_volume_id); + const snapshot_path = driver.getControllerSnapshotPath(snapshot_id); + + await driver.cloneDir(volume_path, snapshot_path); + + return { + snapshot: { + /** + * The purpose of this field is to give CO guidance on how much space + * is needed to create a volume from this snapshot. + */ + size_bytes: 0, + snapshot_id, + source_volume_id: source_volume_id, + //https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto + creation_time: { + seconds: Math.round(new Date().getTime() / 1000), + nanos: 0, + }, + ready_to_use: true, + }, + }; + } + + /** + * In addition, if clones have been created from a snapshot, then they must + * be destroyed before the snapshot can be destroyed. + * + * @param {*} call + */ + async DeleteSnapshot(call) { + const driver = this; + + const snapshot_id = call.request.snapshot_id; + + if (!snapshot_id) { + throw new GrpcError( + grpc.status.INVALID_ARGUMENT, + `snapshot_id is required` + ); + } + + const snapshot_path = driver.getControllerSnapshotPath(snapshot_id); + await driver.deleteDir(snapshot_path); + + return {}; + } + + /** + * + * @param {*} call + */ + async ValidateVolumeCapabilities(call) { + const driver = this; + const result = this.assertCapabilities(call.request.volume_capabilities); + + if (result.valid !== true) { + return { message: result.message }; + } + + return { + confirmed: { + volume_context: call.request.volume_context, + volume_capabilities: call.request.volume_capabilities, // TODO: this is a bit crude, should return *ALL* capabilities, not just what was requested + parameters: call.request.parameters, + }, + }; + } +} + +module.exports.ControllerNfsClientDriver = ControllerNfsClientDriver; diff --git a/src/driver/factory.js b/src/driver/factory.js index 8863e42..a8d581a 100644 --- a/src/driver/factory.js +++ b/src/driver/factory.js @@ -4,6 +4,8 @@ const { ZfsLocalEphemeralInlineDriver, } = require("./zfs-local-ephemeral-inline"); +const { ControllerNfsClientDriver } = require("./controller-nfs-client"); + function factory(ctx, options) { switch (options.driver) { case "freenas-nfs": @@ -16,6 +18,8 @@ function factory(ctx, options) { return new ControllerZfsGenericDriver(ctx, options); case "zfs-local-ephemeral-inline": return new ZfsLocalEphemeralInlineDriver(ctx, options); + case "nfs-client": + return new ControllerNfsClientDriver(ctx, options); default: throw new Error("invalid csi driver: " + options.driver); }