nfs-client driver

README.md

@@ -17,6 +17,7 @@ have access to resizing, snapshots, clones, etc functionality.
   - `zfs-generic-nfs` (works with any ZoL installation...ie: Ubuntu)
   - `zfs-generic-iscsi` (works with any ZoL installation...ie: Ubuntu)
   - `zfs-local-ephemeral-inline` (provisions node-local zfs datasets)
+  - `nfs-client` (crudely provisions storage using a shared nfs share/directory for all volumes)
 - framework for developing `csi` drivers
 
 If you have any interest in providing a `csi` driver, simply open an issue to
@@ -137,3 +138,9 @@ Install `democratic-csi` as usual with `volumeSnapshotClasses` defined as approp
 
 - https://kubernetes.io/docs/concepts/storage/volume-snapshots/
 - https://github.com/kubernetes-csi/external-snapshotter#usage
+
+# Related
+
+- https://github.com/nmaupu/freenas-provisioner
+- https://github.com/travisghansen/freenas-iscsi-provisioner
+- https://datamattsson.tumblr.com/post/624751011659202560/welcome-truenas-core-container-storage-provider

examples/nfs-client.yaml

@@ -0,0 +1,9 @@
+driver: nfs-client
+instance_id:
+nfs:
+  shareHost: server address
+  shareBasePath: "/some/path"
+  controllerBasePath: "/storage"
+  dirPermissionsMode: "0777"
+  dirPermissionsUser: root
+  dirPermissionsGroup: wheel

src/driver/controller-nfs-client/index.js

@@ -0,0 +1,664 @@
+const { CsiBaseDriver } = require("../index");
+const { GrpcError, grpc } = require("../../utils/grpc");
+const cp = require("child_process");
+const { Mount } = require("../../utils/mount");
+
+/**
+ * Crude nfs-client driver which simply creates directories to be mounted
+ * and uses rsync for cloning/snapshots
+ */
+class ControllerNfsClientDriver extends CsiBaseDriver {
+  constructor(ctx, options) {
+    super(...arguments);
+
+    options = options || {};
+    options.service = options.service || {};
+    options.service.identity = options.service.identity || {};
+    options.service.controller = options.service.controller || {};
+    options.service.node = options.service.node || {};
+
+    options.service.identity.capabilities =
+      options.service.identity.capabilities || {};
+
+    options.service.controller.capabilities =
+      options.service.controller.capabilities || {};
+
+    options.service.node.capabilities = options.service.node.capabilities || {};
+
+    if (!("service" in options.service.identity.capabilities)) {
+      this.ctx.logger.debug("setting default identity service caps");
+
+      options.service.identity.capabilities.service = [
+        //"UNKNOWN",
+        "CONTROLLER_SERVICE",
+        //"VOLUME_ACCESSIBILITY_CONSTRAINTS"
+      ];
+    }
+
+    if (!("volume_expansion" in options.service.identity.capabilities)) {
+      this.ctx.logger.debug("setting default identity volume_expansion caps");
+
+      options.service.identity.capabilities.volume_expansion = [
+        //"UNKNOWN",
+        "ONLINE",
+        //"OFFLINE"
+      ];
+    }
+
+    if (!("rpc" in options.service.controller.capabilities)) {
+      this.ctx.logger.debug("setting default controller caps");
+
+      options.service.controller.capabilities.rpc = [
+        //"UNKNOWN",
+        "CREATE_DELETE_VOLUME",
+        //"PUBLISH_UNPUBLISH_VOLUME",
+        //"LIST_VOLUMES",
+        //"GET_CAPACITY",
+        "CREATE_DELETE_SNAPSHOT",
+        //"LIST_SNAPSHOTS",
+        "CLONE_VOLUME",
+        //"PUBLISH_READONLY",
+        //"EXPAND_VOLUME",
+      ];
+    }
+
+    if (!("rpc" in options.service.node.capabilities)) {
+      this.ctx.logger.debug("setting default node caps");
+
+      options.service.node.capabilities.rpc = [
+        //"UNKNOWN",
+        "STAGE_UNSTAGE_VOLUME",
+        "GET_VOLUME_STATS",
+        //"EXPAND_VOLUME"
+      ];
+    }
+  }
+
+  assertCapabilities(capabilities) {
+    this.ctx.logger.verbose("validating capabilities: %j", capabilities);
+
+    let message = null;
+    //[{"access_mode":{"mode":"SINGLE_NODE_WRITER"},"mount":{"mount_flags":["noatime","_netdev"],"fs_type":"nfs"},"access_type":"mount"}]
+    const valid = capabilities.every((capability) => {
+      if (capability.access_type != "mount") {
+        message = `invalid access_type ${capability.access_type}`;
+        return false;
+      }
+
+      if (
+        capability.mount.fs_type &&
+        !["nfs"].includes(capability.mount.fs_type)
+      ) {
+        message = `invalid fs_type ${capability.mount.fs_type}`;
+        return false;
+      }
+
+      if (
+        ![
+          "UNKNOWN",
+          "SINGLE_NODE_WRITER",
+          "SINGLE_NODE_READER_ONLY",
+          "MULTI_NODE_READER_ONLY",
+          "MULTI_NODE_SINGLE_WRITER",
+          "MULTI_NODE_MULTI_WRITER",
+        ].includes(capability.access_mode.mode)
+      ) {
+        message = `invalid access_mode, ${capability.access_mode.mode}`;
+        return false;
+      }
+
+      return true;
+    });
+
+    return { valid, message };
+  }
+
+  // path helpers
+  getVolumeExtraPath() {
+    return "/v";
+  }
+
+  getSnapshotExtraPath() {
+    return "/s";
+  }
+
+  // share paths
+  getShareBasePath() {
+    let path = this.options.nfs.shareBasePath;
+    if (!path) {
+      throw new GrpcError(
+        grpc.status.FAILED_PRECONDITION,
+        `invalid configuration: missing shareBasePath`
+      );
+    }
+
+    path = path.replace(/\/$/, "");
+    if (!path) {
+      throw new GrpcError(
+        grpc.status.FAILED_PRECONDITION,
+        `invalid configuration: missing shareBasePath`
+      );
+    }
+
+    return path;
+  }
+
+  getShareVolumeBasePath() {
+    return this.getShareBasePath() + this.getVolumeExtraPath();
+  }
+
+  getShareSnapshotBasePath() {
+    return this.getShareBasePath() + this.getSnapshotExtraPath();
+  }
+
+  getShareVolumePath(volume_id) {
+    return this.getShareVolumeBasePath() + "/" + volume_id;
+  }
+
+  getShareSnapshotPath(snapshot_id) {
+    return this.getShareSnapshotBasePath() + "/" + snapshot_id;
+  }
+
+  // controller paths
+  getControllerBasePath() {
+    let path = this.options.nfs.controllerBasePath;
+    if (!path) {
+      throw new GrpcError(
+        grpc.status.FAILED_PRECONDITION,
+        `invalid configuration: missing controllerBasePath`
+      );
+    }
+
+    path = path.replace(/\/$/, "");
+    if (!path) {
+      throw new GrpcError(
+        grpc.status.FAILED_PRECONDITION,
+        `invalid configuration: missing controllerBasePath`
+      );
+    }
+
+    return path;
+  }
+
+  getControllerVolumeBasePath() {
+    return this.getControllerBasePath() + this.getVolumeExtraPath();
+  }
+
+  getControllerSnapshotBasePath() {
+    return this.getControllerBasePath() + this.getSnapshotExtraPath();
+  }
+
+  getControllerVolumePath(volume_id) {
+    return this.getControllerVolumeBasePath() + "/" + volume_id;
+  }
+
+  getControllerSnapshotPath(snapshot_id) {
+    return this.getControllerSnapshotBasePath() + "/" + snapshot_id;
+  }
+
+  exec(command, args, options = {}) {
+    args = args || [];
+
+    let timeout;
+    let stdout = "";
+    let stderr = "";
+
+    if (options.sudo) {
+      args.unshift(command);
+      command = "sudo";
+    }
+    console.log("executing command: %s %s", command, args.join(" "));
+    const child = cp.spawn(command, args, options);
+
+    let didTimeout = false;
+    if (options && options.timeout) {
+      timeout = setTimeout(() => {
+        didTimeout = true;
+        child.kill(options.killSignal || "SIGTERM");
+      }, options.timeout);
+    }
+
+    return new Promise((resolve, reject) => {
+      child.stdout.on("data", function (data) {
+        stdout = stdout + data;
+      });
+
+      child.stderr.on("data", function (data) {
+        stderr = stderr + data;
+      });
+
+      child.on("close", function (code) {
+        const result = { code, stdout, stderr };
+        if (timeout) {
+          clearTimeout(timeout);
+        }
+        if (code) {
+          reject(result);
+        } else {
+          resolve(result);
+        }
+      });
+    });
+  }
+
+  stripTrailingSlash(s) {
+    if (s.length > 1) {
+      return s.replace(/\/$/, "");
+    }
+
+    return s;
+  }
+
+  async cloneDir(source_path, target_path) {
+    await this.exec("mkdir", ["-p", target_path]);
+
+    /**
+     * trailing / is important
+     * rsync -a /mnt/storage/s/foo/ /mnt/storage/v/PVC-111/
+     */
+    await this.exec("rsync", [
+      "-a",
+      this.stripTrailingSlash(source_path) + "/",
+      this.stripTrailingSlash(target_path) + "/",
+    ]);
+  }
+
+  async getAvailableSpaceAtPath(path) {
+    //df --output=avail /mnt/storage/
+    //     Avail
+    //1481334328
+
+    const response = await this.exec("df", ["--output=avail", path]);
+
+    return response.stdout.split("\n")[1].trim();
+  }
+
+  async deleteDir(path) {
+    await this.exec("rm", ["-rf", path]);
+
+    return;
+
+    /**
+     * trailing / is important
+     * rsync -a /mnt/storage/s/foo/ /mnt/storage/v/PVC-111/
+     */
+    await this.exec("rsync", [
+      "-a",
+      "--delete",
+      this.stripTrailingSlash(empty_path) + "/",
+      this.stripTrailingSlash(path) + "/",
+    ]);
+  }
+
+  /**
+   * Create a volume doing in essence the following:
+   * 1. create directory
+   *
+   * Should return 2 parameters
+   * 1. `server` - host/ip of the nfs server
+   * 2. `share` - path of the mount shared
+   *
+   * @param {*} call
+   */
+  async CreateVolume(call) {
+    const driver = this;
+
+    let name = call.request.name;
+    let volume_content_source = call.request.volume_content_source;
+
+    if (!name) {
+      throw new GrpcError(
+        grpc.status.INVALID_ARGUMENT,
+        `volume name is required`
+      );
+    }
+
+    if (call.request.volume_capabilities) {
+      const result = this.assertCapabilities(call.request.volume_capabilities);
+      if (result.valid !== true) {
+        throw new GrpcError(grpc.status.INVALID_ARGUMENT, result.message);
+      }
+    }
+
+    if (
+      call.request.capacity_range.required_bytes > 0 &&
+      call.request.capacity_range.limit_bytes > 0 &&
+      call.request.capacity_range.required_bytes >
+        call.request.capacity_range.limit_bytes
+    ) {
+      throw new GrpcError(
+        grpc.status.OUT_OF_RANGE,
+        `required_bytes is greather than limit_bytes`
+      );
+    }
+
+    let capacity_bytes =
+      call.request.capacity_range.required_bytes ||
+      call.request.capacity_range.limit_bytes;
+
+    if (!capacity_bytes) {
+      //should never happen, value must be set
+      throw new GrpcError(
+        grpc.status.INVALID_ARGUMENT,
+        `volume capacity is required (either required_bytes or limit_bytes)`
+      );
+    }
+
+    // ensure *actual* capacity is not greater than limit
+    if (
+      call.request.capacity_range.limit_bytes &&
+      call.request.capacity_range.limit_bytes > 0 &&
+      capacity_bytes > call.request.capacity_range.limit_bytes
+    ) {
+      throw new GrpcError(
+        grpc.status.OUT_OF_RANGE,
+        `required volume capacity is greater than limit`
+      );
+    }
+
+    const volume_path = driver.getControllerVolumePath(name);
+
+    let response;
+    let source_path;
+    //let volume_content_source_snapshot_id;
+    //let volume_content_source_volume_id;
+
+    // create target dir
+    response = await driver.exec("mkdir", ["-p", volume_path]);
+
+    // create dataset
+    if (volume_content_source) {
+      switch (volume_content_source.type) {
+        // must be available when adverstising CREATE_DELETE_SNAPSHOT
+        // simply clone
+        case "snapshot":
+          source_path = driver.getControllerSnapshotPath(
+            volume_content_source.snapshot.snapshot_id
+          );
+          break;
+        // must be available when adverstising CLONE_VOLUME
+        // create snapshot first, then clone
+        case "volume":
+          source_path = driver.getControllerVolumePath(
+            volume_content_source.volume.volume_id
+          );
+          break;
+        default:
+          throw new GrpcError(
+            grpc.status.INVALID_ARGUMENT,
+            `invalid volume_content_source type: ${volume_content_source.type}`
+          );
+          break;
+      }
+
+      driver.ctx.logger.debug("controller source path: %s", source_path);
+      response = await driver.cloneDir(source_path, volume_path);
+    }
+
+    // set mode
+    if (this.options.nfs.dirPermissionsMode) {
+      driver.ctx.logger.verbose(
+        "setting dir mode to: %s on dir: %s",
+        this.options.nfs.dirPermissionsMode,
+        volume_path
+      );
+      response = await driver.exec("chmod", [
+        this.options.nfs.dirPermissionsMode,
+        volume_path,
+      ]);
+    }
+
+    // set ownership
+    if (
+      this.options.nfs.dirPermissionsUser ||
+      this.options.nfs.dirPermissionsGroup
+    ) {
+      driver.ctx.logger.verbose(
+        "setting ownership to: %s:%s on dir: %s",
+        this.options.nfs.dirPermissionsUser,
+        this.options.nfs.dirPermissionsGroup,
+        volume_path
+      );
+      response = await driver.exec("chown", [
+        (this.options.nfs.dirPermissionsUser
+          ? this.options.nfs.dirPermissionsUser
+          : "") +
+          ":" +
+          (this.options.nfs.dirPermissionsGroup
+            ? this.options.nfs.dirPermissionsGroup
+            : ""),
+        volume_path,
+      ]);
+    }
+
+    let volume_context = {
+      node_attach_driver: "nfs",
+      server: this.options.nfs.shareHost,
+      share: driver.getShareVolumePath(name),
+    };
+
+    volume_context["provisioner_driver"] = driver.options.driver;
+    if (driver.options.instance_id) {
+      volume_context["provisioner_driver_instance_id"] =
+        driver.options.instance_id;
+    }
+
+    const res = {
+      volume: {
+        volume_id: name,
+        //capacity_bytes: capacity_bytes, // kubernetes currently pukes if capacity is returned as 0
+        capacity_bytes: 0,
+        content_source: volume_content_source,
+        volume_context,
+      },
+    };
+
+    return res;
+  }
+
+  /**
+   * Delete a volume
+   *
+   * Deleting a volume consists of the following steps:
+   * 1. delete directory
+   *
+   * @param {*} call
+   */
+  async DeleteVolume(call) {
+    const driver = this;
+
+    let name = call.request.volume_id;
+
+    if (!name) {
+      throw new GrpcError(
+        grpc.status.INVALID_ARGUMENT,
+        `volume_id is required`
+      );
+    }
+
+    const volume_path = driver.getControllerVolumePath(name);
+    await driver.deleteDir(volume_path);
+
+    return {};
+  }
+
+  /**
+   *
+   * @param {*} call
+   */
+  async ControllerExpandVolume(call) {
+    throw new GrpcError(
+      grpc.status.UNIMPLEMENTED,
+      `operation not supported by driver`
+    );
+  }
+
+  /**
+   * TODO: consider volume_capabilities?
+   *
+   * @param {*} call
+   */
+  async GetCapacity(call) {
+    // really capacity is not used at all with nfs in this fashion, so no reason to enable
+    // here even though it is technically feasible.
+    throw new GrpcError(
+      grpc.status.UNIMPLEMENTED,
+      `operation not supported by driver`
+    );
+
+    const driver = this;
+
+    if (call.request.volume_capabilities) {
+      const result = this.assertCapabilities(call.request.volume_capabilities);
+
+      if (result.valid !== true) {
+        return { available_capacity: 0 };
+      }
+    }
+
+    const available_capacity = await driver.getAvailableSpaceAtPath(
+      driver.getControllerBasePath()
+    );
+    return { available_capacity };
+  }
+
+  /**
+   *
+   * TODO: check capability to ensure not asking about block volumes
+   *
+   * @param {*} call
+   */
+  async ListVolumes(call) {
+    throw new GrpcError(
+      grpc.status.UNIMPLEMENTED,
+      `operation not supported by driver`
+    );
+  }
+
+  /**
+   *
+   * @param {*} call
+   */
+  async ListSnapshots(call) {
+    throw new GrpcError(
+      grpc.status.UNIMPLEMENTED,
+      `operation not supported by driver`
+    );
+  }
+
+  /**
+   *
+   * @param {*} call
+   */
+  async CreateSnapshot(call) {
+    const driver = this;
+
+    // both these are required
+    let source_volume_id = call.request.source_volume_id;
+    let name = call.request.name;
+
+    if (!source_volume_id) {
+      throw new GrpcError(
+        grpc.status.INVALID_ARGUMENT,
+        `snapshot source_volume_id is required`
+      );
+    }
+
+    if (!name) {
+      throw new GrpcError(
+        grpc.status.INVALID_ARGUMENT,
+        `snapshot name is required`
+      );
+    }
+
+    driver.ctx.logger.verbose("requested snapshot name: %s", name);
+
+    let invalid_chars;
+    invalid_chars = name.match(/[^a-z0-9_\-:.+]+/gi);
+    if (invalid_chars) {
+      invalid_chars = String.prototype.concat(
+        ...new Set(invalid_chars.join(""))
+      );
+      throw new GrpcError(
+        grpc.status.INVALID_ARGUMENT,
+        `snapshot name contains invalid characters: ${invalid_chars}`
+      );
+    }
+
+    // https://stackoverflow.com/questions/32106243/regex-to-remove-all-non-alpha-numeric-and-replace-spaces-with/32106277
+    name = name.replace(/[^a-z0-9_\-:.+]+/gi, "");
+
+    driver.ctx.logger.verbose("cleansed snapshot name: %s", name);
+
+    const snapshot_id = `${source_volume_id}-${name}`;
+    const volume_path = driver.getControllerVolumePath(source_volume_id);
+    const snapshot_path = driver.getControllerSnapshotPath(snapshot_id);
+
+    await driver.cloneDir(volume_path, snapshot_path);
+
+    return {
+      snapshot: {
+        /**
+         * The purpose of this field is to give CO guidance on how much space
+         * is needed to create a volume from this snapshot.
+         */
+        size_bytes: 0,
+        snapshot_id,
+        source_volume_id: source_volume_id,
+        //https://github.com/protocolbuffers/protobuf/blob/master/src/google/protobuf/timestamp.proto
+        creation_time: {
+          seconds: Math.round(new Date().getTime() / 1000),
+          nanos: 0,
+        },
+        ready_to_use: true,
+      },
+    };
+  }
+
+  /**
+   * In addition, if clones have been created from a snapshot, then they must
+   * be destroyed before the snapshot can be destroyed.
+   *
+   * @param {*} call
+   */
+  async DeleteSnapshot(call) {
+    const driver = this;
+
+    const snapshot_id = call.request.snapshot_id;
+
+    if (!snapshot_id) {
+      throw new GrpcError(
+        grpc.status.INVALID_ARGUMENT,
+        `snapshot_id is required`
+      );
+    }
+
+    const snapshot_path = driver.getControllerSnapshotPath(snapshot_id);
+    await driver.deleteDir(snapshot_path);
+
+    return {};
+  }
+
+  /**
+   *
+   * @param {*} call
+   */
+  async ValidateVolumeCapabilities(call) {
+    const driver = this;
+    const result = this.assertCapabilities(call.request.volume_capabilities);
+
+    if (result.valid !== true) {
+      return { message: result.message };
+    }
+
+    return {
+      confirmed: {
+        volume_context: call.request.volume_context,
+        volume_capabilities: call.request.volume_capabilities, // TODO: this is a bit crude, should return *ALL* capabilities, not just what was requested
+        parameters: call.request.parameters,
+      },
+    };
+  }
+}
+
+module.exports.ControllerNfsClientDriver = ControllerNfsClientDriver;

src/driver/factory.js

@@ -4,6 +4,8 @@ const {
   ZfsLocalEphemeralInlineDriver,
 } = require("./zfs-local-ephemeral-inline");
 
+const { ControllerNfsClientDriver } = require("./controller-nfs-client");
+
 function factory(ctx, options) {
   switch (options.driver) {
     case "freenas-nfs":
@@ -16,6 +18,8 @@ function factory(ctx, options) {
       return new ControllerZfsGenericDriver(ctx, options);
     case "zfs-local-ephemeral-inline":
       return new ZfsLocalEphemeralInlineDriver(ctx, options);
+    case "nfs-client":
+      return new ControllerNfsClientDriver(ctx, options);
     default:
       throw new Error("invalid csi driver: " + options.driver);
   }