introduce -nvmeof drivers for TrueNAS

Signed-off-by: Travis Glenn Hansen <travisghansen@yahoo.com>

.github/workflows/main.yml

@@ -125,6 +125,7 @@ jobs:
         config:
           - truenas/scale/25.10/scale-iscsi.yaml
           - truenas/scale/25.10/scale-nfs.yaml
+          - truenas/scale/25.10/scale-nvmeof.yaml
           # 80 char limit
           - truenas/scale/25.10/scale-smb.yaml
     runs-on:

Dockerfile

@@ -5,7 +5,7 @@
 ######################
 # golang builder
 ######################
-FROM golang:1.25.3-bookworm as ctrbuilder
+FROM golang:1.25.3-bookworm AS ctrbuilder
 
 # /go/containerd/ctr
 ADD docker/ctr-mount-labels.diff /tmp
@@ -61,6 +61,7 @@ WORKDIR /home/csi/app
 USER csi
 
 # prevent need to build re2 module
+# https://github.com/uhop/install-artifact-from-github/wiki/Making-local-mirror
 ENV RE2_DOWNLOAD_MIRROR="https://grpc-uds-binaries.s3-us-west-2.amazonaws.com/re2"
 ENV RE2_DOWNLOAD_SKIP_PATH=1
 

ci/configs/truenas/scale/25.10/scale-nvmeof.yaml

@@ -0,0 +1,32 @@
+driver: freenas-api-nvmeof
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey:
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize:
+
+nvmeof:
+  transports:
+    - tcp://${TRUENAS_HOST}:4420
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  ports:
+    - 1
+
+# https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

src/driver/factory.js

@@ -24,13 +24,20 @@ function factory(ctx, options) {
     case "freenas-nfs":
     case "freenas-smb":
     case "freenas-iscsi":
+    case "freenas-nvmeof":
     case "truenas-nfs":
     case "truenas-smb":
     case "truenas-iscsi":
+    case "truenas-nvmeof":
       return new FreeNASSshDriver(ctx, options);
-    case "freenas-api-iscsi":
     case "freenas-api-nfs":
     case "freenas-api-smb":
+    case "freenas-api-iscsi":
+    case "freenas-api-nvmeof":
+    case "truenas-api-nfs":
+    case "truenas-api-smb":
+    case "truenas-api-iscsi":
+    case "truenas-api-nvmeof":
       return new FreeNASApiDriver(ctx, options);
     case "synology-nfs":
     case "synology-smb":

src/driver/freenas/http/api.js

@@ -645,6 +645,356 @@ class Api {
     throw new Error(JSON.stringify(response.body));
   }
 
+  async NvmetSubsysList(data = {}) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = "/nvmet/subsys";
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.get(endpoint, data);
+
+    if (response.statusCode == 200) {
+      return response.body;
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetSubsysCreate(subsysName, data = {}) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    data.name = subsysName;
+    data.allow_any_host = true;
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = "/nvmet/subsys";
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.post(endpoint, data);
+
+    if (response.statusCode == 200) {
+      return response.body;
+    }
+
+    if (
+      response.statusCode == 422 &&
+      JSON.stringify(response.body).includes("already exists")
+    ) {
+      return this.NvmetSubsysGetByName(subsysName);
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetSubsysGetByName(subsysName, data = {}) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    data.name = subsysName;
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = "/nvmet/subsys";
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.get(endpoint, data);
+
+    if (response.statusCode == 200) {
+      for (const subsys of response.body) {
+        if (subsys.name == subsysName) {
+          return subsys;
+        }
+      }
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetSubsysGetById(id, data = {}) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = `/nvmet/subsys/id/${id}`;
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.get(endpoint, data);
+
+    if (response.statusCode == 200) {
+      return response.body;
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetSubsysDeleteById(id, data = {}) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = `/nvmet/subsys/id/${id}`;
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.delete(endpoint, data);
+
+    if (response.statusCode == 200) {
+      return;
+    }
+
+    if (
+      response.statusCode == 422 &&
+      JSON.stringify(response.body).includes("does not exist")
+    ) {
+      return;
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetPortList(data = {}) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = "/nvmet/port";
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.get(endpoint, data);
+
+    if (response.statusCode == 200) {
+      return response.body;
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetPortSubsysList(data = {}) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = "/nvmet/port_subsys";
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.get(endpoint, data);
+
+    if (response.statusCode == 200) {
+      return response.body;
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetPortSubsysCreate(port_id, subsys_id) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    let data = {
+      port_id,
+      subsys_id,
+    };
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = "/nvmet/port_subsys";
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.post(endpoint, data);
+
+    if (response.statusCode == 200) {
+      return response.body;
+    }
+
+    //already exists
+    if (
+      response.statusCode == 422 &&
+      JSON.stringify(response.body).includes("already exists")
+    ) {
+      response = await this.NvmetPortSubsysList({ port_id, subsys_id });
+      if (Array.isArray(response) && response.length == 1) {
+        return response[0];
+      }
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetNamespaceCreate(zvol, subsysId, data = {}) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    zvol = String(zvol);
+    if (zvol.startsWith("/dev/")) {
+      zvol = zvol.substring(5);
+    }
+
+    if (zvol.startsWith("/")) {
+      zvol = zvol.substring(1);
+    }
+
+    if (!zvol.startsWith("zvol/")) {
+      zvol = `zvol/${zvol}`;
+    }
+
+    data.device_type = "ZVOL";
+    data.device_path = zvol;
+    data.subsys_id = subsysId;
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = "/nvmet/namespace";
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.post(endpoint, data);
+
+    if (response.statusCode == 200) {
+      return response.body;
+    }
+
+    if (
+      response.statusCode == 422 &&
+      JSON.stringify(response.body).includes("already exists")
+    ) {
+      return this.NvmetSubsysGetByName(subsysName);
+    }
+
+    if (
+      response.statusCode == 422 &&
+      JSON.stringify(response.body).includes("already used by subsystem")
+    ) {
+      //This device_path already used by subsystem: csi-pvc-111-clustera
+      return this.NvmetNamespaceGetByDeivcePath(zvol);
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetNamespaceGetByDeivcePath(zvol) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    zvol = String(zvol);
+    if (zvol.startsWith("/dev/")) {
+      zvol = zvol.substring(5);
+    }
+
+    if (zvol.startsWith("/")) {
+      zvol = zvol.substring(1);
+    }
+
+    if (!zvol.startsWith("zvol/")) {
+      zvol = `zvol/${zvol}`;
+    }
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = "/nvmet/namespace";
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    let data = {
+      device_path: zvol,
+    };
+
+    response = await httpClient.get(endpoint, data);
+
+    if (response.statusCode == 200) {
+      if (Array.isArray(response.body) && response.body.length == 1) {
+        return response.body[0];
+      }
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
+  async NvmetNamespaceDeleteById(id) {
+    const httpClient = await this.getHttpClient(false);
+    const zb = await this.getZetabyte();
+    const systemVersionSemver = await this.getSystemVersionSemver();
+
+    let response;
+    let endpoint;
+
+    if (semver.satisfies(systemVersionSemver, ">=25.10")) {
+      endpoint = `/nvmet/namespace/id/${id}`;
+    } else {
+      throw new Error("nvmet is unavailable with TrueNAS versions <25.10");
+    }
+
+    response = await httpClient.delete(endpoint);
+
+    if (response.statusCode == 200) {
+      return;
+    }
+
+    if (
+      response.statusCode == 422 &&
+      JSON.stringify(response.body).includes("does not exist")
+    ) {
+      return;
+    }
+
+    throw new Error(JSON.stringify(response.body));
+  }
+
   async CloneCreate(snapshotName, datasetName, data = {}) {
     const httpClient = await this.getHttpClient(false);
     const zb = await this.getZetabyte();

src/driver/index.js

@@ -124,10 +124,13 @@ class CsiBaseDriver {
    * @returns Mount
    */
   getDefaultMountInstance() {
-    return this.ctx.registry.get(`${__REGISTRY_NS__}:default_mount_instance`, () => {
-      const filesystem = this.getDefaultFilesystemInstance();
-      return new Mount({ filesystem });
-    });
+    return this.ctx.registry.get(
+      `${__REGISTRY_NS__}:default_mount_instance`,
+      () => {
+        const filesystem = this.getDefaultFilesystemInstance();
+        return new Mount({ filesystem });
+      }
+    );
   }
 
   /**
@@ -136,9 +139,12 @@ class CsiBaseDriver {
    * @returns ISCSI
    */
   getDefaultISCSIInstance() {
-    return this.ctx.registry.get(`${__REGISTRY_NS__}:default_iscsi_instance`, () => {
-      return new ISCSI();
-    });
+    return this.ctx.registry.get(
+      `${__REGISTRY_NS__}:default_iscsi_instance`,
+      () => {
+        return new ISCSI();
+      }
+    );
   }
 
   /**
@@ -148,37 +154,46 @@ class CsiBaseDriver {
    */
   getDefaultNVMEoFInstance() {
     const driver = this;
-    return this.ctx.registry.get(`${__REGISTRY_NS__}:default_nvmeof_instance`, () => {
-      return new NVMEoF({ logger: driver.ctx.logger });
-    });
+    return this.ctx.registry.get(
+      `${__REGISTRY_NS__}:default_nvmeof_instance`,
+      () => {
+        return new NVMEoF({ logger: driver.ctx.logger });
+      }
+    );
   }
 
   getDefaultZetabyteInstance() {
-    return this.ctx.registry.get(`${__REGISTRY_NS__}:default_zb_instance`, () => {
-      return new Zetabyte({
-        idempotent: true,
-        paths: {
-          zfs: "zfs",
-          zpool: "zpool",
-          sudo: "sudo",
-          chroot: "chroot",
-        },
-        //logger: driver.ctx.logger,
-        executor: {
-          spawn: function () {
-            const command = `${arguments[0]} ${arguments[1].join(" ")}`;
-            return cp.exec(command);
+    return this.ctx.registry.get(
+      `${__REGISTRY_NS__}:default_zb_instance`,
+      () => {
+        return new Zetabyte({
+          idempotent: true,
+          paths: {
+            zfs: "zfs",
+            zpool: "zpool",
+            sudo: "sudo",
+            chroot: "chroot",
           },
-        },
-        log_commands: true,
-      });
-    });
+          //logger: driver.ctx.logger,
+          executor: {
+            spawn: function () {
+              const command = `${arguments[0]} ${arguments[1].join(" ")}`;
+              return cp.exec(command);
+            },
+          },
+          log_commands: true,
+        });
+      }
+    );
   }
 
   getDefaultOneClientInstance() {
-    return this.ctx.registry.get(`${__REGISTRY_NS__}:default_oneclient_instance`, () => {
-      return new OneClient();
-    });
+    return this.ctx.registry.get(
+      `${__REGISTRY_NS__}:default_oneclient_instance`,
+      () => {
+        return new OneClient();
+      }
+    );
   }
 
   getDefaultObjectiveFSInstance() {
@@ -198,11 +213,14 @@ class CsiBaseDriver {
    * @returns CsiProxyClient
    */
   getDefaultCsiProxyClientInstance() {
-    return this.ctx.registry.get(`${__REGISTRY_NS__}:default_csi_proxy_instance`, () => {
-      const options = {};
-      options.services = _.get(this.options, "node.csiProxy.services", {});
-      return new CsiProxyClient(options);
-    });
+    return this.ctx.registry.get(
+      `${__REGISTRY_NS__}:default_csi_proxy_instance`,
+      () => {
+        const options = {};
+        options.services = _.get(this.options, "node.csiProxy.services", {});
+        return new CsiProxyClient(options);
+      }
+    );
   }
 
   getDefaultKubernetsConfigInstance() {
@@ -1054,7 +1072,7 @@ class CsiBaseDriver {
               for (let nvmeofConnection of nvmeofConnections) {
                 // connect
                 try {
-                  await GeneralUtils.retry(15, 2000, async () => {
+                  await GeneralUtils.retry(30, 2000, async () => {
                     await nvmeof.connectByNQNTransport(
                       nvmeofConnection.nqn,
                       nvmeofConnection.transport
@@ -1069,15 +1087,36 @@ class CsiBaseDriver {
                   continue;
                 }
 
+                // wait for connection to actually be connected
+                try {
+                  await GeneralUtils.retry(30, 2000, async () => {
+                    let state = await nvmeof.getSubsystemStateByNQNTransport(
+                      nvmeofConnection.nqn,
+                      nvmeofConnection.transport
+                    );
+                    if (state != "live") {
+                      throw new Error("nvmeof connection is not live");
+                    }
+                  });
+                } catch (err) {
+                  driver.ctx.logger.warn(
+                    `error: ${JSON.stringify(
+                      err
+                    )} transport never became live: ${
+                      nvmeofConnection.transport
+                    }`
+                  );
+                  continue;
+                }
+
                 // find controller device
                 let controllerDevice;
                 try {
-                  await GeneralUtils.retry(15, 2000, async () => {
+                  await GeneralUtils.retry(30, 2000, async () => {
                     controllerDevice =
                       await nvmeof.controllerDevicePathByTransportNQN(
                         nvmeofConnection.transport,
-                        nvmeofConnection.nqn,
-                        nvmeofConnection.nsid
+                        nvmeofConnection.nqn
                       );
 
                     if (!controllerDevice) {
@@ -1488,11 +1527,13 @@ class CsiBaseDriver {
                 // format
                 result = await filesystem.deviceIsFormatted(device);
                 if (!result) {
-                  let formatOptions = _.get(
-                    driver.options.node.format,
-                    [fs_type, "customOptions"],
-                    []
-                  );
+                  let formatOptions = [
+                    ..._.get(
+                      driver.options.node.format,
+                      [fs_type, "customOptions"],
+                      []
+                    ),
+                  ];
                   if (!Array.isArray(formatOptions)) {
                     formatOptions = [];
                   }

src/utils/nvmeof.js

@@ -218,6 +218,50 @@ class NVMEoF {
     return false;
   }
 
+  async parseTransportFromPath(path) {
+    let address;
+    let service;
+    switch (path.Transport) {
+      case "fc":
+      case "rdma":
+      case "tcp":
+        let controllerAddress = path.Address;
+        /**
+         * For backwards compatibility with older nvme-cli versions (at least < 2.2.1)
+         * old: "Address":"traddr=127.0.0.1 trsvcid=4420"
+         * new: "Address":"traddr=127.0.0.1,trsvcid=4420"
+         */
+        controllerAddress = controllerAddress.replace(
+          new RegExp(/ ([a-z_]*=)/, "g"),
+          ",$1"
+        );
+        let parts = controllerAddress.split(",");
+
+        for (let i_part of parts) {
+          let i_parts = i_part.split("=");
+          switch (i_parts[0].trim()) {
+            case "traddr":
+              address = i_parts[1].trim();
+              break;
+            case "trsvcid":
+              service = i_parts[1].trim();
+              break;
+          }
+        }
+
+        break;
+      case "pcie":
+        address = path.Address;
+        break;
+    }
+
+    return {
+      type: path.Transport,
+      address,
+      service,
+    };
+  }
+
   async parseTransport(transport) {
     if (typeof transport === "object") {
       return transport;
@@ -338,11 +382,17 @@ class NVMEoF {
 
   async controllerDevicePathByTransportNQN(transport, nqn) {
     const nvmeof = this;
+
     transport = await nvmeof.parseTransport(transport);
-    let controller = await nvmeof.getControllerByTransportNQN(transport, nqn);
-    if (controller) {
-      return `/dev/${controller.Controller}`;
+    let path = await nvmeof.getSubsystemPathByNQNTransport(nqn, transport);
+    if (path) {
+      return `/dev/${path.Name}`;
     }
+
+    // let controller = await nvmeof.getControllerByTransportNQN(transport, nqn);
+    // if (controller) {
+    //   return `/dev/${controller.Controller}`;
+    // }
   }
 
   async getSubsystems() {
@@ -396,7 +446,7 @@ class NVMEoF {
       for (let subsystem of subsystems) {
         if (subsystem.Namespaces) {
           for (let namespace of subsystem.Namespaces) {
-            if (namespace.NameSpace == name) {
+            if (namespace.NameSpace == name && subsystem.Controllers) {
               return subsystem.Controllers;
             }
           }
@@ -433,37 +483,18 @@ class NVMEoF {
           continue;
         }
 
-        let controllerAddress = controller.Address;
-        /**
-         * For backwards compatibility with older nvme-cli versions (at least < 2.2.1)
-         * old: "Address":"traddr=127.0.0.1 trsvcid=4420"
-         * new: "Address":"traddr=127.0.0.1,trsvcid=4420"
-         */
-        controllerAddress = controllerAddress.replace(
-          new RegExp(/ ([a-z_]*=)/, "g"),
-          ",$1"
+        let controller_transport = await nvmeof.parseTransportFromPath(
+          controller
         );
-        let parts = controllerAddress.split(",");
 
-        let traddr;
-        let trsvcid;
-        for (let i_part of parts) {
-          let i_parts = i_part.split("=");
-          switch (i_parts[0].trim()) {
-            case "traddr":
-              traddr = i_parts[1].trim();
-              break;
-            case "trsvcid":
-              trsvcid = i_parts[1].trim();
-              break;
-          }
-        }
-
-        if (traddr != transport.address) {
+        if (controller_transport.address != transport.address) {
           continue;
         }
 
-        if (transport.service && trsvcid != transport.service) {
+        if (
+          transport.service &&
+          controller_transport.service != transport.service
+        ) {
           continue;
         }
 
@@ -515,6 +546,39 @@ class NVMEoF {
     nvmeof.logger.warn(`failed to find nqn for device: ${name}`);
   }
 
+  async getSubsystemStateByNQNTransport(nqn, transport) {
+    const nvmeof = this;
+    transport = await nvmeof.parseTransport(transport);
+    const path = await nvmeof.getSubsystemPathByNQNTransport(nqn, transport);
+    return path?.State;
+  }
+
+  async getSubsystemPathByNQNTransport(nqn, transport) {
+    const nvmeof = this;
+    transport = await nvmeof.parseTransport(transport);
+    const subsysList = await nvmeof.listSubsys(["-v"]);
+    host_label: for (const host of subsysList) {
+      subsys_label: for (const subsys of host.Subsystems) {
+        if (subsys.NQN != nqn) {
+          continue;
+        }
+        path_label: for (const path of subsys.Paths) {
+          let parsed_path_transport = await nvmeof.parseTransportFromPath(path);
+          for (const key of Object.keys(transport)) {
+            if (
+              ["type", "address", "service"].includes(key) &&
+              transport[key] != parsed_path_transport[key]
+            ) {
+              break path_label;
+            }
+          }
+
+          return path;
+        }
+      }
+    }
+  }
+
   devicePathByModelNumberSerialNumber(modelNumber, serialNumber) {
     modelNumber = modelNumber.replaceAll(" ", "_");
     serialNumber = serialNumber.replaceAll(" ", "_");