Support for TrueNAS 26

new ctr binary to work in HPC
updated objectivefs binary
fix newer versions of nvme client tools

Signed-off-by: Travis Glenn Hansen <travisghansen@yahoo.com>

.github/workflows/main.yml

@@ -115,7 +115,7 @@ jobs:
           SYNOLOGY_PASSWORD: ${{ secrets.SANITY_SYNOLOGY_PASSWORD }}
           SYNOLOGY_VOLUME: ${{ secrets.SANITY_SYNOLOGY_VOLUME }}
 
-  csi-sanity-truenas-scale-25_10:
+  csi-sanity-truenas-api-26:
     needs:
       - build-npm-linux-amd64
     strategy:
@@ -123,11 +123,11 @@ jobs:
       max-parallel: 1
       matrix:
         config:
-          - truenas/scale/25.10/scale-iscsi.yaml
-          - truenas/scale/25.10/scale-nfs.yaml
-          - truenas/scale/25.10/scale-nvmeof.yaml
+          - truenas/api/26/iscsi.yaml
+          - truenas/api/26/nfs.yaml
+          - truenas/api/26/nvmeof.yaml
           # 80 char limit
-          - truenas/scale/25.10/scale-smb.yaml
+          - truenas/api/26/smb.yaml
     runs-on:
       - self-hosted
       - Linux
@@ -150,7 +150,7 @@ jobs:
           TRUENAS_PASSWORD: ${{ secrets.SANITY_TRUENAS_PASSWORD }}
 
   # ssh-based drivers
-  csi-sanity-truenas-core-13_0:
+  csi-sanity-truenas-ssh-26:
     needs:
       - build-npm-linux-amd64
     strategy:
@@ -158,10 +158,11 @@ jobs:
       max-parallel: 1
       matrix:
         config:
-          - truenas/core/13.0/core-iscsi.yaml
-          - truenas/core/13.0/core-nfs.yaml
+          - truenas/ssh/26/iscsi.yaml
+          - truenas/ssh/26/nfs.yaml
+          - truenas/ssh/26/nvmeof.yaml
           # 80 char limit
-          - truenas/core/13.0/core-smb.yaml
+          - truenas/ssh/26/smb.yaml
     runs-on:
       - self-hosted
       - Linux
@@ -179,7 +180,8 @@ jobs:
           ci/bin/run.sh
         env:
           TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
-          TRUENAS_HOST: ${{ secrets.SANITY_TRUENAS_CORE_13_0_HOST }}
+          # TRUENAS_HOST: ${{ secrets.SANITY_TRUENAS_CORE_13_0_HOST }}
+          TRUENAS_HOST: ${{ secrets.SANITY_TRUENAS_SCALE_HOST }}
           TRUENAS_USERNAME: ${{ secrets.SANITY_TRUENAS_USERNAME }}
           TRUENAS_PASSWORD: ${{ secrets.SANITY_TRUENAS_PASSWORD }}
 
@@ -465,8 +467,8 @@ jobs:
       - determine-image-tag
       - csi-sanity-synology-dsm6
       - csi-sanity-synology-dsm7
-      - csi-sanity-truenas-scale-25_10
-      - csi-sanity-truenas-core-13_0
+      - csi-sanity-truenas-api-26
+      - csi-sanity-truenas-ssh-26
       - csi-sanity-zfs-generic-targetcli
       - csi-sanity-zfs-generic-pcs
       - csi-sanity-objectivefs
@@ -507,8 +509,8 @@ jobs:
     needs:
       - csi-sanity-synology-dsm6
       - csi-sanity-synology-dsm7
-      - csi-sanity-truenas-scale-25_10
-      - csi-sanity-truenas-core-13_0
+      - csi-sanity-truenas-api-26
+      - csi-sanity-truenas-ssh-26
       - csi-sanity-zfs-generic-targetcli
       - csi-sanity-zfs-generic-pcs
       - csi-sanity-objectivefs

Dockerfile

@@ -147,7 +147,7 @@ RUN chmod +x /usr/local/sbin/ctr-installer.sh && ctr-installer.sh
 #        rm -rf /var/lib/apt/lists/*
 
 # install objectivefs
-ARG OBJECTIVEFS_VERSION=7.3
+ARG OBJECTIVEFS_VERSION=7.5
 ADD docker/objectivefs-installer.sh /usr/local/sbin
 RUN chmod +x /usr/local/sbin/objectivefs-installer.sh && objectivefs-installer.sh
 

Dockerfile.Windows

@@ -100,7 +100,7 @@ LABEL org.opencontainers.image.licenses MIT
 COPY --from=build /app /app
 WORKDIR /app
 
-ADD https://github.com/democratic-csi/democratic-csi/releases/download/v1.0.0/ctr-v2.0.4-windows-amd64.exe ./bin/ctr.exe
+ADD https://github.com/democratic-csi/democratic-csi/releases/download/v1.0.1/ctr-v2.0.4-windows-amd64.exe ./bin/ctr.exe
 COPY --from=build /nodejs/node.exe ./bin
 COPY --from=build /usr/local/bin/ ./bin
 

ci/configs/truenas/ssh/26/nvmeof.yaml

@@ -0,0 +1,32 @@
+driver: freenas-nvmeof
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey:
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize:
+
+nvmeof:
+  transports:
+    - tcp://${TRUENAS_HOST}:4420
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  ports:
+    - 1
+
+# https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

docker/ctr-installer.sh

@@ -6,36 +6,36 @@ set -x
 PLATFORM_TYPE=${1}
 
 if [[ "${PLATFORM_TYPE}" == "build" ]]; then
-	PLATFORM=$BUILDPLATFORM
+  PLATFORM=$BUILDPLATFORM
 else
-	PLATFORM=$TARGETPLATFORM
+  PLATFORM=$TARGETPLATFORM
 fi
 
 if [[ "x${PLATFORM}" == "x" ]]; then
-	PLATFORM="linux/amd64"
+  PLATFORM="linux/amd64"
 fi
 
 # these come from the --platform option of buildx, indirectly from DOCKER_BUILD_PLATFORM in main.yaml
 # linux/amd64,linux/arm64,linux/arm/v7,linux/s390x,linux/ppc64le
 if [ "$PLATFORM" = "linux/amd64" ]; then
-	export PLATFORM_ARCH="amd64"
+  export PLATFORM_ARCH="amd64"
 elif [ "$PLATFORM" = "linux/arm64" ]; then
-	export PLATFORM_ARCH="arm64"
+  export PLATFORM_ARCH="arm64"
 elif [ "$PLATFORM" = "linux/arm/v7" ]; then
-	export PLATFORM_ARCH="arm"
+  export PLATFORM_ARCH="arm"
 elif [ "$PLATFORM" = "linux/s390x" ]; then
-	export PLATFORM_ARCH="s390x"
+  export PLATFORM_ARCH="s390x"
 elif [ "$PLATFORM" = "linux/ppc64le" ]; then
-	export PLATFORM_ARCH="ppc64le"
+  export PLATFORM_ARCH="ppc64le"
 else
-	echo "unsupported/unknown ctr PLATFORM ${PLATFORM}"
-	exit 0
+  echo "unsupported/unknown ctr PLATFORM ${PLATFORM}"
+  exit 0
 fi
 
 echo "I am installing ctr $CTR_VERSION"
 
 export CTR_FILE="ctr-${CTR_VERSION}-linux-${PLATFORM_ARCH}"
-wget -O "${CTR_FILE}" "https://github.com/democratic-csi/democratic-csi/releases/download/v1.0.0/${CTR_FILE}"
+wget -O "${CTR_FILE}" "https://github.com/democratic-csi/democratic-csi/releases/download/v1.0.1/${CTR_FILE}"
 
 mv ${CTR_FILE} /usr/local/bin/ctr
 chown root:root /usr/local/bin/ctr

src/driver/freenas/http/api.js

@@ -30,7 +30,7 @@ class Api {
         executor: {
           spawn: function () {
             throw new Error(
-              "cannot use the zb implementation to execute zfs commands, must use the http api"
+              "cannot use the zb implementation to execute zfs commands, must use the http api",
             );
           },
         },
@@ -103,7 +103,7 @@ class Api {
           "FreeNAS http error - code: " +
             response.statusCode +
             " body: " +
-            JSON.stringify(response.body)
+            JSON.stringify(response.body),
         );
       }
       page++;
@@ -242,7 +242,7 @@ class Api {
       `FreeNAS error getting system version info: ${stringify({
         errors: versionErrors,
         responses: versionResponses,
-      })}`
+      })}`,
     );
   }
 
@@ -393,7 +393,7 @@ class Api {
     response = await httpClient.put(endpoint, {
       ...this.getSystemProperties(properties),
       user_properties_update: this.getPropertiesKeyValueArray(
-        this.getUserProperties(properties)
+        this.getUserProperties(properties),
       ),
     });
 
@@ -479,7 +479,7 @@ class Api {
           {
             "extra.snapshots": "true",
             "extra.retrieve_children": "false",
-          }
+          },
         );
 
         for (const snapshot of _.get(response, "snapshots", [])) {
@@ -530,7 +530,7 @@ class Api {
     response = await httpClient.put(endpoint, {
       //...this.getSystemProperties(properties),
       user_properties_update: this.getPropertiesKeyValueArray(
-        this.getUserProperties(properties)
+        this.getUserProperties(properties),
       ),
     });
 
@@ -905,13 +905,6 @@ class Api {
       return response.body;
     }
 
-    if (
-      response.statusCode == 422 &&
-      JSON.stringify(response.body).includes("already exists")
-    ) {
-      return this.NvmetSubsysGetByName(subsysName);
-    }
-
     if (
       response.statusCode == 422 &&
       JSON.stringify(response.body).includes("already used by subsystem")

src/driver/freenas/http/index.js

@@ -102,7 +102,7 @@ class Client {
     return options;
   }
 
-  log_repsonse(error, response, body, options) {
+  log_response(error, response, body, options) {
     let prop;
     let val;
 
@@ -130,18 +130,18 @@ class Client {
     let duration = parseFloat(
       Math.round((_.get(response, "duration", 0) + Number.EPSILON) * 100) /
         100 /
-        1000
+        1000,
     ).toFixed(2);
 
     this.logger.debug("FREENAS HTTP REQUEST DETAILS: " + stringify(options));
     this.logger.debug("FREENAS HTTP REQUEST DURATION: " + duration + "s");
     this.logger.debug("FREENAS HTTP ERROR: " + error);
     this.logger.debug(
-      "FREENAS HTTP RESPONSE STATUS CODE: " + _.get(response, "statusCode", "")
+      "FREENAS HTTP RESPONSE STATUS CODE: " + _.get(response, "statusCode", ""),
     );
     this.logger.debug(
       "FREENAS HTTP RESPONSE HEADERS: " +
-        stringify(_.get(response, "headers", ""))
+        stringify(_.get(response, "headers", "")),
     );
     this.logger.debug("FREENAS HTTP RESPONSE BODY: " + stringify(body));
   }
@@ -159,7 +159,7 @@ class Client {
       options.params = data;
 
       axios_request(options, function (err, res, body) {
-        client.log_repsonse(...arguments, options);
+        client.log_response(...arguments, options);
         if (err) {
           reject(err);
         }
@@ -181,7 +181,7 @@ class Client {
       options.data = data;
 
       axios_request(options, function (err, res, body) {
-        client.log_repsonse(...arguments, options);
+        client.log_response(...arguments, options);
         if (err) {
           reject(err);
         }
@@ -204,7 +204,7 @@ class Client {
       options.data = data;
 
       axios_request(options, function (err, res, body) {
-        client.log_repsonse(...arguments, options);
+        client.log_response(...arguments, options);
         if (err) {
           reject(err);
         }
@@ -227,7 +227,7 @@ class Client {
       options.data = data;
 
       axios_request(options, function (err, res, body) {
-        client.log_repsonse(...arguments, options);
+        client.log_response(...arguments, options);
         if (err) {
           reject(err);
         }

src/driver/freenas/websocket/index.js

@@ -0,0 +1,393 @@
+/**
+ *
+ * https://www.truenas.com/docs/api/scale_websocket_api.html
+ * https://github.com/truenas/api_client
+ *
+ * base websocket client functionality here
+ */
+
+const _ = require("lodash");
+const EventEmitter = require("node:events");
+const WebSocket = require("ws");
+const ReconnectingWebSocket = require("reconnecting-websocket");
+const uuidv4 = require("uuid").v4;
+
+const GeneralUtils = require("../../../utils/general");
+
+class TrueNASError extends Error {
+  constructor(rpcError, options = {}) {
+    super(rpcError.reason ?? "TrueNAS RPC error", {
+      cause: options.cause,
+    });
+
+    this.name = "TrueNASError";
+
+    // Node.js-style error properties
+    this.code = rpcError.errname; // "ENOENT"
+    this.errno = rpcError.error; // 2
+
+    // TrueNAS-specific properties
+    this.type = rpcError.type; // "VALIDATION"
+    this.reason = rpcError.reason;
+
+    // TrueNAS Python exception information
+    this.remoteClass = rpcError.trace?.class;
+    this.remoteTrace = rpcError.trace?.formatted;
+    this.remoteRepr = rpcError.trace?.repr;
+
+    // Additional TrueNAS error information
+    this.extra = rpcError.extra;
+
+    // Preserve the complete response error
+    this.rpcError = rpcError;
+  }
+}
+
+class TimeoutError extends Error {
+  constructor(message = "Operation timed out", options = {}) {
+    super(message, options);
+
+    this.name = "TimeoutError";
+    this.code = "ETIMEDOUT";
+  }
+}
+
+class Client extends EventEmitter {
+  constructor(options = {}, logger = null) {
+    super();
+
+    const client = this;
+    this.options = JSON.parse(JSON.stringify(options));
+    this.logger = logger;
+    if (!this.logger) {
+      this.logger = console;
+    }
+
+    // rewrite protocol
+    switch (String(this.options.protocol).toLowerCase()) {
+      case "http":
+        this.options.protocol = "ws";
+        break;
+      case "https":
+        this.options.protocol = "wss";
+        break;
+    }
+
+    // legacy endpoint
+    // this.url = `${this.options.protocol}://${this.options.host}:${this.options.port}/websocket`;
+
+    // JSON-RPC 2.0 endpoint
+    this.url = `${this.options.protocol}://${this.options.host}:${this.options.port}/api/current`;
+    this.logger.info(`TrueNAS WebSocket connecting to: ${this.url}`);
+    this.wss = new ReconnectingWebSocket(this.url, [], {
+      WebSocket: class extends WebSocket {
+        constructor(url, protocols) {
+          // https://github.com/websockets/ws/blob/master/doc/ws.md#class-websocket
+          super(url, protocols, {
+            rejectUnauthorized: !!!client.options.allowInsecure,
+          });
+        }
+        send(data) {
+          client.logger.debug(
+            "TrueNAS WebSocket sending message: " +
+              GeneralUtils.stringify(client.log_cleanse(JSON.parse(data))),
+          );
+          super.send(data);
+        }
+      },
+      //connectionTimeout: 1000,
+      startClosed: true,
+      //debug: true,
+    });
+    this.session;
+    this.authenticated = false;
+    this._systemInfo = {};
+    this._systemVersion = null;
+
+    this.wss.addEventListener("open", async () => {
+      this.logger.info("TrueNAS WebSocket opened!");
+      await this.authenticate();
+    });
+
+    this.wss.addEventListener("close", () => {
+      this.logger.warn("TrueNAS WebSocket closed!");
+      this.session = null;
+      this.authenticated = false;
+      this._systemInfo = {};
+      this._systemVersion = null;
+    });
+
+    this.wss.addEventListener("message", (msg) => {
+      const data = JSON.parse(msg.data);
+      client.logger.debug(
+        "TrueNAS WebSocket received message: " +
+          GeneralUtils.stringify(client.log_cleanse(data)),
+      );
+
+      // event pushed from server, not a response to an rpc call
+      if (!data.id) {
+        switch (data.method) {
+          case "collection_update":
+            // data.params.msg = added, changed, or removed
+            this.emit(
+              `event:${data.method}:${data.params.collection}:${data.params.msg}`,
+              data,
+            );
+            this.emit(`event:${data.method}:${data.params.collection}`, data);
+            this.emit(`event:${data.method}`, data);
+            break;
+
+          case "notify_unsubscribed":
+            this.emit(`event:${data.method}:${data.params.collection}`, data);
+            this.emit(`event:${data.method}`, data);
+            break;
+
+          default:
+            this.logger.log(`unhandled event method: ${data.method}`);
+            break;
+        }
+      }
+    });
+
+    this.wss.addEventListener("error", (err) => {
+      this.logger.error("TrueNAS WebSocket received error!", err);
+    });
+
+    this.on("authenticated", () => {
+      // TODO: make this return when all initialization is complete and set a 'ready' parameter
+      let promises = [];
+      this.call("system.info").then((result) => {
+        this._systemInfo = result;
+      });
+
+      this.call("system.version").then((result) => {
+        this._systemVersion = result;
+      });
+
+      //this.call("core.subscribe", ["*"]);
+      //this.call("core.set_options", [{ legacy_jobs: false }]);
+    });
+  }
+
+  log_cleanse(data) {
+    let log = JSON.parse(JSON.stringify(data));
+
+    let method = log?.method || "";
+
+    if (String(method).startsWith("auth.")) {
+      if (log.params && Array.isArray(log.params) && log.params.length > 0) {
+        log.params = log.params.map(() => {
+          return "redacted";
+        });
+      }
+    }
+
+    return log;
+  }
+
+  async systemInfo() {
+    if (!Object.keys(this._systemInfo).length > 0) {
+      this._systemInfo = await this.call("system.info");
+    }
+
+    return this._systemInfo;
+  }
+
+  async systemVersion() {
+    if (!this._systemVersion) {
+      this._systemVersion = await this.call("system.version");
+    }
+    return this._systemVersion;
+  }
+
+  async connect() {
+    await new Promise((resolve, reject) => {
+      this.wss.addEventListener(
+        "open",
+        () => {
+          resolve();
+        },
+        { once: true },
+      );
+
+      this.wss.addEventListener(
+        "close",
+        () => {
+          reject();
+        },
+        { once: true },
+      );
+
+      this.wss.addEventListener(
+        "error",
+        () => {
+          reject();
+        },
+        { once: true },
+      );
+      this.wss.reconnect();
+    });
+
+    // this keeps the connection alive, otherwise TN will drop it after a timeout on their end
+    if (!this.pingInterval) {
+      this.pingInterval = setInterval(() => {
+        if (this.wss.readyState == WebSocket.OPEN) {
+          this.wss._ws.ping();
+        }
+      }, 10 * 1000);
+    }
+  }
+
+  async authenticate() {
+    let method = "auth.login";
+    let params = [this.options.username, this.options.password];
+
+    if (this.options.apiKey && this.options.protocol == "ws") {
+      throw new Error("TrueNAS WebSocket unable to use apiKey over clear text");
+    }
+
+    if (this.options.apiKey) {
+      method = "auth.login_with_api_key";
+      params = [this.options.apiKey];
+    }
+
+    return this.call(method, params).then((result) => {
+      this.authenticated = result;
+      if (result) {
+        this.emit("authenticated");
+      } else {
+        this.wss.close();
+      }
+    });
+  }
+
+  async waitauthenticated(timeout = 3 * 1000) {
+    let timeoutId;
+    return new Promise((resolve, reject) => {
+      if (this.authenticated) {
+        resolve();
+      }
+
+      if (timeout) {
+        timeoutId = setTimeout(() => {
+          reject(
+            new TimeoutError(
+              `TrueNAS timeout waiting for authentication to complete`,
+            ),
+          );
+        }, timeout);
+      }
+
+      setInterval(() => {
+        if (this.authenticated) {
+          resolve();
+        }
+      }, 100);
+    });
+  }
+
+  async subscribe(events, options = {}) {
+    if (!Array.isArray(events)) {
+      events = [events];
+    }
+    return this.call("core.subscribe", events, options);
+  }
+
+  async unsubscribe(id, options = {}) {
+    return this.call("core.unsubscribe", [id], options);
+  }
+
+  async call(method, params = [], options = {}) {
+    const controller = new AbortController();
+    let timeoutId;
+    let timeout = options.timeout;
+    let is_job_method = false;
+    let wait_for_job = true;
+    let job_id;
+
+    const uuid = uuidv4();
+    const req = {};
+    req.jsonrpc = "2.0";
+    req.id = uuid;
+    req.msg = "method";
+    req.method = method;
+    req.params = params;
+
+    switch (method) {
+      case "filesystem.chown":
+      case "filesystem.get":
+      case "pool.scrub.scrub":
+        is_job_method = true;
+        break;
+    }
+
+    return new Promise((resolve, reject) => {
+      if (timeout) {
+        timeoutId = setTimeout(() => {
+          reject(
+            new TimeoutError(
+              `TrueNAS timeout waiting for call response: type=${req.method}, id=${
+                req.id
+              }, params=[${params.join(" ")}]`,
+            ),
+          );
+        }, timeout);
+      }
+
+      if (is_job_method && wait_for_job) {
+        this.on(
+          "event:core.get_jobs:changed",
+          (event) => {
+            if (event.id == job_id) {
+              switch (event.fields.state) {
+                case "FAILED":
+                  reject(new TrueNASError(event.fields.error));
+                  break;
+                case "SUCCESS":
+                case "ABORTED":
+                  resolve(event.fields.result);
+                  break;
+                case "WAITING":
+                case "RUNNING":
+                default:
+                  // ignore
+                  break;
+              }
+            }
+          },
+          {
+            signal: controller.signal,
+          },
+        );
+      }
+
+      this.wss.addEventListener(
+        "message",
+        (msg) => {
+          const data = JSON.parse(msg.data);
+          if (data.id == uuid) {
+            if (data.error) {
+              reject(new TrueNASError(data.error.data));
+            } else {
+              if (is_job_method && wait_for_job) {
+                job_id = data.result;
+              } else {
+                resolve(data.result);
+              }
+            }
+          }
+        },
+        {
+          signal: controller.signal,
+        },
+      );
+
+      this.wss.send(JSON.stringify(req));
+    }).finally(() => {
+      controller.abort();
+      clearTimeout(timeoutId);
+    });
+  }
+}
+
+module.exports.Client = Client;

src/utils/general.js

@@ -11,7 +11,7 @@ axios.interceptors.request.use(
   },
   function (error) {
     return Promise.reject(error);
-  }
+  },
 );
 
 axios.interceptors.response.use(
@@ -26,7 +26,7 @@ axios.interceptors.response.use(
     error.duration =
       error.config.metadata.endTime - error.config.metadata.startTime;
     return Promise.reject(error);
-  }
+  },
 );
 
 function sleep(ms) {
@@ -285,6 +285,19 @@ function expandenv(string, env) {
   });
 }
 
+function IsJsonString(str) {
+  try {
+    JSON.parse(str);
+  } catch (e) {
+    return false;
+  }
+  return true;
+}
+
+function is_object(v) {
+  return typeof v === "object" && !Array.isArray(v) && v !== null;
+}
+
 module.exports.sleep = sleep;
 module.exports.md5 = md5;
 module.exports.crc32 = crc32;
@@ -306,3 +319,5 @@ module.exports.retry = retry;
 module.exports.trimchar = trimchar;
 module.exports.hostname_lookup = hostname_lookup;
 module.exports.expandenv = expandenv;
+module.exports.IsJsonString = IsJsonString;
+module.exports.is_object = is_object;

src/utils/nvmeof.js

@@ -2,6 +2,7 @@ const cp = require("child_process");
 const { hostname_lookup, trimchar } = require("./general");
 const URI = require("uri-js");
 const querystring = require("querystring");
+const semver = require("semver");
 
 const DEFAULT_TIMEOUT = process.env.NVMEOF_DEFAULT_TIMEOUT || 30000;
 
@@ -35,6 +36,16 @@ class NVMEoF {
     }
   }
 
+  async version() {
+    const nvmeof = this;
+    let args = [];
+    args.unshift("version");
+    let result = await nvmeof.exec(nvmeof.options.paths.nvme, args);
+    const match = result.stdout.match(/^nvme version\s+([0-9]+(?:\.[0-9]+)*)/m);
+    const version = match?.[1];
+    return version;
+  }
+
   /**
    * List all NVMe devices and namespaces on machine
    *
@@ -119,6 +130,12 @@ class NVMEoF {
 
     args.unshift("connect", "--nqn", nqn, ...transport_args);
 
+    // version 3.0 added a new flag and changed the stderr output
+    let version = await nvmeof.version();
+    if (semver.satisfies(semver.coerce(version), ">=3.0")) {
+      args.push("--idempotent");
+    }
+
     try {
       await nvmeof.exec(nvmeof.options.paths.nvme, args);
     } catch (err) {
@@ -233,7 +250,7 @@ class NVMEoF {
          */
         controllerAddress = controllerAddress.replace(
           new RegExp(/ ([a-z_]*=)/, "g"),
-          ",$1"
+          ",$1",
         );
         let parts = controllerAddress.split(",");
 
@@ -483,9 +500,8 @@ class NVMEoF {
           continue;
         }
 
-        let controller_transport = await nvmeof.parseTransportFromPath(
-          controller
-        );
+        let controller_transport =
+          await nvmeof.parseTransportFromPath(controller);
 
         if (controller_transport.address != transport.address) {
           continue;
@@ -504,8 +520,8 @@ class NVMEoF {
 
     nvmeof.logger.warn(
       `failed to find controller for transport: ${JSON.stringify(
-        transport
-      )}, nqn: ${nqn}`
+        transport,
+      )}, nqn: ${nqn}`,
     );
   }
 
@@ -601,7 +617,7 @@ class NVMEoF {
     nvmeof.logger.verbose(
       "executing nvmeof command: %s %s",
       command,
-      args.join(" ")
+      args.join(" "),
     );
 
     return new Promise((resolve, reject) => {