Merge pull request #162 from democratic-csi/next

Next

.github/workflows/main.yml

@@ -254,6 +254,7 @@ jobs:
           chmod a+x ~/.docker/cli-plugins/docker-buildx
           docker info
           docker buildx version
+          docker buildx ls
           docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
           docker buildx create --name xbuilder --use
           docker buildx inspect --bootstrap
@@ -265,4 +266,4 @@ jobs:
           GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
           GHCR_PASSWORD: ${{ secrets.GHCR_PASSWORD }}
           DOCKER_CLI_EXPERIMENTAL: enabled
-          DOCKER_BUILD_PLATFORM: linux/amd64,linux/arm64,linux/arm/v7
+          DOCKER_BUILD_PLATFORM: linux/amd64,linux/arm64,linux/arm/v7,linux/s390x,linux/ppc64le

CHANGELOG.md

@@ -1,3 +1,30 @@
+# v1.6.0
+
+Released 2022-03-
+
+This is a **massive** release with substantial changes. Ideally this release
+should be installed with chart version `>=0.11.0`. Make note that due to the
+updated base image from `buster` to `bullseye` that the filesystem tools have
+all been updated as well (`mkfs.foo`, `resize2fs`, `fsck.foo`, etc).
+
+To facilitate the removal `grpc-uds` package a new sister project was created:
+
+https://github.com/democratic-csi/csi-grpc-proxy
+
+Not all environments require the usage of the proxy, but it is enabled by
+default with `helm` chart versions `>=0.11.0`.
+
+- update `nodejs` version to `v16`
+  - remove dependency on `grpc-uds` package (replaced entirely by
+    `@grpc/grpc-js`)
+  - remove dependency on `request` package (replaced by `axios`)
+  - use native `timeout` functionality for `spawn` operations
+- update http clients to use `keep-alive` logic
+- add a default 30s `timeout` to `iscsiadm` commands
+- base docker image on `bullseye`
+- support for `btrfs` as a `fs_type`
+- support `s390x` and `ppc64le` docker images
+
 # v1.5.4
 
 Released 2022-03-03

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:10-slim AS build
+FROM debian:11-slim AS build
 #FROM --platform=$BUILDPLATFORM debian:10-slim AS build
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -12,8 +12,8 @@ RUN apt-get update && apt-get install -y locales && rm -rf /var/lib/apt/lists/*
         && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
 
 ENV LANG=en_US.utf8
-ENV NODE_VERSION=v12.22.6
+#ENV NODE_VERSION=v12.22.6
-#ENV NODE_VERSION=v14.15.1
+ENV NODE_VERSION=v16.14.0
 ENV NODE_ENV=production
 
 # install build deps
@@ -32,7 +32,7 @@ RUN useradd --create-home csi \
 WORKDIR /home/csi/app
 USER csi
 
-COPY package*.json ./
+COPY --chown=csi:csi package*.json ./
 RUN npm install --grpc_node_binary_host_mirror=https://grpc-uds-binaries.s3-us-west-2.amazonaws.com/debian-buster
 COPY --chown=csi:csi . .
 RUN rm -rf docker
@@ -41,7 +41,7 @@ RUN rm -rf docker
 ######################
 # actual image
 ######################
-FROM debian:10-slim
+FROM debian:11-slim
 
 LABEL org.opencontainers.image.source https://github.com/democratic-csi/democratic-csi
 
@@ -74,7 +74,7 @@ COPY --from=build /usr/local/lib/nodejs/bin/node /usr/local/bin/node
 # netbase is required by rpcbind/rpcinfo to work properly
 # /etc/{services,rpc} are required
 RUN apt-get update && \
-        apt-get install -y netbase socat e2fsprogs xfsprogs fatresize dosfstools nfs-common cifs-utils sudo && \
+        apt-get install -y netbase socat e2fsprogs xfsprogs btrfs-progs fatresize dosfstools nfs-common cifs-utils sudo && \
         rm -rf /var/lib/apt/lists/*
 
 # controller requirements

bin/democratic-csi

@@ -2,6 +2,8 @@
 
 const yaml = require("js-yaml");
 const fs = require("fs");
+const { grpc } = require("../src/utils/grpc");
+const { stringify } = require("../src/utils/general");
 
 let options;
 const args = require("yargs")
@@ -78,8 +80,6 @@ if (!args.serverSocket && !args.serverAddress && !args.serverPort) {
 const package = require("../package.json");
 args.version = package.version;
 
-//const grpc = require("grpc");
-const grpc = require("grpc-uds");
 const protoLoader = require("@grpc/proto-loader");
 const LRU = require("lru-cache");
 const cache = new LRU({ max: 500 });
@@ -91,7 +91,7 @@ if (args.logLevel) {
   logger.level = args.logLevel;
 }
 
-const csiVersion = process.env.CSI_VERSION || args.csiVersion || "1.2.0";
+const csiVersion = process.env.CSI_VERSION || args.csiVersion || "1.5.0";
 const PROTO_PATH = __dirname + "/../csi_proto/csi-v" + csiVersion + ".proto";
 
 // Suggested options for similarity to existing grpc.load behavior
@@ -128,7 +128,7 @@ try {
 let operationLock = new Set();
 
 async function requestHandlerProxy(call, callback, serviceMethodName) {
-  const cleansedCall = JSON.parse(JSON.stringify(call));
+  const cleansedCall = JSON.parse(stringify(call));
   for (const key in cleansedCall.request) {
     if (key.includes("secret")) {
       cleansedCall.request[key] = "redacted";
@@ -335,27 +335,71 @@ logger.info(
   bindSocket
 );
 
-if (bindAddress) {
-  csiServer.bind(bindAddress, grpc.ServerCredentials.createInsecure());
-}
-
-if (bindSocket) {
-  csiServer.bind(bindSocket, grpc.ServerCredentials.createInsecure());
-}
-
-csiServer.start();
-
 [`SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`].forEach(
   (eventType) => {
     process.on(eventType, (code) => {
       console.log(`running server shutdown, exit code: ${code}`);
-      let socketPath = args.serverSocket || "";
+      let socketPath = bindSocket;
       socketPath = socketPath.replace(/^unix:\/\//g, "");
       if (socketPath && fs.existsSync(socketPath)) {
-        fs.unlinkSync(socketPath);
+        let fsStat = fs.statSync(socketPath);
+        if (fsStat.isSocket()) {
+          fs.unlinkSync(socketPath);
+        }
       }
 
       process.exit(code);
     });
   }
 );
+
+if (require.main === module) {
+  (async function () {
+    try {
+      if (bindAddress) {
+        await new Promise((resolve, reject) => {
+          csiServer.bindAsync(
+            bindAddress,
+            grpc.ServerCredentials.createInsecure(),
+            (err) => {
+              if (err) {
+                reject(err);
+                return;
+              }
+              resolve();
+            }
+          );
+        });
+      }
+
+      if (bindSocket) {
+        let socketPath = bindSocket;
+        socketPath = socketPath.replace(/^unix:\/\//g, "");
+        if (socketPath && fs.existsSync(socketPath)) {
+          let fsStat = fs.statSync(socketPath);
+          if (fsStat.isSocket()) {
+            fs.unlinkSync(socketPath);
+          }
+        }
+
+        await new Promise((resolve, reject) => {
+          csiServer.bindAsync(
+            bindSocket,
+            grpc.ServerCredentials.createInsecure(),
+            (err) => {
+              if (err) {
+                reject(err);
+                return;
+              }
+              resolve();
+            }
+          );
+        });
+      }
+      csiServer.start();
+    } catch (e) {
+      console.log(e);
+      process.exit(1);
+    }
+  })();
+}

bin/liveness-probe

@@ -1,9 +1,7 @@
 #!/usr/bin/env -S node --nouse-idle-notification --expose-gc
 
-const yaml = require("js-yaml");
+const { grpc } = require("../src/utils/grpc");
-const fs = require("fs");
 
-let options;
 const args = require("yargs")
   .env("DEMOCRATIC_CSI_LIVENESS_PROBE")
   .scriptName("liveness-probe")
@@ -33,10 +31,8 @@ const args = require("yargs")
 const package = require("../package.json");
 args.version = package.version;
 
-//const grpc = require("grpc");
-const grpc = require("grpc-uds");
 const protoLoader = require("@grpc/proto-loader");
-const csiVersion = process.env.CSI_VERSION || args.csiVersion || "1.2.0";
+const csiVersion = process.env.CSI_VERSION || args.csiVersion || "1.5.0";
 const PROTO_PATH = __dirname + "/../csi_proto/csi-v" + csiVersion + ".proto";
 
 // Suggested options for similarity to existing grpc.load behavior
@@ -51,8 +47,18 @@ const packageDefinition = protoLoader.loadSync(PROTO_PATH, {
 const protoDescriptor = grpc.loadPackageDefinition(packageDefinition);
 const csi = protoDescriptor.csi.v1;
 
+let csiAddress = args.csiAddress;
+const tcpRegex = /[^\:]+:[0-9]*$/;
+
+if (
+  !tcpRegex.test(csiAddress) &&
+  !csiAddress.toLowerCase().startsWith("unix://")
+) {
+  csiAddress = "unix://" + csiAddress;
+}
+
 const clientIdentity = new csi.Identity(
-  args.csiAddress,
+  csiAddress,
   grpc.credentials.createInsecure()
 );
 

ci/bin/build.sh

@@ -4,6 +4,10 @@ set -e
 set -x
 
 export PATH="/usr/local/lib/nodejs/bin:${PATH}"
+
+node --version
+npm --version
+
 # install deps
 npm i
 

docker/mount

@@ -5,6 +5,7 @@ container_supported_filesystems=(
   "ext3"
   "ext4"
   "ext4dev"
+  "btrfs"
   "xfs"
   "vfat"
   "nfs"

docker/node-installer.sh

@@ -15,12 +15,17 @@ if [[ "x${PLATFORM}" == "x" ]]; then
   PLATFORM="linux/amd64"
 fi
 
+# these come from the --platform option of buildx, indirectly from DOCKER_BUILD_PLATFORM in main.yaml
 if [ "$PLATFORM" = "linux/amd64" ]; then
   export NODE_DISTRO="linux-x64"
 elif [ "$PLATFORM" = "linux/arm64" ]; then
   export NODE_DISTRO="linux-arm64"
 elif [ "$PLATFORM" = "linux/arm/v7" ]; then
   export NODE_DISTRO="linux-armv7l"
+elif [ "$PLATFORM" = "linux/s390x" ]; then
+  export NODE_DISTRO="linux-s390x"
+elif [ "$PLATFORM" = "linux/ppc64le" ]; then
+  export NODE_DISTRO="linux-ppc64le"
 else
   echo "unsupported/unknown PLATFORM ${PLATFORM}"
   exit 1

docker/umount

@@ -5,6 +5,7 @@ container_supported_filesystems=(
   "ext3"
   "ext4"
   "ext4dev"
+  "btrfs"
   "xfs"
   "vfat"
   "nfs"

examples/node-common.yaml

@@ -11,6 +11,10 @@ node:
         enabled: false
         customOptions: []
         customFilesystemOptions: []
+      btrfs:
+        enabled: false
+        customOptions: []
+        customFilesystemOptions: []
   format:
     xfs:
       customOptions: []
@@ -21,3 +25,5 @@ node:
       #- -E
       #- nodiscard
       # ...
+    btrfs:
+      customOptions: []

package.json

@@ -1,6 +1,6 @@
 {
   "name": "democratic-csi",
-  "version": "1.5.4",
+  "version": "1.6.0",
   "description": "kubernetes csi driver framework",
   "main": "bin/democratic-csi",
   "scripts": {
@@ -22,14 +22,13 @@
     "@grpc/proto-loader": "^0.6.0",
     "@kubernetes/client-node": "^0.16.3",
     "async-mutex": "^0.3.1",
+    "axios": "^0.26.1",
     "bunyan": "^1.8.15",
-    "grpc-uds": "^0.1.6",
     "handlebars": "^4.7.7",
     "js-yaml": "^4.0.0",
     "lodash": "^4.17.21",
     "lru-cache": "^7.4.0",
     "prompt": "^1.2.2",
-    "request": "^2.88.2",
     "semver": "^7.3.4",
     "ssh2": "^1.1.0",
     "uri-js": "^4.4.1",

src/driver/controller-synology/http/index.js

@@ -1,4 +1,7 @@
-const request = require("request");
+const _ = require("lodash");
+const http = require("http");
+const https = require("https");
+const { axios_request, stringify } = require("../../../utils/general");
 const Mutex = require("async-mutex").Mutex;
 
 const USER_AGENT = "democratic-csi";
@@ -18,34 +21,57 @@ class SynologyHttpClient {
     }
   }
 
-  async login() {
+  getHttpAgent() {
-    if (!this.sid) {
+    if (!this.httpAgent) {
-      // See https://global.download.synology.com/download/Document/Software/DeveloperGuide/Os/DSM/All/enu/DSM_Login_Web_API_Guide_enu.pdf
+      this.httpAgent = new http.Agent({
-      const data = {
+        keepAlive: true,
-        api: "SYNO.API.Auth",
+        maxSockets: Infinity,
-        version: "6",
+        rejectUnauthorized: !!!this.options.allowInsecure,
-        method: "login",
+      });
-        account: this.options.username,
-        passwd: this.options.password,
-        session: this.options.session,
-        format: "sid",
-      };
-
-      let response = await this.do_request("GET", "auth.cgi", data);
-      this.sid = response.body.data.sid;
     }
 
-    return this.sid;
+    return this.httpAgent;
+  }
+
+  getHttpsAgent() {
+    if (!this.httpsAgent) {
+      this.httpsAgent = new https.Agent({
+        keepAlive: true,
+        maxSockets: Infinity,
+        rejectUnauthorized: !!!this.options.allowInsecure,
+      });
+    }
+
+    return this.httpsAgent;
   }
 
   log_response(error, response, body, options) {
-    this.logger.debug("SYNOLOGY HTTP REQUEST: " + JSON.stringify(options));
+    let prop;
+    let val;
+
+    prop = "auth.username";
+    val = _.get(options, prop, false);
+    if (val) {
+      _.set(options, prop, "redacted");
+    }
+
+    prop = "auth.password";
+    val = _.get(options, prop, false);
+    if (val) {
+      _.set(options, prop, "redacted");
+    }
+
+    prop = "headers.Authorization";
+    val = _.get(options, prop, false);
+    if (val) {
+      _.set(options, prop, "redacted");
+    }
+
+    this.logger.debug("SYNOLOGY HTTP REQUEST: " + stringify(options));
     this.logger.debug("SYNOLOGY HTTP ERROR: " + error);
     this.logger.debug("SYNOLOGY HTTP STATUS: " + response.statusCode);
-    this.logger.debug(
+    this.logger.debug("SYNOLOGY HTTP HEADERS: " + stringify(response.headers));
-      "SYNOLOGY HTTP HEADERS: " + JSON.stringify(response.headers)
+    this.logger.debug("SYNOLOGY HTTP BODY: " + stringify(body));
-    );
-    this.logger.debug("SYNOLOGY HTTP BODY: " + JSON.stringify(body));
   }
 
   async do_request(method, path, data = {}, options = {}) {
@@ -82,10 +108,10 @@ class SynologyHttpClient {
             ? "application/x-www-form-urlencoded"
             : "application/json",
         },
-        json: invoke_options.use_form_encoded ? false : true,
+        responseType: "json",
-        agentOptions: {
+        httpAgent: this.getHttpAgent(),
-          rejectUnauthorized: !!!client.options.allowInsecure,
+        httpsAgent: this.getHttpsAgent(),
-        },
+        timeout: 60 * 1000,
       };
 
       switch (method) {
@@ -96,20 +122,19 @@ class SynologyHttpClient {
               qsData[p] = JSON.stringify(qsData[p]);
             }
           }
-          options.qs = qsData;
+          options.params = qsData;
           break;
         default:
           if (invoke_options.use_form_encoded) {
-            //options.body = URLSearchParams(data);
+            options.data = URLSearchParams(data).toString();
-            options.form = data;
           } else {
-            options.body = data;
+            options.data = data;
           }
           break;
       }
 
       try {
-        request(options, function (error, response, body) {
+        axios_request(options, function (error, response, body) {
           client.log_response(...arguments, options);
 
           if (error) {
@@ -147,6 +172,26 @@ class SynologyHttpClient {
     });
   }
 
+  async login() {
+    if (!this.sid) {
+      // See https://global.download.synology.com/download/Document/Software/DeveloperGuide/Os/DSM/All/enu/DSM_Login_Web_API_Guide_enu.pdf
+      const data = {
+        api: "SYNO.API.Auth",
+        version: "6",
+        method: "login",
+        account: this.options.username,
+        passwd: this.options.password,
+        session: this.options.session,
+        format: "sid",
+      };
+
+      let response = await this.do_request("GET", "auth.cgi", data);
+      this.sid = response.body.data.sid;
+    }
+
+    return this.sid;
+  }
+
   async GetLuns() {
     const lun_list = {
       api: "SYNO.Core.ISCSI.LUN",

src/driver/controller-synology/index.js

@@ -196,7 +196,7 @@ class ControllerSynologyDriver extends CsiBaseDriver {
           if (capability.access_type == "mount") {
             if (
               capability.mount.fs_type &&
-              !["ext3", "ext4", "ext4dev", "xfs"].includes(
+              !["btrfs", "ext3", "ext4", "ext4dev", "xfs"].includes(
                 capability.mount.fs_type
               )
             ) {

src/driver/controller-zfs-local/index.js

@@ -89,7 +89,7 @@ class ControllerZfsLocalDriver extends ControllerZfsBaseDriver {
       case "filesystem":
         return ["zfs"];
       case "volume":
-        return ["ext3", "ext4", "ext4dev", "xfs"];
+        return ["btrfs", "ext3", "ext4", "ext4dev", "xfs"];
     }
   }
 

src/driver/controller-zfs/index.js

@@ -29,6 +29,8 @@ const VOLUME_CONTEXT_PROVISIONER_DRIVER_PROPERTY_NAME =
 const VOLUME_CONTEXT_PROVISIONER_INSTANCE_ID_PROPERTY_NAME =
   "democratic-csi:volume_context_provisioner_instance_id";
 
+const MAX_ZVOL_NAME_LENGTH_CACHE_KEY = "controller-zfs:max_zvol_name_length";
+
 /**
  * Base driver to provisin zfs assets using zfs cli commands.
  * Derived drivers only need to implement:
@@ -201,7 +203,7 @@ class ControllerZfsBaseDriver extends CsiBaseDriver {
       case "filesystem":
         return ["nfs", "cifs"];
       case "volume":
-        return ["ext3", "ext4", "ext4dev", "xfs"];
+        return ["btrfs", "ext3", "ext4", "ext4dev", "xfs"];
     }
   }
 
@@ -413,6 +415,14 @@ class ControllerZfsBaseDriver extends CsiBaseDriver {
     let kernel;
     let kernel_release;
 
+    const cachedValue = await driver.ctx.cache.get(
+      MAX_ZVOL_NAME_LENGTH_CACHE_KEY
+    );
+
+    if (cachedValue) {
+      return cachedValue;
+    }
+
     // get kernel
     command = "uname -s";
     driver.ctx.logger.verbose("uname command: %s", command);
@@ -423,12 +433,14 @@ class ControllerZfsBaseDriver extends CsiBaseDriver {
       kernel = response.stdout.trim();
     }
 
+    let max;
     switch (kernel.toLowerCase().trim()) {
       // Linux is 255 (probably larger 4096) but scst may have a 255 limit
       // https://ngelinux.com/what-is-the-maximum-file-name-length-in-linux-and-how-to-see-this-is-this-really-255-characters-answer-is-no/
       // https://github.com/dmeister/scst/blob/master/iscsi-scst/include/iscsi_scst.h#L28
       case "linux":
-        return 255;
+        max = 255;
+        break;
       case "freebsd":
         // get kernel_release
         command = "uname -r";
@@ -444,14 +456,20 @@ class ControllerZfsBaseDriver extends CsiBaseDriver {
           let kernel_release_major = parts[0];
 
           if (kernel_release_major >= 13) {
-            return 255;
+            max = 255;
           } else {
-            return 63;
+            max = 63;
           }
         }
+        break;
       default:
         throw new Error(`unknown kernel: ${kernel}`);
     }
+
+    await driver.ctx.cache.set(MAX_ZVOL_NAME_LENGTH_CACHE_KEY, max, {
+      ttl: 60 * 1000,
+    });
+    return max;
   }
 
   async setFilesystemMode(path, mode) {

src/driver/freenas/api.js

@@ -1,3 +1,4 @@
+const _ = require("lodash");
 const { GrpcError, grpc } = require("../../utils/grpc");
 const { CsiBaseDriver } = require("../index");
 const HttpClient = require("./http").Client;
@@ -1539,7 +1540,30 @@ class FreeNASApiDriver extends CsiBaseDriver {
                 }
 
                 if (deleteAsset) {
+                  let retries = 0;
+                  let maxRetries = 5;
+                  let retryWait = 1000;
                   response = await httpClient.delete(endpoint);
+
+                  // sometimes after an initiator has detached it takes a moment for TrueNAS to settle
+                  // code: 422 body: {\"message\":\"Target csi-ci-55877e95sanity-node-expand-volume-e54f81fa-cd38e798 is in use.\",\"errno\":14}
+                  while (
+                    response.statusCode == 422 &&
+                    retries < maxRetries &&
+                    _.get(response, "body.message").includes("Target") &&
+                    _.get(response, "body.message").includes("is in use") &&
+                    _.get(response, "body.errno") == 14
+                  ) {
+                    retries++;
+                    this.ctx.logger.debug(
+                      "target: %s is in use, retry %s shortly",
+                      targetId,
+                      retries
+                    );
+                    await sleep(retryWait);
+                    response = await httpClient.delete(endpoint);
+                  }
+
                   if (![200, 204, 404].includes(response.statusCode)) {
                     throw new GrpcError(
                       grpc.status.UNKNOWN,
@@ -1929,7 +1953,7 @@ class FreeNASApiDriver extends CsiBaseDriver {
           if (capability.access_type == "mount") {
             if (
               capability.mount.fs_type &&
-              !["ext3", "ext4", "ext4dev", "xfs"].includes(
+              !["btrfs", "ext3", "ext4", "ext4dev", "xfs"].includes(
                 capability.mount.fs_type
               )
             ) {

src/driver/freenas/http/api.js

@@ -208,11 +208,9 @@ class Api {
   }
 
   async setVersionInfoCache(versionInfo) {
-    await this.cache.set(
+    await this.cache.set(FREENAS_SYSTEM_VERSION_CACHE_KEY, versionInfo, {
-      FREENAS_SYSTEM_VERSION_CACHE_KEY,
+      ttl: 60 * 1000,
-      versionInfo,
+    });
-      60 * 1000
-    );
   }
 
   async getSystemVersion() {

src/driver/freenas/http/index.js

@@ -1,5 +1,8 @@
-const request = require("request");
+const _ = require("lodash");
+const http = require("http");
+const https = require("https");
 const URI = require("uri-js");
+const { axios_request, stringify } = require("../../../utils/general");
 const USER_AGENT = "democratic-csi-driver";
 
 class Client {
@@ -12,6 +15,31 @@ class Client {
       this.options.apiVersion = 1;
     }
   }
+
+  getHttpAgent() {
+    if (!this.httpAgent) {
+      this.httpAgent = new http.Agent({
+        keepAlive: true,
+        maxSockets: Infinity,
+        rejectUnauthorized: !!!this.options.allowInsecure,
+      });
+    }
+
+    return this.httpAgent;
+  }
+
+  getHttpsAgent() {
+    if (!this.httpsAgent) {
+      this.httpsAgent = new https.Agent({
+        keepAlive: true,
+        maxSockets: Infinity,
+        rejectUnauthorized: !!!this.options.allowInsecure,
+      });
+    }
+
+    return this.httpsAgent;
+  }
+
   getBaseURL() {
     const server = this.options;
     if (!server.protocol) {
@@ -46,14 +74,59 @@ class Client {
     return this.options.apiVersion;
   }
 
+  getRequestCommonOptions() {
+    const client = this;
+    const options = {
+      headers: {
+        Accept: "application/json",
+        "User-Agent": USER_AGENT,
+        "Content-Type": "application/json",
+      },
+      responseType: "json",
+      httpAgent: this.getHttpAgent(),
+      httpsAgent: this.getHttpsAgent(),
+      timeout: 60 * 1000,
+    };
+
+    if (client.options.apiKey) {
+      options.headers.Authorization = `Bearer ${client.options.apiKey}`;
+    } else if (client.options.username && client.options.password) {
+      options.auth = {
+        username: client.options.username,
+        password: client.options.password,
+      };
+    }
+
+    return options;
+  }
+
   log_repsonse(error, response, body, options) {
-    this.logger.debug("FREENAS HTTP REQUEST: " + JSON.stringify(options));
+    let prop;
+    let val;
+
+    prop = "auth.username";
+    val = _.get(options, prop, false);
+    if (val) {
+      _.set(options, prop, "redacted");
+    }
+
+    prop = "auth.password";
+    val = _.get(options, prop, false);
+    if (val) {
+      _.set(options, prop, "redacted");
+    }
+
+    prop = "headers.Authorization";
+    val = _.get(options, prop, false);
+    if (val) {
+      _.set(options, prop, "redacted");
+    }
+
+    this.logger.debug("FREENAS HTTP REQUEST: " + stringify(options));
     this.logger.debug("FREENAS HTTP ERROR: " + error);
     this.logger.debug("FREENAS HTTP STATUS: " + response.statusCode);
-    this.logger.debug(
+    this.logger.debug("FREENAS HTTP HEADERS: " + stringify(response.headers));
-      "FREENAS HTTP HEADERS: " + JSON.stringify(response.headers)
+    this.logger.debug("FREENAS HTTP BODY: " + stringify(body));
-    );
-    this.logger.debug("FREENAS HTTP BODY: " + JSON.stringify(body));
   }
 
   async get(endpoint, data) {
@@ -63,33 +136,18 @@ class Client {
     }
 
     return new Promise((resolve, reject) => {
-      const options = {
+      const options = client.getRequestCommonOptions();
-        method: "GET",
+      options.method = "GET";
-        url: this.getBaseURL() + endpoint,
+      options.url = this.getBaseURL() + endpoint;
-        headers: {
+      options.params = data;
-          Accept: "application/json",
+
-          "User-Agent": USER_AGENT,
+      axios_request(options, function (err, res, body) {
-          "Content-Type": "application/json",
-        },
-        json: true,
-        qs: data,
-        agentOptions: {
-          rejectUnauthorized: !!!client.options.allowInsecure,
-        },
-      };
-      request(options, function (err, res, body) {
         client.log_repsonse(...arguments, options);
         if (err) {
           reject(err);
         }
-
         resolve(res);
-      }).auth(
+      });
-        client.options.username,
-        client.options.password,
-        true,
-        client.options.apiKey
-      );
     });
   }
 
@@ -100,33 +158,19 @@ class Client {
     }
 
     return new Promise((resolve, reject) => {
-      const options = {
+      const options = client.getRequestCommonOptions();
-        method: "POST",
+      options.method = "POST";
-        url: this.getBaseURL() + endpoint,
+      options.url = this.getBaseURL() + endpoint;
-        headers: {
+      options.data = data;
-          Accept: "application/json",
+
-          "User-Agent": USER_AGENT,
+      axios_request(options, function (err, res, body) {
-          "Content-Type": "application/json",
-        },
-        json: true,
-        body: data,
-        agentOptions: {
-          rejectUnauthorized: !!!client.options.allowInsecure,
-        },
-      };
-      request(options, function (err, res, body) {
         client.log_repsonse(...arguments, options);
         if (err) {
           reject(err);
         }
 
         resolve(res);
-      }).auth(
+      });
-        client.options.username,
-        client.options.password,
-        true,
-        client.options.apiKey
-      );
     });
   }
 
@@ -137,33 +181,19 @@ class Client {
     }
 
     return new Promise((resolve, reject) => {
-      const options = {
+      const options = client.getRequestCommonOptions();
-        method: "PUT",
+      options.method = "PUT";
-        url: this.getBaseURL() + endpoint,
+      options.url = this.getBaseURL() + endpoint;
-        headers: {
+      options.data = data;
-          Accept: "application/json",
+
-          "User-Agent": USER_AGENT,
+      axios_request(options, function (err, res, body) {
-          "Content-Type": "application/json",
-        },
-        json: true,
-        body: data,
-        agentOptions: {
-          rejectUnauthorized: !!!client.options.allowInsecure,
-        },
-      };
-      request(options, function (err, res, body) {
         client.log_repsonse(...arguments, options);
         if (err) {
           reject(err);
         }
 
         resolve(res);
-      }).auth(
+      });
-        client.options.username,
-        client.options.password,
-        true,
-        client.options.apiKey
-      );
     });
   }
 
@@ -174,33 +204,19 @@ class Client {
     }
 
     return new Promise((resolve, reject) => {
-      const options = {
+      const options = client.getRequestCommonOptions();
-        method: "DELETE",
+      options.method = "DELETE";
-        url: this.getBaseURL() + endpoint,
+      options.url = this.getBaseURL() + endpoint;
-        headers: {
+      options.data = data;
-          Accept: "application/json",
+
-          "User-Agent": USER_AGENT,
+      axios_request(options, function (err, res, body) {
-          "Content-Type": "application/json",
-        },
-        json: true,
-        body: data,
-        agentOptions: {
-          rejectUnauthorized: !!!client.options.allowInsecure,
-        },
-      };
-      request(options, function (err, res, body) {
         client.log_repsonse(...arguments, options);
         if (err) {
           reject(err);
         }
 
         resolve(res);
-      }).auth(
+      });
-        client.options.username,
-        client.options.password,
-        true,
-        client.options.apiKey
-      );
     });
   }
 }

src/driver/freenas/ssh.js

@@ -4,6 +4,7 @@ const { GrpcError, grpc } = require("../../utils/grpc");
 const SshClient = require("../../utils/ssh").SshClient;
 const HttpClient = require("./http").Client;
 const { Zetabyte, ZfsSshProcessManager } = require("../../utils/zfs");
+const sleep = require("../../utils/general").sleep;
 
 const Handlebars = require("handlebars");
 
@@ -1572,7 +1573,30 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver {
                 }
 
                 if (deleteAsset) {
+                  let retries = 0;
+                  let maxRetries = 5;
+                  let retryWait = 1000;
                   response = await httpClient.delete(endpoint);
+
+                  // sometimes after an initiator has detached it takes a moment for TrueNAS to settle
+                  // code: 422 body: {\"message\":\"Target csi-ci-55877e95sanity-node-expand-volume-e54f81fa-cd38e798 is in use.\",\"errno\":14}
+                  while (
+                    response.statusCode == 422 &&
+                    retries < maxRetries &&
+                    _.get(response, "body.message").includes("Target") &&
+                    _.get(response, "body.message").includes("is in use") &&
+                    _.get(response, "body.errno") == 14
+                  ) {
+                    retries++;
+                    this.ctx.logger.debug(
+                      "target: %s is in use, retry %s shortly",
+                      targetId,
+                      retries
+                    );
+                    await sleep(retryWait);
+                    response = await httpClient.delete(endpoint);
+                  }
+
                   if (![200, 204].includes(response.statusCode)) {
                     throw new GrpcError(
                       grpc.status.UNKNOWN,
@@ -2011,11 +2035,9 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver {
   async setVersionInfoCache(versionInfo) {
     const driver = this;
 
-    await driver.ctx.cache.set(
+    await driver.ctx.cache.set(FREENAS_SYSTEM_VERSION_CACHE_KEY, versionInfo, {
-      FREENAS_SYSTEM_VERSION_CACHE_KEY,
+      ttl: 60 * 1000,
-      versionInfo,
+    });
-      60 * 1000
-    );
   }
 
   async getSystemVersion() {

src/driver/index.js

@@ -461,6 +461,8 @@ class CsiBaseDriver {
                 normalizedSecrets[key];
             }
           }
+
+          // create 'DB' entry
           await iscsi.iscsiadm.createNodeDBEntry(
             iscsiConnection.iqn,
             iscsiConnection.portal,
@@ -804,6 +806,7 @@ class CsiBaseDriver {
                 }
               }
               break;
+            case "btrfs":
             case "xfs":
               //await filesystem.checkFilesystem(device, fs_info.type);
               await filesystem.expandFilesystem(staging_target_path, fs_type);
@@ -1493,9 +1496,10 @@ class CsiBaseDriver {
               //await filesystem.checkFilesystem(device, fs_info.type);
               await filesystem.expandFilesystem(device, fs_type);
               break;
+            case "btrfs":
             case "xfs":
               let mount_info = await mount.getMountDetails(device_path);
-              if (mount_info.fstype == "xfs") {
+              if (["btrfs", "xfs"].includes(mount_info.fstype)) {
                 //await filesystem.checkFilesystem(device, fs_info.type);
                 await filesystem.expandFilesystem(device_path, fs_type);
               }

src/driver/node-manual/index.js

@@ -130,11 +130,11 @@ class NodeManualDriver extends CsiBaseDriver {
         break;
       case "iscsi":
         driverResourceType = "volume";
-        fs_types = ["ext3", "ext4", "ext4dev", "xfs"];
+        fs_types = ["btrfs", "ext3", "ext4", "ext4dev", "xfs"];
         break;
       case "zfs-local":
         driverResourceType = "volume";
-        fs_types = ["ext3", "ext4", "ext4dev", "xfs", "zfs"];
+        fs_types = ["btrfs", "ext3", "ext4", "ext4dev", "xfs", "zfs"];
         access_modes = [
           "UNKNOWN",
           "SINGLE_NODE_WRITER",

src/driver/zfs-local-ephemeral-inline/index.js

@@ -209,7 +209,7 @@ class ZfsLocalEphemeralInlineDriver extends CsiBaseDriver {
           if (capability.access_type == "mount") {
             if (
               capability.mount.fs_type &&
-              !["ext3", "ext4", "ext4dev", "xfs"].includes(
+              !["btrfs", "ext3", "ext4", "ext4dev", "xfs"].includes(
                 capability.mount.fs_type
               )
             ) {

src/utils/filesystem.js

@@ -1,6 +1,8 @@
 const cp = require("child_process");
 const fs = require("fs");
 
+const DEFAULT_TIMEOUT = process.env.FILESYSTEM_DEFAULT_TIMEOUT || 30000;
+
 /**
  * https://github.com/kubernetes/kubernetes/tree/master/pkg/util/mount
  * https://github.com/kubernetes/kubernetes/blob/master/pkg/util/mount/mount_linux.go
@@ -16,10 +18,6 @@ class Filesystem {
       options.paths.sudo = "/usr/bin/sudo";
     }
 
-    if (!options.timeout) {
-      options.timeout = 10 * 60 * 1000;
-    }
-
     if (!options.executor) {
       options.executor = {
         spawn: cp.spawn,
@@ -454,6 +452,12 @@ class Filesystem {
     let result;
 
     switch (fstype.toLowerCase()) {
+      case "btrfs":
+        command = "btrfs";
+        //args = args.concat(options);
+        args = args.concat(["filesystem", "resize", "max"]);
+        args.push(device); // in this case should be a mounted path
+        break;
       case "ext4":
       case "ext3":
       case "ext4dev":
@@ -500,6 +504,12 @@ class Filesystem {
     let result;
 
     switch (fstype.toLowerCase()) {
+      case "btrfs":
+        command = "btrfs";
+        args = args.concat(options);
+        args.push("check");
+        args.push(device);
+        break;
       case "ext4":
       case "ext3":
       case "ext4dev":
@@ -591,11 +601,15 @@ class Filesystem {
     return true;
   }
 
-  exec(command, args, options) {
+  exec(command, args, options = {}) {
+    if (!options.hasOwnProperty("timeout")) {
+      // TODO: cannot use this as fsck etc are too risky to kill
+      //options.timeout = DEFAULT_TIMEOUT;
+    }
+
     const filesystem = this;
     args = args || [];
 
-    let timeout;
     let stdout = "";
     let stderr = "";
 
@@ -606,14 +620,6 @@ class Filesystem {
     console.log("executing filesystem command: %s %s", command, args.join(" "));
     const child = filesystem.options.executor.spawn(command, args, options);
 
-    let didTimeout = false;
-    if (options && options.timeout) {
-      timeout = setTimeout(() => {
-        didTimeout = true;
-        child.kill(options.killSignal || "SIGTERM");
-      }, options.timeout);
-    }
-
     return new Promise((resolve, reject) => {
       child.stdout.on("data", function (data) {
         stdout = stdout + data;
@@ -625,10 +631,8 @@ class Filesystem {
 
       child.on("close", function (code) {
         const result = { code, stdout, stderr, timeout: false };
-        if (timeout) {
-          clearTimeout(timeout);
-        }
 
+        // timeout scenario
         if (code === null) {
           result.timeout = true;
           reject(result);

src/utils/general.js

@@ -1,3 +1,5 @@
+const axios = require("axios");
+
 function sleep(ms) {
   return new Promise((resolve) => {
     setTimeout(resolve, ms);
@@ -51,6 +53,65 @@ function getLargestNumber() {
   return number;
 }
 
+/**
+ * transition function to replicate `request` style requests using axios
+ * 
+ * @param {*} options 
+ * @param {*} callback 
+ */
+function axios_request(options, callback = function () {}) {
+  function prep_response(res) {
+    res["statusCode"] = res["status"];
+    delete res["status"];
+
+    res["body"] = res["data"];
+    delete res["data"];
+
+    return res;
+  }
+
+  axios(options)
+    .then((res) => {
+      res = prep_response(res);
+      callback(null, res, res.body);
+    })
+    .catch((err) => {
+      if (err.response) {
+        // The request was made and the server responded with a status code
+        // that falls out of the range of 2xx
+        let res = prep_response(err.response);
+        callback(null, res, res.body);
+      } else if (err.request) {
+        // The request was made but no response was received
+        // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
+        // http.ClientRequest in node.js
+        callback(err, null, null);
+      } else {
+        // Something happened in setting up the request that triggered an Error
+        callback(err, null, null);
+      }
+    });
+}
+
+function stringify(value) {
+  const getCircularReplacer = () => {
+    const seen = new WeakSet();
+    return (key, value) => {
+      if (typeof value === "object" && value !== null) {
+        if (seen.has(value)) {
+          return;
+        }
+        seen.add(value);
+      }
+      return value;
+    };
+  };
+
+  return JSON.stringify(value, getCircularReplacer());
+}
+
 module.exports.sleep = sleep;
 module.exports.lockKeysFromRequest = lockKeysFromRequest;
 module.exports.getLargestNumber = getLargestNumber;
+module.exports.stringify = stringify;
+module.exports.axios_request = axios_request;

src/utils/grpc.js

@@ -7,4 +7,7 @@ class GrpcError {
 }
 
 module.exports.GrpcError = GrpcError;
-module.exports.grpc = require("grpc-uds");
+
+const grpcImplementation = process.env.GRPC_IMPLEMENTATION || "@grpc/grpc-js";
+console.log(`grpc implementation: ${grpcImplementation}`);
+module.exports.grpc = require(grpcImplementation);

src/utils/iscsi.js

@@ -1,10 +1,13 @@
 const cp = require("child_process");
+const { sleep } = require("./general");
 
 function getIscsiValue(value) {
   if (value == "<empty>") return null;
   return value;
 }
 
+const DEFAULT_TIMEOUT = process.env.ISCSI_DEFAULT_TIMEOUT || 30000;
+
 class ISCSI {
   constructor(options = {}) {
     const iscsi = this;
@@ -19,10 +22,6 @@ class ISCSI {
       options.paths.sudo = "/usr/bin/sudo";
     }
 
-    if (!options.timeout) {
-      options.timeout = 10 * 60 * 1000;
-    }
-
     if (!options.executor) {
       options.executor = {
         spawn: cp.spawn,
@@ -105,7 +104,10 @@ class ISCSI {
           "-o",
           "new",
         ]);
+        // create DB entry
         await iscsi.exec(options.paths.iscsiadm, args);
+
+        // update attributes 1 by 1
         for (let attribute in attributes) {
           let args = [];
           args = args.concat([
@@ -122,7 +124,30 @@ class ISCSI {
             "--value",
             attributes[attribute],
           ]);
-          await iscsi.exec(options.paths.iscsiadm, args);
+          // https://bugzilla.redhat.com/show_bug.cgi?id=884427
+          // Could not execute operation on all records: encountered iSCSI database failure
+          let retries = 0;
+          let maxRetries = 5;
+          let retryWait = 1000;
+          while (retries < maxRetries) {
+            retries++;
+            try {
+              //throw {stderr: "Could not execute operation on all records: encountered iSCSI database failure"};
+              await iscsi.exec(options.paths.iscsiadm, args);
+              break;
+            } catch (err) {
+              if (
+                retries < maxRetries &&
+                err.stderr.includes(
+                  "Could not execute operation on all records: encountered iSCSI database failure"
+                )
+              ) {
+                await sleep(retryWait);
+              } else {
+                throw err;
+              }
+            }
+          }
         }
       },
 
@@ -155,10 +180,7 @@ class ISCSI {
 
         let session = false;
         sessions.every((i_session) => {
-          if (
+          if (`${i_session.iqn}` == tgtIQN && portal == i_session.portal) {
-            `${i_session.iqn}` == tgtIQN &&
-            portal == i_session.portal
-          ) {
             session = i_session;
             return false;
           }
@@ -493,11 +515,14 @@ class ISCSI {
     };
   }
 
-  exec(command, args, options) {
+  exec(command, args, options = {}) {
+    if (!options.hasOwnProperty("timeout")) {
+      options.timeout = DEFAULT_TIMEOUT;
+    }
+
     const iscsi = this;
     args = args || [];
 
-    let timeout;
     let stdout = "";
     let stderr = "";
 
@@ -508,14 +533,6 @@ class ISCSI {
     console.log("executing iscsi command: %s %s", command, args.join(" "));
     const child = iscsi.options.executor.spawn(command, args, options);
 
-    let didTimeout = false;
-    if (options && options.timeout) {
-      timeout = setTimeout(() => {
-        didTimeout = true;
-        child.kill(options.killSignal || "SIGTERM");
-      }, options.timeout);
-    }
-
     return new Promise((resolve, reject) => {
       child.stdout.on("data", function (data) {
         stdout = stdout + data;
@@ -526,10 +543,14 @@ class ISCSI {
       });
 
       child.on("close", function (code) {
-        const result = { code, stdout, stderr };
+        const result = { code, stdout, stderr, timeout: false };
-        if (timeout) {
+
-          clearTimeout(timeout);
+        // timeout scenario
+        if (code === null) {
+          result.timeout = true;
+          reject(result);
         }
+
         if (code) {
           reject(result);
         } else {

src/utils/mount.js

@@ -10,7 +10,7 @@ FINDMNT_COMMON_OPTIONS = [
   "--nofsroot", // prevents unwanted behavior with cifs volumes
 ];
 
-DEFAULT_TIMEOUT = process.env.MOUNT_DEFAULT_TIMEOUT || 30000;
+const DEFAULT_TIMEOUT = process.env.MOUNT_DEFAULT_TIMEOUT || 30000;
 
 class Mount {
   constructor(options = {}) {
@@ -38,10 +38,6 @@ class Mount {
       options.paths.chroot = "/usr/sbin/chroot";
     }
 
-    if (!options.timeout) {
-      options.timeout = 10 * 60 * 1000;
-    }
-
     if (!options.executor) {
       options.executor = {
         spawn: cp.spawn,
@@ -389,7 +385,6 @@ class Mount {
     const mount = this;
     args = args || [];
 
-    let timeout;
     let stdout = "";
     let stderr = "";
 
@@ -409,18 +404,6 @@ class Mount {
     console.log("executing mount command: %s", cleansedLog);
     const child = mount.options.executor.spawn(command, args, options);
 
-    /**
-     * timeout option natively supported since v16
-     * TODO: properly handle this based on nodejs version
-     */
-    let didTimeout = false;
-    if (options && options.timeout) {
-      timeout = setTimeout(() => {
-        didTimeout = true;
-        child.kill(options.killSignal || "SIGTERM");
-      }, options.timeout);
-    }
-
     return new Promise((resolve, reject) => {
       child.stdout.on("data", function (data) {
         stdout = stdout + data;
@@ -433,10 +416,6 @@ class Mount {
       child.on("close", function (code) {
         const result = { code, stdout, stderr, timeout: false };
 
-        if (timeout) {
-          clearTimeout(timeout);
-        }
-
         // timeout scenario
         if (code === null) {
           result.timeout = true;

src/utils/oneclient.js

@@ -1,6 +1,6 @@
 const cp = require("child_process");
 
-DEFAULT_TIMEOUT = process.env.MOUNT_DEFAULT_TIMEOUT || 30000;
+const DEFAULT_TIMEOUT = process.env.MOUNT_DEFAULT_TIMEOUT || 30000;
 
 /**
  * - https://github.com/onedata/oneclient
@@ -23,10 +23,6 @@ class OneClient {
       options.paths.chroot = "/usr/sbin/chroot";
     }
 
-    if (!options.timeout) {
-      options.timeout = 10 * 60 * 1000;
-    }
-
     if (!options.executor) {
       options.executor = {
         spawn: cp.spawn,
@@ -81,7 +77,6 @@ class OneClient {
     const oneclient = this;
     args = args || [];
 
-    let timeout;
     let stdout = "";
     let stderr = "";
 
@@ -100,18 +95,6 @@ class OneClient {
     console.log("executing oneclient command: %s", cleansedLog);
     const child = oneclient.options.executor.spawn(command, args, options);
 
-    /**
-     * timeout option natively supported since v16
-     * TODO: properly handle this based on nodejs version
-     */
-    let didTimeout = false;
-    if (options && options.timeout) {
-      timeout = setTimeout(() => {
-        didTimeout = true;
-        child.kill(options.killSignal || "SIGTERM");
-      }, options.timeout);
-    }
-
     return new Promise((resolve, reject) => {
       child.stdout.on("data", function (data) {
         stdout = stdout + data;
@@ -124,10 +107,6 @@ class OneClient {
       child.on("close", function (code) {
         const result = { code, stdout, stderr, timeout: false };
 
-        if (timeout) {
-          clearTimeout(timeout);
-        }
-
         // timeout scenario
         if (code === null) {
           result.timeout = true;