improved logic for the various race conditions from FreeNAS api

Dockerfile

@@ -59,8 +59,10 @@ ENV PATH=/usr/local/lib/nodejs/bin:$PATH
 COPY --from=build /usr/local/lib/nodejs /usr/local/lib/nodejs
 
 # node service requirements
+# netbase is required by rpcbind/rpcinfo to work properly
+# /etc/{services,rpc} are required
 RUN apt-get update && \
-        apt-get install -y e2fsprogs xfsprogs fatresize dosfstools nfs-common cifs-utils sudo && \
+        apt-get install -y netbase e2fsprogs xfsprogs fatresize dosfstools nfs-common cifs-utils sudo && \
         rm -rf /var/lib/apt/lists/*
 
 # controller requirements
@@ -75,6 +77,14 @@ RUN chmod +x /usr/local/sbin/iscsiadm
 ADD docker/multipath /usr/local/sbin
 RUN chmod +x /usr/local/sbin/multipath
 
+## USE_HOST_MOUNT_TOOLS=1
+ADD docker/mount /usr/local/bin/mount
+RUN chmod +x /usr/local/bin/mount
+
+## USE_HOST_MOUNT_TOOLS=1
+ADD docker/umount /usr/local/bin/umount
+RUN chmod +x /usr/local/bin/umount
+
 # Run as a non-root user
 RUN useradd --create-home csi \
         && chown -R csi: /home/csi

contrib/ctld-config-watchdog.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# under certain circumstances high concurrency requests to the FreeNAS/TrueNAS
+# API can result in an invalid /etc/ctl.conf written to disk
+# this script attempts to mitigate those failures by forcing a rebuild of the
+# file using info strictly from the sqlite DB
+
+# can test with this
+# logger -t ctld "error in configuration file"
+
+while [ 1 ]; do
+  egrep -m 1 "ctld.*error in configuration file" <(tail -n 0 -F /var/log/messages) &>/dev/null
+
+  echo "regen ctld config"
+  midclt call etc.generate ctld &>/dev/null
+
+  echo "reload ctld service"
+  /etc/rc.d/ctld reload &>/dev/null
+done

contrib/ctld-service-watchdog.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+# watch the ctld pid file and ensure the service is actually running
+
+while [ 1 ]; do
+  sleep 5
+  ps -p $(cat /var/run/ctld.pid) | grep ctld &>/dev/null || {
+    echo "ctld not running, restarting"
+
+    echo "regen ctld config"
+    midclt call etc.generate ctld &>/dev/null
+
+    echo "restart ctld service"
+    /etc/rc.d/ctld restart &>/dev/null
+  }
+done

docker/mount

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ ${USE_HOST_MOUNT_TOOLS} -eq 1 ]];then
+  chroot /host /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" mount "${@:1}"
+else
+  /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" mount "${@:1}"
+fi

docker/umount

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ ${USE_HOST_MOUNT_TOOLS} -eq 1 ]];then
+  chroot /host /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" umount "${@:1}"
+else
+  /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" umount "${@:1}"
+fi

src/driver/index.js

@@ -356,15 +356,7 @@ class CsiBaseDriver {
             nodeDB
           );
           // login
-          try {
+          await iscsi.iscsiadm.login(volume_context.iqn, portal);
-            await iscsi.iscsiadm.login(volume_context.iqn, portal);
-          } catch (err) {
-            if (typeof this.failedAttachHelper === "function") {
-              // no need to await this
-              this.failedAttachHelper(call, err);
-            }
-            throw err;
-          }
 
           // find device name
           device = `/dev/disk/by-path/ip-${portal}-iscsi-${volume_context.iqn}-lun-${volume_context.lun}`;
@@ -386,16 +378,6 @@ class CsiBaseDriver {
 
             let current_time = Math.round(new Date().getTime() / 1000);
             if (!result && current_time - timer_start > timer_max) {
-              if (typeof this.failedAttachHelper === "function") {
-                // no need to await this
-                this.failedAttachHelper(
-                  call,
-                  new Error(
-                    `hit timeout waiting for device node to appear: ${device}`
-                  )
-                );
-              }
-
               driver.ctx.logger.warn(
                 `hit timeout waiting for device node to appear: ${device}`
               );