more robust unpublish and unstage logic

Signed-off-by: Travis Glenn Hansen <travisghansen@yahoo.com>

.github/workflows/main.yml

@@ -113,7 +113,6 @@ jobs:
           SYNOLOGY_PASSWORD: ${{ secrets.SANITY_SYNOLOGY_PASSWORD }}
           SYNOLOGY_VOLUME: ${{ secrets.SANITY_SYNOLOGY_VOLUME }}
 
-
   # api-based drivers
   csi-sanity-truenas-scale-22_02:
     needs:
@@ -376,6 +375,7 @@ jobs:
       - csi-sanity-truenas-core-12_0
       - csi-sanity-truenas-core-13_0
       - csi-sanity-zfs-generic
+      - csi-sanity-client
       - csi-sanity-zfs-local
       - csi-sanity-local-hostpath
       - csi-sanity-windows-node

src/driver/index.js

@@ -2043,7 +2043,21 @@ class CsiBaseDriver {
         result = await mount.pathIsMounted(normalized_staging_path);
         if (result) {
           try {
-            result = await mount.umount(normalized_staging_path, umount_args);
+            result = await GeneralUtils.retry(
+              10,
+              0,
+              async () => {
+                return await mount.umount(normalized_staging_path, umount_args);
+              },
+              {
+                minExecutionTime: 1000,
+                retryCondition: (err) => {
+                  if (_.get(err, "stderr", "").includes("busy")) {
+                    return true;
+                  }
+                },
+              }
+            );
           } catch (err) {
             if (err.timeout) {
               driver.ctx.logger.warn(
@@ -2188,13 +2202,41 @@ class CsiBaseDriver {
           // remove touched file
           result = await filesystem.pathExists(block_path);
           if (result) {
-            result = await filesystem.rm(block_path);
+            result = await GeneralUtils.retry(
+              10,
+              0,
+              async () => {
+                return await filesystem.rm(block_path);
+              },
+              {
+                minExecutionTime: 1000,
+                retryCondition: (err) => {
+                  if (_.get(err, "stderr", "").includes("busy")) {
+                    return true;
+                  }
+                },
+              }
+            );
           }
         }
 
         result = await filesystem.pathExists(staging_target_path);
         if (result) {
-          result = await filesystem.rmdir(staging_target_path);
+          result = await GeneralUtils.retry(
+            10,
+            0,
+            async () => {
+              return await filesystem.rmdir(staging_target_path);
+            },
+            {
+              minExecutionTime: 1000,
+              retryCondition: (err) => {
+                if (_.get(err, "stderr", "").includes("busy")) {
+                  return true;
+                }
+              },
+            }
+          );
         }
         break;
       case NODE_OS_DRIVER_WINDOWS: {
@@ -2777,7 +2819,21 @@ class CsiBaseDriver {
 
         if (result) {
           try {
-            result = await mount.umount(target_path, umount_args);
+            result = await GeneralUtils.retry(
+              10,
+              0,
+              async () => {
+                return await mount.umount(target_path, umount_args);
+              },
+              {
+                minExecutionTime: 1000,
+                retryCondition: (err) => {
+                  if (_.get(err, "stderr", "").includes("busy")) {
+                    return true;
+                  }
+                },
+              }
+            );
           } catch (err) {
             if (err.timeout) {
               driver.ctx.logger.warn(
@@ -2808,9 +2864,30 @@ class CsiBaseDriver {
         result = await filesystem.pathExists(target_path);
         if (result) {
           if (fs.lstatSync(target_path).isDirectory()) {
-            result = await filesystem.rmdir(target_path);
+            result = await GeneralUtils.retry(
+              10,
+              0,
+              async () => {
+                return await filesystem.rmdir(target_path);
+              },
+              {
+                minExecutionTime: 1000,
+                retryCondition: (err) => {
+                  if (_.get(err, "stderr", "").includes("busy")) {
+                    return true;
+                  }
+                },
+              }
+            );
           } else {
-            result = await filesystem.rm([target_path]);
+            result = await GeneralUtils.retry(
+              10,
+              0,
+              async () => {
+                return await filesystem.rm([target_path]);
+              },
+              { minExecutionTime: 1000 }
+            );
           }
         }
 

src/utils/general.js

@@ -145,10 +145,8 @@ function axios_request(options, callback = function () {}) {
           options.validateStatus &&
           typeof options.validateStatus == "function"
         ) {
-          if (!options.validateStatus(res.statusCode)) {
           senderr = true;
         }
-        }
         callback(senderr ? err : null, res, res.body);
       } else if (err.request) {
         // The request was made but no response was received
@@ -214,12 +212,14 @@ async function retry(retries, retriesDelay, code, options = {}) {
         console.log(`retry - err:`, err);
       }
     }
+
     // handle minExecutionTime
     if (options.minExecutionTime > 0) {
-      let minDelayTime =
-        options.minExecutionTime - (Date.now() - executeStartTime);
-      if (minDelayTime > 0) {
-        await sleep(minDelayTime);
+      let executionElapsedTIme = Date.now() - executeStartTime;
+      let minExecutionDelayTime =
+        options.minExecutionTime - executionElapsedTIme;
+      if (minExecutionDelayTime > 0) {
+        await sleep(minExecutionDelayTime);
       }
     }
 
@@ -234,8 +234,10 @@ async function retry(retries, retriesDelay, code, options = {}) {
         sleep_time = maxwait;
       }
     }
+    if (sleep_time > 0) {
       console.log(`retry - waiting ${sleep_time}ms before trying again`);
       await sleep(sleep_time);
+    }
   } while (true);
 }