Merge 79ce55a4f4 into 8193b689ed

bin/democratic-csi

@@ -486,6 +486,21 @@ const signalMapping = {
         }
       }
 
+      if (driver && typeof driver.getCleanupHandlers === 'function') {
+        const cleanup = driver.getCleanupHandlers();
+        console.log(`running driver ${cleanup.length} cleanup handlers`);
+        for (const c of cleanup) {
+          try {
+            c();
+          } catch (e) {
+            console.log("cleanup handler failed");
+            console.log(e);
+          }
+        }
+      } else {
+        console.log(`current driver does not support cleanup`);
+      }
+
       console.log("server fully shutdown, exiting");
       process.exit(codeNumber);
     });

src/driver/controller-client-common/index.js

@@ -113,19 +113,27 @@ class ControllerClientCommonDriver extends CsiBaseDriver {
       }
     }
 
-    if (this.ctx.args.csiMode.includes("controller")) {
-      setInterval(() => {
+    if (this.ctx.args.csiMode.includes("controller") && !options.disableBackgroundSnapshotChecks) {
+      let cutCheckIndex = setInterval(() => {
+        if (SNAPSHOTS_CUT_IN_FLIGHT.size == 0) {
+          return;
+        }
         this.ctx.logger.info("snapshots cut in flight", {
           names: [...SNAPSHOTS_CUT_IN_FLIGHT],
           count: SNAPSHOTS_CUT_IN_FLIGHT.size,
         });
       }, 30 * 1000);
-      setInterval(() => {
+      this.cleanup.push(() => clearInterval(cutCheckIndex));
+      let restoreCheckIndex = setInterval(() => {
+        if (SNAPSHOTS_RESTORE_IN_FLIGHT.size == 0) {
+          return;
+        }
         this.ctx.logger.info("snapshots restore in flight", {
           names: [...SNAPSHOTS_RESTORE_IN_FLIGHT],
           count: SNAPSHOTS_RESTORE_IN_FLIGHT.size,
         });
       }, 30 * 1000);
+      this.cleanup.push(() => clearInterval(restoreCheckIndex));
     }
   }
 

src/driver/controller-zfs-generic/index.js

@@ -16,10 +16,12 @@ class ControllerZfsGenericDriver extends ControllerZfsBaseDriver {
   getExecClient() {
     return this.ctx.registry.get(`${__REGISTRY_NS__}:exec_client`, () => {
       if (this.options.sshConnection) {
-        return new SshClient({
+        const sshClient = new SshClient({
           logger: this.ctx.logger,
           connection: this.options.sshConnection,
         });
+        this.cleanup.push(() => sshClient.finalize());
+        return sshClient;
       } else {
         return new LocalCliExecClient({
           logger: this.ctx.logger,

src/driver/freenas/ssh.js

@@ -57,10 +57,12 @@ class FreeNASSshDriver extends ControllerZfsBaseDriver {
 
   getExecClient() {
     return this.ctx.registry.get(`${__REGISTRY_NS__}:exec_client`, () => {
-      return new SshClient({
+      const sshClient = new SshClient({
         logger: this.ctx.logger,
         connection: this.options.sshConnection,
       });
+      this.cleanup.push(() => sshClient.finalize());
+      return sshClient;
     });
   }
 

src/driver/index.js

@@ -33,6 +33,7 @@ class CsiBaseDriver {
   constructor(ctx, options) {
     this.ctx = ctx;
     this.options = options || {};
+    this.cleanup = [];
 
     if (!this.options.hasOwnProperty("node")) {
       this.options.node = {};
@@ -51,6 +52,10 @@ class CsiBaseDriver {
     }
   }
 
+  getCleanupHandlers() {
+    return this.cleanup;
+  }
+
   /**
    * abstract way of retrieving values from parameters/secrets
    * in order of preference:

src/driver/zfs-local-ephemeral-inline/index.js

@@ -125,10 +125,12 @@ class ZfsLocalEphemeralInlineDriver extends CsiBaseDriver {
 
   getSshClient() {
     return this.ctx.registry.get(`${__REGISTRY_NS__}:ssh_client`, () => {
-      return new SshClient({
+      const sshClient = new SshClient({
         logger: this.ctx.logger,
         connection: this.options.sshConnection,
       });
+      this.cleanup.push(() => sshClient.finalize());
+      return sshClient;
     });
   }
 

src/utils/zfs_ssh_exec_client.js

@@ -75,8 +75,14 @@ class SshClient {
     const start_error_event_count = this.error_event_count;
     try {
       await this.conn_mutex.runExclusive(async () => {
+        if (this.finalized) {
+          throw 'using finalized ssh client';
+        }
         this.conn.connect(this.options.connection);
         do {
+          if (this.finalized) {
+            throw 'ssh client finalized during connection';
+          }
           if (start_error_event_count != this.error_event_count) {
             throw this.conn_err;
           }
@@ -104,6 +110,12 @@ class SshClient {
     return this._connect();
   }
 
+  finalize() {
+    this.finalized = true;
+    const conn = this.conn;
+    conn.end();
+  }
+
   async exec(command, options = {}, stream_proxy = null) {
     // default is to reuse
     if (process.env.SSH_REUSE_CONNECTION == "0") {