From 31dcf845fc419d19bf9cae84b825f9be41108010 Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Sat, 22 Mar 2025 19:36:00 +0200
Subject: [PATCH 01/16] Add iSCSI targets/LUNs through Pacemaker clusters

Support pcs for zfs-generic to configure iSCSI
targets and LUNs. It uses the targetcli implementation
since it's the default pcs resources and also to
keep compatibility with the rest of the code.

closes #462
---
 examples/zfs-generic-iscsi.yaml            |  12 ++
 src/driver/controller-zfs-generic/index.js | 165 ++++++++++++++++++++-
 2 files changed, 175 insertions(+), 2 deletions(-)

diff --git a/examples/zfs-generic-iscsi.yaml b/examples/zfs-generic-iscsi.yaml
index 8e38e42..e62e2c6 100644
--- a/examples/zfs-generic-iscsi.yaml
+++ b/examples/zfs-generic-iscsi.yaml
@@ -46,6 +46,7 @@ zfs:
 
 iscsi:
   shareStrategy: "targetCli"
+  #shareStrategy: "pcs"
 
   # https://kifarunix.com/how-to-install-and-configure-iscsi-storage-server-on-ubuntu-18-04/
   # https://kifarunix.com/how-install-and-configure-iscsi-storage-server-on-centos-7/
@@ -77,6 +78,17 @@ iscsi:
       attributes:
         # set to 1 to enable Thin Provisioning Unmap
         emulate_tpu: 0
+
+  shareStrategyPcs:
+    #sudoEnabled: true
+    pcs_group: "group-nas"
+    basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664"
+    auth:
+      enabled: 0
+      # CHAP – incoming only, mutual not supported by the Pacemaker resource agent
+      incoming_username: "foo"
+      incoming_password: "bar"
+
   targetPortal: "server[:port]"
   # for multipath
   targetPortals: [] # [ "server[:port]", "server[:port]", ... ]
diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js
index 1c7c32f..c2e15e8 100644
--- a/src/driver/controller-zfs-generic/index.js
+++ b/src/driver/controller-zfs-generic/index.js
@@ -303,8 +303,70 @@ create /backstores/block/${assetName}
               }
             );
             break;
+          case "pcs":
+              basename = this.options.iscsi.shareStrategyPcs.basename;
+              pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group;
+
+              let groupText = `group ${pcs_group}`;
+              let createTargetText = [
+                `resource create --future target-${assetName} iSCSITarget`,
+                'implementation="lio-t"',
+                `iqn="${basename}:${assetName}"`
+              ];
+
+              if (this.options.iscsi.shareStrategyPcs.auth.enabled) {
+                createTargetText.push(`incoming_username="${this.options.iscsi.shareStrategyPcs.auth.incoming_username}"`);
+                createTargetText.push(`incoming_password="${this.options.iscsi.shareStrategyPcs.auth.incoming_password}"`);
+              }
+
+              createTargetText.push(groupText);
+
+              await GeneralUtils.retry(
+                3,
+                2000,
+                async () => {
+                  await this.pcsCommand(createTargetText);
+                },
+                {
+                  retryCondition: (err) => {
+                    if (err.stdout && err.stdout.includes("Timed Out")) {
+                      return true;
+                    }
+                    return false;
+                  },
+                }
+              );
+
+              let createLunText = [
+                `resource create --future lun-${assetName} iSCSILogicalUnit`,
+                'implementation="lio-t"',
+                `target_iqn="${basename}:${assetName}" lun="1"`,
+                `path="/dev/${extentDiskName}"`,
+                groupText
+              ];
+
+              await GeneralUtils.retry(
+                3,
+                2000,
+                async () => {
+                  await this.pcsCommand(createLunText);
+                },
+                {
+                  retryCondition: (err) => {
+                    if (err.stdout && err.stdout.includes("Timed Out")) {
+                      return true;
+                    }
+                    return false;
+                  },
+                }
+              );
+              break;
+
           default:
-            break;
+            throw new GrpcError(
+              grpc.status.FAILED_PRECONDITION,
+              `invalid configuration: unknown shareStrategy ${this.options.iscsi.shareStrategy}`
+            );
         }
 
         // iqn = target
@@ -695,8 +757,54 @@ delete ${assetName}
             );
 
             break;
-          default:
+          case "pcs":
+            let deleteLunText = [
+              `resource delete lun-${assetName}`
+            ];
+
+            await GeneralUtils.retry(
+              3,
+              2000,
+              async () => {
+                await this.pcsCommand(deleteLunText);
+              },
+              {
+                retryCondition: (err) => {
+                  if (err.stdout && err.stdout.includes("Timed Out")) {
+                    return true;
+                  }
+                  return false;
+                },
+              }
+            );
+
+            let deleteTargetText = [
+              `resource delete target-${assetName}`
+            ];
+
+            await GeneralUtils.retry(
+              3,
+              2000,
+              async () => {
+                await this.pcsCommand(deleteTargetText);
+              },
+              {
+                retryCondition: (err) => {
+                  if (err.stdout && err.stdout.includes("Timed Out")) {
+                    return true;
+                  }
+                  return false;
+                },
+              }
+            );
+
             break;
+
+          default:
+            throw new GrpcError(
+              grpc.status.FAILED_PRECONDITION,
+              `invalid configuration: unknown shareStrategy ${this.options.iscsi.shareStrategy}`
+            );
         }
         break;
       }
@@ -846,6 +954,9 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
           case "targetCli":
             // nothing required, just need to rescan on the node
             break;
+          case "pcs":
+            // nothing required, just need to rescan on the node
+            break;
           default:
             break;
         }
@@ -856,6 +967,56 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
     }
   }
 
+  async pcsCommand(commandLines) {
+    const execClient = this.getExecClient();
+    const driver = this;
+
+    let command = "sh";
+    let args = ["-c"];
+
+    let cliArgs = ["pcs"];
+    if (
+      _.get(this.options, "iscsi.shareStrategyPcs.sudoEnabled", false)
+    ) {
+      cliArgs.unshift("sudo");
+    }
+
+    let cliCommand = [];
+    cliCommand.push(cliArgs.join(" "));
+    cliCommand.push(commandLines.join(" "));
+    args.push("'" + cliCommand.join(" ") + "'");
+
+    let logCommandTmp = command + " " + args.join(" ");
+    let logCommand = "";
+
+    logCommandTmp.split(" ").forEach((term) => {
+      logCommand += " ";
+      
+      if (term.startsWith("incoming_password=")) {
+        logCommand += "incoming_password=<redacted>";
+      } else {
+        logCommand += term;
+      }
+    });
+
+    driver.ctx.logger.verbose("Pcs command:" + logCommand);
+
+    let options = {
+      pty: true,
+    };
+    let response = await execClient.exec(
+      execClient.buildCommand(command, args),
+      options
+    );
+    driver.ctx.logger.verbose(
+      "Pcs response: " + JSON.stringify(response)
+    );
+    if (response.code != 0) {
+      throw response;
+    }
+    return response;
+  }
+
   async targetCliCommand(data) {
     const execClient = this.getExecClient();
     const driver = this;

From 373a7c7f79909a1fe17a732eef321fd8146152a7 Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Sun, 23 Mar 2025 10:48:49 +0200
Subject: [PATCH 02/16] Small CR and preparatory work for CI

---
 .github/workflows/main.yml                    |  3 +-
 ci/configs/zfs-generic/iscsi-pcs.yaml         | 28 +++++++++++++
 .../{iscsi.yaml => iscsi-targetcli.yaml}      |  0
 src/driver/controller-zfs-generic/index.js    | 41 ++++++++-----------
 4 files changed, 48 insertions(+), 24 deletions(-)
 create mode 100644 ci/configs/zfs-generic/iscsi-pcs.yaml
 rename ci/configs/zfs-generic/{iscsi.yaml => iscsi-targetcli.yaml} (100%)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 16cdb78..81567cb 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -192,7 +192,8 @@ jobs:
       max-parallel: 1
       matrix:
         config:
-          - zfs-generic/iscsi.yaml
+          - zfs-generic/iscsi-pcs.yaml
+          - zfs-generic/iscsi-targetcli.yaml
           - zfs-generic/nfs.yaml
           - zfs-generic/smb.yaml
           - zfs-generic/nvmeof.yaml
diff --git a/ci/configs/zfs-generic/iscsi-pcs.yaml b/ci/configs/zfs-generic/iscsi-pcs.yaml
new file mode 100644
index 0000000..baabb69
--- /dev/null
+++ b/ci/configs/zfs-generic/iscsi-pcs.yaml
@@ -0,0 +1,28 @@
+driver: zfs-generic-iscsi
+
+sshConnection:
+  host: ${SERVER_HOST}
+  port: 22
+  username: ${SERVER_USERNAME}
+  password: ${SERVER_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize:
+
+iscsi:
+  targetPortal: ${SERVER_HOST}
+  interface: ""
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+  shareStrategy: "pcs"
+  shareStrategyPcs:
+    pcs_group: "group-nas"
+    basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664"
+    auth:
+      enabled: 0
diff --git a/ci/configs/zfs-generic/iscsi.yaml b/ci/configs/zfs-generic/iscsi-targetcli.yaml
similarity index 100%
rename from ci/configs/zfs-generic/iscsi.yaml
rename to ci/configs/zfs-generic/iscsi-targetcli.yaml
diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js
index c2e15e8..cf86920 100644
--- a/src/driver/controller-zfs-generic/index.js
+++ b/src/driver/controller-zfs-generic/index.js
@@ -307,25 +307,22 @@ create /backstores/block/${assetName}
               basename = this.options.iscsi.shareStrategyPcs.basename;
               pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group;
 
-              let groupText = `group ${pcs_group}`;
-              let createTargetText = [
-                `resource create --future target-${assetName} iSCSITarget`,
-                'implementation="lio-t"',
-                `iqn="${basename}:${assetName}"`
+              let groupTerms = ['group', `${pcs_group}`];
+              let createTargetTerms = [
+                'resource', 'create', '--future', `target-${assetName}`, 'iSCSITarget',
+                'implementation="lio-t"', `iqn="${basename}:${assetName}"`
               ];
 
               if (this.options.iscsi.shareStrategyPcs.auth.enabled) {
-                createTargetText.push(`incoming_username="${this.options.iscsi.shareStrategyPcs.auth.incoming_username}"`);
-                createTargetText.push(`incoming_password="${this.options.iscsi.shareStrategyPcs.auth.incoming_password}"`);
+                createTargetTerms.push(`incoming_username="${this.options.iscsi.shareStrategyPcs.auth.incoming_username}"`);
+                createTargetTerms.push(`incoming_password="${this.options.iscsi.shareStrategyPcs.auth.incoming_password}"`);
               }
 
-              createTargetText.push(groupText);
-
               await GeneralUtils.retry(
                 3,
                 2000,
                 async () => {
-                  await this.pcsCommand(createTargetText);
+                  await this.pcsCommand(createTargetTerms.concat(groupTerms));
                 },
                 {
                   retryCondition: (err) => {
@@ -337,19 +334,17 @@ create /backstores/block/${assetName}
                 }
               );
 
-              let createLunText = [
-                `resource create --future lun-${assetName} iSCSILogicalUnit`,
-                'implementation="lio-t"',
-                `target_iqn="${basename}:${assetName}" lun="1"`,
-                `path="/dev/${extentDiskName}"`,
-                groupText
+              let createLunTerms = [
+                'resource', 'create', '--future', `lun-${assetName}`, 'iSCSILogicalUnit',
+                'implementation="lio-t"', `target_iqn="${basename}:${assetName}" lun="1"`,
+                `path="/dev/${extentDiskName}"`
               ];
 
               await GeneralUtils.retry(
                 3,
                 2000,
                 async () => {
-                  await this.pcsCommand(createLunText);
+                  await this.pcsCommand(createLunTerms.concat(groupTerms));
                 },
                 {
                   retryCondition: (err) => {
@@ -759,7 +754,7 @@ delete ${assetName}
             break;
           case "pcs":
             let deleteLunText = [
-              `resource delete lun-${assetName}`
+              'resource', 'delete', `lun-${assetName}`
             ];
 
             await GeneralUtils.retry(
@@ -779,7 +774,7 @@ delete ${assetName}
             );
 
             let deleteTargetText = [
-              `resource delete target-${assetName}`
+              'resource', 'delete', `target-${assetName}`
             ];
 
             await GeneralUtils.retry(
@@ -967,7 +962,7 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
     }
   }
 
-  async pcsCommand(commandLines) {
+  async pcsCommand(commandTerms) {
     const execClient = this.getExecClient();
     const driver = this;
 
@@ -983,7 +978,7 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
 
     let cliCommand = [];
     cliCommand.push(cliArgs.join(" "));
-    cliCommand.push(commandLines.join(" "));
+    cliCommand.push(commandTerms.join(" "));
     args.push("'" + cliCommand.join(" ") + "'");
 
     let logCommandTmp = command + " " + args.join(" ");
@@ -999,7 +994,7 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
       }
     });
 
-    driver.ctx.logger.verbose("Pcs command:" + logCommand);
+    driver.ctx.logger.verbose("pcs command:" + logCommand);
 
     let options = {
       pty: true,
@@ -1009,7 +1004,7 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
       options
     );
     driver.ctx.logger.verbose(
-      "Pcs response: " + JSON.stringify(response)
+      "pcs response: " + JSON.stringify(response)
     );
     if (response.code != 0) {
       throw response;

From 3931bd9ef0305c03ea9d364303e41ccc95e75e4e Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Sun, 23 Mar 2025 11:29:53 +0200
Subject: [PATCH 03/16] Adding some sleeps to allow things to settle down

---
 src/driver/controller-zfs-generic/index.js | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js
index cf86920..3899792 100644
--- a/src/driver/controller-zfs-generic/index.js
+++ b/src/driver/controller-zfs-generic/index.js
@@ -355,6 +355,9 @@ create /backstores/block/${assetName}
                   },
                 }
               );
+  
+              await GeneralUtils.sleep(2000); // let things settle
+
               break;
 
           default:
@@ -793,6 +796,8 @@ delete ${assetName}
               }
             );
 
+            await GeneralUtils.sleep(2000); // let things settle
+
             break;
 
           default:

From 53560cd8b1e9f847877336cba368a5a6dbf95b12 Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Sun, 23 Mar 2025 11:32:32 +0200
Subject: [PATCH 04/16] small fix

---
 src/driver/controller-zfs-generic/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js
index 3899792..98a7145 100644
--- a/src/driver/controller-zfs-generic/index.js
+++ b/src/driver/controller-zfs-generic/index.js
@@ -336,7 +336,7 @@ create /backstores/block/${assetName}
 
               let createLunTerms = [
                 'resource', 'create', '--future', `lun-${assetName}`, 'iSCSILogicalUnit',
-                'implementation="lio-t"', `target_iqn="${basename}:${assetName}" lun="1"`,
+                'implementation="lio-t"', `target_iqn="${basename}:${assetName}"`, 'lun="1"',
                 `path="/dev/${extentDiskName}"`
               ];
 

From 34452b2f3a72200582d11f1242f58aacce9569d2 Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Mon, 24 Mar 2025 23:11:00 +0000
Subject: [PATCH 05/16] Added vagrant and devcontainers + fixed all tests

---
 .devcontainer/devcontainer.json            |  38 ++++++++
 .github/dependabot.yml                     |  12 +++
 .gitignore                                 |   4 +-
 CONTRIBUTING.md                            | 108 +++++++++++++++++++++
 Vagrantfile                                |  71 ++++++++++++++
 ci/bin/run.sh                              |   2 +-
 dev/run.sh                                 |  34 +++++++
 src/driver/controller-zfs-generic/index.js |  27 +++---
 8 files changed, 282 insertions(+), 14 deletions(-)
 create mode 100644 .devcontainer/devcontainer.json
 create mode 100644 .github/dependabot.yml
 create mode 100644 CONTRIBUTING.md
 create mode 100644 Vagrantfile
 create mode 100755 dev/run.sh

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
new file mode 100644
index 0000000..2835179
--- /dev/null
+++ b/.devcontainer/devcontainer.json
@@ -0,0 +1,38 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node
+{
+	"name": "democratic-csi",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/go:1": {} // To compile csi-sanity during postCreateCommand
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "npm install",
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"terminal.integrated.shell.linux": "/bin/bash"
+			},
+			"extensions": [
+				"dbaeumer.vscode-eslint",
+				"esbenp.prettier-vscode",
+				"waderyan.nodejs-extension-pack",
+				"ms-vscode.node-debug2",
+				"christian-kohler.npm-intellisense",
+				"christian-kohler.path-intellisense",
+				"HashiCorp.terraform"
+			]
+		}
+	}
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
new file mode 100644
index 0000000..f33a02c
--- /dev/null
+++ b/.github/dependabot.yml
@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly
diff --git a/.gitignore b/.gitignore
index 2855d28..b945e95 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
 **~
 node_modules
-dev
+dev/*
 /ci/bin/*dev*
+.vagrant
+!dev/run.sh
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..a770271
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,108 @@
+# Contributing to democratic-csi
+
+## Development Environment Setup
+
+This project uses a hybrid development approach with devcontainers for IDE configuration and Vagrant for system-level testing.
+
+### Prerequisites
+
+Before you begin, ensure you have the following installed:
+- [Visual Studio Code](https://code.visualstudio.com/)
+- [Docker](https://www.docker.com/get-started)
+- [Vagrant](https://www.vagrantup.com/downloads)
+- Virtualization Provider:
+  - For Intel/AMD Machines: VirtualBox
+  - For Apple Silicon: Qemu (`brew install qemu vagrant` and `vagrant plugin install vagrant-qemu`)
+
+### Development Workflow
+
+#### 1. Local Development with Devcontainers
+
+Devcontainers provide a consistent development environment with:
+- Configured VSCode extensions
+- Necessary development tools
+- Integrated development experience
+
+To use the devcontainer:
+1. Open the project in VSCode
+2. Install the "Dev Containers" extension
+3. Click "Reopen in Container" when prompted
+4. Start coding with pre-configured environment
+
+#### 2. System Testing with Vagrant
+
+Vagrant provides a full virtual machine environment for:
+- System-level testing
+- Running code with kernel dependencies
+- Simulating production-like environments
+
+Workflow:
+```bash
+# Navigate to project directory
+cd ~/democratic-csi
+
+# Start the Vagrant VM
+vagrant up
+
+# Connect to the VM
+vagrant ssh
+
+# Inside the VM, navigate to the project
+cd ~/democratic-csi
+
+# Run project tests, the config.yaml can be any from the examples folders
+# just configured for your own environment.
+# You can also create a file `dev/secrets.env` that has `export VARIABLE=VALUE`
+# and reference those in your `config.yaml`
+./dev/run.sh -c ./dev/config.yaml
+```
+
+##### Keeping Files in Sync
+
+Use these methods to keep your local files synchronized with the Vagrant VM:
+
+###### Manual Sync
+```bash
+# Sync files from local to Vagrant VM
+vagrant rsync
+```
+
+###### Continuous Sync
+```bash
+# Automatically sync files as they change
+vagrant rsync-auto
+```
+
+### Best Practices
+
+- Use devcontainer for day-to-day development and coding
+- Use Vagrant for comprehensive system testing
+- Always run `vagrant rsync` before running tests in the VM
+- Commit and push changes frequently
+- If encountering issues, try:
+  1. Recreating the devcontainer
+  2. Reprovisioning the Vagrant VM with `vagrant reload --provision` or `vagrant destroy -f && vagrant up`
+
+### Troubleshooting
+
+#### Devcontainer Issues
+- Ensure Docker is running
+- Rebuild the container if extensions fail to load
+- Check VSCode Dev Containers extension logs
+
+#### Vagrant Issues
+- Verify virtualization is enabled in your BIOS
+- Ensure you have the latest Vagrant and virtualization provider
+- For Apple Silicon, use Parallels or Lima
+
+### Contribution Guidelines
+
+1. Create a new branch for your feature targetting `next`
+2. Write clear, concise commit messages
+3. Include coverage for tests of csi-sanity for new functionality
+4. Run tests in Vagrant VM
+5. Submit a pull request with a clear description of changes
+
+### Contact
+
+For any questions or issues, please [open an issue](https://github.com/democratic-csi/democratic-csi/issues) on the project repository.
\ No newline at end of file
diff --git a/Vagrantfile b/Vagrantfile
new file mode 100644
index 0000000..294ceab
--- /dev/null
+++ b/Vagrantfile
@@ -0,0 +1,71 @@
+Vagrant.configure("2") do |config|
+    # Check the host's architecture
+    host_arch = `uname -m`.strip
+
+    # Use a different box for ARM vs x86_64
+    if host_arch == "arm64"
+        # requires qemu, install qemu and then:
+        # vagrant plugin install vagrant-qemu
+        config.vm.box = "perk/ubuntu-24.04-arm64" 
+    else
+        # Use the x86_64 compatible Ubuntu box
+        config.vm.box = "ubuntu/jammy64"
+    end
+  
+    config.vm.provider "virtualbox" do |vb|
+      vb.memory = "2048"
+      vb.cpus = 2
+    end
+  
+    config.vm.provision "shell", inline: <<-SHELL
+      sudo apt-get update -y
+      sudo apt-get install -y open-iscsi nodejs git make
+  
+      # Enable and start iscsid service
+      sudo systemctl enable --now iscsid
+  
+      # Verify installation
+      systemctl status iscsid --no-pager
+
+      ####
+      # Install golang
+      ####
+      GO_VERSION="1.24.1"
+      ARCH=$(uname -m)
+      GO_TAR_URL=""
+
+      if [[ "$ARCH" == "aarch64" ]]; then
+          GO_TAR_URL="https://go.dev/dl/go${GO_VERSION}.linux-arm64.tar.gz"
+      elif [[ "$ARCH" == "x86_64" ]]; then
+          GO_TAR_URL="https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz"
+      else
+          echo "Unsupported architecture: $ARCH"
+          exit 1
+      fi
+
+      echo "Downloading Go version $GO_VERSION for $ARCH..."
+      wget -q "$GO_TAR_URL" -O go.tar.gz
+      tar -C /usr/local -xzf go.tar.gz
+      rm go.tar.gz
+      echo "export PATH=\$PATH:/usr/local/go/bin" >> /etc/profile
+      source /etc/profile
+
+      ####
+      # Install csi-test
+      ####
+      echo "Installing csi-test"
+      git clone https://github.com/kubernetes-csi/csi-test /tmp/csi-test
+      pushd /tmp/csi-test/cmd/csi-sanity
+      make csi-sanity
+      sudo cp csi-sanity /usr/local/bin
+      popd
+      #rm -rf /tmp/csi-test
+    SHELL
+  
+    # Sync project directory for seamless workflow
+    config.vm.synced_folder ".", "/home/vagrant/democratic-csi", type: "rsync",
+        rsync__exclude: ".git/"
+    
+    # Allow SSH access with default key
+    config.ssh.insert_key = false
+  end  
\ No newline at end of file
diff --git a/ci/bin/run.sh b/ci/bin/run.sh
index 1d4eaf8..fd929f9 100755
--- a/ci/bin/run.sh
+++ b/ci/bin/run.sh
@@ -20,7 +20,7 @@ if [[ -f "node_modules-linux-amd64.tar.gz" && ! -d "node_modules" ]];then
 fi
 
 # generate key for paths etc
-export CI_BUILD_KEY=$(uuidgen | cut -d "-" -f 1)
+export CI_BUILD_KEY=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8)
 
 # launch the server
 sudo -E ci/bin/launch-server.sh &
diff --git a/dev/run.sh b/dev/run.sh
new file mode 100755
index 0000000..c38e310
--- /dev/null
+++ b/dev/run.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+set -x
+TEMPLATE_CONFIG=""
+
+ROOT_DIR="$(dirname "$(realpath "$0")")"
+
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    -c|--config) TEMPLATE_CONFIG="$(realpath "$2")"; shift ;;
+    *) echo "Unknown parameter passed: $1"; exit 1 ;;
+  esac
+  shift
+done
+
+if [ -z "${TEMPLATE_CONFIG}" ]; then
+  echo "Error: --config or -c parameter is required."
+  exit 1
+fi
+
+if [ ! -f $ROOT_DIR/secrets.env ]; then
+  echo "Error: secrets.env file not found."
+  exit 1
+fi
+
+source $ROOT_DIR/secrets.env # needs to have exported variables
+
+# generate key for paths etc
+export CI_BUILD_KEY=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8)
+
+export TEMPLATE_CONFIG_FILE=${TEMPLATE_CONFIG}
+
+$ROOT_DIR/../ci/bin/run.sh
\ No newline at end of file
diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js
index 98a7145..9e8d8c2 100644
--- a/src/driver/controller-zfs-generic/index.js
+++ b/src/driver/controller-zfs-generic/index.js
@@ -305,12 +305,12 @@ create /backstores/block/${assetName}
             break;
           case "pcs":
               basename = this.options.iscsi.shareStrategyPcs.basename;
-              pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group;
+              let pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group;
 
-              let groupTerms = ['group', `${pcs_group}`];
+              let extraTerms = ['group', `${pcs_group}`, '--wait']; // The wait is important to avoid race conditions
               let createTargetTerms = [
-                'resource', 'create', '--future', `target-${assetName}`, 'iSCSITarget',
-                'implementation="lio-t"', `iqn="${basename}:${assetName}"`
+                'resource', 'create', '--future', `target-${assetName}`, 'ocf:heartbeat:iSCSITarget',
+                'implementation="lio-t"', 'portals=":::3260"', `iqn="${basename}:${assetName}"`
               ];
 
               if (this.options.iscsi.shareStrategyPcs.auth.enabled) {
@@ -322,7 +322,7 @@ create /backstores/block/${assetName}
                 3,
                 2000,
                 async () => {
-                  await this.pcsCommand(createTargetTerms.concat(groupTerms));
+                  await this.pcsCommand(createTargetTerms.concat(extraTerms));
                 },
                 {
                   retryCondition: (err) => {
@@ -335,8 +335,8 @@ create /backstores/block/${assetName}
               );
 
               let createLunTerms = [
-                'resource', 'create', '--future', `lun-${assetName}`, 'iSCSILogicalUnit',
-                'implementation="lio-t"', `target_iqn="${basename}:${assetName}"`, 'lun="1"',
+                'resource', 'create', '--future', `lun-${assetName}`, 'ocf:heartbeat:iSCSILogicalUnit',
+                'implementation="lio-t"', `target_iqn="${basename}:${assetName}"`, 'lun="0"',
                 `path="/dev/${extentDiskName}"`
               ];
 
@@ -344,7 +344,7 @@ create /backstores/block/${assetName}
                 3,
                 2000,
                 async () => {
-                  await this.pcsCommand(createLunTerms.concat(groupTerms));
+                  await this.pcsCommand(createLunTerms.concat(extraTerms));
                 },
                 {
                   retryCondition: (err) => {
@@ -356,8 +356,6 @@ create /backstores/block/${assetName}
                 }
               );
   
-              await GeneralUtils.sleep(2000); // let things settle
-
               break;
 
           default:
@@ -796,8 +794,6 @@ delete ${assetName}
               }
             );
 
-            await GeneralUtils.sleep(2000); // let things settle
-
             break;
 
           default:
@@ -1011,6 +1007,13 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
     driver.ctx.logger.verbose(
       "pcs response: " + JSON.stringify(response)
     );
+        
+    // Handle idempotence for create commands
+    if (response.code == 1 && response.stdout.includes("already exists")) {
+      driver.ctx.logger.verbose("pcs resource already exists, ignoring error (setting response.code=0)");
+      response.code = 0;
+    }
+
     if (response.code != 0) {
       throw response;
     }

From 7867bb467271925a50a9085f147544f53fa669a2 Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Mon, 24 Mar 2025 23:31:17 +0000
Subject: [PATCH 06/16] Minor changes

---
 .devcontainer/devcontainer.json |  2 +-
 .devcontainer/postCreate.sh     | 12 ++++++++++++
 .github/workflows/main.yml      |  2 +-
 CONTRIBUTING.md                 |  5 +++++
 4 files changed, 19 insertions(+), 2 deletions(-)
 create mode 100755 .devcontainer/postCreate.sh

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 2835179..22e0c6b 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -14,7 +14,7 @@
 	// "forwardPorts": [],
 
 	// Use 'postCreateCommand' to run commands after the container is created.
-	"postCreateCommand": "npm install",
+	"postCreateCommand": "/bin/bash .devcontainer/postCreate.sh",
 
 	// Configure tool-specific properties.
 	"customizations": {
diff --git a/.devcontainer/postCreate.sh b/.devcontainer/postCreate.sh
new file mode 100755
index 0000000..1f1cee7
--- /dev/null
+++ b/.devcontainer/postCreate.sh
@@ -0,0 +1,12 @@
+#!/bin/env bash
+
+npm install
+
+git clone https://github.com/kubernetes-csi/csi-test /tmp/csi-test
+pushd /tmp/csi-test 
+make
+sudo cp /tmp/csi-test/cmd/csi-sanity/csi-sanity /usr/local/bin
+popd
+rm -rf /tmp/csi-test
+
+sudo apt update
\ No newline at end of file
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 81567cb..ecf7467 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -192,7 +192,7 @@ jobs:
       max-parallel: 1
       matrix:
         config:
-          - zfs-generic/iscsi-pcs.yaml
+          #- zfs-generic/iscsi-pcs.yaml # TODO: enable this once the server is setup
           - zfs-generic/iscsi-targetcli.yaml
           - zfs-generic/nfs.yaml
           - zfs-generic/smb.yaml
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a770271..f9965b6 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -29,6 +29,11 @@ To use the devcontainer:
 3. Click "Reopen in Container" when prompted
 4. Start coding with pre-configured environment
 
+> [!Note]
+> For `iSCSI` it's mandatory to use the Vagrant VM, due to the need of a kernel driver.
+> However for other tests the container is probably enough. It's possible to run the `dev/run.sh`
+> as explained below in the devcontainer and see if it's possible, before spinning up a full VM.
+
 #### 2. System Testing with Vagrant
 
 Vagrant provides a full virtual machine environment for:

From 536f0d487fe23f539e3dd0a24bf6d8583b0f0f2c Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Tue, 25 Mar 2025 07:52:43 +0000
Subject: [PATCH 07/16] CR fixes

---
 .gitignore           |  5 +++--
 CONTRIBUTING.md      |  4 ++--
 Vagrantfile          | 20 +++++++++++++++++++-
 {dev => hack}/run.sh |  0
 4 files changed, 24 insertions(+), 5 deletions(-)
 rename {dev => hack}/run.sh (100%)

diff --git a/.gitignore b/.gitignore
index b945e95..d19bbc7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 **~
 node_modules
-dev/*
+dev
 /ci/bin/*dev*
 .vagrant
-!dev/run.sh
\ No newline at end of file
+hack/*
+!hack/run.sh
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f9965b6..1da1090 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -31,7 +31,7 @@ To use the devcontainer:
 
 > [!Note]
 > For `iSCSI` it's mandatory to use the Vagrant VM, due to the need of a kernel driver.
-> However for other tests the container is probably enough. It's possible to run the `dev/run.sh`
+> However for other tests the container is probably enough. It's possible to run the `hack/run.sh`
 > as explained below in the devcontainer and see if it's possible, before spinning up a full VM.
 
 #### 2. System Testing with Vagrant
@@ -59,7 +59,7 @@ cd ~/democratic-csi
 # just configured for your own environment.
 # You can also create a file `dev/secrets.env` that has `export VARIABLE=VALUE`
 # and reference those in your `config.yaml`
-./dev/run.sh -c ./dev/config.yaml
+./hack/run.sh -c ./hack/config.yaml
 ```
 
 ##### Keeping Files in Sync
diff --git a/Vagrantfile b/Vagrantfile
index 294ceab..e4bf1e4 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -19,7 +19,25 @@ Vagrant.configure("2") do |config|
   
     config.vm.provision "shell", inline: <<-SHELL
       sudo apt-get update -y
-      sudo apt-get install -y open-iscsi nodejs git make
+
+      # for building dependecies and executing node
+      sudo apt-get install -y nodejs git make
+
+      # for app functionality
+      sudo apt-get install -y netbase socat e2fsprogs xfsprogs fatresize dosfstools nfs-common cifs-utils
+
+       # Install the following system packages
+      sudo apt-get install -y open-iscsi lsscsi sg3-utils multipath-tools scsitools nvme-cli
+
+      # Enable multipathing
+      sudo tee /etc/multipath.conf << EOF
+      defaults {
+          user_friendly_names yes
+          find_multipaths yes
+      }
+      EOF
+
+      sudo systemctl enable multipath-tools.service
   
       # Enable and start iscsid service
       sudo systemctl enable --now iscsid
diff --git a/dev/run.sh b/hack/run.sh
similarity index 100%
rename from dev/run.sh
rename to hack/run.sh

From 37b1726460aff45cb08b63bd2edb9696f8e2cead Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Tue, 25 Mar 2025 07:57:21 +0000
Subject: [PATCH 08/16] remove unneeded rm

---
 Vagrantfile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Vagrantfile b/Vagrantfile
index e4bf1e4..b69d2cf 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -77,7 +77,6 @@ Vagrant.configure("2") do |config|
       make csi-sanity
       sudo cp csi-sanity /usr/local/bin
       popd
-      #rm -rf /tmp/csi-test
     SHELL
   
     # Sync project directory for seamless workflow

From 490f93e665f2933e4a0ced289f571a3976e7dccc Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Tue, 25 Mar 2025 10:06:47 +0200
Subject: [PATCH 09/16] fix vagrant provisioning script

---
 Vagrantfile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Vagrantfile b/Vagrantfile
index b69d2cf..0d5c5d5 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -31,11 +31,11 @@ Vagrant.configure("2") do |config|
 
       # Enable multipathing
       sudo tee /etc/multipath.conf << EOF
-      defaults {
-          user_friendly_names yes
-          find_multipaths yes
-      }
-      EOF
+defaults {
+    user_friendly_names yes
+    find_multipaths yes
+}
+EOF
 
       sudo systemctl enable multipath-tools.service
   

From 0c74b90e01ff187647ba3f738010b7bfe993d4a4 Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Sat, 5 Apr 2025 09:12:47 +0000
Subject: [PATCH 10/16] Add build_push.sh

---
 .gitignore         |  3 ++-
 CONTRIBUTING.md    | 25 +++++++++++++++++++++++++
 hack/build_push.sh | 18 ++++++++++++++++++
 3 files changed, 45 insertions(+), 1 deletion(-)
 create mode 100644 hack/build_push.sh

diff --git a/.gitignore b/.gitignore
index d19bbc7..5bafd17 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,4 +4,5 @@ dev
 /ci/bin/*dev*
 .vagrant
 hack/*
-!hack/run.sh
\ No newline at end of file
+!hack/run.sh
+!hack/build_push.sh
\ No newline at end of file
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 1da1090..a0fef36 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -78,6 +78,31 @@ vagrant rsync
 vagrant rsync-auto
 ```
 
+#### 3. Deploy development version to K8s cluster
+
+Deployment provides a good environment for:
+- Final testing in a real world scenario
+- Run the final version until included in a release
+
+> [!Note]
+> Make sure to do the build on the architecture you will be running it.
+> For example, don't build in Apple Silicon if your cluster runs in amd64.
+
+
+1. Login to your github container registry
+```bash
+docker login ghcr.io
+```
+
+> [!Important]
+> Login to the container registry is stored plain text, use a PAT instead of your Github password. [Create a PAT with write:packages](https://github.com/settings/tokens/new?scopes=write:packages).
+
+2. Compile and push to your github container registry.
+```bash
+./hack/build_push.sh 
+```
+2. 
+
 ### Best Practices
 
 - Use devcontainer for day-to-day development and coding
diff --git a/hack/build_push.sh b/hack/build_push.sh
new file mode 100644
index 0000000..609bd28
--- /dev/null
+++ b/hack/build_push.sh
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+set -x
+
+ROOT_DIR="$(dirname "$(realpath "$0")")"
+
+GITHUB_USER=${GITHUB_USER:-$(jq -r '.auths."ghcr.io".auth' ~/.docker/config.json|base64 -d|cut -d':' -f1)}
+GITHUB_REPO=${GITHUB_REPO:-$(basename -s ${ROOT_DIR}/../.git $(git remote get-url origin))}
+DOCKER_TAG=${DOCKER_TAG:-$(git branch --show-current)-$(git rev-parse --short HEAD)}
+
+if [ -z "${GITHUB_USER}" ]; then
+  echo "Error: Need to login to ghcr.io ; execute docker login ghcr.io"
+  exit 1
+fi
+
+docker build $ROOT_DIR/.. --push -t ghcr.io/${GITHUB_USER}/${GITHUB_REPO}:${DOCKER_TAG}
+echo "Image pushed to ghcr.io/${GITHUB_USER}/${GITHUB_REPO}:${DOCKER_TAG}"
\ No newline at end of file

From 0f04c7733010dc8dca2af48284ec1d1fdd1be60e Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Sat, 5 Apr 2025 10:45:38 +0000
Subject: [PATCH 11/16] small typo

---
 hack/build_push.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/build_push.sh b/hack/build_push.sh
index 609bd28..2aa4e2e 100644
--- a/hack/build_push.sh
+++ b/hack/build_push.sh
@@ -6,7 +6,7 @@ set -x
 ROOT_DIR="$(dirname "$(realpath "$0")")"
 
 GITHUB_USER=${GITHUB_USER:-$(jq -r '.auths."ghcr.io".auth' ~/.docker/config.json|base64 -d|cut -d':' -f1)}
-GITHUB_REPO=${GITHUB_REPO:-$(basename -s ${ROOT_DIR}/../.git $(git remote get-url origin))}
+GITHUB_REPO=${GITHUB_REPO:-$(basename -s .git $(git remote get-url origin))}
 DOCKER_TAG=${DOCKER_TAG:-$(git branch --show-current)-$(git rev-parse --short HEAD)}
 
 if [ -z "${GITHUB_USER}" ]; then

From b1adbcd854f8f03912a281193e4b1606ea482f78 Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Sat, 5 Apr 2025 11:27:31 +0000
Subject: [PATCH 12/16] fixed doc

---
 CONTRIBUTING.md | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a0fef36..9eabd8a 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -101,7 +101,16 @@ docker login ghcr.io
 ```bash
 ./hack/build_push.sh 
 ```
-2. 
+
+3. When you deploy, in the `values.yaml` add the following, using the output from the script
+```yaml
+controller:
+  driver:
+    image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4
+node:
+  driver:
+    image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4
+``` 
 
 ### Best Practices
 

From 126ab5356c46a94f1569a8535596c55c32d10bfb Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Sat, 5 Apr 2025 11:43:52 +0000
Subject: [PATCH 13/16] explain to put visibility public to package

---
 CONTRIBUTING.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 9eabd8a..0ae44c5 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -112,6 +112,14 @@ node:
     image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4
 ``` 
 
+4. Make the Image Public
+
+   By default, images pushed to GHCR are private. To make it public:
+   1. Go to GitHub → Your Repository → Packages (or directly github.com/USERNAME?tab=packages)
+   2. Select the package
+   3. Click Package Settings
+   4. Change Visibility to Public
+
 ### Best Practices
 
 - Use devcontainer for day-to-day development and coding

From ff206518edd7e4cdb290e226846c0a5f00ce9969 Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Thu, 1 May 2025 10:09:02 +0000
Subject: [PATCH 14/16] add force to target creation to allow multiple
 non-unique usernames

---
 src/driver/controller-zfs-generic/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js
index 9e8d8c2..b234078 100644
--- a/src/driver/controller-zfs-generic/index.js
+++ b/src/driver/controller-zfs-generic/index.js
@@ -309,7 +309,7 @@ create /backstores/block/${assetName}
 
               let extraTerms = ['group', `${pcs_group}`, '--wait']; // The wait is important to avoid race conditions
               let createTargetTerms = [
-                'resource', 'create', '--future', `target-${assetName}`, 'ocf:heartbeat:iSCSITarget',
+                'resource', 'create', '--future', '--force', `target-${assetName}`, 'ocf:heartbeat:iSCSITarget',
                 'implementation="lio-t"', 'portals=":::3260"', `iqn="${basename}:${assetName}"`
               ];
 

From 822889ed1d5491109e95e3d2355a99b51026cf86 Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Thu, 1 May 2025 15:09:19 +0300
Subject: [PATCH 15/16] make build_push.sh executable

---
 hack/build_push.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 hack/build_push.sh

diff --git a/hack/build_push.sh b/hack/build_push.sh
old mode 100644
new mode 100755

From 00c025df8e51b5c010e88b1e15eb875051db940c Mon Sep 17 00:00:00 2001
From: Michel Peterson <michelp@microsoft.com>
Date: Sun, 4 Jan 2026 08:53:55 +0000
Subject: [PATCH 16/16] update devcontainer image, enhance contributing guide,
 and implement mutex for PCS commands

---
 .devcontainer/devcontainer.json            |  4 ++-
 CONTRIBUTING.md                            |  4 +++
 Vagrantfile                                |  3 ++
 src/driver/controller-zfs-generic/index.js | 38 ++++++++++++----------
 4 files changed, 31 insertions(+), 18 deletions(-)

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
index 22e0c6b..fd97ce9 100644
--- a/.devcontainer/devcontainer.json
+++ b/.devcontainer/devcontainer.json
@@ -3,7 +3,9 @@
 {
 	"name": "democratic-csi",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm",
+	// regarding versioning the first number is the devcontainer image version,
+	// the second is the Node.js version, and the third is the OS version.
+	"image": "mcr.microsoft.com/devcontainers/typescript-node:4-20-bookworm",
 
 	// Features to add to the dev container. More info: https://containers.dev/features.
 	"features": {
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 0ae44c5..277b4f8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -62,6 +62,10 @@ cd ~/democratic-csi
 ./hack/run.sh -c ./hack/config.yaml
 ```
 
+>![Note]
+> For running tests with democratic-csi the authentication needs to be disabled, as
+> it always initiates connections to the share without authentication.
+
 ##### Keeping Files in Sync
 
 Use these methods to keep your local files synchronized with the Vagrant VM:
diff --git a/Vagrantfile b/Vagrantfile
index 0d5c5d5..0238103 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -18,6 +18,9 @@ Vagrant.configure("2") do |config|
     end
   
     config.vm.provision "shell", inline: <<-SHELL
+      # force version 20.x of nodejs
+      curl -fsSL https://deb.nodesource.com/setup_20.x | sudo -E bash -
+
       sudo apt-get update -y
 
       # for building dependecies and executing node
diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js
index b234078..084b440 100644
--- a/src/driver/controller-zfs-generic/index.js
+++ b/src/driver/controller-zfs-generic/index.js
@@ -20,6 +20,7 @@ class ControllerZfsGenericDriver extends ControllerZfsBaseDriver {
     this.targetCliMutex = new Mutex();
     this.nvmetCliMutex = new Mutex();
     this.spdkCliMutex = new Mutex();
+    this.pcsMutex = new Mutex();
   }
 
   getExecClient() {
@@ -1000,24 +1001,27 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
     let options = {
       pty: true,
     };
-    let response = await execClient.exec(
-      execClient.buildCommand(command, args),
-      options
-    );
-    driver.ctx.logger.verbose(
-      "pcs response: " + JSON.stringify(response)
-    );
-        
-    // Handle idempotence for create commands
-    if (response.code == 1 && response.stdout.includes("already exists")) {
-      driver.ctx.logger.verbose("pcs resource already exists, ignoring error (setting response.code=0)");
-      response.code = 0;
-    }
 
-    if (response.code != 0) {
-      throw response;
-    }
-    return response;
+    return driver.pcsMutex.runExclusive(async () => {
+      let response = await execClient.exec(
+        execClient.buildCommand(command, args),
+        options
+      );
+      driver.ctx.logger.verbose(
+        "pcs response: " + JSON.stringify(response)
+      );
+          
+      // Handle idempotence for create commands
+      if (response.code == 1 && response.stdout.includes("already exists")) {
+        driver.ctx.logger.verbose("pcs resource already exists, ignoring error (setting response.code=0)");
+        response.code = 0;
+      }
+
+      if (response.code != 0) {
+        throw response;
+      }
+      return response;
+    });
   }
 
   async targetCliCommand(data) {