Merge 0d928774ce into 73af26298c

Support pcs for zfs-generic to configure iSCSI
targets and LUNs. It uses the targetcli implementation
since it's the default pcs resources and also to
keep compatibility with the rest of the code.

closes #462

.devcontainer/devcontainer.json

@@ -0,0 +1,38 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node
+{
+	"name": "democratic-csi",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/go:1": {} // To compile csi-sanity during postCreateCommand
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": "/bin/bash .devcontainer/postCreate.sh",
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"settings": {
+				"terminal.integrated.shell.linux": "/bin/bash"
+			},
+			"extensions": [
+				"dbaeumer.vscode-eslint",
+				"esbenp.prettier-vscode",
+				"waderyan.nodejs-extension-pack",
+				"ms-vscode.node-debug2",
+				"christian-kohler.npm-intellisense",
+				"christian-kohler.path-intellisense",
+				"HashiCorp.terraform"
+			]
+		}
+	}
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.devcontainer/postCreate.sh

@@ -0,0 +1,12 @@
+#!/bin/env bash
+
+npm install
+
+git clone https://github.com/kubernetes-csi/csi-test /tmp/csi-test
+pushd /tmp/csi-test 
+make
+sudo cp /tmp/csi-test/cmd/csi-sanity/csi-sanity /usr/local/bin
+popd
+rm -rf /tmp/csi-test
+
+sudo apt update

.github/dependabot.yml

@@ -0,0 +1,12 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for more information:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+# https://containers.dev/guide/dependabot
+
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly

.github/workflows/main.yml

@@ -191,7 +191,8 @@ jobs:
       max-parallel: 1
       matrix:
         config:
-          - zfs-generic/iscsi.yaml
+          #- zfs-generic/iscsi-pcs.yaml # TODO: enable this once the server is setup
+          - zfs-generic/iscsi-targetcli.yaml
           - zfs-generic/nfs.yaml
           - zfs-generic/smb.yaml
           - zfs-generic/nvmeof.yaml

.gitignore

@@ -2,3 +2,7 @@
 node_modules
 dev
 /ci/bin/*dev*
+.vagrant
+hack/*
+!hack/run.sh
+!hack/build_push.sh

CONTRIBUTING.md

@@ -0,0 +1,155 @@
+# Contributing to democratic-csi
+
+## Development Environment Setup
+
+This project uses a hybrid development approach with devcontainers for IDE configuration and Vagrant for system-level testing.
+
+### Prerequisites
+
+Before you begin, ensure you have the following installed:
+- [Visual Studio Code](https://code.visualstudio.com/)
+- [Docker](https://www.docker.com/get-started)
+- [Vagrant](https://www.vagrantup.com/downloads)
+- Virtualization Provider:
+  - For Intel/AMD Machines: VirtualBox
+  - For Apple Silicon: Qemu (`brew install qemu vagrant` and `vagrant plugin install vagrant-qemu`)
+
+### Development Workflow
+
+#### 1. Local Development with Devcontainers
+
+Devcontainers provide a consistent development environment with:
+- Configured VSCode extensions
+- Necessary development tools
+- Integrated development experience
+
+To use the devcontainer:
+1. Open the project in VSCode
+2. Install the "Dev Containers" extension
+3. Click "Reopen in Container" when prompted
+4. Start coding with pre-configured environment
+
+> [!Note]
+> For `iSCSI` it's mandatory to use the Vagrant VM, due to the need of a kernel driver.
+> However for other tests the container is probably enough. It's possible to run the `hack/run.sh`
+> as explained below in the devcontainer and see if it's possible, before spinning up a full VM.
+
+#### 2. System Testing with Vagrant
+
+Vagrant provides a full virtual machine environment for:
+- System-level testing
+- Running code with kernel dependencies
+- Simulating production-like environments
+
+Workflow:
+```bash
+# Navigate to project directory
+cd ~/democratic-csi
+
+# Start the Vagrant VM
+vagrant up
+
+# Connect to the VM
+vagrant ssh
+
+# Inside the VM, navigate to the project
+cd ~/democratic-csi
+
+# Run project tests, the config.yaml can be any from the examples folders
+# just configured for your own environment.
+# You can also create a file `dev/secrets.env` that has `export VARIABLE=VALUE`
+# and reference those in your `config.yaml`
+./hack/run.sh -c ./hack/config.yaml
+```
+
+##### Keeping Files in Sync
+
+Use these methods to keep your local files synchronized with the Vagrant VM:
+
+###### Manual Sync
+```bash
+# Sync files from local to Vagrant VM
+vagrant rsync
+```
+
+###### Continuous Sync
+```bash
+# Automatically sync files as they change
+vagrant rsync-auto
+```
+
+#### 3. Deploy development version to K8s cluster
+
+Deployment provides a good environment for:
+- Final testing in a real world scenario
+- Run the final version until included in a release
+
+> [!Note]
+> Make sure to do the build on the architecture you will be running it.
+> For example, don't build in Apple Silicon if your cluster runs in amd64.
+
+
+1. Login to your github container registry
+```bash
+docker login ghcr.io
+```
+
+> [!Important]
+> Login to the container registry is stored plain text, use a PAT instead of your Github password. [Create a PAT with write:packages](https://github.com/settings/tokens/new?scopes=write:packages).
+
+2. Compile and push to your github container registry.
+```bash
+./hack/build_push.sh 
+```
+
+3. When you deploy, in the `values.yaml` add the following, using the output from the script
+```yaml
+controller:
+  driver:
+    image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4
+node:
+  driver:
+    image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4
+``` 
+
+4. Make the Image Public
+
+   By default, images pushed to GHCR are private. To make it public:
+   1. Go to GitHub → Your Repository → Packages (or directly github.com/USERNAME?tab=packages)
+   2. Select the package
+   3. Click Package Settings
+   4. Change Visibility to Public
+
+### Best Practices
+
+- Use devcontainer for day-to-day development and coding
+- Use Vagrant for comprehensive system testing
+- Always run `vagrant rsync` before running tests in the VM
+- Commit and push changes frequently
+- If encountering issues, try:
+  1. Recreating the devcontainer
+  2. Reprovisioning the Vagrant VM with `vagrant reload --provision` or `vagrant destroy -f && vagrant up`
+
+### Troubleshooting
+
+#### Devcontainer Issues
+- Ensure Docker is running
+- Rebuild the container if extensions fail to load
+- Check VSCode Dev Containers extension logs
+
+#### Vagrant Issues
+- Verify virtualization is enabled in your BIOS
+- Ensure you have the latest Vagrant and virtualization provider
+- For Apple Silicon, use Parallels or Lima
+
+### Contribution Guidelines
+
+1. Create a new branch for your feature targetting `next`
+2. Write clear, concise commit messages
+3. Include coverage for tests of csi-sanity for new functionality
+4. Run tests in Vagrant VM
+5. Submit a pull request with a clear description of changes
+
+### Contact
+
+For any questions or issues, please [open an issue](https://github.com/democratic-csi/democratic-csi/issues) on the project repository.

Vagrantfile

@@ -0,0 +1,88 @@
+Vagrant.configure("2") do |config|
+    # Check the host's architecture
+    host_arch = `uname -m`.strip
+
+    # Use a different box for ARM vs x86_64
+    if host_arch == "arm64"
+        # requires qemu, install qemu and then:
+        # vagrant plugin install vagrant-qemu
+        config.vm.box = "perk/ubuntu-24.04-arm64" 
+    else
+        # Use the x86_64 compatible Ubuntu box
+        config.vm.box = "ubuntu/jammy64"
+    end
+  
+    config.vm.provider "virtualbox" do |vb|
+      vb.memory = "2048"
+      vb.cpus = 2
+    end
+  
+    config.vm.provision "shell", inline: <<-SHELL
+      sudo apt-get update -y
+
+      # for building dependecies and executing node
+      sudo apt-get install -y nodejs git make
+
+      # for app functionality
+      sudo apt-get install -y netbase socat e2fsprogs xfsprogs fatresize dosfstools nfs-common cifs-utils
+
+       # Install the following system packages
+      sudo apt-get install -y open-iscsi lsscsi sg3-utils multipath-tools scsitools nvme-cli
+
+      # Enable multipathing
+      sudo tee /etc/multipath.conf << EOF
+defaults {
+    user_friendly_names yes
+    find_multipaths yes
+}
+EOF
+
+      sudo systemctl enable multipath-tools.service
+  
+      # Enable and start iscsid service
+      sudo systemctl enable --now iscsid
+  
+      # Verify installation
+      systemctl status iscsid --no-pager
+
+      ####
+      # Install golang
+      ####
+      GO_VERSION="1.24.1"
+      ARCH=$(uname -m)
+      GO_TAR_URL=""
+
+      if [[ "$ARCH" == "aarch64" ]]; then
+          GO_TAR_URL="https://go.dev/dl/go${GO_VERSION}.linux-arm64.tar.gz"
+      elif [[ "$ARCH" == "x86_64" ]]; then
+          GO_TAR_URL="https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz"
+      else
+          echo "Unsupported architecture: $ARCH"
+          exit 1
+      fi
+
+      echo "Downloading Go version $GO_VERSION for $ARCH..."
+      wget -q "$GO_TAR_URL" -O go.tar.gz
+      tar -C /usr/local -xzf go.tar.gz
+      rm go.tar.gz
+      echo "export PATH=\$PATH:/usr/local/go/bin" >> /etc/profile
+      source /etc/profile
+
+      ####
+      # Install csi-test
+      ####
+      echo "Installing csi-test"
+      git clone https://github.com/kubernetes-csi/csi-test /tmp/csi-test
+      pushd /tmp/csi-test/cmd/csi-sanity
+      make csi-sanity
+      sudo cp csi-sanity /usr/local/bin
+      popd
+    SHELL
+  
+    # Sync project directory for seamless workflow
+    config.vm.synced_folder ".", "/home/vagrant/democratic-csi", type: "rsync",
+        rsync__exclude: ".git/"
+    
+    # Allow SSH access with default key
+    config.ssh.insert_key = false
+  end  

ci/bin/run.sh

@@ -20,7 +20,7 @@ if [[ -f "node_modules-linux-amd64.tar.gz" && ! -d "node_modules" ]];then
 fi
 
 # generate key for paths etc
-export CI_BUILD_KEY=$(uuidgen | cut -d "-" -f 1)
+export CI_BUILD_KEY=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8)
 
 # launch the server
 sudo -E ci/bin/launch-server.sh &

ci/configs/zfs-generic/iscsi-pcs.yaml

@@ -0,0 +1,28 @@
+driver: zfs-generic-iscsi
+
+sshConnection:
+  host: ${SERVER_HOST}
+  port: 22
+  username: ${SERVER_USERNAME}
+  password: ${SERVER_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize:
+
+iscsi:
+  targetPortal: ${SERVER_HOST}
+  interface: ""
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+  shareStrategy: "pcs"
+  shareStrategyPcs:
+    pcs_group: "group-nas"
+    basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664"
+    auth:
+      enabled: 0

examples/zfs-generic-iscsi.yaml

@@ -44,6 +44,7 @@ zfs:
 
 iscsi:
   shareStrategy: "targetCli"
+  #shareStrategy: "pcs"
 
   # https://kifarunix.com/how-to-install-and-configure-iscsi-storage-server-on-ubuntu-18-04/
   # https://kifarunix.com/how-install-and-configure-iscsi-storage-server-on-centos-7/
@@ -75,6 +76,17 @@ iscsi:
       attributes:
         # set to 1 to enable Thin Provisioning Unmap
         emulate_tpu: 0
+
+  shareStrategyPcs:
+    #sudoEnabled: true
+    pcs_group: "group-nas"
+    basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664"
+    auth:
+      enabled: 0
+      # CHAP – incoming only, mutual not supported by the Pacemaker resource agent
+      incoming_username: "foo"
+      incoming_password: "bar"
+
   targetPortal: "server[:port]"
   # for multipath
   targetPortals: [] # [ "server[:port]", "server[:port]", ... ]

hack/build_push.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+set -x
+
+ROOT_DIR="$(dirname "$(realpath "$0")")"
+
+GITHUB_USER=${GITHUB_USER:-$(jq -r '.auths."ghcr.io".auth' ~/.docker/config.json|base64 -d|cut -d':' -f1)}
+GITHUB_REPO=${GITHUB_REPO:-$(basename -s .git $(git remote get-url origin))}
+DOCKER_TAG=${DOCKER_TAG:-$(git branch --show-current)-$(git rev-parse --short HEAD)}
+
+if [ -z "${GITHUB_USER}" ]; then
+  echo "Error: Need to login to ghcr.io ; execute docker login ghcr.io"
+  exit 1
+fi
+
+docker build $ROOT_DIR/.. --push -t ghcr.io/${GITHUB_USER}/${GITHUB_REPO}:${DOCKER_TAG}
+echo "Image pushed to ghcr.io/${GITHUB_USER}/${GITHUB_REPO}:${DOCKER_TAG}"

hack/run.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+set -e
+set -x
+TEMPLATE_CONFIG=""
+
+ROOT_DIR="$(dirname "$(realpath "$0")")"
+
+while [[ "$#" -gt 0 ]]; do
+  case $1 in
+    -c|--config) TEMPLATE_CONFIG="$(realpath "$2")"; shift ;;
+    *) echo "Unknown parameter passed: $1"; exit 1 ;;
+  esac
+  shift
+done
+
+if [ -z "${TEMPLATE_CONFIG}" ]; then
+  echo "Error: --config or -c parameter is required."
+  exit 1
+fi
+
+if [ ! -f $ROOT_DIR/secrets.env ]; then
+  echo "Error: secrets.env file not found."
+  exit 1
+fi
+
+source $ROOT_DIR/secrets.env # needs to have exported variables
+
+# generate key for paths etc
+export CI_BUILD_KEY=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8)
+
+export TEMPLATE_CONFIG_FILE=${TEMPLATE_CONFIG}
+
+$ROOT_DIR/../ci/bin/run.sh

src/driver/controller-zfs-generic/index.js

@@ -297,8 +297,66 @@ create /backstores/block/${assetName}
               }
             );
             break;
+          case "pcs":
+              basename = this.options.iscsi.shareStrategyPcs.basename;
+              let pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group;
+
+              let extraTerms = ['group', `${pcs_group}`, '--wait']; // The wait is important to avoid race conditions
+              let createTargetTerms = [
+                'resource', 'create', '--future', '--force', `target-${assetName}`, 'ocf:heartbeat:iSCSITarget',
+                'implementation="lio-t"', 'portals=":::3260"', `iqn="${basename}:${assetName}"`
+              ];
+
+              if (this.options.iscsi.shareStrategyPcs.auth.enabled) {
+                createTargetTerms.push(`incoming_username="${this.options.iscsi.shareStrategyPcs.auth.incoming_username}"`);
+                createTargetTerms.push(`incoming_password="${this.options.iscsi.shareStrategyPcs.auth.incoming_password}"`);
+              }
+
+              await GeneralUtils.retry(
+                3,
+                2000,
+                async () => {
+                  await this.pcsCommand(createTargetTerms.concat(extraTerms));
+                },
+                {
+                  retryCondition: (err) => {
+                    if (err.stdout && err.stdout.includes("Timed Out")) {
+                      return true;
+                    }
+                    return false;
+                  },
+                }
+              );
+
+              let createLunTerms = [
+                'resource', 'create', '--future', `lun-${assetName}`, 'ocf:heartbeat:iSCSILogicalUnit',
+                'implementation="lio-t"', `target_iqn="${basename}:${assetName}"`, 'lun="0"',
+                `path="/dev/${extentDiskName}"`
+              ];
+
+              await GeneralUtils.retry(
+                3,
+                2000,
+                async () => {
+                  await this.pcsCommand(createLunTerms.concat(extraTerms));
+                },
+                {
+                  retryCondition: (err) => {
+                    if (err.stdout && err.stdout.includes("Timed Out")) {
+                      return true;
+                    }
+                    return false;
+                  },
+                }
+              );
+  
+              break;
+
           default:
-            break;
+            throw new GrpcError(
+              grpc.status.FAILED_PRECONDITION,
+              `invalid configuration: unknown shareStrategy ${this.options.iscsi.shareStrategy}`
+            );
         }
 
         // iqn = target
@@ -689,8 +747,54 @@ delete ${assetName}
             );
 
             break;
-          default:
+          case "pcs":
+            let deleteLunText = [
+              'resource', 'delete', `lun-${assetName}`
+            ];
+
+            await GeneralUtils.retry(
+              3,
+              2000,
+              async () => {
+                await this.pcsCommand(deleteLunText);
+              },
+              {
+                retryCondition: (err) => {
+                  if (err.stdout && err.stdout.includes("Timed Out")) {
+                    return true;
+                  }
+                  return false;
+                },
+              }
+            );
+
+            let deleteTargetText = [
+              'resource', 'delete', `target-${assetName}`
+            ];
+
+            await GeneralUtils.retry(
+              3,
+              2000,
+              async () => {
+                await this.pcsCommand(deleteTargetText);
+              },
+              {
+                retryCondition: (err) => {
+                  if (err.stdout && err.stdout.includes("Timed Out")) {
+                    return true;
+                  }
+                  return false;
+                },
+              }
+            );
+
             break;
+
+          default:
+            throw new GrpcError(
+              grpc.status.FAILED_PRECONDITION,
+              `invalid configuration: unknown shareStrategy ${this.options.iscsi.shareStrategy}`
+            );
         }
         break;
       }
@@ -840,6 +944,9 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
           case "targetCli":
             // nothing required, just need to rescan on the node
             break;
+          case "pcs":
+            // nothing required, just need to rescan on the node
+            break;
           default:
             break;
         }
@@ -850,6 +957,63 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath}
     }
   }
 
+  async pcsCommand(commandTerms) {
+    const execClient = this.getExecClient();
+    const driver = this;
+
+    let command = "sh";
+    let args = ["-c"];
+
+    let cliArgs = ["pcs"];
+    if (
+      _.get(this.options, "iscsi.shareStrategyPcs.sudoEnabled", false)
+    ) {
+      cliArgs.unshift("sudo");
+    }
+
+    let cliCommand = [];
+    cliCommand.push(cliArgs.join(" "));
+    cliCommand.push(commandTerms.join(" "));
+    args.push("'" + cliCommand.join(" ") + "'");
+
+    let logCommandTmp = command + " " + args.join(" ");
+    let logCommand = "";
+
+    logCommandTmp.split(" ").forEach((term) => {
+      logCommand += " ";
+      
+      if (term.startsWith("incoming_password=")) {
+        logCommand += "incoming_password=<redacted>";
+      } else {
+        logCommand += term;
+      }
+    });
+
+    driver.ctx.logger.verbose("pcs command:" + logCommand);
+
+    let options = {
+      pty: true,
+    };
+    let response = await execClient.exec(
+      execClient.buildCommand(command, args),
+      options
+    );
+    driver.ctx.logger.verbose(
+      "pcs response: " + JSON.stringify(response)
+    );
+        
+    // Handle idempotence for create commands
+    if (response.code == 1 && response.stdout.includes("already exists")) {
+      driver.ctx.logger.verbose("pcs resource already exists, ignoring error (setting response.code=0)");
+      response.code = 0;
+    }
+
+    if (response.code != 0) {
+      throw response;
+    }
+    return response;
+  }
+
   async targetCliCommand(data) {
     const execClient = this.getExecClient();
     const driver = this;