From 9252b4e779e7ad10247da7f2075befb42270f132 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Sat, 22 Mar 2025 19:36:00 +0200 Subject: [PATCH 01/15] Add iSCSI targets/LUNs through Pacemaker clusters Support pcs for zfs-generic to configure iSCSI targets and LUNs. It uses the targetcli implementation since it's the default pcs resources and also to keep compatibility with the rest of the code. closes #462 --- examples/zfs-generic-iscsi.yaml | 12 ++ src/driver/controller-zfs-generic/index.js | 165 ++++++++++++++++++++- 2 files changed, 175 insertions(+), 2 deletions(-) diff --git a/examples/zfs-generic-iscsi.yaml b/examples/zfs-generic-iscsi.yaml index af5df37..f8aa270 100644 --- a/examples/zfs-generic-iscsi.yaml +++ b/examples/zfs-generic-iscsi.yaml @@ -44,6 +44,7 @@ zfs: iscsi: shareStrategy: "targetCli" + #shareStrategy: "pcs" # https://kifarunix.com/how-to-install-and-configure-iscsi-storage-server-on-ubuntu-18-04/ # https://kifarunix.com/how-install-and-configure-iscsi-storage-server-on-centos-7/ @@ -75,6 +76,17 @@ iscsi: attributes: # set to 1 to enable Thin Provisioning Unmap emulate_tpu: 0 + + shareStrategyPcs: + #sudoEnabled: true + pcs_group: "group-nas" + basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664" + auth: + enabled: 0 + # CHAP – incoming only, mutual not supported by the Pacemaker resource agent + incoming_username: "foo" + incoming_password: "bar" + targetPortal: "server[:port]" # for multipath targetPortals: [] # [ "server[:port]", "server[:port]", ... ] diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index d3080ae..380b674 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -298,8 +298,70 @@ create /backstores/block/${assetName} } ); break; + case "pcs": + basename = this.options.iscsi.shareStrategyPcs.basename; + pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group; + + let groupText = `group ${pcs_group}`; + let createTargetText = [ + `resource create --future target-${assetName} iSCSITarget`, + 'implementation="lio-t"', + `iqn="${basename}:${assetName}"` + ]; + + if (this.options.iscsi.shareStrategyPcs.auth.enabled) { + createTargetText.push(`incoming_username="${this.options.iscsi.shareStrategyPcs.auth.incoming_username}"`); + createTargetText.push(`incoming_password="${this.options.iscsi.shareStrategyPcs.auth.incoming_password}"`); + } + + createTargetText.push(groupText); + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.pcsCommand(createTargetText); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Timed Out")) { + return true; + } + return false; + }, + } + ); + + let createLunText = [ + `resource create --future lun-${assetName} iSCSILogicalUnit`, + 'implementation="lio-t"', + `target_iqn="${basename}:${assetName}" lun="1"`, + `path="/dev/${extentDiskName}"`, + groupText + ]; + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.pcsCommand(createLunText); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Timed Out")) { + return true; + } + return false; + }, + } + ); + break; + default: - break; + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown shareStrategy ${this.options.iscsi.shareStrategy}` + ); } // iqn = target @@ -690,8 +752,54 @@ delete ${assetName} ); break; - default: + case "pcs": + let deleteLunText = [ + `resource delete lun-${assetName}` + ]; + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.pcsCommand(deleteLunText); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Timed Out")) { + return true; + } + return false; + }, + } + ); + + let deleteTargetText = [ + `resource delete target-${assetName}` + ]; + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.pcsCommand(deleteTargetText); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Timed Out")) { + return true; + } + return false; + }, + } + ); + break; + + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown shareStrategy ${this.options.iscsi.shareStrategy}` + ); } break; } @@ -841,6 +949,9 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} case "targetCli": // nothing required, just need to rescan on the node break; + case "pcs": + // nothing required, just need to rescan on the node + break; default: break; } @@ -851,6 +962,56 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} } } + async pcsCommand(commandLines) { + const execClient = this.getExecClient(); + const driver = this; + + let command = "sh"; + let args = ["-c"]; + + let cliArgs = ["pcs"]; + if ( + _.get(this.options, "iscsi.shareStrategyPcs.sudoEnabled", false) + ) { + cliArgs.unshift("sudo"); + } + + let cliCommand = []; + cliCommand.push(cliArgs.join(" ")); + cliCommand.push(commandLines.join(" ")); + args.push("'" + cliCommand.join(" ") + "'"); + + let logCommandTmp = command + " " + args.join(" "); + let logCommand = ""; + + logCommandTmp.split(" ").forEach((term) => { + logCommand += " "; + + if (term.startsWith("incoming_password=")) { + logCommand += "incoming_password="; + } else { + logCommand += term; + } + }); + + driver.ctx.logger.verbose("Pcs command:" + logCommand); + + let options = { + pty: true, + }; + let response = await execClient.exec( + execClient.buildCommand(command, args), + options + ); + driver.ctx.logger.verbose( + "Pcs response: " + JSON.stringify(response) + ); + if (response.code != 0) { + throw response; + } + return response; + } + async targetCliCommand(data) { const execClient = this.getExecClient(); const driver = this; From da04033abc96f33ce3fb3b72fff96d6a3a7ef1e7 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Sun, 23 Mar 2025 10:48:49 +0200 Subject: [PATCH 02/15] Small CR and preparatory work for CI --- .github/workflows/main.yml | 3 +- ci/configs/zfs-generic/iscsi-pcs.yaml | 28 +++++++++++++ .../{iscsi.yaml => iscsi-targetcli.yaml} | 0 src/driver/controller-zfs-generic/index.js | 41 ++++++++----------- 4 files changed, 48 insertions(+), 24 deletions(-) create mode 100644 ci/configs/zfs-generic/iscsi-pcs.yaml rename ci/configs/zfs-generic/{iscsi.yaml => iscsi-targetcli.yaml} (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 47b6c73..f9b1bc8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -191,7 +191,8 @@ jobs: max-parallel: 1 matrix: config: - - zfs-generic/iscsi.yaml + - zfs-generic/iscsi-pcs.yaml + - zfs-generic/iscsi-targetcli.yaml - zfs-generic/nfs.yaml - zfs-generic/smb.yaml - zfs-generic/nvmeof.yaml diff --git a/ci/configs/zfs-generic/iscsi-pcs.yaml b/ci/configs/zfs-generic/iscsi-pcs.yaml new file mode 100644 index 0000000..baabb69 --- /dev/null +++ b/ci/configs/zfs-generic/iscsi-pcs.yaml @@ -0,0 +1,28 @@ +driver: zfs-generic-iscsi + +sshConnection: + host: ${SERVER_HOST} + port: 22 + username: ${SERVER_USERNAME} + password: ${SERVER_PASSWORD} + +zfs: + datasetParentName: tank/ci/${CI_BUILD_KEY}/v + detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s + + zvolCompression: + zvolDedup: + zvolEnableReservation: false + zvolBlocksize: + +iscsi: + targetPortal: ${SERVER_HOST} + interface: "" + namePrefix: "csi-ci-${CI_BUILD_KEY}-" + nameSuffix: "" + shareStrategy: "pcs" + shareStrategyPcs: + pcs_group: "group-nas" + basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664" + auth: + enabled: 0 diff --git a/ci/configs/zfs-generic/iscsi.yaml b/ci/configs/zfs-generic/iscsi-targetcli.yaml similarity index 100% rename from ci/configs/zfs-generic/iscsi.yaml rename to ci/configs/zfs-generic/iscsi-targetcli.yaml diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index 380b674..2ec3965 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -302,25 +302,22 @@ create /backstores/block/${assetName} basename = this.options.iscsi.shareStrategyPcs.basename; pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group; - let groupText = `group ${pcs_group}`; - let createTargetText = [ - `resource create --future target-${assetName} iSCSITarget`, - 'implementation="lio-t"', - `iqn="${basename}:${assetName}"` + let groupTerms = ['group', `${pcs_group}`]; + let createTargetTerms = [ + 'resource', 'create', '--future', `target-${assetName}`, 'iSCSITarget', + 'implementation="lio-t"', `iqn="${basename}:${assetName}"` ]; if (this.options.iscsi.shareStrategyPcs.auth.enabled) { - createTargetText.push(`incoming_username="${this.options.iscsi.shareStrategyPcs.auth.incoming_username}"`); - createTargetText.push(`incoming_password="${this.options.iscsi.shareStrategyPcs.auth.incoming_password}"`); + createTargetTerms.push(`incoming_username="${this.options.iscsi.shareStrategyPcs.auth.incoming_username}"`); + createTargetTerms.push(`incoming_password="${this.options.iscsi.shareStrategyPcs.auth.incoming_password}"`); } - createTargetText.push(groupText); - await GeneralUtils.retry( 3, 2000, async () => { - await this.pcsCommand(createTargetText); + await this.pcsCommand(createTargetTerms.concat(groupTerms)); }, { retryCondition: (err) => { @@ -332,19 +329,17 @@ create /backstores/block/${assetName} } ); - let createLunText = [ - `resource create --future lun-${assetName} iSCSILogicalUnit`, - 'implementation="lio-t"', - `target_iqn="${basename}:${assetName}" lun="1"`, - `path="/dev/${extentDiskName}"`, - groupText + let createLunTerms = [ + 'resource', 'create', '--future', `lun-${assetName}`, 'iSCSILogicalUnit', + 'implementation="lio-t"', `target_iqn="${basename}:${assetName}" lun="1"`, + `path="/dev/${extentDiskName}"` ]; await GeneralUtils.retry( 3, 2000, async () => { - await this.pcsCommand(createLunText); + await this.pcsCommand(createLunTerms.concat(groupTerms)); }, { retryCondition: (err) => { @@ -754,7 +749,7 @@ delete ${assetName} break; case "pcs": let deleteLunText = [ - `resource delete lun-${assetName}` + 'resource', 'delete', `lun-${assetName}` ]; await GeneralUtils.retry( @@ -774,7 +769,7 @@ delete ${assetName} ); let deleteTargetText = [ - `resource delete target-${assetName}` + 'resource', 'delete', `target-${assetName}` ]; await GeneralUtils.retry( @@ -962,7 +957,7 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} } } - async pcsCommand(commandLines) { + async pcsCommand(commandTerms) { const execClient = this.getExecClient(); const driver = this; @@ -978,7 +973,7 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} let cliCommand = []; cliCommand.push(cliArgs.join(" ")); - cliCommand.push(commandLines.join(" ")); + cliCommand.push(commandTerms.join(" ")); args.push("'" + cliCommand.join(" ") + "'"); let logCommandTmp = command + " " + args.join(" "); @@ -994,7 +989,7 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} } }); - driver.ctx.logger.verbose("Pcs command:" + logCommand); + driver.ctx.logger.verbose("pcs command:" + logCommand); let options = { pty: true, @@ -1004,7 +999,7 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} options ); driver.ctx.logger.verbose( - "Pcs response: " + JSON.stringify(response) + "pcs response: " + JSON.stringify(response) ); if (response.code != 0) { throw response; From 1d7f8eb865276bcc92beb4c902e3a34b8c6f181a Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Sun, 23 Mar 2025 11:29:53 +0200 Subject: [PATCH 03/15] Adding some sleeps to allow things to settle down --- src/driver/controller-zfs-generic/index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index 2ec3965..a58c6e1 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -350,6 +350,9 @@ create /backstores/block/${assetName} }, } ); + + await GeneralUtils.sleep(2000); // let things settle + break; default: @@ -788,6 +791,8 @@ delete ${assetName} } ); + await GeneralUtils.sleep(2000); // let things settle + break; default: From 0d8ec13f3cdb04cb7b9939e6a720c22136ef038c Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Sun, 23 Mar 2025 11:32:32 +0200 Subject: [PATCH 04/15] small fix --- src/driver/controller-zfs-generic/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index a58c6e1..d560dd2 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -331,7 +331,7 @@ create /backstores/block/${assetName} let createLunTerms = [ 'resource', 'create', '--future', `lun-${assetName}`, 'iSCSILogicalUnit', - 'implementation="lio-t"', `target_iqn="${basename}:${assetName}" lun="1"`, + 'implementation="lio-t"', `target_iqn="${basename}:${assetName}"`, 'lun="1"', `path="/dev/${extentDiskName}"` ]; From 703e897424e6264c49935f42bc72f937d157c069 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Mon, 24 Mar 2025 23:11:00 +0000 Subject: [PATCH 05/15] Added vagrant and devcontainers + fixed all tests --- .devcontainer/devcontainer.json | 38 ++++++++ .github/dependabot.yml | 12 +++ .gitignore | 4 +- CONTRIBUTING.md | 108 +++++++++++++++++++++ Vagrantfile | 71 ++++++++++++++ ci/bin/run.sh | 2 +- dev/run.sh | 34 +++++++ src/driver/controller-zfs-generic/index.js | 27 +++--- 8 files changed, 282 insertions(+), 14 deletions(-) create mode 100644 .devcontainer/devcontainer.json create mode 100644 .github/dependabot.yml create mode 100644 CONTRIBUTING.md create mode 100644 Vagrantfile create mode 100755 dev/run.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..2835179 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node +{ + "name": "democratic-csi", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm", + + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers/features/go:1": {} // To compile csi-sanity during postCreateCommand + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "npm install", + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + "extensions": [ + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "waderyan.nodejs-extension-pack", + "ms-vscode.node-debug2", + "christian-kohler.npm-intellisense", + "christian-kohler.path-intellisense", + "HashiCorp.terraform" + ] + } + } + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f33a02c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for more information: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +# https://containers.dev/guide/dependabot + +version: 2 +updates: + - package-ecosystem: "devcontainers" + directory: "/" + schedule: + interval: weekly diff --git a/.gitignore b/.gitignore index 2855d28..b945e95 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ **~ node_modules -dev +dev/* /ci/bin/*dev* +.vagrant +!dev/run.sh \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..a770271 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,108 @@ +# Contributing to democratic-csi + +## Development Environment Setup + +This project uses a hybrid development approach with devcontainers for IDE configuration and Vagrant for system-level testing. + +### Prerequisites + +Before you begin, ensure you have the following installed: +- [Visual Studio Code](https://code.visualstudio.com/) +- [Docker](https://www.docker.com/get-started) +- [Vagrant](https://www.vagrantup.com/downloads) +- Virtualization Provider: + - For Intel/AMD Machines: VirtualBox + - For Apple Silicon: Qemu (`brew install qemu vagrant` and `vagrant plugin install vagrant-qemu`) + +### Development Workflow + +#### 1. Local Development with Devcontainers + +Devcontainers provide a consistent development environment with: +- Configured VSCode extensions +- Necessary development tools +- Integrated development experience + +To use the devcontainer: +1. Open the project in VSCode +2. Install the "Dev Containers" extension +3. Click "Reopen in Container" when prompted +4. Start coding with pre-configured environment + +#### 2. System Testing with Vagrant + +Vagrant provides a full virtual machine environment for: +- System-level testing +- Running code with kernel dependencies +- Simulating production-like environments + +Workflow: +```bash +# Navigate to project directory +cd ~/democratic-csi + +# Start the Vagrant VM +vagrant up + +# Connect to the VM +vagrant ssh + +# Inside the VM, navigate to the project +cd ~/democratic-csi + +# Run project tests, the config.yaml can be any from the examples folders +# just configured for your own environment. +# You can also create a file `dev/secrets.env` that has `export VARIABLE=VALUE` +# and reference those in your `config.yaml` +./dev/run.sh -c ./dev/config.yaml +``` + +##### Keeping Files in Sync + +Use these methods to keep your local files synchronized with the Vagrant VM: + +###### Manual Sync +```bash +# Sync files from local to Vagrant VM +vagrant rsync +``` + +###### Continuous Sync +```bash +# Automatically sync files as they change +vagrant rsync-auto +``` + +### Best Practices + +- Use devcontainer for day-to-day development and coding +- Use Vagrant for comprehensive system testing +- Always run `vagrant rsync` before running tests in the VM +- Commit and push changes frequently +- If encountering issues, try: + 1. Recreating the devcontainer + 2. Reprovisioning the Vagrant VM with `vagrant reload --provision` or `vagrant destroy -f && vagrant up` + +### Troubleshooting + +#### Devcontainer Issues +- Ensure Docker is running +- Rebuild the container if extensions fail to load +- Check VSCode Dev Containers extension logs + +#### Vagrant Issues +- Verify virtualization is enabled in your BIOS +- Ensure you have the latest Vagrant and virtualization provider +- For Apple Silicon, use Parallels or Lima + +### Contribution Guidelines + +1. Create a new branch for your feature targetting `next` +2. Write clear, concise commit messages +3. Include coverage for tests of csi-sanity for new functionality +4. Run tests in Vagrant VM +5. Submit a pull request with a clear description of changes + +### Contact + +For any questions or issues, please [open an issue](https://github.com/democratic-csi/democratic-csi/issues) on the project repository. \ No newline at end of file diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..294ceab --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,71 @@ +Vagrant.configure("2") do |config| + # Check the host's architecture + host_arch = `uname -m`.strip + + # Use a different box for ARM vs x86_64 + if host_arch == "arm64" + # requires qemu, install qemu and then: + # vagrant plugin install vagrant-qemu + config.vm.box = "perk/ubuntu-24.04-arm64" + else + # Use the x86_64 compatible Ubuntu box + config.vm.box = "ubuntu/jammy64" + end + + config.vm.provider "virtualbox" do |vb| + vb.memory = "2048" + vb.cpus = 2 + end + + config.vm.provision "shell", inline: <<-SHELL + sudo apt-get update -y + sudo apt-get install -y open-iscsi nodejs git make + + # Enable and start iscsid service + sudo systemctl enable --now iscsid + + # Verify installation + systemctl status iscsid --no-pager + + #### + # Install golang + #### + GO_VERSION="1.24.1" + ARCH=$(uname -m) + GO_TAR_URL="" + + if [[ "$ARCH" == "aarch64" ]]; then + GO_TAR_URL="https://go.dev/dl/go${GO_VERSION}.linux-arm64.tar.gz" + elif [[ "$ARCH" == "x86_64" ]]; then + GO_TAR_URL="https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" + else + echo "Unsupported architecture: $ARCH" + exit 1 + fi + + echo "Downloading Go version $GO_VERSION for $ARCH..." + wget -q "$GO_TAR_URL" -O go.tar.gz + tar -C /usr/local -xzf go.tar.gz + rm go.tar.gz + echo "export PATH=\$PATH:/usr/local/go/bin" >> /etc/profile + source /etc/profile + + #### + # Install csi-test + #### + echo "Installing csi-test" + git clone https://github.com/kubernetes-csi/csi-test /tmp/csi-test + pushd /tmp/csi-test/cmd/csi-sanity + make csi-sanity + sudo cp csi-sanity /usr/local/bin + popd + #rm -rf /tmp/csi-test + SHELL + + # Sync project directory for seamless workflow + config.vm.synced_folder ".", "/home/vagrant/democratic-csi", type: "rsync", + rsync__exclude: ".git/" + + # Allow SSH access with default key + config.ssh.insert_key = false + end \ No newline at end of file diff --git a/ci/bin/run.sh b/ci/bin/run.sh index 1d4eaf8..fd929f9 100755 --- a/ci/bin/run.sh +++ b/ci/bin/run.sh @@ -20,7 +20,7 @@ if [[ -f "node_modules-linux-amd64.tar.gz" && ! -d "node_modules" ]];then fi # generate key for paths etc -export CI_BUILD_KEY=$(uuidgen | cut -d "-" -f 1) +export CI_BUILD_KEY=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8) # launch the server sudo -E ci/bin/launch-server.sh & diff --git a/dev/run.sh b/dev/run.sh new file mode 100755 index 0000000..c38e310 --- /dev/null +++ b/dev/run.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e +set -x +TEMPLATE_CONFIG="" + +ROOT_DIR="$(dirname "$(realpath "$0")")" + +while [[ "$#" -gt 0 ]]; do + case $1 in + -c|--config) TEMPLATE_CONFIG="$(realpath "$2")"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +if [ -z "${TEMPLATE_CONFIG}" ]; then + echo "Error: --config or -c parameter is required." + exit 1 +fi + +if [ ! -f $ROOT_DIR/secrets.env ]; then + echo "Error: secrets.env file not found." + exit 1 +fi + +source $ROOT_DIR/secrets.env # needs to have exported variables + +# generate key for paths etc +export CI_BUILD_KEY=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8) + +export TEMPLATE_CONFIG_FILE=${TEMPLATE_CONFIG} + +$ROOT_DIR/../ci/bin/run.sh \ No newline at end of file diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index d560dd2..b7ed07b 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -300,12 +300,12 @@ create /backstores/block/${assetName} break; case "pcs": basename = this.options.iscsi.shareStrategyPcs.basename; - pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group; + let pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group; - let groupTerms = ['group', `${pcs_group}`]; + let extraTerms = ['group', `${pcs_group}`, '--wait']; // The wait is important to avoid race conditions let createTargetTerms = [ - 'resource', 'create', '--future', `target-${assetName}`, 'iSCSITarget', - 'implementation="lio-t"', `iqn="${basename}:${assetName}"` + 'resource', 'create', '--future', `target-${assetName}`, 'ocf:heartbeat:iSCSITarget', + 'implementation="lio-t"', 'portals=":::3260"', `iqn="${basename}:${assetName}"` ]; if (this.options.iscsi.shareStrategyPcs.auth.enabled) { @@ -317,7 +317,7 @@ create /backstores/block/${assetName} 3, 2000, async () => { - await this.pcsCommand(createTargetTerms.concat(groupTerms)); + await this.pcsCommand(createTargetTerms.concat(extraTerms)); }, { retryCondition: (err) => { @@ -330,8 +330,8 @@ create /backstores/block/${assetName} ); let createLunTerms = [ - 'resource', 'create', '--future', `lun-${assetName}`, 'iSCSILogicalUnit', - 'implementation="lio-t"', `target_iqn="${basename}:${assetName}"`, 'lun="1"', + 'resource', 'create', '--future', `lun-${assetName}`, 'ocf:heartbeat:iSCSILogicalUnit', + 'implementation="lio-t"', `target_iqn="${basename}:${assetName}"`, 'lun="0"', `path="/dev/${extentDiskName}"` ]; @@ -339,7 +339,7 @@ create /backstores/block/${assetName} 3, 2000, async () => { - await this.pcsCommand(createLunTerms.concat(groupTerms)); + await this.pcsCommand(createLunTerms.concat(extraTerms)); }, { retryCondition: (err) => { @@ -351,8 +351,6 @@ create /backstores/block/${assetName} } ); - await GeneralUtils.sleep(2000); // let things settle - break; default: @@ -791,8 +789,6 @@ delete ${assetName} } ); - await GeneralUtils.sleep(2000); // let things settle - break; default: @@ -1006,6 +1002,13 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} driver.ctx.logger.verbose( "pcs response: " + JSON.stringify(response) ); + + // Handle idempotence for create commands + if (response.code == 1 && response.stdout.includes("already exists")) { + driver.ctx.logger.verbose("pcs resource already exists, ignoring error (setting response.code=0)"); + response.code = 0; + } + if (response.code != 0) { throw response; } From 446239b6a0013353ff2608840f6229210875204c Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Mon, 24 Mar 2025 23:31:17 +0000 Subject: [PATCH 06/15] Minor changes --- .devcontainer/devcontainer.json | 2 +- .devcontainer/postCreate.sh | 12 ++++++++++++ .github/workflows/main.yml | 2 +- CONTRIBUTING.md | 5 +++++ 4 files changed, 19 insertions(+), 2 deletions(-) create mode 100755 .devcontainer/postCreate.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2835179..22e0c6b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -14,7 +14,7 @@ // "forwardPorts": [], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "npm install", + "postCreateCommand": "/bin/bash .devcontainer/postCreate.sh", // Configure tool-specific properties. "customizations": { diff --git a/.devcontainer/postCreate.sh b/.devcontainer/postCreate.sh new file mode 100755 index 0000000..1f1cee7 --- /dev/null +++ b/.devcontainer/postCreate.sh @@ -0,0 +1,12 @@ +#!/bin/env bash + +npm install + +git clone https://github.com/kubernetes-csi/csi-test /tmp/csi-test +pushd /tmp/csi-test +make +sudo cp /tmp/csi-test/cmd/csi-sanity/csi-sanity /usr/local/bin +popd +rm -rf /tmp/csi-test + +sudo apt update \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f9b1bc8..50dbc88 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -191,7 +191,7 @@ jobs: max-parallel: 1 matrix: config: - - zfs-generic/iscsi-pcs.yaml + #- zfs-generic/iscsi-pcs.yaml # TODO: enable this once the server is setup - zfs-generic/iscsi-targetcli.yaml - zfs-generic/nfs.yaml - zfs-generic/smb.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a770271..f9965b6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,6 +29,11 @@ To use the devcontainer: 3. Click "Reopen in Container" when prompted 4. Start coding with pre-configured environment +> [!Note] +> For `iSCSI` it's mandatory to use the Vagrant VM, due to the need of a kernel driver. +> However for other tests the container is probably enough. It's possible to run the `dev/run.sh` +> as explained below in the devcontainer and see if it's possible, before spinning up a full VM. + #### 2. System Testing with Vagrant Vagrant provides a full virtual machine environment for: From 3fb905cacb1db36f9c2b22bf52cbef04d7452139 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Tue, 25 Mar 2025 07:52:43 +0000 Subject: [PATCH 07/15] CR fixes --- .gitignore | 5 +++-- CONTRIBUTING.md | 4 ++-- Vagrantfile | 20 +++++++++++++++++++- {dev => hack}/run.sh | 0 4 files changed, 24 insertions(+), 5 deletions(-) rename {dev => hack}/run.sh (100%) diff --git a/.gitignore b/.gitignore index b945e95..d19bbc7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ **~ node_modules -dev/* +dev /ci/bin/*dev* .vagrant -!dev/run.sh \ No newline at end of file +hack/* +!hack/run.sh \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f9965b6..1da1090 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ To use the devcontainer: > [!Note] > For `iSCSI` it's mandatory to use the Vagrant VM, due to the need of a kernel driver. -> However for other tests the container is probably enough. It's possible to run the `dev/run.sh` +> However for other tests the container is probably enough. It's possible to run the `hack/run.sh` > as explained below in the devcontainer and see if it's possible, before spinning up a full VM. #### 2. System Testing with Vagrant @@ -59,7 +59,7 @@ cd ~/democratic-csi # just configured for your own environment. # You can also create a file `dev/secrets.env` that has `export VARIABLE=VALUE` # and reference those in your `config.yaml` -./dev/run.sh -c ./dev/config.yaml +./hack/run.sh -c ./hack/config.yaml ``` ##### Keeping Files in Sync diff --git a/Vagrantfile b/Vagrantfile index 294ceab..e4bf1e4 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -19,7 +19,25 @@ Vagrant.configure("2") do |config| config.vm.provision "shell", inline: <<-SHELL sudo apt-get update -y - sudo apt-get install -y open-iscsi nodejs git make + + # for building dependecies and executing node + sudo apt-get install -y nodejs git make + + # for app functionality + sudo apt-get install -y netbase socat e2fsprogs xfsprogs fatresize dosfstools nfs-common cifs-utils + + # Install the following system packages + sudo apt-get install -y open-iscsi lsscsi sg3-utils multipath-tools scsitools nvme-cli + + # Enable multipathing + sudo tee /etc/multipath.conf << EOF + defaults { + user_friendly_names yes + find_multipaths yes + } + EOF + + sudo systemctl enable multipath-tools.service # Enable and start iscsid service sudo systemctl enable --now iscsid diff --git a/dev/run.sh b/hack/run.sh similarity index 100% rename from dev/run.sh rename to hack/run.sh From 931e255bd9302bdc8da7e9a9d7bd72b8dc437a78 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Tue, 25 Mar 2025 07:57:21 +0000 Subject: [PATCH 08/15] remove unneeded rm --- Vagrantfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Vagrantfile b/Vagrantfile index e4bf1e4..b69d2cf 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -77,7 +77,6 @@ Vagrant.configure("2") do |config| make csi-sanity sudo cp csi-sanity /usr/local/bin popd - #rm -rf /tmp/csi-test SHELL # Sync project directory for seamless workflow From fc02fc4dc1efbb764569892c19e6e0ef733c6005 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Tue, 25 Mar 2025 10:06:47 +0200 Subject: [PATCH 09/15] fix vagrant provisioning script --- Vagrantfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index b69d2cf..0d5c5d5 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -31,11 +31,11 @@ Vagrant.configure("2") do |config| # Enable multipathing sudo tee /etc/multipath.conf << EOF - defaults { - user_friendly_names yes - find_multipaths yes - } - EOF +defaults { + user_friendly_names yes + find_multipaths yes +} +EOF sudo systemctl enable multipath-tools.service From 2b656d9b2245fc066df113834d8bd041577d58b8 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Sat, 5 Apr 2025 09:12:47 +0000 Subject: [PATCH 10/15] Add build_push.sh --- .gitignore | 3 ++- CONTRIBUTING.md | 25 +++++++++++++++++++++++++ hack/build_push.sh | 18 ++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 hack/build_push.sh diff --git a/.gitignore b/.gitignore index d19bbc7..5bafd17 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,5 @@ dev /ci/bin/*dev* .vagrant hack/* -!hack/run.sh \ No newline at end of file +!hack/run.sh +!hack/build_push.sh \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1da1090..a0fef36 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -78,6 +78,31 @@ vagrant rsync vagrant rsync-auto ``` +#### 3. Deploy development version to K8s cluster + +Deployment provides a good environment for: +- Final testing in a real world scenario +- Run the final version until included in a release + +> [!Note] +> Make sure to do the build on the architecture you will be running it. +> For example, don't build in Apple Silicon if your cluster runs in amd64. + + +1. Login to your github container registry +```bash +docker login ghcr.io +``` + +> [!Important] +> Login to the container registry is stored plain text, use a PAT instead of your Github password. [Create a PAT with write:packages](https://github.com/settings/tokens/new?scopes=write:packages). + +2. Compile and push to your github container registry. +```bash +./hack/build_push.sh +``` +2. + ### Best Practices - Use devcontainer for day-to-day development and coding diff --git a/hack/build_push.sh b/hack/build_push.sh new file mode 100644 index 0000000..609bd28 --- /dev/null +++ b/hack/build_push.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +set -x + +ROOT_DIR="$(dirname "$(realpath "$0")")" + +GITHUB_USER=${GITHUB_USER:-$(jq -r '.auths."ghcr.io".auth' ~/.docker/config.json|base64 -d|cut -d':' -f1)} +GITHUB_REPO=${GITHUB_REPO:-$(basename -s ${ROOT_DIR}/../.git $(git remote get-url origin))} +DOCKER_TAG=${DOCKER_TAG:-$(git branch --show-current)-$(git rev-parse --short HEAD)} + +if [ -z "${GITHUB_USER}" ]; then + echo "Error: Need to login to ghcr.io ; execute docker login ghcr.io" + exit 1 +fi + +docker build $ROOT_DIR/.. --push -t ghcr.io/${GITHUB_USER}/${GITHUB_REPO}:${DOCKER_TAG} +echo "Image pushed to ghcr.io/${GITHUB_USER}/${GITHUB_REPO}:${DOCKER_TAG}" \ No newline at end of file From 8700b49f6916d92c34526a52cd0ccbbff3c441a5 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Sat, 5 Apr 2025 10:45:38 +0000 Subject: [PATCH 11/15] small typo --- hack/build_push.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/build_push.sh b/hack/build_push.sh index 609bd28..2aa4e2e 100644 --- a/hack/build_push.sh +++ b/hack/build_push.sh @@ -6,7 +6,7 @@ set -x ROOT_DIR="$(dirname "$(realpath "$0")")" GITHUB_USER=${GITHUB_USER:-$(jq -r '.auths."ghcr.io".auth' ~/.docker/config.json|base64 -d|cut -d':' -f1)} -GITHUB_REPO=${GITHUB_REPO:-$(basename -s ${ROOT_DIR}/../.git $(git remote get-url origin))} +GITHUB_REPO=${GITHUB_REPO:-$(basename -s .git $(git remote get-url origin))} DOCKER_TAG=${DOCKER_TAG:-$(git branch --show-current)-$(git rev-parse --short HEAD)} if [ -z "${GITHUB_USER}" ]; then From a01818316b8d46c5f5134b5a34c40707c2f8c07a Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Sat, 5 Apr 2025 11:27:31 +0000 Subject: [PATCH 12/15] fixed doc --- CONTRIBUTING.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a0fef36..9eabd8a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -101,7 +101,16 @@ docker login ghcr.io ```bash ./hack/build_push.sh ``` -2. + +3. When you deploy, in the `values.yaml` add the following, using the output from the script +```yaml +controller: + driver: + image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4 +node: + driver: + image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4 +``` ### Best Practices From 5868b224bd8db08eb7d294928db1b3dfe1f77b60 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Sat, 5 Apr 2025 11:43:52 +0000 Subject: [PATCH 13/15] explain to put visibility public to package --- CONTRIBUTING.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9eabd8a..0ae44c5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -112,6 +112,14 @@ node: image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4 ``` +4. Make the Image Public + + By default, images pushed to GHCR are private. To make it public: + 1. Go to GitHub → Your Repository → Packages (or directly github.com/USERNAME?tab=packages) + 2. Select the package + 3. Click Package Settings + 4. Change Visibility to Public + ### Best Practices - Use devcontainer for day-to-day development and coding From 853bf291675c8418d9acd1967e1a9ea8fe1177a9 Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Thu, 1 May 2025 10:09:02 +0000 Subject: [PATCH 14/15] add force to target creation to allow multiple non-unique usernames --- src/driver/controller-zfs-generic/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index b7ed07b..53429b0 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -304,7 +304,7 @@ create /backstores/block/${assetName} let extraTerms = ['group', `${pcs_group}`, '--wait']; // The wait is important to avoid race conditions let createTargetTerms = [ - 'resource', 'create', '--future', `target-${assetName}`, 'ocf:heartbeat:iSCSITarget', + 'resource', 'create', '--future', '--force', `target-${assetName}`, 'ocf:heartbeat:iSCSITarget', 'implementation="lio-t"', 'portals=":::3260"', `iqn="${basename}:${assetName}"` ]; From 0d928774ce0a9393792e190f2bab3fdb9635edae Mon Sep 17 00:00:00 2001 From: Michel Peterson Date: Thu, 1 May 2025 15:09:19 +0300 Subject: [PATCH 15/15] make build_push.sh executable --- hack/build_push.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 hack/build_push.sh diff --git a/hack/build_push.sh b/hack/build_push.sh old mode 100644 new mode 100755