diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..22e0c6b --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,38 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/typescript-node +{ + "name": "democratic-csi", + // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile + "image": "mcr.microsoft.com/devcontainers/typescript-node:1-22-bookworm", + + // Features to add to the dev container. More info: https://containers.dev/features. + "features": { + "ghcr.io/devcontainers/features/go:1": {} // To compile csi-sanity during postCreateCommand + }, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "/bin/bash .devcontainer/postCreate.sh", + + // Configure tool-specific properties. + "customizations": { + "vscode": { + "settings": { + "terminal.integrated.shell.linux": "/bin/bash" + }, + "extensions": [ + "dbaeumer.vscode-eslint", + "esbenp.prettier-vscode", + "waderyan.nodejs-extension-pack", + "ms-vscode.node-debug2", + "christian-kohler.npm-intellisense", + "christian-kohler.path-intellisense", + "HashiCorp.terraform" + ] + } + } + // Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "root" +} diff --git a/.devcontainer/postCreate.sh b/.devcontainer/postCreate.sh new file mode 100755 index 0000000..1f1cee7 --- /dev/null +++ b/.devcontainer/postCreate.sh @@ -0,0 +1,12 @@ +#!/bin/env bash + +npm install + +git clone https://github.com/kubernetes-csi/csi-test /tmp/csi-test +pushd /tmp/csi-test +make +sudo cp /tmp/csi-test/cmd/csi-sanity/csi-sanity /usr/local/bin +popd +rm -rf /tmp/csi-test + +sudo apt update \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..f33a02c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,12 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for more information: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +# https://containers.dev/guide/dependabot + +version: 2 +updates: + - package-ecosystem: "devcontainers" + directory: "/" + schedule: + interval: weekly diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a64234a..3e10643 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -191,7 +191,8 @@ jobs: max-parallel: 1 matrix: config: - - zfs-generic/iscsi.yaml + #- zfs-generic/iscsi-pcs.yaml # TODO: enable this once the server is setup + - zfs-generic/iscsi-targetcli.yaml - zfs-generic/nfs.yaml - zfs-generic/smb.yaml - zfs-generic/nvmeof.yaml diff --git a/.gitignore b/.gitignore index 2855d28..5bafd17 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ node_modules dev /ci/bin/*dev* +.vagrant +hack/* +!hack/run.sh +!hack/build_push.sh \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..0ae44c5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,155 @@ +# Contributing to democratic-csi + +## Development Environment Setup + +This project uses a hybrid development approach with devcontainers for IDE configuration and Vagrant for system-level testing. + +### Prerequisites + +Before you begin, ensure you have the following installed: +- [Visual Studio Code](https://code.visualstudio.com/) +- [Docker](https://www.docker.com/get-started) +- [Vagrant](https://www.vagrantup.com/downloads) +- Virtualization Provider: + - For Intel/AMD Machines: VirtualBox + - For Apple Silicon: Qemu (`brew install qemu vagrant` and `vagrant plugin install vagrant-qemu`) + +### Development Workflow + +#### 1. Local Development with Devcontainers + +Devcontainers provide a consistent development environment with: +- Configured VSCode extensions +- Necessary development tools +- Integrated development experience + +To use the devcontainer: +1. Open the project in VSCode +2. Install the "Dev Containers" extension +3. Click "Reopen in Container" when prompted +4. Start coding with pre-configured environment + +> [!Note] +> For `iSCSI` it's mandatory to use the Vagrant VM, due to the need of a kernel driver. +> However for other tests the container is probably enough. It's possible to run the `hack/run.sh` +> as explained below in the devcontainer and see if it's possible, before spinning up a full VM. + +#### 2. System Testing with Vagrant + +Vagrant provides a full virtual machine environment for: +- System-level testing +- Running code with kernel dependencies +- Simulating production-like environments + +Workflow: +```bash +# Navigate to project directory +cd ~/democratic-csi + +# Start the Vagrant VM +vagrant up + +# Connect to the VM +vagrant ssh + +# Inside the VM, navigate to the project +cd ~/democratic-csi + +# Run project tests, the config.yaml can be any from the examples folders +# just configured for your own environment. +# You can also create a file `dev/secrets.env` that has `export VARIABLE=VALUE` +# and reference those in your `config.yaml` +./hack/run.sh -c ./hack/config.yaml +``` + +##### Keeping Files in Sync + +Use these methods to keep your local files synchronized with the Vagrant VM: + +###### Manual Sync +```bash +# Sync files from local to Vagrant VM +vagrant rsync +``` + +###### Continuous Sync +```bash +# Automatically sync files as they change +vagrant rsync-auto +``` + +#### 3. Deploy development version to K8s cluster + +Deployment provides a good environment for: +- Final testing in a real world scenario +- Run the final version until included in a release + +> [!Note] +> Make sure to do the build on the architecture you will be running it. +> For example, don't build in Apple Silicon if your cluster runs in amd64. + + +1. Login to your github container registry +```bash +docker login ghcr.io +``` + +> [!Important] +> Login to the container registry is stored plain text, use a PAT instead of your Github password. [Create a PAT with write:packages](https://github.com/settings/tokens/new?scopes=write:packages). + +2. Compile and push to your github container registry. +```bash +./hack/build_push.sh +``` + +3. When you deploy, in the `values.yaml` add the following, using the output from the script +```yaml +controller: + driver: + image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4 +node: + driver: + image: ghcr.io/your_user/democratic-csi:your_branch-fc02fc4 +``` + +4. Make the Image Public + + By default, images pushed to GHCR are private. To make it public: + 1. Go to GitHub → Your Repository → Packages (or directly github.com/USERNAME?tab=packages) + 2. Select the package + 3. Click Package Settings + 4. Change Visibility to Public + +### Best Practices + +- Use devcontainer for day-to-day development and coding +- Use Vagrant for comprehensive system testing +- Always run `vagrant rsync` before running tests in the VM +- Commit and push changes frequently +- If encountering issues, try: + 1. Recreating the devcontainer + 2. Reprovisioning the Vagrant VM with `vagrant reload --provision` or `vagrant destroy -f && vagrant up` + +### Troubleshooting + +#### Devcontainer Issues +- Ensure Docker is running +- Rebuild the container if extensions fail to load +- Check VSCode Dev Containers extension logs + +#### Vagrant Issues +- Verify virtualization is enabled in your BIOS +- Ensure you have the latest Vagrant and virtualization provider +- For Apple Silicon, use Parallels or Lima + +### Contribution Guidelines + +1. Create a new branch for your feature targetting `next` +2. Write clear, concise commit messages +3. Include coverage for tests of csi-sanity for new functionality +4. Run tests in Vagrant VM +5. Submit a pull request with a clear description of changes + +### Contact + +For any questions or issues, please [open an issue](https://github.com/democratic-csi/democratic-csi/issues) on the project repository. \ No newline at end of file diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 0000000..0d5c5d5 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,88 @@ +Vagrant.configure("2") do |config| + # Check the host's architecture + host_arch = `uname -m`.strip + + # Use a different box for ARM vs x86_64 + if host_arch == "arm64" + # requires qemu, install qemu and then: + # vagrant plugin install vagrant-qemu + config.vm.box = "perk/ubuntu-24.04-arm64" + else + # Use the x86_64 compatible Ubuntu box + config.vm.box = "ubuntu/jammy64" + end + + config.vm.provider "virtualbox" do |vb| + vb.memory = "2048" + vb.cpus = 2 + end + + config.vm.provision "shell", inline: <<-SHELL + sudo apt-get update -y + + # for building dependecies and executing node + sudo apt-get install -y nodejs git make + + # for app functionality + sudo apt-get install -y netbase socat e2fsprogs xfsprogs fatresize dosfstools nfs-common cifs-utils + + # Install the following system packages + sudo apt-get install -y open-iscsi lsscsi sg3-utils multipath-tools scsitools nvme-cli + + # Enable multipathing + sudo tee /etc/multipath.conf << EOF +defaults { + user_friendly_names yes + find_multipaths yes +} +EOF + + sudo systemctl enable multipath-tools.service + + # Enable and start iscsid service + sudo systemctl enable --now iscsid + + # Verify installation + systemctl status iscsid --no-pager + + #### + # Install golang + #### + GO_VERSION="1.24.1" + ARCH=$(uname -m) + GO_TAR_URL="" + + if [[ "$ARCH" == "aarch64" ]]; then + GO_TAR_URL="https://go.dev/dl/go${GO_VERSION}.linux-arm64.tar.gz" + elif [[ "$ARCH" == "x86_64" ]]; then + GO_TAR_URL="https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" + else + echo "Unsupported architecture: $ARCH" + exit 1 + fi + + echo "Downloading Go version $GO_VERSION for $ARCH..." + wget -q "$GO_TAR_URL" -O go.tar.gz + tar -C /usr/local -xzf go.tar.gz + rm go.tar.gz + echo "export PATH=\$PATH:/usr/local/go/bin" >> /etc/profile + source /etc/profile + + #### + # Install csi-test + #### + echo "Installing csi-test" + git clone https://github.com/kubernetes-csi/csi-test /tmp/csi-test + pushd /tmp/csi-test/cmd/csi-sanity + make csi-sanity + sudo cp csi-sanity /usr/local/bin + popd + SHELL + + # Sync project directory for seamless workflow + config.vm.synced_folder ".", "/home/vagrant/democratic-csi", type: "rsync", + rsync__exclude: ".git/" + + # Allow SSH access with default key + config.ssh.insert_key = false + end \ No newline at end of file diff --git a/ci/bin/run.sh b/ci/bin/run.sh index 1d4eaf8..fd929f9 100755 --- a/ci/bin/run.sh +++ b/ci/bin/run.sh @@ -20,7 +20,7 @@ if [[ -f "node_modules-linux-amd64.tar.gz" && ! -d "node_modules" ]];then fi # generate key for paths etc -export CI_BUILD_KEY=$(uuidgen | cut -d "-" -f 1) +export CI_BUILD_KEY=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8) # launch the server sudo -E ci/bin/launch-server.sh & diff --git a/ci/configs/zfs-generic/iscsi-pcs.yaml b/ci/configs/zfs-generic/iscsi-pcs.yaml new file mode 100644 index 0000000..baabb69 --- /dev/null +++ b/ci/configs/zfs-generic/iscsi-pcs.yaml @@ -0,0 +1,28 @@ +driver: zfs-generic-iscsi + +sshConnection: + host: ${SERVER_HOST} + port: 22 + username: ${SERVER_USERNAME} + password: ${SERVER_PASSWORD} + +zfs: + datasetParentName: tank/ci/${CI_BUILD_KEY}/v + detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s + + zvolCompression: + zvolDedup: + zvolEnableReservation: false + zvolBlocksize: + +iscsi: + targetPortal: ${SERVER_HOST} + interface: "" + namePrefix: "csi-ci-${CI_BUILD_KEY}-" + nameSuffix: "" + shareStrategy: "pcs" + shareStrategyPcs: + pcs_group: "group-nas" + basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664" + auth: + enabled: 0 diff --git a/ci/configs/zfs-generic/iscsi.yaml b/ci/configs/zfs-generic/iscsi-targetcli.yaml similarity index 100% rename from ci/configs/zfs-generic/iscsi.yaml rename to ci/configs/zfs-generic/iscsi-targetcli.yaml diff --git a/examples/zfs-generic-iscsi.yaml b/examples/zfs-generic-iscsi.yaml index af5df37..f8aa270 100644 --- a/examples/zfs-generic-iscsi.yaml +++ b/examples/zfs-generic-iscsi.yaml @@ -44,6 +44,7 @@ zfs: iscsi: shareStrategy: "targetCli" + #shareStrategy: "pcs" # https://kifarunix.com/how-to-install-and-configure-iscsi-storage-server-on-ubuntu-18-04/ # https://kifarunix.com/how-install-and-configure-iscsi-storage-server-on-centos-7/ @@ -75,6 +76,17 @@ iscsi: attributes: # set to 1 to enable Thin Provisioning Unmap emulate_tpu: 0 + + shareStrategyPcs: + #sudoEnabled: true + pcs_group: "group-nas" + basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664" + auth: + enabled: 0 + # CHAP – incoming only, mutual not supported by the Pacemaker resource agent + incoming_username: "foo" + incoming_password: "bar" + targetPortal: "server[:port]" # for multipath targetPortals: [] # [ "server[:port]", "server[:port]", ... ] diff --git a/hack/build_push.sh b/hack/build_push.sh new file mode 100755 index 0000000..2aa4e2e --- /dev/null +++ b/hack/build_push.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e +set -x + +ROOT_DIR="$(dirname "$(realpath "$0")")" + +GITHUB_USER=${GITHUB_USER:-$(jq -r '.auths."ghcr.io".auth' ~/.docker/config.json|base64 -d|cut -d':' -f1)} +GITHUB_REPO=${GITHUB_REPO:-$(basename -s .git $(git remote get-url origin))} +DOCKER_TAG=${DOCKER_TAG:-$(git branch --show-current)-$(git rev-parse --short HEAD)} + +if [ -z "${GITHUB_USER}" ]; then + echo "Error: Need to login to ghcr.io ; execute docker login ghcr.io" + exit 1 +fi + +docker build $ROOT_DIR/.. --push -t ghcr.io/${GITHUB_USER}/${GITHUB_REPO}:${DOCKER_TAG} +echo "Image pushed to ghcr.io/${GITHUB_USER}/${GITHUB_REPO}:${DOCKER_TAG}" \ No newline at end of file diff --git a/hack/run.sh b/hack/run.sh new file mode 100755 index 0000000..c38e310 --- /dev/null +++ b/hack/run.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -e +set -x +TEMPLATE_CONFIG="" + +ROOT_DIR="$(dirname "$(realpath "$0")")" + +while [[ "$#" -gt 0 ]]; do + case $1 in + -c|--config) TEMPLATE_CONFIG="$(realpath "$2")"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +if [ -z "${TEMPLATE_CONFIG}" ]; then + echo "Error: --config or -c parameter is required." + exit 1 +fi + +if [ ! -f $ROOT_DIR/secrets.env ]; then + echo "Error: secrets.env file not found." + exit 1 +fi + +source $ROOT_DIR/secrets.env # needs to have exported variables + +# generate key for paths etc +export CI_BUILD_KEY=$(head /dev/urandom | tr -dc A-Za-z0-9 | head -c 8) + +export TEMPLATE_CONFIG_FILE=${TEMPLATE_CONFIG} + +$ROOT_DIR/../ci/bin/run.sh \ No newline at end of file diff --git a/src/driver/controller-zfs-generic/index.js b/src/driver/controller-zfs-generic/index.js index 3e4836f..049134b 100644 --- a/src/driver/controller-zfs-generic/index.js +++ b/src/driver/controller-zfs-generic/index.js @@ -297,8 +297,66 @@ create /backstores/block/${assetName} } ); break; + case "pcs": + basename = this.options.iscsi.shareStrategyPcs.basename; + let pcs_group = this.options.iscsi.shareStrategyPcs.pcs_group; + + let extraTerms = ['group', `${pcs_group}`, '--wait']; // The wait is important to avoid race conditions + let createTargetTerms = [ + 'resource', 'create', '--future', '--force', `target-${assetName}`, 'ocf:heartbeat:iSCSITarget', + 'implementation="lio-t"', 'portals=":::3260"', `iqn="${basename}:${assetName}"` + ]; + + if (this.options.iscsi.shareStrategyPcs.auth.enabled) { + createTargetTerms.push(`incoming_username="${this.options.iscsi.shareStrategyPcs.auth.incoming_username}"`); + createTargetTerms.push(`incoming_password="${this.options.iscsi.shareStrategyPcs.auth.incoming_password}"`); + } + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.pcsCommand(createTargetTerms.concat(extraTerms)); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Timed Out")) { + return true; + } + return false; + }, + } + ); + + let createLunTerms = [ + 'resource', 'create', '--future', `lun-${assetName}`, 'ocf:heartbeat:iSCSILogicalUnit', + 'implementation="lio-t"', `target_iqn="${basename}:${assetName}"`, 'lun="0"', + `path="/dev/${extentDiskName}"` + ]; + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.pcsCommand(createLunTerms.concat(extraTerms)); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Timed Out")) { + return true; + } + return false; + }, + } + ); + + break; + default: - break; + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown shareStrategy ${this.options.iscsi.shareStrategy}` + ); } // iqn = target @@ -689,8 +747,54 @@ delete ${assetName} ); break; - default: + case "pcs": + let deleteLunText = [ + 'resource', 'delete', `lun-${assetName}` + ]; + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.pcsCommand(deleteLunText); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Timed Out")) { + return true; + } + return false; + }, + } + ); + + let deleteTargetText = [ + 'resource', 'delete', `target-${assetName}` + ]; + + await GeneralUtils.retry( + 3, + 2000, + async () => { + await this.pcsCommand(deleteTargetText); + }, + { + retryCondition: (err) => { + if (err.stdout && err.stdout.includes("Timed Out")) { + return true; + } + return false; + }, + } + ); + break; + + default: + throw new GrpcError( + grpc.status.FAILED_PRECONDITION, + `invalid configuration: unknown shareStrategy ${this.options.iscsi.shareStrategy}` + ); } break; } @@ -840,6 +944,9 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} case "targetCli": // nothing required, just need to rescan on the node break; + case "pcs": + // nothing required, just need to rescan on the node + break; default: break; } @@ -850,6 +957,63 @@ save_config filename=${this.options.nvmeof.shareStrategySpdkCli.configPath} } } + async pcsCommand(commandTerms) { + const execClient = this.getExecClient(); + const driver = this; + + let command = "sh"; + let args = ["-c"]; + + let cliArgs = ["pcs"]; + if ( + _.get(this.options, "iscsi.shareStrategyPcs.sudoEnabled", false) + ) { + cliArgs.unshift("sudo"); + } + + let cliCommand = []; + cliCommand.push(cliArgs.join(" ")); + cliCommand.push(commandTerms.join(" ")); + args.push("'" + cliCommand.join(" ") + "'"); + + let logCommandTmp = command + " " + args.join(" "); + let logCommand = ""; + + logCommandTmp.split(" ").forEach((term) => { + logCommand += " "; + + if (term.startsWith("incoming_password=")) { + logCommand += "incoming_password="; + } else { + logCommand += term; + } + }); + + driver.ctx.logger.verbose("pcs command:" + logCommand); + + let options = { + pty: true, + }; + let response = await execClient.exec( + execClient.buildCommand(command, args), + options + ); + driver.ctx.logger.verbose( + "pcs response: " + JSON.stringify(response) + ); + + // Handle idempotence for create commands + if (response.code == 1 && response.stdout.includes("already exists")) { + driver.ctx.logger.verbose("pcs resource already exists, ignoring error (setting response.code=0)"); + response.code = 0; + } + + if (response.code != 0) { + throw response; + } + return response; + } + async targetCliCommand(data) { const execClient = this.getExecClient(); const driver = this;