Merge pull request #452 from d-uzlov/fix-global-registry

refactor: stop using global variable for cache registry

.dockerignore

@@ -1,9 +1,9 @@
-chart
-dev
-examples
-contrib
-node_modules
-Dockerfile*
-TODO.md
-.git
-/ci
+**
+
+!/bin
+!/csi_proto
+!/csi_proxy_proto
+!/docker
+!/LICENSE
+!/package*.json
+!/src

.github/bin/docker-release-windows.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+
+echo "$DOCKER_PASSWORD" | docker login docker.io -u "$DOCKER_USERNAME" --password-stdin
+echo "$GHCR_PASSWORD"   | docker login ghcr.io   -u "$GHCR_USERNAME"   --password-stdin
+
+export DOCKER_ORG="democraticcsi"
+export DOCKER_PROJECT="democratic-csi"
+export DOCKER_REPO="docker.io/${DOCKER_ORG}/${DOCKER_PROJECT}"
+
+export GHCR_ORG="democratic-csi"
+export GHCR_PROJECT="democratic-csi"
+export GHCR_REPO="ghcr.io/${GHCR_ORG}/${GHCR_PROJECT}"
+
+export MANIFEST_NAME="democratic-csi-combined:${IMAGE_TAG}"
+
+if [[ -n "${IMAGE_TAG}" ]]; then
+  # create local manifest to work with
+  buildah manifest rm "${MANIFEST_NAME}" || true
+  buildah manifest create "${MANIFEST_NAME}"
+  
+  # all all the existing linux data to the manifest
+  buildah manifest add "${MANIFEST_NAME}" --all "${DOCKER_REPO}:${IMAGE_TAG}"
+  buildah manifest inspect "${MANIFEST_NAME}"
+  
+  # import pre-built images
+  buildah pull docker-archive:democratic-csi-windows-ltsc2019.tar
+  buildah pull docker-archive:democratic-csi-windows-ltsc2022.tar
+
+  # add pre-built images to manifest
+  buildah manifest add "${MANIFEST_NAME}" democratic-csi-windows:${GITHUB_RUN_ID}-ltsc2019
+  buildah manifest add "${MANIFEST_NAME}" democratic-csi-windows:${GITHUB_RUN_ID}-ltsc2022
+  buildah manifest inspect "${MANIFEST_NAME}"
+
+  # push manifest
+  buildah manifest push --all "${MANIFEST_NAME}" docker://${DOCKER_REPO}:${IMAGE_TAG}
+  buildah manifest push --all "${MANIFEST_NAME}" docker://${GHCR_REPO}:${IMAGE_TAG}
+
+  # cleanup
+  buildah manifest rm "${MANIFEST_NAME}" || true
+else
+  :
+fi

.github/bin/docker-release.sh

@@ -11,20 +11,13 @@ export GHCR_ORG="democratic-csi"
 export GHCR_PROJECT="democratic-csi"
 export GHCR_REPO="ghcr.io/${GHCR_ORG}/${GHCR_PROJECT}"
 
-if [[ $GITHUB_REF == refs/tags/* ]]; then
-  export GIT_TAG=${GITHUB_REF#refs/tags/}
-else
-  export GIT_BRANCH=${GITHUB_REF#refs/heads/}
-fi
-
-if [[ -n "${GIT_TAG}" ]]; then
-  docker buildx build --progress plain --pull --push --platform "${DOCKER_BUILD_PLATFORM}" -t ${DOCKER_REPO}:${GIT_TAG} -t ${GHCR_REPO}:${GIT_TAG} .
-elif [[ -n "${GIT_BRANCH}" ]]; then
-  if [[ "${GIT_BRANCH}" == "master" ]]; then
-    docker buildx build --progress plain --pull --push --platform "${DOCKER_BUILD_PLATFORM}" -t ${DOCKER_REPO}:latest -t ${GHCR_REPO}:latest .
-  else
-    docker buildx build --progress plain --pull --push --platform "${DOCKER_BUILD_PLATFORM}" -t ${DOCKER_REPO}:${GIT_BRANCH} -t ${GHCR_REPO}:${GIT_BRANCH} .
-  fi
+if [[ -n "${IMAGE_TAG}" ]]; then
+  # -t ${GHCR_REPO}:${IMAGE_TAG}
+  docker buildx build --progress plain --pull --push --platform "${DOCKER_BUILD_PLATFORM}" -t ${DOCKER_REPO}:${IMAGE_TAG} \
+  --label "org.opencontainers.image.created=$(date -u --iso-8601=seconds)" \
+  --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
+  --build-arg OBJECTIVEFS_DOWNLOAD_ID=${OBJECTIVEFS_DOWNLOAD_ID} \
+  .
 else
   :
 fi

.github/workflows/main.yml

@@ -1,3 +1,5 @@
+# https://www.truenas.com/software-status/
+
 name: CI
 
 on:
@@ -13,140 +15,163 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Cancel Previous Runs
-        uses: styfle/cancel-workflow-action@0.6.0
+        uses: styfle/cancel-workflow-action@0.12.1
         with:
           access_token: ${{ github.token }}
 
-  build-npm:
-    name: build-npm
-    runs-on:
-      - self-hosted
+  build-npm-linux-amd64:
+    runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
       - shell: bash
         name: npm install
         run: |
           ci/bin/build.sh
       - name: upload build
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v4
         with:
-          name: node-modules
-          #path: node_modules/
-          path: node_modules.tar.gz
-          retention-days: 7
+          name: node-modules-linux-amd64
+          path: node_modules-linux-amd64.tar.gz
+          retention-days: 1
 
-  csi-sanity-synology:
+  build-npm-windows-amd64:
+    runs-on: windows-2022
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - shell: pwsh
+        name: npm install
+        run: |
+          ci\bin\build.ps1
+      - name: upload build
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-modules-windows-amd64
+          path: node_modules-windows-amd64.tar.gz
+          retention-days: 1
+
+  csi-sanity-synology-dsm6:
     needs:
-      - build-npm
+      - build-npm-linux-amd64
     strategy:
       fail-fast: false
       matrix:
         config:
-          - synlogy/iscsi.yaml
+          - synlogy/dsm6/iscsi.yaml
     runs-on:
       - self-hosted
+      - Linux
+      - X64
       - csi-sanity-synology
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
-          name: node-modules
+          name: node-modules-linux-amd64
       - name: csi-sanity
         run: |
           # run tests
           ci/bin/run.sh
         env:
           TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
-          SYNOLOGY_HOST: ${{ secrets.SANITY_SYNOLOGY_HOST }}
-          SYNOLOGY_PORT: ${{ secrets.SANITY_SYNOLOGY_PORT }}
+          SYNOLOGY_HOST: ${{ secrets.SANITY_SYNOLOGY_DSM6_HOST }}
+          SYNOLOGY_PORT: ${{ secrets.SANITY_SYNOLOGY_DSM6_PORT }}
           SYNOLOGY_USERNAME: ${{ secrets.SANITY_SYNOLOGY_USERNAME }}
           SYNOLOGY_PASSWORD: ${{ secrets.SANITY_SYNOLOGY_PASSWORD }}
           SYNOLOGY_VOLUME: ${{ secrets.SANITY_SYNOLOGY_VOLUME }}
 
-  # api-based drivers
-  csi-sanity-truenas-scale-22_02:
+  csi-sanity-synology-dsm7:
     needs:
-      - build-npm
+      - build-npm-linux-amd64
     strategy:
       fail-fast: false
       matrix:
         config:
-          - truenas/scale/22.02/scale-iscsi.yaml
-          - truenas/scale/22.02/scale-nfs.yaml
-          # 80 char limit
-          #- truenas/scale-smb.yaml
+          - synlogy/dsm7/iscsi.yaml
     runs-on:
       - self-hosted
-      - csi-sanity-zfs-local
-      #- csi-sanity-truenas-scale
+      - Linux
+      - X64
+      - csi-sanity-synology
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
-          name: node-modules
+          name: node-modules-linux-amd64
       - name: csi-sanity
         run: |
           # run tests
           ci/bin/run.sh
         env:
           TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
-          TRUENAS_HOST: ${{ secrets.SANITY_TRUENAS_SCALE_22_02_HOST }}
-          TRUENAS_USERNAME: ${{ secrets.SANITY_TRUENAS_USERNAME }}
-          TRUENAS_PASSWORD: ${{ secrets.SANITY_TRUENAS_PASSWORD }}
+          SYNOLOGY_HOST: ${{ secrets.SANITY_SYNOLOGY_DSM7_HOST }}
+          SYNOLOGY_PORT: ${{ secrets.SANITY_SYNOLOGY_DSM7_PORT }}
+          SYNOLOGY_USERNAME: ${{ secrets.SANITY_SYNOLOGY_USERNAME }}
+          SYNOLOGY_PASSWORD: ${{ secrets.SANITY_SYNOLOGY_PASSWORD }}
+          SYNOLOGY_VOLUME: ${{ secrets.SANITY_SYNOLOGY_VOLUME }}
 
-  # ssh-based drivers
-  csi-sanity-truenas-core-12_0:
+  csi-sanity-truenas-scale-24_04:
     needs:
-      - build-npm
+      - build-npm-linux-amd64
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         config:
-          # 63 char limit
-          #- truenas/core-iscsi.yaml
-          - truenas/core/12.0/core-nfs.yaml
+          - truenas/scale/24.04/scale-iscsi.yaml
+          - truenas/scale/24.04/scale-nfs.yaml
           # 80 char limit
-          #- truenas/core-smb.yaml
+          - truenas/scale/24.04/scale-smb.yaml
     runs-on:
       - self-hosted
-      - csi-sanity-zfs-local
-      #- csi-sanity-truenas-core
+      - Linux
+      - X64
+      #- csi-sanity-truenas
+      - csi-sanity-zfs-generic
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
-          name: node-modules
+          name: node-modules-linux-amd64
       - name: csi-sanity
         run: |
           # run tests
           ci/bin/run.sh
         env:
           TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
-          TRUENAS_HOST: ${{ secrets.SANITY_TRUENAS_CORE_12_0_HOST }}
+          TRUENAS_HOST: ${{ secrets.SANITY_TRUENAS_SCALE_24_04_HOST }}
           TRUENAS_USERNAME: ${{ secrets.SANITY_TRUENAS_USERNAME }}
           TRUENAS_PASSWORD: ${{ secrets.SANITY_TRUENAS_PASSWORD }}
 
   # ssh-based drivers
   csi-sanity-truenas-core-13_0:
     needs:
-      - build-npm
+      - build-npm-linux-amd64
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         config:
           - truenas/core/13.0/core-iscsi.yaml
           - truenas/core/13.0/core-nfs.yaml
           # 80 char limit
-          #- truenas/core-smb.yaml
+          - truenas/core/13.0/core-smb.yaml
     runs-on:
       - self-hosted
-      - csi-sanity-zfs-local
-      #- csi-sanity-truenas-core
+      - Linux
+      - X64
+      #- csi-sanity-truenas
+      - csi-sanity-zfs-generic
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
-          name: node-modules
+          name: node-modules-linux-amd64
       - name: csi-sanity
         run: |
           # run tests
@@ -160,21 +185,26 @@ jobs:
   # ssh-based drivers
   csi-sanity-zfs-generic:
     needs:
-      - build-npm
+      - build-npm-linux-amd64
     strategy:
       fail-fast: false
+      max-parallel: 1
       matrix:
         config:
           - zfs-generic/iscsi.yaml
           - zfs-generic/nfs.yaml
+          - zfs-generic/smb.yaml
+          - zfs-generic/nvmeof.yaml
     runs-on:
       - self-hosted
+      - Linux
+      - X64
       - csi-sanity-zfs-generic
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
-          name: node-modules
+          name: node-modules-linux-amd64
       - name: csi-sanity
         run: |
           # run tests
@@ -185,10 +215,107 @@ jobs:
           SERVER_USERNAME: ${{ secrets.SANITY_ZFS_GENERIC_USERNAME }}
           SERVER_PASSWORD: ${{ secrets.SANITY_ZFS_GENERIC_PASSWORD }}
 
+  # client drivers
+  csi-sanity-objectivefs:
+    needs:
+      - build-npm-linux-amd64
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - objectivefs/objectivefs.yaml
+    runs-on:
+      - self-hosted
+      - Linux
+      - X64
+      - csi-sanity-client
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: node-modules-linux-amd64
+      - name: csi-sanity
+        run: |
+          # run tests
+          ci/bin/run.sh
+        env:
+          TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
+          OBJECTIVEFS_POOL: ${{ secrets.SANITY_OBJECTIVEFS_POOL }}
+          OBJECTIVEFS_LICENSE: ${{ secrets.SANITY_OBJECTIVEFS_LICENSE }}
+          OBJECTIVEFS_OBJECTSTORE: ${{ secrets.SANITY_OBJECTIVEFS_OBJECTSTORE }}
+          OBJECTIVEFS_ENDPOINT_PROTOCOL: ${{ secrets.SANITY_OBJECTIVEFS_ENDPOINT_PROTOCOL }}
+          OBJECTIVEFS_ENDPOINT_HOST: ${{ secrets.SANITY_OBJECTIVEFS_ENDPOINT_HOST }}
+          OBJECTIVEFS_ENDPOINT_PORT: ${{ secrets.SANITY_OBJECTIVEFS_ENDPOINT_PORT }}
+          OBJECTIVEFS_SECRET_KEY: ${{ secrets.SANITY_OBJECTIVEFS_SECRET_KEY }}
+          OBJECTIVEFS_ACCESS_KEY: ${{ secrets.SANITY_OBJECTIVEFS_ACCESS_KEY }}
+          OBJECTIVEFS_PASSPHRASE: ${{ secrets.SANITY_OBJECTIVEFS_PASSPHRASE }}
+
+          # these secrets need to match the above secrets for staging/etc
+          CSI_SANITY_SECRETS: /root/csi-secrets/objectivefs-secrets.yaml
+          CSI_SANITY_SKIP: "should fail when requesting to create a snapshot with already existing name and different source volume ID|should fail when requesting to create a volume with already existing name and different capacity"
+
+  # client drivers
+  csi-sanity-client:
+    needs:
+      - build-npm-linux-amd64
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - client/nfs.yaml
+          - client/smb.yaml
+    runs-on:
+      - self-hosted
+      - Linux
+      - X64
+      - csi-sanity-client
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: node-modules-linux-amd64
+      - name: csi-sanity
+        run: |
+          # run tests
+          ci/bin/run.sh
+        env:
+          TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
+          SERVER_HOST: ${{ secrets.SANITY_ZFS_GENERIC_HOST }}
+          SHARE_NAME: tank_client_smb
+          CSI_SANITY_SKIP: "should fail when requesting to create a snapshot with already existing name and different source volume ID|should fail when requesting to create a volume with already existing name and different capacity"
+
+  csi-sanity-client-windows:
+    needs:
+      - build-npm-windows-amd64
+    strategy:
+      fail-fast: false
+      matrix:
+        config:
+          - client\smb.yaml
+    runs-on:
+      - self-hosted
+      - Windows
+      - X64
+      - csi-sanity-client
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: node-modules-windows-amd64
+      - name: csi-sanity
+        run: |
+          # run tests
+          ci\bin\run.ps1
+        env:
+          TEMPLATE_CONFIG_FILE: ".\\ci\\configs\\${{ matrix.config }}"
+          SERVER_HOST: ${{ secrets.SANITY_ZFS_GENERIC_HOST }}
+          SHARE_NAME: tank_client_smb
+          CSI_SANITY_SKIP: "should fail when requesting to create a snapshot with already existing name and different source volume ID|should fail when requesting to create a volume with already existing name and different capacity"
+
   # zfs-local drivers
   csi-sanity-zfs-local:
     needs:
-      - build-npm
+      - build-npm-linux-amd64
     strategy:
       fail-fast: false
       matrix:
@@ -197,12 +324,14 @@ jobs:
           - zfs-local/dataset.yaml
     runs-on:
       - self-hosted
+      - Linux
+      - X64
       - csi-sanity-zfs-local
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
-          name: node-modules
+          name: node-modules-linux-amd64
       - name: csi-sanity
         run: |
           # run tests
@@ -213,44 +342,116 @@ jobs:
   # local-hostpath driver
   csi-sanity-local-hostpath:
     needs:
-      - build-npm
+      - build-npm-linux-amd64
+      - build-npm-windows-amd64
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [Linux, Windows]
+        include:
+          - os: Linux
+            npmartifact: node-modules-linux-amd64
+            template: "./ci/configs/local-hostpath/basic.yaml"
+            run: |
+              # run tests
+              ci/bin/run.sh
+          - os: Windows
+            npmartifact: node-modules-windows-amd64
+            template: ".\\ci\\configs\\local-hostpath\\basic.yaml"
+            run: |
+              # run tests
+              ci\bin\run.ps1
+    runs-on:
+      - self-hosted
+      - ${{ matrix.os }}
+      - X64
+      - csi-sanity-local-hostpath
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: ${{ matrix.npmartifact }}
+      - name: csi-sanity
+        run: ${{ matrix.run }}
+        env:
+          TEMPLATE_CONFIG_FILE: "${{ matrix.template }}"
+          CSI_SANITY_SKIP: "should fail when requesting to create a snapshot with already existing name and different source volume ID|should fail when requesting to create a volume with already existing name and different capacity"
+
+  csi-sanity-windows-node:
+    needs:
+      - build-npm-windows-amd64
     strategy:
       fail-fast: false
       matrix:
         config:
-          - local-hostpath/basic.yaml
+          - windows\iscsi.yaml
+          - windows\smb.yaml
     runs-on:
       - self-hosted
+      - Windows
+      - X64
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/download-artifact@v2
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
-          name: node-modules
+          name: node-modules-windows-amd64
       - name: csi-sanity
         run: |
           # run tests
-          ci/bin/run.sh
+          ci\bin\run.ps1
         env:
-          TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
-          CSI_SANITY_SKIP: "should fail when requesting to create a snapshot with already existing name and different source volume ID|should fail when requesting to create a volume with already existing name and different capacity"
+          TEMPLATE_CONFIG_FILE: ".\\ci\\configs\\${{ matrix.config }}"
+          SERVER_HOST: ${{ secrets.SANITY_ZFS_GENERIC_HOST }}
+          SERVER_USERNAME: ${{ secrets.SANITY_ZFS_GENERIC_USERNAME }}
+          SERVER_PASSWORD: ${{ secrets.SANITY_ZFS_GENERIC_PASSWORD }}
+          CSI_SANITY_FOCUS: "Node Service"
 
-  build-docker:
+  determine-image-tag:
+    runs-on: ubuntu-latest
+    outputs:
+      tag: ${{ steps.tag.outputs.tag }}
+    steps:
+      - id: tag
+        run: |
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            export GIT_TAG=${GITHUB_REF#refs/tags/}
+          else
+            export GIT_BRANCH=${GITHUB_REF#refs/heads/}
+          fi
+          if [[ -n "${GIT_TAG}" ]]; then
+            echo "::set-output name=tag::${GIT_TAG}"
+          elif [[ -n "${GIT_BRANCH}" ]]; then
+            if [[ "${GIT_BRANCH}" == "master" ]]; then
+              echo "::set-output name=tag::latest"
+            else
+              echo "::set-output name=tag::${GIT_BRANCH}"
+            fi
+          else
+            :
+          fi
+
+  build-docker-linux:
     needs:
-      - csi-sanity-synology
-      - csi-sanity-truenas-scale-22_02
-      - csi-sanity-truenas-core-12_0
+      - determine-image-tag
+      - csi-sanity-synology-dsm6
+      - csi-sanity-synology-dsm7
+      - csi-sanity-truenas-scale-24_04
       - csi-sanity-truenas-core-13_0
       - csi-sanity-zfs-generic
+      - csi-sanity-objectivefs
+      - csi-sanity-client
+      - csi-sanity-client-windows
       - csi-sanity-zfs-local
       - csi-sanity-local-hostpath
+      - csi-sanity-windows-node
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: docker build
         run: |
           export ARCH=$([ $(uname -m) = "x86_64" ] && echo "amd64" || echo "arm64")
           mkdir -p ~/.docker/cli-plugins/
-          wget -qO ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.5.1/buildx-v0.5.1.linux-${ARCH}
+          wget -qO ~/.docker/cli-plugins/docker-buildx https://github.com/docker/buildx/releases/download/v0.8.2/buildx-v0.8.2.linux-${ARCH}
           chmod a+x ~/.docker/cli-plugins/docker-buildx
           docker info
           docker buildx version
@@ -265,5 +466,81 @@ jobs:
           DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
           GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
           GHCR_PASSWORD: ${{ secrets.GHCR_PASSWORD }}
+          OBJECTIVEFS_DOWNLOAD_ID: ${{ secrets.OBJECTIVEFS_DOWNLOAD_ID }}
           DOCKER_CLI_EXPERIMENTAL: enabled
           DOCKER_BUILD_PLATFORM: linux/amd64,linux/arm64,linux/arm/v7,linux/s390x,linux/ppc64le
+          IMAGE_TAG: ${{needs.determine-image-tag.outputs.tag}}
+
+  build-docker-windows:
+    needs:
+      - csi-sanity-synology-dsm6
+      - csi-sanity-synology-dsm7
+      - csi-sanity-truenas-scale-24_04
+      - csi-sanity-truenas-core-13_0
+      - csi-sanity-zfs-generic
+      - csi-sanity-objectivefs
+      - csi-sanity-client
+      - csi-sanity-client-windows
+      - csi-sanity-zfs-local
+      - csi-sanity-local-hostpath
+      - csi-sanity-windows-node
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-2019, windows-2022]
+        include:
+          - os: windows-2019
+            core_base_tag: ltsc2019
+            nano_base_tag: "1809"
+            file: Dockerfile.Windows
+          - os: windows-2022
+            core_base_tag: ltsc2022
+            nano_base_tag: ltsc2022
+            file: Dockerfile.Windows
+    steps:
+      - uses: actions/checkout@v4
+      - name: docker build
+        shell: bash
+        run: |
+          docker info
+          docker build --pull -f ${{ matrix.file }} --build-arg NANO_BASE_TAG=${{ matrix.nano_base_tag }} --build-arg CORE_BASE_TAG=${{ matrix.core_base_tag }} -t democratic-csi-windows:${GITHUB_RUN_ID}-${{ matrix.core_base_tag }} \
+          --label "org.opencontainers.image.created=$(date -u --iso-8601=seconds)" \
+          --label "org.opencontainers.image.revision=${GITHUB_SHA}" \
+          .
+          docker inspect democratic-csi-windows:${GITHUB_RUN_ID}-${{ matrix.core_base_tag }}
+          docker save democratic-csi-windows:${GITHUB_RUN_ID}-${{ matrix.core_base_tag }} -o democratic-csi-windows-${{ matrix.core_base_tag }}.tar
+      - name: upload image tar
+        uses: actions/upload-artifact@v4
+        with:
+          name: democratic-csi-windows-${{ matrix.core_base_tag }}.tar
+          path: democratic-csi-windows-${{ matrix.core_base_tag }}.tar
+          retention-days: 1
+
+  push-docker-windows:
+    needs:
+      - build-docker-linux
+      - build-docker-windows
+      - determine-image-tag
+    runs-on:
+      - self-hosted
+      - buildah
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: democratic-csi-windows-ltsc2019.tar
+      - uses: actions/download-artifact@v4
+        with:
+          name: democratic-csi-windows-ltsc2022.tar
+      - name: push windows images with buildah
+        run: |
+          #.github/bin/install_latest_buildah.sh
+          buildah version
+          .github/bin/docker-release-windows.sh
+        env:
+          DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
+          GHCR_USERNAME: ${{ secrets.GHCR_USERNAME }}
+          GHCR_PASSWORD: ${{ secrets.GHCR_PASSWORD }}
+          DOCKER_CLI_EXPERIMENTAL: enabled
+          IMAGE_TAG: ${{needs.determine-image-tag.outputs.tag}}

.gitignore

@@ -1,2 +1,4 @@
+**~
 node_modules
 dev
+/ci/bin/*dev*

CHANGELOG.md

@@ -1,9 +1,182 @@
+# v1.9.4
+
+Release 2024-07-06
+
+- minor doc updates
+
+# v1.9.3
+
+Released 2024-06-01
+
+- minor fixes for objectivefs and iscsi
+
+# v1.9.2
+
+Released 2024-05-23
+
+- minor fixes for objectivefs and iscsi
+
+# v1.9.1
+
+Released 2024-05-06
+
+- fix iscsi hostname lookup regression (#393)
+- fix resize issue (#390)
+- fix Probe issue (#385)
+
+# v1.9.0
+
+Released 2024-03-26
+
+- new `objectivefs` driver (https://objectivefs.com) support available for x86_64 and arm64
+- TrueNAS
+  - SCALE 24.04 support
+  - fix `sudo` issue during resize operations (see #295)
+  - fix version detection logic and default to api version 2 (see #351)
+  - more robust `Probe` implementation
+- contaimer images
+  - various fixes, improvements, dep upgrades, etc
+  - update container images to `debian:12` (bookworm)
+  - bump to nodejs-lts-iron from nodejs-lts-hydrogen
+- support csi v1.6.0-v1.9.0
+- allow `noop` delete operations (dangerous, only use if you _really_ know what you are doing, see #289)
+- properly adhere to the `zvolDedup` and `zvolCompression` settings (see #322)
+- `restic` and `kopia` support as a snapshot solution for `local-hostpath` and `*-client` drivers
+
+# v1.8.4
+
+Released 2023-11-09
+
+- allow templatized `volume_id` (dangerous, only use if you _really_ know what you are doing)
+- fix TrueNAS SCALE iscsi resize issue
+- TrueNAS SCALE 23.10 support
+- minor improvements/fixes throughout
+- dependency updates
+
+# v1.8.3
+
+Released 2023-04-05
+
+- fix invalid `access_mode` logic (see #287)
+
+# v1.8.2
+
+Released 2023-04-02
+
+- more comprehensive support to manually set `access_modes`
+- more intelligent handling of `access_modes` when `access_type=block`
+  - https://github.com/ceph/ceph-csi/blob/devel/examples/README.md#how-to-test-rbd-multi_node_multi_writer-block-feature
+  - others? allow this by default
+- remove older versions of TrueNAS from ci
+
+# v1.8.1
+
+Released 2023-02-25
+
+- minor fixes
+- updated `nvmeof` docs
+
+# v1.8.0
+
+Released 2023-02-23
+
+- `nvmeof` support
+
+# v1.7.7
+
+Released 2022-10-17
+
+- support `csi.access_modes` config value in all zfs-based drivers
+- bump deps
+
+# v1.7.6
+
+Released 2022-08-06
+
+- support fo `talos.dev` clusters
+- dep bumps
+
+# v1.7.5
+
+Released 2022-08-02
+
+- improved ipv6 iscsi support
+- allow using `blkid` for filesystem detection on block devices
+
+# v1.7.4
+
+Released 2022-07-29
+
+- improved ipv6 iscsi support
+
+# v1.7.3
+
+Released 2022-07-28
+
+- more stringent block device lookup logic (see #215)
+- ipv6 iscsi support
+- dependency bumps
+- minor fixes throughout
+
+# v1.7.2
+
+Released 2022-06-28
+
+- support for inode stats
+- doc updates
+- bump deps
+
+# v1.7.1
+
+Released 2022-06-14
+
+- support for the alpha TrueNAS SCALE 22.12
+- Fix invalid class reference
+
+# v1.7.0
+
+Released 2022-06-08
+
+The windows release.
+
+- windows smb, iscsi, and local-hostpath support (requires chart `v0.13.0+`)
+- ntfs, exfat, vfat fs support
+- `zfs-generic-smb` driver
+- synology improvements
+  - DSM7 support
+  - synology enhancements to allow templates to be configured at various
+    'levels'
+- testing improvements
+  - support (for testing) generating volume_id from name
+  - test all the smb variants
+  - test all nfs/smb client drivers
+- misc fixes
+  - wait for chown/chmod jobs to complete (freenas)
+  - general improvement to smb behavior throughout
+  - better logging
+  - better sudo logic throughout
+  - minor fixes throughout
+- more robust logic for connecting to iscsi devices with partition tables
+- massive performance improvement for ssh-based drivers (reusing existing
+  connection instead of new connection per-command)
+- dep bumps
+- trimmed container images
+- windows container images for 2019 and 2022
+
+# v1.6.3
+
+Released 2022-04-08
+
+- dep bumps
+- more secure permissions on the socket file
+
 # v1.6.2
 
 Released 2022-04-06
 
 - dep bumps
 - optimize via object instance reuse of various clients etc
+- graceful shutdown of the grpc server
 
 # v1.6.1
 

Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:11-slim AS build
+FROM debian:12-slim AS build
 #FROM --platform=$BUILDPLATFORM debian:10-slim AS build
 
 ENV DEBIAN_FRONTEND=noninteractive
@@ -9,14 +9,14 @@ ARG BUILDPLATFORM
 RUN echo "I am running build on $BUILDPLATFORM, building for $TARGETPLATFORM"
 
 RUN apt-get update && apt-get install -y locales && rm -rf /var/lib/apt/lists/* \
-        && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+  && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
 
 ENV LANG=en_US.utf8
-ENV NODE_VERSION=v16.14.2
+ENV NODE_VERSION=v20.11.1
 ENV NODE_ENV=production
 
 # install build deps
-RUN apt-get update && apt-get install -y python make cmake gcc g++
+RUN apt-get update && apt-get install -y python3 make cmake gcc g++
 
 # install node
 RUN apt-get update && apt-get install -y wget xz-utils
@@ -26,8 +26,8 @@ ENV PATH=/usr/local/lib/nodejs/bin:$PATH
 
 # Run as a non-root user
 RUN useradd --create-home csi \
-        && mkdir /home/csi/app \
-        && chown -R csi: /home/csi
+  && mkdir /home/csi/app \
+  && chown -R csi: /home/csi
 WORKDIR /home/csi/app
 USER csi
 
@@ -40,29 +40,33 @@ RUN rm -rf docker
 ######################
 # actual image
 ######################
-FROM debian:11-slim
+FROM debian:12-slim
 
 LABEL org.opencontainers.image.source https://github.com/democratic-csi/democratic-csi
+LABEL org.opencontainers.image.url https://github.com/democratic-csi/democratic-csi
+LABEL org.opencontainers.image.licenses MIT
 
 ENV DEBIAN_FRONTEND=noninteractive
+ENV DEMOCRATIC_CSI_IS_CONTAINER=true
 
 ARG TARGETPLATFORM
 ARG BUILDPLATFORM
+ARG OBJECTIVEFS_DOWNLOAD_ID
 
 RUN echo "I am running on final $BUILDPLATFORM, building for $TARGETPLATFORM"
 
 RUN apt-get update && apt-get install -y locales && rm -rf /var/lib/apt/lists/* \
-        && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
+  && localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
 
 ENV LANG=en_US.utf8
 ENV NODE_ENV=production
 
 # Workaround for https://github.com/nodejs/node/issues/37219
 RUN test $(uname -m) != armv7l || ( \
-                apt-get update \
-                && apt-get install -y libatomic1 \
-                && rm -rf /var/lib/apt/lists/* \
-        )
+  apt-get update \
+  && apt-get install -y libatomic1 \
+  && rm -rf /var/lib/apt/lists/* \
+  )
 
 # install node
 #ENV PATH=/usr/local/lib/nodejs/bin:$PATH
@@ -73,14 +77,31 @@ COPY --from=build /usr/local/lib/nodejs/bin/node /usr/local/bin/node
 # netbase is required by rpcbind/rpcinfo to work properly
 # /etc/{services,rpc} are required
 RUN apt-get update && \
-        apt-get install -y netbase socat e2fsprogs xfsprogs btrfs-progs fatresize dosfstools nfs-common cifs-utils sudo rsync && \
-        rm -rf /var/lib/apt/lists/*
+  apt-get install -y wget netbase zip bzip2 socat e2fsprogs exfatprogs xfsprogs btrfs-progs fatresize dosfstools ntfs-3g nfs-common cifs-utils fdisk gdisk cloud-guest-utils sudo rsync procps util-linux nvme-cli fuse3 && \
+  rm -rf /var/lib/apt/lists/*
+
+ARG RCLONE_VERSION=1.66.0
+ADD docker/rclone-installer.sh /usr/local/sbin
+RUN chmod +x /usr/local/sbin/rclone-installer.sh && rclone-installer.sh
+
+ARG RESTIC_VERSION=0.16.4
+ADD docker/restic-installer.sh /usr/local/sbin
+RUN chmod +x /usr/local/sbin/restic-installer.sh && restic-installer.sh
+
+ARG KOPIA_VERSION=0.16.1
+ADD docker/kopia-installer.sh /usr/local/sbin
+RUN chmod +x /usr/local/sbin/kopia-installer.sh && kopia-installer.sh
 
 # controller requirements
 #RUN apt-get update && \
 #        apt-get install -y ansible && \
 #        rm -rf /var/lib/apt/lists/*
 
+# install objectivefs
+ARG OBJECTIVEFS_VERSION=7.2
+ADD docker/objectivefs-installer.sh /usr/local/sbin
+RUN chmod +x /usr/local/sbin/objectivefs-installer.sh && objectivefs-installer.sh
+
 # install wrappers
 ADD docker/iscsiadm /usr/local/sbin
 RUN chmod +x /usr/local/sbin/iscsiadm
@@ -105,7 +126,7 @@ RUN chmod +x /usr/local/bin/oneclient
 
 # Run as a non-root user
 RUN useradd --create-home csi \
-        && chown -R csi: /home/csi
+  && chown -R csi: /home/csi
 
 COPY --from=build --chown=csi:csi /home/csi/app /home/csi/app
 

Dockerfile.Windows

@@ -0,0 +1,100 @@
+#
+# https://github.com/kubernetes/kubernetes/blob/master/test/images/windows/powershell-helper/Dockerfile_windows
+# https://github.com/kubernetes/kubernetes/blob/master/test/images/busybox/Dockerfile_windows
+# https://github.com/kubernetes/kubernetes/tree/master/test/images#windows-test-images-considerations
+# https://stefanscherer.github.io/find-dependencies-in-windows-containers/
+# 
+# docker build --build-arg NANO_BASE_TAG=1809 --build-arg CORE_BASE_TAG=ltsc2019 -t foobar -f Dockerfile.Windows .
+# docker run --rm -ti --entrypoint powershell foobar
+# docker run --rm foobar
+# docker save foobar -o foobar.tar
+# buildah pull docker-archive:foobar.tar
+
+# mcr.microsoft.com/windows/servercore:ltsc2019
+# mcr.microsoft.com/windows/nanoserver:1809
+
+ARG NANO_BASE_TAG
+ARG CORE_BASE_TAG
+
+FROM mcr.microsoft.com/windows/servercore:${CORE_BASE_TAG} as powershell
+
+# install powershell
+ENV PS_VERSION=6.2.7
+ADD https://github.com/PowerShell/PowerShell/releases/download/v$PS_VERSION/PowerShell-$PS_VERSION-win-x64.zip /PowerShell/powershell.zip
+
+RUN cd C:\PowerShell &\
+    tar.exe -xf powershell.zip &\
+    del powershell.zip &\
+    mklink powershell.exe pwsh.exe
+
+
+FROM mcr.microsoft.com/windows/servercore:${CORE_BASE_TAG} as build
+
+SHELL ["powershell", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+#ENV GPG_VERSION 4.0.2
+ENV GPG_VERSION 2.3.4
+
+RUN Invoke-WebRequest $('https://files.gpg4win.org/gpg4win-vanilla-{0}.exe' -f $env:GPG_VERSION) -OutFile 'gpg4win.exe' -UseBasicParsing ; \
+    Start-Process .\gpg4win.exe -ArgumentList '/S' -NoNewWindow -Wait
+
+# https://github.com/nodejs/node#release-keys
+RUN @( \
+    '4ED778F539E3634C779C87C6D7062848A1AB005C', \
+    '141F07595B7B3FFE74309A937405533BE57C7D57', \
+    '94AE36675C464D64BAFA68DD7434390BDBE9B9C5', \
+    '74F12602B6F1C4E913FAA37AD3A89613643B6201', \
+    '71DCFD284A79C3B38668286BC97EC7A07EDE3FC1', \
+    '61FC681DFB92A079F1685E77973F295594EC4689', \
+    '8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600', \
+    'C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8', \
+    'C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C', \
+    'DD8F2338BAE7501E3DD5AC78C273792F7D83545D', \
+    'A48C2BEE680E841632CD4E44F07496B3EB3C1762', \
+    '108F52B48DB57BB0CC439B2997B01419BD92F80A', \
+    'B9E2F5981AA6E0CD28160D9FF13993A75599653C' \
+    ) | foreach { \
+      gpg --keyserver hkps://keys.openpgp.org --recv-keys $_ ; \
+    }
+
+ENV NODE_VERSION 16.18.0
+
+RUN Invoke-WebRequest $('https://nodejs.org/dist/v{0}/SHASUMS256.txt.asc' -f $env:NODE_VERSION) -OutFile 'SHASUMS256.txt.asc' -UseBasicParsing ;
+#RUN Invoke-WebRequest $('https://nodejs.org/dist/v{0}/SHASUMS256.txt.asc' -f $env:NODE_VERSION) -OutFile 'SHASUMS256.txt.asc' -UseBasicParsing ; \
+#    gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc
+#gpg --verify SHASUMS256.txt.sig SHASUMS256.txt
+
+RUN Invoke-WebRequest $('https://nodejs.org/dist/v{0}/node-v{0}-win-x64.zip' -f $env:NODE_VERSION) -OutFile 'node.zip' -UseBasicParsing ; \
+    $sum = $(cat SHASUMS256.txt.asc | sls $('  node-v{0}-win-x64.zip' -f $env:NODE_VERSION)) -Split ' ' ; \
+    if ((Get-FileHash node.zip -Algorithm sha256).Hash -ne $sum[0]) { Write-Error 'SHA256 mismatch' } ; \
+    Expand-Archive node.zip -DestinationPath C:\ ; \
+    Rename-Item -Path $('C:\node-v{0}-win-x64' -f $env:NODE_VERSION) -NewName 'C:\nodejs'
+
+#RUN setx /M PATH "%PATH%;C:\nodejs"
+RUN setx /M PATH $(${Env:PATH} + \";C:\nodejs\")
+
+RUN node --version; npm --version;
+
+RUN mkdir /app
+WORKDIR /app
+
+COPY package*.json ./
+RUN npm install --only=production; ls /
+COPY . .
+
+FROM mcr.microsoft.com/windows/nanoserver:${NANO_BASE_TAG}
+
+LABEL org.opencontainers.image.source https://github.com/democratic-csi/democratic-csi
+LABEL org.opencontainers.image.url https://github.com/democratic-csi/democratic-csi
+LABEL org.opencontainers.image.licenses MIT
+
+# if additional dlls are required can copy like this
+#COPY --from=build /Windows/System32/nltest.exe /Windows/System32/nltest.exe
+
+COPY --from=build /app /app
+WORKDIR /app
+
+# this works for both host-process and non-host-process container semantics
+COPY --from=build /nodejs/node.exe ./bin
+
+ENTRYPOINT [ "bin/node.exe", "--expose-gc", "bin/democratic-csi" ]

README.md

@@ -1,5 +1,6 @@
 ![Image](https://img.shields.io/docker/pulls/democraticcsi/democratic-csi.svg)
-![Image](https://img.shields.io/github/workflow/status/democratic-csi/democratic-csi/CI?style=flat-square)
+![Image](https://img.shields.io/github/actions/workflow/status/democratic-csi/democratic-csi/main.yml?branch=master&style=flat-square)
+[![Artifact Hub](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/democratic-csi)](https://artifacthub.io/packages/search?repo=democratic-csi)
 
 # Introduction
 
@@ -23,10 +24,13 @@ have access to resizing, snapshots, clones, etc functionality.
   - `freenas-api-smb` experimental use with SCALE only (manages zfs datasets to share over smb)
   - `zfs-generic-nfs` (works with any ZoL installation...ie: Ubuntu)
   - `zfs-generic-iscsi` (works with any ZoL installation...ie: Ubuntu)
+  - `zfs-generic-smb` (works with any ZoL installation...ie: Ubuntu)
+  - `zfs-generic-nvmeof` (works with any ZoL installation...ie: Ubuntu)
   - `zfs-local-ephemeral-inline` (provisions node-local zfs datasets)
   - `zfs-local-dataset` (provision node-local volume as dataset)
   - `zfs-local-zvol` (provision node-local volume as zvol)
   - `synology-iscsi` experimental (manages volumes to share over iscsi)
+  - `objectivefs` (manages objectivefs volumes)
   - `lustre-client` (crudely provisions storage using a shared lustre
     share/directory for all volumes)
   - `nfs-client` (crudely provisions storage using a shared nfs share/directory
@@ -35,7 +39,8 @@ have access to resizing, snapshots, clones, etc functionality.
     for all volumes)
   - `local-hostpath` (crudely provisions node-local directories)
   - `node-manual` (allows connecting to manually created smb, nfs, lustre,
-    oneclient, and iscsi volumes, see sample PVs in the `examples` directory)
+    oneclient, nvmeof, and iscsi volumes, see sample PVs in the `examples`
+    directory)
 - framework for developing `csi` drivers
 
 If you have any interest in providing a `csi` driver, simply open an issue to
@@ -59,18 +64,29 @@ Predominantly 3 things are needed:
   from `nfs-client-provisioner` to `democratic-csi`)
 - https://gist.github.com/deefdragon/d58a4210622ff64088bd62a5d8a4e8cc
   (migrating between storage classes using `velero`)
+- https://github.com/fenio/k8s-truenas (NFS/iSCSI over API with TrueNAS Scale)
 
 ## Node Prep
 
 You should install/configure the requirements for both nfs and iscsi.
 
+### cifs
+
+```bash
+# RHEL / CentOS
+sudo yum install -y cifs-utils
+
+# Ubuntu / Debian
+sudo apt-get install -y cifs-utils
+```
+
 ### nfs
 
-```
-RHEL / CentOS
+```bash
+# RHEL / CentOS
 sudo yum install -y nfs-utils
 
-Ubuntu / Debian
+# Ubuntu / Debian
 sudo apt-get install -y nfs-common
 ```
 
@@ -83,9 +99,9 @@ If you are running Kubernetes with rancher/rke please see the following:
 
 - https://github.com/rancher/rke/issues/1846
 
-```
-RHEL / CentOS
+#### RHEL / CentOS
 
+```bash
 # Install the following system packages
 sudo yum install -y lsscsi iscsi-initiator-utils sg3_utils device-mapper-multipath
 
@@ -99,10 +115,11 @@ sudo systemctl start iscsid multipathd
 # Start and enable iscsi
 sudo systemctl enable iscsi
 sudo systemctl start iscsi
+```
 
+#### Ubuntu / Debian
 
-Ubuntu / Debian
-
+```
 # Install the following system packages
 sudo apt-get install -y open-iscsi lsscsi sg3-utils multipath-tools scsitools
 
@@ -124,14 +141,93 @@ sudo service open-iscsi start
 sudo systemctl status open-iscsi
 ```
 
-### freenas-smb
+#### [Talos](https://www.talos.dev/)
 
-If using with Windows based machines you may need to enable guest access (even
-if you are connecting with credentials)
+To use iscsi storage in kubernetes cluster in talos these steps are needed which are similar to the ones explained in https://www.talos.dev/v1.1/kubernetes-guides/configuration/replicated-local-storage-with-openebs-jiva/#patching-the-jiva-installation
+
+##### Patch nodes
+
+since talos does not have iscsi support by default, the iscsi extension is needed
+create a `patch.yaml` file with
+
+```yaml
+- op: add
+  path: /machine/install/extensions
+  value:
+    - image: ghcr.io/siderolabs/iscsi-tools:v0.1.1
+```
+
+and apply the patch across all of your nodes
+
+```bash
+talosctl -e <endpoint ip/hostname> -n <node ip/hostname> patch mc -p @patch.yaml
+```
+
+the extension will not activate until you "upgrade" the nodes, even if there is no update, use the latest version of talos installer.
+VERIFY THE TALOS VERSION IN THIS COMMAND BEFORE RUNNING IT AND READ THE [OpenEBS Jiva](https://www.talos.dev/v1.1/kubernetes-guides/configuration/replicated-local-storage-with-openebs-jiva/#patching-the-jiva-installation).
+upgrade all of the nodes in the cluster to get the extension
+
+```bash
+talosctl -e <endpoint ip/hostname> -n <node ip/hostname> upgrade --image=ghcr.io/siderolabs/installer:v1.1.1
+```
+
+in your `values.yaml` file make sure to enable these settings
+
+```yaml
+node:
+  hostPID: true
+  driver:
+    extraEnv:
+      - name: ISCSIADM_HOST_STRATEGY
+        value: nsenter
+      - name: ISCSIADM_HOST_PATH
+        value: /usr/local/sbin/iscsiadm
+    iscsiDirHostPath: /usr/local/etc/iscsi
+    iscsiDirHostPathType: ""
+```
+
+and continue your democratic installation as usuall with other iscsi drivers.
+
+#### Privileged Namespace
+
+democratic-csi requires privileged access to the nodes, so the namespace should allow for privileged pods. One way of doing it is via [namespace labels](https://kubernetes.io/docs/tasks/configure-pod-container/enforce-standards-namespace-labels/).
+Add the followin label to the democratic-csi installation namespace `pod-security.kubernetes.io/enforce=privileged`
 
 ```
-Set-ItemProperty HKLM:\SYSTEM\CurrentControlSet\Services\LanmanWorkstation\Parameters AllowInsecureGuestAuth -Value 1
-Restart-Service LanmanWorkstation -Force
+kubectl label --overwrite namespace democratic-csi pod-security.kubernetes.io/enforce=privileged
+```
+
+### nvmeof
+
+```bash
+# not required but likely helpful (tools are included in the democratic images
+# so not needed on the host)
+apt-get install -y nvme-cli
+
+# get the nvme fabric modules
+apt-get install linux-generic
+
+# ensure the nvmeof modules get loaded at boot
+cat <<EOF > /etc/modules-load.d/nvme.conf
+nvme
+nvme-tcp
+nvme-fc
+nvme-rdma
+EOF
+
+# load the modules immediately
+modprobe nvme
+modprobe nvme-tcp
+modprobe nvme-fc
+modprobe nvme-rdma
+
+# nvme has native multipath or can use DM multipath
+# democratic-csi will gracefully handle either configuration
+# RedHat recommends DM multipath (nvme_core.multipath=N)
+cat /sys/module/nvme_core/parameters/multipath
+
+# kernel arg to enable/disable native multipath
+nvme_core.multipath=N
 ```
 
 ### zfs-local-ephemeral-inline
@@ -176,6 +272,58 @@ volume is/was provisioned.
 The nature of this `driver` also prevents the enforcement of quotas. In short
 the requested volume size is generally ignored.
 
+### windows
+
+Support for Windows was introduced in `v1.7.0`. Currently support is limited
+to kubernetes nodes capabale of running `HostProcess` containers. Support was
+tested against `Windows Server 2019` using `rke2-v1.24`. Currently any of the
+`-smb` and `-iscsi` drivers will work. Support for `ntfs` was added to the
+linux nodes as well (using the `ntfs3` driver) so volumes created can be
+utilized by nodes with either operating system (in the case of `cifs` by both
+simultaneously).
+
+If using any `-iscsi` driver be sure your iqns are always fully lower-case by
+default (https://github.com/PowerShell/PowerShell/issues/17306).
+
+Due to current limits in the kubernetes tooling it is not possible to use the
+`local-hostpath` driver but support is implemented in this project and will
+work as soon as kubernetes support is available.
+
+```powershell
+# ensure all updates are installed
+
+# enable the container feature
+Enable-WindowsOptionalFeature -Online -FeatureName Containers –All
+
+# install a HostProcess compatible kubernetes
+
+# smb support
+# If using with Windows based machines you may need to enable guest access
+# (even if you are connecting with credentials)
+Set-ItemProperty HKLM:\SYSTEM\CurrentControlSet\Services\LanmanWorkstation\Parameters AllowInsecureGuestAuth -Value 1
+Restart-Service LanmanWorkstation -Force
+
+# iscsi
+# enable iscsi service and mpio as appropriate
+Get-Service -Name MSiSCSI
+Set-Service -Name MSiSCSI -StartupType Automatic
+Start-Service -Name MSiSCSI
+Get-Service -Name MSiSCSI
+
+# mpio
+Get-WindowsFeature -Name 'Multipath-IO'
+Add-WindowsFeature -Name 'Multipath-IO'
+
+Enable-MSDSMAutomaticClaim -BusType "iSCSI"
+Disable-MSDSMAutomaticClaim -BusType "iSCSI"
+
+Get-MSDSMGlobalDefaultLoadBalancePolicy
+Set-MSDSMGlobalLoadBalancePolicy -Policy RR
+```
+
+- https://kubernetes.io/blog/2021/08/16/windows-hostprocess-containers/
+- https://kubernetes.io/docs/tasks/configure-pod-container/create-hostprocess-pod/
+
 ## Server Prep
 
 Server preparation depends slightly on which `driver` you are using.
@@ -188,8 +336,9 @@ with much older versions as well.
 The various `freenas-api-*` drivers are currently EXPERIMENTAL and can only be
 used with SCALE 21.08+. Fundamentally these drivers remove the need for `ssh`
 connections and do all operations entirely with the TrueNAS api. With that in
-mind, any ssh/shell/etc requirements below can be safely ignored. Also note the
-following known issues:
+mind, any ssh/shell/etc requirements below can be safely ignored. The minimum
+volume size through the api is `1G` so beware that requested volumes with a
+size small will be increased to `1G`. Also note the following known issues:
 
 - https://jira.ixsystems.com/browse/NAS-111870
 - https://github.com/democratic-csi/democratic-csi/issues/112
@@ -198,9 +347,12 @@ following known issues:
 Ensure the following services are configurged and running:
 
 - ssh (if you use a password for authentication make sure it is allowed)
+  - https://www.truenas.com/community/threads/ssh-access-ssh-rsa-not-in-pubkeyacceptedalgorithms.101715/
+  - `PubkeyAcceptedAlgorithms +ssh-rsa`
 - ensure `zsh`, `bash`, or `sh` is set as the root shell, `csh` gives false errors due to quoting
 - nfs
 - iscsi
+
   - (fixed in 12.0-U2+) when using the FreeNAS API concurrently the
     `/etc/ctl.conf` file on the server can become invalid, some sample scripts
     are provided in the `contrib` directory to clean things up ie: copy the
@@ -215,6 +367,9 @@ Ensure the following services are configurged and running:
       - `curl --header "Accept: application/json" --user root:<password> 'http(s)://<ip>/api/v2.0/iscsi/portal'`
       - `curl --header "Accept: application/json" --user root:<password> 'http(s)://<ip>/api/v2.0/iscsi/initiator'`
       - `curl --header "Accept: application/json" --user root:<password> 'http(s)://<ip>/api/v2.0/iscsi/auth'`
+  - The maximum number of volumes is limited to 255 by default on FreeBSD (physical devices such as disks and CD-ROM drives count against this value).
+    Be sure to properly adjust both [tunables](https://www.freebsd.org/cgi/man.cgi?query=ctl&sektion=4#end) `kern.cam.ctl.max_ports` and `kern.cam.ctl.max_luns` to avoid running out of resources when dynamically provisioning iSCSI volumes on FreeNAS or TrueNAS Core.
+
 - smb
 
 If you would prefer you can configure `democratic-csi` to use a
@@ -270,21 +425,144 @@ Issues to review:
 - https://jira.ixsystems.com/browse/NAS-108522
 - https://jira.ixsystems.com/browse/NAS-107219
 
-### ZoL (zfs-generic-nfs, zfs-generic-iscsi)
+### ZoL (zfs-generic-nfs, zfs-generic-iscsi, zfs-generic-smb, zfs-generic-nvmeof)
 
 Ensure ssh and zfs is installed on the nfs/iscsi server and that you have installed
 `targetcli`.
 
-- `sudo yum install targetcli -y`
-- `sudo apt-get -y install targetcli-fb`
+The driver executes many commands over an ssh connection. You may consider
+disabling all the `motd` details for the ssh user as it can spike the cpu
+unecessarily:
+
+- https://askubuntu.com/questions/318592/how-can-i-remove-the-landscape-canonical-com-greeting-from-motd
+- https://linuxconfig.org/disable-dynamic-motd-and-news-on-ubuntu-20-04-focal-fossa-linux
+- https://github.com/democratic-csi/democratic-csi/issues/151 (some notes on
+  using delegated zfs permissions)
+
+```bash
+####### nfs
+yum install -y nfs-utils
+systemctl enable --now nfs-server.service
+
+apt-get install -y nfs-kernel-server
+systemctl enable --now nfs-kernel-server.service
+
+####### iscsi
+yum install targetcli -y
+apt-get -y install targetcli-fb
+
+####### smb
+apt-get install -y samba smbclient
+
+# create posix user
+groupadd -g 1001 smbroot
+useradd -u 1001 -g 1001 -M -N -s /sbin/nologin smbroot
+passwd smbroot (optional)
+
+# create smb user and set password
+smbpasswd -L -a smbroot
+
+####### nvmeof
+# ensure nvmeof target modules are loaded at startup
+cat <<EOF > /etc/modules-load.d/nvmet.conf
+nvmet
+nvmet-tcp
+nvmet-fc
+nvmet-rdma
+EOF
+
+# load the modules immediately
+modprobe nvmet
+modprobe nvmet-tcp
+modprobe nvmet-fc
+modprobe nvmet-rdma
+
+# install nvmetcli and systemd services
+git clone git://git.infradead.org/users/hch/nvmetcli.git
+cd nvmetcli
+
+## install globally
+python3 setup.py install --prefix=/usr
+pip install configshell_fb
+
+## install to root home dir
+python3 setup.py install --user
+pip install configshell_fb --user
+
+# prevent log files from filling up disk
+ln -sf /dev/null ~/.nvmetcli/log.txt
+ln -sf /dev/null ~/.nvmetcli/history.txt
+
+# install systemd unit and enable/start
+## optionally to ensure the config file is loaded before we start
+## reading/writing to it add an ExecStartPost= to the unit file
+##
+## ExecStartPost=/usr/bin/touch /var/run/nvmet-config-loaded
+##
+## in your dirver config set nvmeof.shareStrategyNvmetCli.configIsImportedFilePath=/var/run/nvmet-config-loaded
+## which will prevent the driver from making any changes until the configured
+## file is present
+vi nvmet.service
+
+cp nvmet.service /etc/systemd/system/
+mkdir -p /etc/nvmet
+systemctl daemon-reload
+systemctl enable --now nvmet.service
+systemctl status nvmet.service
+
+# create the port(s) configuration manually
+echo "
+cd /
+ls
+" | nvmetcli
+
+# do this multiple times altering as appropriate if you have/want multipath
+# change the port to 2, 3.. each additional path
+# the below example creates a tcp port listening on all IPs on port 4420
+echo "
+cd /ports
+create 1
+cd 1
+set addr adrfam=ipv4 trtype=tcp traddr=0.0.0.0 trsvcid=4420
+
+saveconfig /etc/nvmet/config.json
+" | nvmetcli
+
+# if running TrueNAS SCALE you can skip the above and simply copy
+# contrib/scale-nvmet-start.sh to your machine and add it as a startup script
+# to launch POSTINIT type COMMAND
+# and then create the port(s) as mentioned above
+```
 
 ### Synology (synology-iscsi)
 
-Ensure iscsi manager has been installed and is generally setup/configured.
+Ensure iscsi manager has been installed and is generally setup/configured. DSM 6.3+ is supported.
+
+### objectivefs (objectivefs)
+
+ObjectiveFS requires the use of an _Admin Key_ to properly automate the
+lifecycle of filesystems. Each deployment of the driver will point to a single
+`pool` (bucket) and create individual `filesystems` within that bucket
+on-demand.
+
+Ensure the config value used for `pool` is an existing bucket. Be sure the
+bucket is _NOT_ being used in fs mode (ie: the whole bucket is a single fs).
+
+The `democratic-csi` `node` container will host the fuse mount process so
+be careful to only upgrade when all relevant workloads have been drained from
+the respective node. Also beware that any cpu/memory limits placed on the
+container by the orchestration system will impact any ability to use the
+caching, etc features of objectivefs.
+
+- https://objectivefs.com/howto/csi-driver-objectivefs
+- https://objectivefs.com/howto/csi-driver-objectivefs-kubernetes-managed
+- https://objectivefs.com/howto/objectivefs-admin-key-setup
+- https://objectivefs.com/features#filesystem-pool
+- https://objectivefs.com/howto/how-to-create-a-filesystem-with-an-existing-empty-bucket
 
 ## Helm Installation
 
-```
+```bash
 helm repo add democratic-csi https://democratic-csi.github.io/charts/
 helm repo update
 # helm v2
@@ -328,13 +606,14 @@ microk8s helm upgrade \
 
 - microk8s - `/var/snap/microk8s/common/var/lib/kubelet`
 - pivotal - `/var/vcap/data/kubelet`
+- k0s - `/var/lib/k0s/kubelet`
 
 ### openshift
 
 `democratic-csi` generally works fine with openshift. Some special parameters
 need to be set with helm (support added in chart version `0.6.1`):
 
-```
+```bash
 # for sure required
 --set node.rbac.openshift.privileged=true
 --set node.driver.localtimeHostPath=false
@@ -345,38 +624,40 @@ need to be set with helm (support added in chart version `0.6.1`):
 
 ### Nomad
 
-`democratic-csi` works with Nomad in a functioning but limted capacity. See the [Nomad docs](docs/nomad.md) for details.
+`democratic-csi` works with Nomad in a functioning but limted capacity. See the
+[Nomad docs](docs/nomad.md) for details.
+
+### Docker Swarm
+
+- https://github.com/moby/moby/blob/master/docs/cluster_volumes.md
+- https://github.com/olljanat/csi-plugins-for-docker-swarm
 
 ## Multiple Deployments
 
-You may install multiple deployments of each/any driver. It requires the following:
+You may install multiple deployments of each/any driver. It requires the
+following:
 
 - Use a new helm release name for each deployment
-- Make sure you have a unique `csiDriver.name` in the values file
+- Make sure you have a unique `csiDriver.name` in the values file (within the
+  same cluster)
 - Use unqiue names for your storage classes (per cluster)
-- Use a unique parent dataset (ie: don't try to use the same parent across deployments or clusters)
+- Use a unique parent dataset (ie: don't try to use the same parent across
+  deployments or clusters)
+- For `iscsi` and `smb` be aware that the names of assets/shares are _global_
+  and so collisions are possible/probable. Appropriate use of the respective
+  `nameTemplate`, `namePrefix`, and `nameSuffix` configuration options will
+  mitigate the issue [#210](https://github.com/democratic-csi/democratic-csi/issues/210).
 
 # Snapshot Support
 
-Install beta (v1.17+) CRDs (once per cluster):
-
-- https://github.com/kubernetes-csi/external-snapshotter/tree/master/client/config/crd
-
-```
-kubectl apply -f snapshot.storage.k8s.io_volumesnapshotclasses.yaml
-kubectl apply -f snapshot.storage.k8s.io_volumesnapshotcontents.yaml
-kubectl apply -f snapshot.storage.k8s.io_volumesnapshots.yaml
-```
-
 Install snapshot controller (once per cluster):
 
-- https://github.com/kubernetes-csi/external-snapshotter/tree/master/deploy/kubernetes/snapshot-controller
+- https://github.com/democratic-csi/charts/tree/master/stable/snapshot-controller
 
-```
-# replace namespace references to your liking
-kubectl apply -f rbac-snapshot-controller.yaml
-kubectl apply -f setup-snapshot-controller.yaml
-```
+OR
+
+- https://github.com/kubernetes-csi/external-snapshotter/tree/master/client/config/crd
+- https://github.com/kubernetes-csi/external-snapshotter/tree/master/deploy/kubernetes/snapshot-controller
 
 Install `democratic-csi` as usual with `volumeSnapshotClasses` defined as appropriate.
 
@@ -393,12 +674,6 @@ Copy the `contrib/freenas-provisioner-to-democratic-csi.sh` script from the
 project to your workstation, read the script in detail, and edit the variables
 to your needs to start migrating!
 
-# Sponsors
-
-A special shout out to the wonderful sponsors of the project!
-
-[![ixSystems](https://www.ixsystems.com/wp-content/uploads/2021/06/ix_logo_200x47.png "ixSystems")](http://ixsystems.com/)
-
 # Related
 
 - https://github.com/nmaupu/freenas-provisioner

bin/democratic-csi

@@ -1,10 +1,18 @@
-#!/usr/bin/env -S node --expose-gc ${NODE_OPTIONS_CSI_1} ${NODE_OPTIONS_CSI_2} ${NODE_OPTIONS_CSI_3} ${NODE_OPTIONS_CSI_4} ${NODE_OPTIONS_CSI_5}
+#!/usr/bin/env -S node --expose-gc ${NODE_OPTIONS_CSI_1} ${NODE_OPTIONS_CSI_2} ${NODE_OPTIONS_CSI_3} ${NODE_OPTIONS_CSI_4}
 
+/**
+ * keep the shebang line length under 128
+ * https://github.com/democratic-csi/democratic-csi/issues/171
+ */
+
+// polyfills
+require("../src/utils/polyfills");
 const yaml = require("js-yaml");
 const fs = require("fs");
 const { grpc } = require("../src/utils/grpc");
-const { stringify } = require("../src/utils/general");
+const { stringify, stripWindowsDriveLetter } = require("../src/utils/general");
 
+let driverConfigFile;
 let options;
 const args = require("yargs")
   .env("DEMOCRATIC_CSI")
@@ -14,17 +22,29 @@ const args = require("yargs")
     describe: "provide a path to driver config file",
     config: true,
     configParser: (path) => {
-      try {
-        options = JSON.parse(fs.readFileSync(path, "utf-8"));
-        return true;
-      } catch (e) {}
+      // normalize path for host-process containers
+      // CONTAINER_SANDBOX_MOUNT_POINT  C:\C\0eac9a8da76f6d7119c5d9f86c8b3106d67dbbf01dbeb22fdc0192476b7e31cb\
+      // path is injected as C:\config\driver-config-file.yaml
+      if (process.env.CONTAINER_SANDBOX_MOUNT_POINT) {
+        path = `${
+          process.env.CONTAINER_SANDBOX_MOUNT_POINT
+        }${stripWindowsDriveLetter(path)}`;
+      }
 
       try {
         options = yaml.load(fs.readFileSync(path, "utf8"));
-        return true;
-      } catch (e) {}
+        try {
+          driverConfigFile = fs.realpathSync(path);
+        } catch (e) {
+          console.log("failed finding config file realpath: " + e.toString());
+          driverConfigFile = path;
+        }
 
-      throw new Error("failed parsing config file: " + path);
+        return true;
+      } catch (e) {
+        console.log("failed parsing config file: " + path);
+        throw e;
+      }
     },
   })
   .demandOption(["driver-config-file"], "driver-config-file is required")
@@ -43,6 +63,10 @@ const args = require("yargs")
       "1.3.0",
       "1.4.0",
       "1.5.0",
+      "1.6.0",
+      "1.7.0",
+      "1.8.0",
+      "1.9.0",
     ],
   })
   .demandOption(["csi-version"], "csi-version is required")
@@ -69,6 +93,11 @@ const args = require("yargs")
     describe: "listen socket for the server",
     type: "string",
   })
+  .option("server-socket-permissions-mode", {
+    describe: "permissions on the socket file for the server",
+    type: "string",
+    default: "0600", // os default is 755
+  })
   .version()
   .help().argv;
 
@@ -77,6 +106,9 @@ if (!args.serverSocket && !args.serverAddress && !args.serverPort) {
   process.exit(1);
 }
 
+//console.log(args);
+//console.log(process.env);
+
 const package = require("../package.json");
 args.version = package.version;
 
@@ -108,10 +140,13 @@ const csi = protoDescriptor.csi.v1;
 
 logger.info("initializing csi driver: %s", options.driver);
 
+const { Registry } = require("../src/utils/registry");
+let globalRegistry = new Registry();
+
 let driver;
 try {
   driver = require("../src/driver/factory").factory(
-    { logger, args, cache, package, csiVersion },
+    { logger, args, cache, package, csiVersion, registry: globalRegistry },
     options
   );
 } catch (err) {
@@ -129,6 +164,14 @@ let operationLock = new Set();
 
 async function requestHandlerProxy(call, callback, serviceMethodName) {
   const cleansedCall = JSON.parse(stringify(call));
+
+  delete cleansedCall.call;
+  delete cleansedCall.canceled;
+  for (const key in cleansedCall) {
+    if (key.startsWith("_")) {
+      delete cleansedCall[key];
+    }
+  }
   for (const key in cleansedCall.request) {
     if (key.includes("secret")) {
       cleansedCall.request[key] = "redacted";
@@ -158,6 +201,18 @@ async function requestHandlerProxy(call, callback, serviceMethodName) {
       });
     }
 
+    // for testing purposes
+    //await GeneralUtils.sleep(10000);
+    //throw new Error("fake error");
+
+    // for CI/testing purposes
+    if (["NodePublishVolume", "NodeStageVolume"].includes(serviceMethodName)) {
+      await driver.setVolumeContextCache(
+        call.request.volume_id,
+        call.request.volume_context
+      );
+    }
+
     let response;
     let responseError;
     try {
@@ -183,12 +238,21 @@ async function requestHandlerProxy(call, callback, serviceMethodName) {
       throw responseError;
     }
 
+    // for CI/testing purposes
+    if (serviceMethodName == "CreateVolume") {
+      await driver.setVolumeContextCache(
+        response.volume.volume_id,
+        response.volume.volume_context
+      );
+    }
+
     logger.info(
       "new response - driver: %s method: %s response: %j",
       driver.constructor.name,
       serviceMethodName,
       response
     );
+
     callback(null, response);
   } catch (e) {
     let message;
@@ -198,7 +262,7 @@ async function requestHandlerProxy(call, callback, serviceMethodName) {
         message += ` ${e.stack}`;
       }
     } else {
-      message = JSON.stringify(e);
+      message = stringify(e);
     }
 
     logger.error(
@@ -329,9 +393,11 @@ if (args.serverSocket) {
 }
 
 logger.info(
-  "starting csi server - name: %s, version: %s, driver: %s, mode: %s, csi version: %s, address: %s, socket: %s",
-  args.csiName,
+  "starting csi server - node version: %s, package version: %s, config file: %s, csi-name: %s, csi-driver: %s, csi-mode: %s, csi-version: %s, address: %s, socket: %s",
+  process.version,
   args.version,
+  driverConfigFile,
+  args.csiName,
   options.driver,
   args.csiMode.join(","),
   args.csiVersion,
@@ -339,10 +405,58 @@ logger.info(
   bindSocket
 );
 
-[`SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`].forEach(
+const signalMapping = {
+  1: "SIGHUP",
+  2: "SIGINT",
+  3: "SIGQUIT",
+  4: "SIGILL",
+  5: "SIGTRAP",
+  6: "SIGABRT",
+  7: "SIGEMT",
+  8: "SIGFPE",
+  9: "SIGKILL",
+  10: "SIGBUS",
+  11: "SIGSEGV",
+  12: "SIGSYS",
+  13: "SIGPIPE",
+  14: "SIGALRM",
+  15: "SIGTERM",
+  16: "SIGURG",
+  17: "SIGSTOP",
+  18: "SIGTSTP",
+  19: "SIGCONT",
+  20: "SIGCHLD",
+  21: "SIGTTIN",
+  22: "SIGTTOU",
+  23: "SIGIO",
+  24: "SIGXCPU",
+  25: "SIGXFSZ",
+  26: "SIGVTALRM",
+  27: "SIGPROF",
+  28: "SIGWINCH",
+  29: "SIGINFO",
+  30: "SIGUSR1",
+  31: "SIGUSR2",
+};
+
+[(`SIGINT`, `SIGUSR1`, `SIGUSR2`, `uncaughtException`, `SIGTERM`)].forEach(
   (eventType) => {
     process.on(eventType, async (code) => {
-      console.log(`running server shutdown, exit code: ${code}`);
+      let codeNumber = null;
+      let codeName = null;
+      if (code > 0) {
+        codeNumber = code;
+        codeName = signalMapping[code];
+      } else {
+        codeNumber = Object.keys(signalMapping).find(
+          (key) => signalMapping[key] === code
+        );
+        codeName = code;
+      }
+
+      console.log(
+        `running server shutdown, exit code: ${codeNumber} (${codeName})`
+      );
 
       // attempt clean shutdown of in-flight requests
       try {
@@ -373,7 +487,7 @@ logger.info(
       }
 
       console.log("server fully shutdown, exiting");
-      process.exit(code);
+      process.exit(codeNumber);
     });
   }
 );
@@ -455,7 +569,7 @@ if (require.main === module) {
           );
         });
 
-        fs.chmodSync(socketPath, 0o666);
+        fs.chmodSync(socketPath, args["server-socket-permissions-mode"]);
       }
       csiServer.start();
     } catch (e) {

bin/k8s-csi-cleaner

@@ -127,6 +127,7 @@ async function main() {
 
   for (let csiVolume of csiVolumes) {
     let volume_id = csiVolume.volume.volume_id;
+    let volume_context = JSON.stringify(csiVolume.volume.volume_context) || "Unknown";
     //console.log(`processing csi volume ${volume_id}`);
     let k8sVolume = k8sVolumes.find((i_k8sVolume) => {
       let volume_handle = _.get(i_k8sVolume, "spec.csi.volumeHandle", null);
@@ -134,7 +135,7 @@ async function main() {
     });
 
     if (!k8sVolume) {
-      console.log(`volume ${volume_id} is NOT in k8s`);
+      console.log(`volume ${volume_id} (${volume_context}) is NOT in k8s`);
       if (process.env.DRY_RUN == "1") {
         continue;
       }
@@ -159,7 +160,7 @@ async function main() {
         console.log(`skipping delete of csi volume ${volume_id}`);
       }
     } else {
-      console.log(`volume ${volume_id} is in k8s`);
+      console.log(`volume ${volume_id} (${volume_context}) is in k8s`);
     }
   }
 

ci/bin/build.ps1

@@ -0,0 +1,19 @@
+Write-Output "current user"
+whoami
+Write-Output "current working directory"
+(Get-Location).Path
+Write-Output "current PATH"
+$Env:PATH
+
+Write-Output "node version"
+node --version
+Write-Output "npm version"
+npm --version
+
+# install deps
+Write-Output "running npm i"
+npm i
+
+Write-Output "creating tar.gz"
+# tar node_modules to keep the number of files low to upload
+tar -zcf node_modules-windows-amd64.tar.gz node_modules

ci/bin/build.sh

@@ -12,4 +12,4 @@ npm --version
 npm i
 
 # tar node_modules to keep the number of files low to upload
-tar -zcf node_modules.tar.gz node_modules
+tar -zcf node_modules-linux-amd64.tar.gz node_modules

ci/bin/helper.ps1

@@ -0,0 +1,16 @@
+#Set-StrictMode -Version Latest
+#$ErrorActionPreference = "Stop"
+#$PSDefaultParameterValues['*:ErrorAction'] = "Stop"
+function ThrowOnNativeFailure {
+  if (-not $?) {
+    throw 'Native Failure'
+  }
+}
+
+function psenvsubstr($data) {
+  foreach($v in Get-ChildItem env:) {
+    $key = '${' + $v.Name + '}'
+    $data = $data.Replace($key, $v.Value)
+  }
+  return $data
+}

ci/bin/launch-csi-grpc-proxy.ps1

@@ -0,0 +1,15 @@
+if (! $PSScriptRoot) {
+  $PSScriptRoot = $args[0]
+}
+
+. "${PSScriptRoot}\helper.ps1"
+
+Set-Location $env:PWD
+
+Write-Output "launching csi-grpc-proxy"
+
+$env:PROXY_TO = "npipe://" + $env:NPIPE_ENDPOINT
+$env:BIND_TO = "unix://" + $env:CSI_ENDPOINT
+
+# https://stackoverflow.com/questions/2095088/error-when-calling-3rd-party-executable-from-powershell-when-using-an-ide
+csi-grpc-proxy.exe 2>&1 | % { "$_" }

ci/bin/launch-csi-sanity.ps1

@@ -0,0 +1,69 @@
+if (! $PSScriptRoot) {
+  $PSScriptRoot = $args[0]
+}
+
+. "${PSScriptRoot}\helper.ps1"
+
+Set-Location $env:PWD
+
+$exit_code = 0
+$tmpdir = New-Item -ItemType Directory -Path ([System.IO.Path]::GetTempPath()) -Name ([System.IO.Path]::GetRandomFileName())
+$env:CSI_SANITY_TEMP_DIR = $tmpdir.FullName
+
+# cleanse endpoint to something csi-sanity plays nicely with
+$endpoint = ${env:CSI_ENDPOINT}
+$endpoint = $endpoint.replace("C:\", "/")
+$endpoint = $endpoint.replace("\", "/")
+
+if (! $env:CSI_SANITY_FAILFAST) {
+  $env:CSI_SANITY_FAILFAST = "false"
+}
+
+Write-Output "launching csi-sanity"
+Write-Output "connecting to: ${endpoint}"
+Write-Output "failfast: ${env:CSI_SANITY_FAILFAST}"
+Write-Output "skip: ${env:CSI_SANITY_SKIP}"
+Write-Output "focus: ${env:CSI_SANITY_FOCUS}"
+Write-Output "csi.mountdir: ${env:CSI_SANITY_TEMP_DIR}\mnt"
+Write-Output "csi.stagingdir: ${env:CSI_SANITY_TEMP_DIR}\stage"
+
+$exe = "csi-sanity.exe"
+$exeargs = @()
+$exeargs += "-csi.endpoint", "unix://${endpoint}"
+$exeargs += "-csi.mountdir", "${env:CSI_SANITY_TEMP_DIR}\mnt"
+$exeargs += "-csi.stagingdir", "${env:CSI_SANITY_TEMP_DIR}\stage"
+$exeargs += "-csi.testvolumeexpandsize", "2147483648"
+$exeargs += "-csi.testvolumesize", "1073741824"
+$exeargs += "--csi.secrets", "${env:CSI_SANITY_SECRETS}"
+$exeargs += "-ginkgo.skip", "${env:CSI_SANITY_SKIP}"
+$exeargs += "-ginkgo.focus", "${env:CSI_SANITY_FOCUS}"
+
+if ($env:CSI_SANITY_FAILFAST -eq "true") {
+  $exeargs += "-ginkgo.fail-fast"
+}
+
+Write-Output "csi-sanity command: $exe $($exeargs -join ' ')"
+
+&$exe $exeargs
+
+if (-not $?) {
+  $exit_code = $LASTEXITCODE
+  Write-Output "csi-sanity exit code: ${exit_code}"
+  if ($exit_code -gt 0) {
+    $exit_code = 1
+  }
+}
+
+# remove tmp dir
+Remove-Item -Path "$env:CSI_SANITY_TEMP_DIR" -Force -Recurse
+
+#Exit $exit_code
+Write-Output "exiting with exit code: ${exit_code}"
+
+if ($exit_code -gt 0) {
+  throw "csi-sanity failed"
+}
+
+# these do not work for whatever reason
+#Exit $exit_code
+#[System.Environment]::Exit($exit_code)

ci/bin/launch-csi-sanity.sh

@@ -7,7 +7,7 @@ set -x
 : ${CSI_ENDPOINT:=/tmp/csi-${CI_BUILD_KEY}.sock}
 : ${CSI_SANITY_TEMP_DIR:=$(mktemp -d -t ci-csi-sanity-tmp-XXXXXXXX)}
 
-if [[ ! -S "${CSI_ENDPOINT}" ]];then
+if [[ ! -S "${CSI_ENDPOINT}" ]]; then
   echo "csi socket: ${CSI_ENDPOINT} does not exist"
   exit 1
 fi
@@ -15,27 +15,29 @@ fi
 trap ctrl_c INT
 
 function ctrl_c() {
-        echo "Trapped CTRL-C"
-        exit 1
+  echo "Trapped CTRL-C"
+  exit 1
 }
 
-chmod g+w,o+w "${CSI_ENDPOINT}";
-mkdir -p "${CSI_SANITY_TEMP_DIR}";
-rm -rf "${CSI_SANITY_TEMP_DIR}"/*;
-chmod -R 777 "${CSI_SANITY_TEMP_DIR}";
+chmod g+w,o+w "${CSI_ENDPOINT}"
+mkdir -p "${CSI_SANITY_TEMP_DIR}"
+rm -rf "${CSI_SANITY_TEMP_DIR}"/*
+chmod -R 777 "${CSI_SANITY_TEMP_DIR}"
 
 # https://github.com/kubernetes-csi/csi-test/tree/master/cmd/csi-sanity
 # FOR DEBUG: --ginkgo.v
 # --csi.secrets=<path to secrets file>
+#
 # expand size 2073741824 to have mis-alignments
 # expand size 2147483648 to have everything line up nicely
 
 csi-sanity --csi.endpoint "unix://${CSI_ENDPOINT}" \
---csi.mountdir "${CSI_SANITY_TEMP_DIR}/mnt" \
---csi.stagingdir "${CSI_SANITY_TEMP_DIR}/stage" \
---csi.testvolumeexpandsize 2147483648 \
---csi.testvolumesize 1073741824 \
--ginkgo.skip "${CSI_SANITY_SKIP}" \
--ginkgo.focus "${CSI_SANITY_FOCUS}"
+  --csi.mountdir "${CSI_SANITY_TEMP_DIR}/mnt" \
+  --csi.stagingdir "${CSI_SANITY_TEMP_DIR}/stage" \
+  --csi.testvolumeexpandsize 2147483648 \
+  --csi.testvolumesize 1073741824 \
+  --csi.secrets="${CSI_SANITY_SECRETS}" \
+  -ginkgo.skip "${CSI_SANITY_SKIP}" \
+  -ginkgo.focus "${CSI_SANITY_FOCUS}"
 
 rm -rf "${CSI_SANITY_TEMP_DIR}"

ci/bin/launch-server.ps1

@@ -0,0 +1,29 @@
+if (! $PSScriptRoot) {
+  $PSScriptRoot = $args[0]
+}
+
+. "${PSScriptRoot}\helper.ps1"
+
+Set-Location $env:PWD
+Write-Output "launching server"
+
+$env:LOG_LEVEL = "debug"
+$env:CSI_VERSION = "1.9.0"
+$env:CSI_NAME = "driver-test"
+$env:CSI_SANITY = "1"
+
+if (! ${env:CONFIG_FILE}) {
+  $env:CONFIG_FILE = $env:TEMP + "\csi-config-" + $env:CI_BUILD_KEY + ".yaml"
+  if ($env:TEMPLATE_CONFIG_FILE) {
+    $config_data = Get-Content "${env:TEMPLATE_CONFIG_FILE}" -Raw
+    $config_data = psenvsubstr($config_data)
+    $config_data | Set-Content "${env:CONFIG_FILE}"
+  }
+}
+
+node "${PSScriptRoot}\..\..\bin\democratic-csi" `
+  --log-level "$env:LOG_LEVEL" `
+  --driver-config-file "$env:CONFIG_FILE" `
+  --csi-version "$env:CSI_VERSION" `
+  --csi-name "$env:CSI_NAME" `
+  --server-socket "${env:NPIPE_ENDPOINT}" 2>&1 | % { "$_" }

ci/bin/launch-server.sh

@@ -3,24 +3,27 @@
 set -e
 set -x
 
+export PATH="/usr/local/lib/nodejs/bin:${PATH}"
+echo "current launch-server PATH: ${PATH}"
+
 : ${CI_BUILD_KEY:="local"}
 : ${TEMPLATE_CONFIG_FILE:=${1}}
 : ${CSI_MODE:=""}
-: ${CSI_VERSION:="1.5.0"}
+: ${CSI_VERSION:="1.9.0"}
 : ${CSI_ENDPOINT:=/tmp/csi-${CI_BUILD_KEY}.sock}
 : ${LOG_PATH:=/tmp/csi-${CI_BUILD_KEY}.log}
 
-if [[ "x${CONFIG_FILE}" == "x" ]];then
+if [[ "x${CONFIG_FILE}" == "x" ]]; then
   : ${CONFIG_FILE:=/tmp/csi-config-${CI_BUILD_KEY}.yaml}
 
-  if [[ "x${TEMPLATE_CONFIG_FILE}" != "x" ]];then
-    envsubst < "${TEMPLATE_CONFIG_FILE}" > "${CONFIG_FILE}"
+  if [[ "x${TEMPLATE_CONFIG_FILE}" != "x" ]]; then
+    envsubst <"${TEMPLATE_CONFIG_FILE}" >"${CONFIG_FILE}"
   fi
 fi
 
-if [[ "x${CSI_MODE}" != "x" ]];then
+if [[ "x${CSI_MODE}" != "x" ]]; then
   EXTRA_ARGS="--csi-mode ${CSI_MODE} ${EXTRA_ARGS}"
 fi
 
-# > "${LOG_PATH}" 2>&1 
+# > "${LOG_PATH}" 2>&1
 exec ./bin/democratic-csi --log-level debug --driver-config-file "${CONFIG_FILE}" --csi-version "${CSI_VERSION}" --csi-name "driver-test" --server-socket "${CSI_ENDPOINT}" ${EXTRA_ARGS}

ci/bin/run.ps1

@@ -0,0 +1,133 @@
+# https://stackoverflow.com/questions/2095088/error-when-calling-3rd-party-executable-from-powershell-when-using-an-ide
+# 
+# Examples:
+#
+# $mypath = $MyInvocation.MyCommand.Path
+# Get-ChildItem env:\
+# Get-Job | Where-Object -Property State -eq “Running”
+# Get-Location (like pwd)
+# if ($null -eq $env:FOO) { $env:FOO = 'bar' }
+
+. "${PSScriptRoot}\helper.ps1"
+
+#Set-PSDebug -Trace 2
+
+Write-Output "current user"
+whoami
+Write-Output "current working directory"
+(Get-Location).Path
+Write-Output "current PATH"
+$Env:PATH
+
+function Job-Cleanup() {
+  Get-Job | Stop-Job
+  Get-Job | Remove-Job
+}
+
+# start clean
+Job-Cleanup
+
+# install from artifacts
+if ((Test-Path "node_modules-windows-amd64.tar.gz") -and !(Test-Path "node_modules")) {
+  Write-Output "extracting node_modules-windows-amd64.tar.gz"
+  tar -zxf node_modules-windows-amd64.tar.gz
+}
+
+# setup env
+$env:PWD = (Get-Location).Path
+$env:CI_BUILD_KEY = ([guid]::NewGuid() -Split "-")[0]
+$env:CSI_ENDPOINT = $env:TEMP + "\csi-sanity-" + $env:CI_BUILD_KEY + ".sock"
+$env:NPIPE_ENDPOINT = "//./pipe/csi-sanity-" + $env:CI_BUILD_KEY + "csi.sock"
+
+# testing values
+if (Test-Path "${PSScriptRoot}\run-dev.ps1") {
+  . "${PSScriptRoot}\run-dev.ps1"
+}
+
+# launch server
+$server_job = Start-Job -FilePath .\ci\bin\launch-server.ps1 -InitializationScript {} -ArgumentList $PSScriptRoot
+
+# launch csi-grpc-proxy
+$csi_grpc_proxy_job = Start-Job -FilePath .\ci\bin\launch-csi-grpc-proxy.ps1 -InitializationScript {} -ArgumentList $PSScriptRoot
+
+# wait for socket to appear
+$iter = 0
+$max_iter = 60
+$started = 1
+while (!(Test-Path "${env:CSI_ENDPOINT}")) {
+  $iter++
+  Write-Output "Waiting for ${env:CSI_ENDPOINT} to appear"
+  Start-Sleep 1
+  try {
+    Get-Job | Receive-Job
+  } catch {}
+  if ($iter -gt $max_iter) {
+    Write-Output "${env:CSI_ENDPOINT} failed to appear"
+    $started = 0
+    break
+  }
+}
+
+# launch csi-sanity
+if ($started -eq 1) {
+  $csi_sanity_job = Start-Job -FilePath .\ci\bin\launch-csi-sanity.ps1 -InitializationScript {} -ArgumentList $PSScriptRoot
+}
+
+# https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/get-job?view=powershell-7.2
+# -ChildJobState
+$iter = 0
+while ($csi_sanity_job -and ($csi_sanity_job.State -eq "Running" -or $csi_sanity_job.State -eq "NotStarted")) {
+  $iter++
+  foreach ($job in Get-Job) {
+    if (($job -eq $csi_grpc_proxy_job) -and ($iter -gt 20)) {
+      continue
+    }
+    if (!$job.HasMoreData) {
+      continue
+    }
+    try {
+      $job | Receive-Job
+    }
+    catch {
+      if ($job.State -ne "Failed") {
+        Write-Output "failure receiving job data: ${_}"
+        # just swallow the errors as it seems there are various reasons errors
+        # may show up (perhaps no data currently, etc)
+        #$job | fl
+        #throw $_
+      }
+    }
+  }
+}
+
+# spew any remaining job output to the console
+foreach ($job in Get-Job) {
+  if ($job -eq $csi_grpc_proxy_job) {
+    continue
+  }
+  try {
+    $job | Receive-Job
+  }
+  catch {}
+}
+
+# wait for good measure
+if ($csi_sanity_job) {
+  Wait-Job -Job $csi_sanity_job
+}
+
+#Get-Job | fl
+
+$exit_code = 0
+
+if (! $csi_sanity_job) {
+  $exit_code = 1
+}
+
+if ($csi_sanity_job -and $csi_sanity_job.State -eq "Failed") {
+  $exit_code = 1
+}
+
+# cleanup after ourselves
+Job-Cleanup
+Exit $exit_code

ci/bin/run.sh

@@ -15,8 +15,8 @@ export PATH="/usr/local/lib/nodejs/bin:${PATH}"
 # install deps
 #npm i
 # install from artifacts
-if [[ -f "node_modules.tar.gz" ]];then
-  tar -zxf node_modules.tar.gz
+if [[ -f "node_modules-linux-amd64.tar.gz" && ! -d "node_modules" ]];then
+  tar -zxf node_modules-linux-amd64.tar.gz
 fi
 
 # generate key for paths etc

ci/configs/client/nfs.yaml

@@ -0,0 +1,10 @@
+driver: nfs-client
+instance_id:
+nfs:
+  shareHost: ${SERVER_HOST}
+  shareBasePath: "/mnt/tank/client/nfs/${CI_BUILD_KEY}"
+  # shareHost:shareBasePath should be mounted at this location in the controller container
+  controllerBasePath: "/mnt/client/nfs/${CI_BUILD_KEY}"
+  dirPermissionsMode: "0777"
+  dirPermissionsUser: 0
+  dirPermissionsGroup: 0

ci/configs/client/smb.yaml

@@ -0,0 +1,14 @@
+driver: smb-client
+instance_id:
+smb:
+  shareHost: ${SERVER_HOST}
+  shareBasePath: "${SHARE_NAME}/${CI_BUILD_KEY}"
+  # shareHost:shareBasePath should be mounted at this location in the controller container
+  controllerBasePath: "/mnt/client/smb/${CI_BUILD_KEY}"
+  dirPermissionsMode: "0777"
+  dirPermissionsUser: 0
+  dirPermissionsGroup: 0
+
+node:
+  mount:
+    mount_flags: "username=smbroot,password=smbroot"

ci/configs/objectivefs/objectivefs.yaml

@@ -0,0 +1,20 @@
+driver: objectivefs
+
+objectivefs:
+  pool: ${OBJECTIVEFS_POOL}
+  cli:
+    sudoEnabled: false
+  env:
+    OBJECTIVEFS_LICENSE: ${OBJECTIVEFS_LICENSE}
+    OBJECTSTORE: ${OBJECTIVEFS_OBJECTSTORE}
+    ENDPOINT: ${OBJECTIVEFS_ENDPOINT_PROTOCOL}://${OBJECTIVEFS_ENDPOINT_HOST}:${OBJECTIVEFS_ENDPOINT_PORT}
+    SECRET_KEY: ${OBJECTIVEFS_SECRET_KEY}
+    ACCESS_KEY: ${OBJECTIVEFS_ACCESS_KEY}
+    OBJECTIVEFS_PASSPHRASE: ${OBJECTIVEFS_PASSPHRASE}
+
+_private:
+  csi:
+    volume:
+      idHash:
+        # max volume name length is 63
+        strategy: crc32

ci/configs/synlogy/dsm6/iscsi.yaml

@@ -16,8 +16,8 @@ iscsi:
   targetPortal: ${SYNOLOGY_HOST}
   targetPortals: []
   baseiqn: "iqn.2000-01.com.synology:XpenoDsm62x."
-  namePrefix: "csi-${CI_BUILD_KEY}-"
-  nameSuffix: "-ci"
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
 
   lunTemplate:
     # btrfs thin provisioning

ci/configs/synlogy/dsm7/iscsi.yaml

@@ -0,0 +1,77 @@
+driver: synology-iscsi
+httpConnection:
+  protocol: http
+  host: ${SYNOLOGY_HOST}
+  port: ${SYNOLOGY_PORT}
+  username: ${SYNOLOGY_USERNAME}
+  password: ${SYNOLOGY_PASSWORD}
+  allowInsecure: true
+  session: "democratic-csi-${CI_BUILD_KEY}"
+  serialize: true
+
+synology:
+  volume: ${SYNOLOGY_VOLUME}
+
+iscsi:
+  targetPortal: ${SYNOLOGY_HOST}
+  targetPortals: []
+  baseiqn: "iqn.2000-01.com.synology:XpenoDsm62x."
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+
+  lunTemplate:
+    # btrfs thin provisioning
+    type: "BLUN"
+    # tpws = Hardware-assisted zeroing
+    # caw = Hardware-assisted locking
+    # 3pc = Hardware-assisted data transfer
+    # tpu = Space reclamation
+    # can_snapshot = Snapshot
+    #dev_attribs:
+    #- dev_attrib: emulate_tpws
+    #  enable: 1
+    #- dev_attrib: emulate_caw
+    #  enable: 1
+    #- dev_attrib: emulate_3pc
+    #  enable: 1
+    #- dev_attrib: emulate_tpu
+    #  enable: 0
+    #- dev_attrib: can_snapshot
+    #  enable: 1
+
+    # btfs thick provisioning
+    # only zeroing and locking supported
+    #type: "BLUN_THICK"
+    # tpws = Hardware-assisted zeroing
+    # caw = Hardware-assisted locking
+    #dev_attribs:
+    #- dev_attrib: emulate_tpws
+    #  enable: 1
+    #- dev_attrib: emulate_caw
+    #  enable: 1
+
+    # ext4 thinn provisioning UI sends everything with enabled=0
+    #type: "THIN"
+
+    # ext4 thin with advanced legacy features set
+    # can only alter tpu (all others are set as enabled=1)
+    #type: "ADV"
+    #dev_attribs:
+    #- dev_attrib: emulate_tpu
+    #  enable: 1
+
+    # ext4 thick
+    # can only alter caw
+    #type: "FILE"
+    #dev_attribs:
+    #- dev_attrib: emulate_caw
+    #  enable: 1
+
+  lunSnapshotTemplate:
+    is_locked: true
+    # https://kb.synology.com/en-me/DSM/tutorial/What_is_file_system_consistent_snapshot
+    is_app_consistent: true
+
+  targetTemplate:
+    auth_type: 0
+    max_sessions: 0

ci/configs/truenas/core/12.0/core-iscsi.yaml

@@ -26,7 +26,7 @@ zfs:
 iscsi:
   targetPortal: ${TRUENAS_HOST}
   interface: ""
-  namePrefix: "csi-ci-${CI_BUILD_KEY}"
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
   nameSuffix: ""
   targetGroups:
     - targetGroupPortalGroup: 1
@@ -35,3 +35,10 @@ iscsi:
       targetGroupAuthGroup:
   # 0-100 (0 == ignore)
   extentAvailThreshold: 0
+
+# overcome the 63 char limit for testing purposes only
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/truenas/core/12.0/core-nfs.yaml

@@ -19,7 +19,7 @@ zfs:
   detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
 
   datasetEnableQuotas: true
-  datasetEnableReservation: true
+  datasetEnableReservation: false
   datasetPermissionsMode: "0777"
   datasetPermissionsUser: 0
   datasetPermissionsGroup: 0

ci/configs/truenas/core/12.0/core-smb.yaml

@@ -17,33 +17,29 @@ sshConnection:
 zfs:
   datasetProperties:
     # smb options
-    #aclmode: restricted
-    #casesensitivity: mixed
+    aclmode: restricted
+    aclinherit: passthrough
+    acltype: nfsv4
+    casesensitivity: insensitive
 
   datasetParentName: tank/ci/${CI_BUILD_KEY}/v
   detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
 
   datasetEnableQuotas: true
-  datasetEnableReservation: true
-  datasetPermissionsMode: "0777"
-  datasetPermissionsUser: 0
-  datasetPermissionsGroup: 0
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: 1001
+  datasetPermissionsGroup: 1001
   
-  # for smb with guest
-  #datasetPermissionsUser: nobody
-  #datasetPermissionsGroup: nobody
-  
-  #datasetPermissionsGroup: root
-  #datasetPermissionsAcls:
-  #- "-m everyone@:full_set:allow"
-
-  #datasetPermissionsAcls:
-  #- "-m u:kube:full_set:allow"
+  datasetPermissionsAcls:
+  - "-m g:builtin_users:full_set:fd:allow"
+  - "-m group@:modify_set:fd:allow"
+  - "-m owner@:full_set:fd:allow"
 
 smb:
   shareHost: ${TRUENAS_HOST}
   #nameTemplate: ""
-  namePrefix: "csi-ci-${CI_BUILD_KEY}"
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
   nameSuffix: ""
   shareAuxiliaryConfigurationTemplate: |
     #guest ok = yes
@@ -52,11 +48,21 @@ smb:
   shareAllowedHosts: []
   shareDeniedHosts: []
   #shareDefaultPermissions: true
-  shareGuestOk: true
+  shareGuestOk: false
   #shareGuestOnly: true
   #shareShowHiddenFiles: true
-  shareRecycleBin: true
+  shareRecycleBin: false
   shareBrowsable: false
   shareAccessBasedEnumeration: true
   shareTimeMachine: false
   #shareStorageTask:
+
+node:
+  mount:
+    mount_flags: "username=smbroot,password=smbroot"
+
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/truenas/core/13.0/core-iscsi.yaml

@@ -26,7 +26,7 @@ zfs:
 iscsi:
   targetPortal: ${TRUENAS_HOST}
   interface: ""
-  namePrefix: "csi-ci-${CI_BUILD_KEY}"
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
   nameSuffix: ""
   targetGroups:
     - targetGroupPortalGroup: 1

ci/configs/truenas/core/13.0/core-nfs.yaml

@@ -19,7 +19,7 @@ zfs:
   detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
 
   datasetEnableQuotas: true
-  datasetEnableReservation: true
+  datasetEnableReservation: false
   datasetPermissionsMode: "0777"
   datasetPermissionsUser: 0
   datasetPermissionsGroup: 0

ci/configs/truenas/core/13.0/core-smb.yaml

@@ -17,33 +17,29 @@ sshConnection:
 zfs:
   datasetProperties:
     # smb options
-    #aclmode: restricted
-    #casesensitivity: mixed
+    aclmode: restricted
+    aclinherit: passthrough
+    acltype: nfsv4
+    casesensitivity: insensitive
 
   datasetParentName: tank/ci/${CI_BUILD_KEY}/v
   detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
 
   datasetEnableQuotas: true
-  datasetEnableReservation: true
-  datasetPermissionsMode: "0777"
-  datasetPermissionsUser: 0
-  datasetPermissionsGroup: 0
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: 1001
+  datasetPermissionsGroup: 1001
   
-  # for smb with guest
-  #datasetPermissionsUser: nobody
-  #datasetPermissionsGroup: nobody
-  
-  #datasetPermissionsGroup: root
-  #datasetPermissionsAcls:
-  #- "-m everyone@:full_set:allow"
-
-  #datasetPermissionsAcls:
-  #- "-m u:kube:full_set:allow"
+  datasetPermissionsAcls:
+  - "-m g:builtin_users:full_set:fd:allow"
+  - "-m group@:modify_set:fd:allow"
+  - "-m owner@:full_set:fd:allow"
 
 smb:
   shareHost: ${TRUENAS_HOST}
   #nameTemplate: ""
-  namePrefix: "csi-ci-${CI_BUILD_KEY}"
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
   nameSuffix: ""
   shareAuxiliaryConfigurationTemplate: |
     #guest ok = yes
@@ -52,11 +48,21 @@ smb:
   shareAllowedHosts: []
   shareDeniedHosts: []
   #shareDefaultPermissions: true
-  shareGuestOk: true
+  shareGuestOk: false
   #shareGuestOnly: true
   #shareShowHiddenFiles: true
-  shareRecycleBin: true
+  shareRecycleBin: false
   shareBrowsable: false
   shareAccessBasedEnumeration: true
   shareTimeMachine: false
   #shareStorageTask:
+
+node:
+  mount:
+    mount_flags: "username=smbroot,password=smbroot"
+
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/truenas/scale/22.02/scale-iscsi.yaml

@@ -20,7 +20,7 @@ zfs:
 iscsi:
   targetPortal: ${TRUENAS_HOST}
   interface: ""
-  namePrefix: "csi-ci-${CI_BUILD_KEY}"
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
   nameSuffix: ""
   targetGroups:
     - targetGroupPortalGroup: 1

ci/configs/truenas/scale/22.02/scale-nfs.yaml

@@ -13,7 +13,7 @@ zfs:
   detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
 
   datasetEnableQuotas: true
-  datasetEnableReservation: true
+  datasetEnableReservation: false
   datasetPermissionsMode: "0777"
   datasetPermissionsUser: 0
   datasetPermissionsGroup: 0

ci/configs/truenas/scale/22.02/scale-smb.yaml

@@ -9,35 +9,19 @@ httpConnection:
   password: ${TRUENAS_PASSWORD}
 
 zfs:
-  datasetProperties:
-    # smb options
-    #aclmode: restricted
-    #casesensitivity: mixed
-
   datasetParentName: tank/ci/${CI_BUILD_KEY}/v
   detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
 
   datasetEnableQuotas: true
-  datasetEnableReservation: true
-  datasetPermissionsMode: "0777"
-  datasetPermissionsUser: 0
-  datasetPermissionsGroup: 0
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: 1001
+  datasetPermissionsGroup: 1001
   
-  # for smb with guest
-  #datasetPermissionsUser: nobody
-  #datasetPermissionsGroup: nobody
-  
-  #datasetPermissionsGroup: root
-  #datasetPermissionsAcls:
-  #- "-m everyone@:full_set:allow"
-
-  #datasetPermissionsAcls:
-  #- "-m u:kube:full_set:allow"
-
 smb:
   shareHost: ${TRUENAS_HOST}
   #nameTemplate: ""
-  namePrefix: "csi-ci-${CI_BUILD_KEY}"
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
   nameSuffix: ""
   shareAuxiliaryConfigurationTemplate: |
     #guest ok = yes
@@ -46,11 +30,21 @@ smb:
   shareAllowedHosts: []
   shareDeniedHosts: []
   #shareDefaultPermissions: true
-  shareGuestOk: true
+  shareGuestOk: false
   #shareGuestOnly: true
   #shareShowHiddenFiles: true
-  shareRecycleBin: true
+  shareRecycleBin: false
   shareBrowsable: false
   shareAccessBasedEnumeration: true
   shareTimeMachine: false
   #shareStorageTask:
+
+node:
+  mount:
+    mount_flags: "username=smbroot,password=smbroot"
+
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/truenas/scale/22.12/scale-iscsi.yaml

@@ -0,0 +1,38 @@
+driver: freenas-api-iscsi
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey: 
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize:
+
+iscsi:
+  targetPortal: ${TRUENAS_HOST}
+  interface: ""
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+  targetGroups:
+    - targetGroupPortalGroup: 1
+      targetGroupInitiatorGroup: 1
+      targetGroupAuthType: None
+      targetGroupAuthGroup:
+  # 0-100 (0 == ignore)
+  extentAvailThreshold: 0
+
+# https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/truenas/scale/22.12/scale-nfs.yaml

@@ -0,0 +1,29 @@
+driver: freenas-api-nfs
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey: 
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  datasetEnableQuotas: true
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0777"
+  datasetPermissionsUser: 0
+  datasetPermissionsGroup: 0
+
+nfs:
+  shareHost: ${TRUENAS_HOST}
+  shareAlldirs: false
+  shareAllowedHosts: []
+  shareAllowedNetworks: []
+  shareMaprootUser: root
+  shareMaprootGroup: root
+  shareMapallUser: ""
+  shareMapallGroup: ""

ci/configs/truenas/scale/22.12/scale-smb.yaml

@@ -0,0 +1,50 @@
+driver: freenas-api-smb
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey: 
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  datasetEnableQuotas: true
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: 1001
+  datasetPermissionsGroup: 1001
+  
+smb:
+  shareHost: ${TRUENAS_HOST}
+  #nameTemplate: ""
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+  shareAuxiliaryConfigurationTemplate: |
+    #guest ok = yes
+    #guest only = yes
+  shareHome: false
+  shareAllowedHosts: []
+  shareDeniedHosts: []
+  #shareDefaultPermissions: true
+  shareGuestOk: false
+  #shareGuestOnly: true
+  #shareShowHiddenFiles: true
+  shareRecycleBin: false
+  shareBrowsable: false
+  shareAccessBasedEnumeration: true
+  shareTimeMachine: false
+  #shareStorageTask:
+
+node:
+  mount:
+    mount_flags: "username=smbroot,password=smbroot"
+
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/truenas/scale/23.10/scale-iscsi.yaml

@@ -0,0 +1,38 @@
+driver: freenas-api-iscsi
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey: 
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize:
+
+iscsi:
+  targetPortal: ${TRUENAS_HOST}
+  interface: ""
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+  targetGroups:
+    - targetGroupPortalGroup: 1
+      targetGroupInitiatorGroup: 1
+      targetGroupAuthType: None
+      targetGroupAuthGroup:
+  # 0-100 (0 == ignore)
+  extentAvailThreshold: 0
+
+# https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/truenas/scale/23.10/scale-nfs.yaml

@@ -0,0 +1,29 @@
+driver: freenas-api-nfs
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey: 
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  datasetEnableQuotas: true
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0777"
+  datasetPermissionsUser: 0
+  datasetPermissionsGroup: 0
+
+nfs:
+  shareHost: ${TRUENAS_HOST}
+  shareAlldirs: false
+  shareAllowedHosts: []
+  shareAllowedNetworks: []
+  shareMaprootUser: root
+  shareMaprootGroup: root
+  shareMapallUser: ""
+  shareMapallGroup: ""

ci/configs/truenas/scale/23.10/scale-smb.yaml

@@ -0,0 +1,50 @@
+driver: freenas-api-smb
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey: 
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  datasetEnableQuotas: true
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: 1001
+  datasetPermissionsGroup: 1001
+  
+smb:
+  shareHost: ${TRUENAS_HOST}
+  #nameTemplate: ""
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+  shareAuxiliaryConfigurationTemplate: |
+    #guest ok = yes
+    #guest only = yes
+  shareHome: false
+  shareAllowedHosts: []
+  shareDeniedHosts: []
+  #shareDefaultPermissions: true
+  shareGuestOk: false
+  #shareGuestOnly: true
+  #shareShowHiddenFiles: true
+  shareRecycleBin: false
+  shareBrowsable: false
+  shareAccessBasedEnumeration: true
+  shareTimeMachine: false
+  #shareStorageTask:
+
+node:
+  mount:
+    mount_flags: "username=smbroot,password=smbroot"
+
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/truenas/scale/24.04/scale-iscsi.yaml

@@ -0,0 +1,38 @@
+driver: freenas-api-iscsi
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey: 
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize:
+
+iscsi:
+  targetPortal: ${TRUENAS_HOST}
+  interface: ""
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+  targetGroups:
+    - targetGroupPortalGroup: 1
+      targetGroupInitiatorGroup: 1
+      targetGroupAuthType: None
+      targetGroupAuthGroup:
+  # 0-100 (0 == ignore)
+  extentAvailThreshold: 0
+
+# https://github.com/SCST-project/scst/blob/master/scst/src/dev_handlers/scst_vdisk.c#L203
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/truenas/scale/24.04/scale-nfs.yaml

@@ -0,0 +1,29 @@
+driver: freenas-api-nfs
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey: 
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  datasetEnableQuotas: true
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0777"
+  datasetPermissionsUser: 0
+  datasetPermissionsGroup: 0
+
+nfs:
+  shareHost: ${TRUENAS_HOST}
+  shareAlldirs: false
+  shareAllowedHosts: []
+  shareAllowedNetworks: []
+  shareMaprootUser: root
+  shareMaprootGroup: root
+  shareMapallUser: ""
+  shareMapallGroup: ""

ci/configs/truenas/scale/24.04/scale-smb.yaml

@@ -0,0 +1,50 @@
+driver: freenas-api-smb
+
+httpConnection:
+  protocol: http
+  host: ${TRUENAS_HOST}
+  port: 80
+  #apiKey: 
+  username: ${TRUENAS_USERNAME}
+  password: ${TRUENAS_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  datasetEnableQuotas: true
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: 1001
+  datasetPermissionsGroup: 1001
+  
+smb:
+  shareHost: ${TRUENAS_HOST}
+  #nameTemplate: ""
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+  shareAuxiliaryConfigurationTemplate: |
+    #guest ok = yes
+    #guest only = yes
+  shareHome: false
+  shareAllowedHosts: []
+  shareDeniedHosts: []
+  #shareDefaultPermissions: true
+  shareGuestOk: false
+  #shareGuestOnly: true
+  #shareShowHiddenFiles: true
+  shareRecycleBin: false
+  shareBrowsable: false
+  shareAccessBasedEnumeration: true
+  shareTimeMachine: false
+  #shareStorageTask:
+
+node:
+  mount:
+    mount_flags: "username=smbroot,password=smbroot"
+
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/windows/iscsi.yaml

@@ -0,0 +1,31 @@
+driver: zfs-generic-iscsi
+
+sshConnection:
+  host: ${SERVER_HOST}
+  port: 22
+  username: ${SERVER_USERNAME}
+  password: ${SERVER_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize:
+
+iscsi:
+  targetPortal: ${SERVER_HOST}
+  interface: ""
+  namePrefix: "csi-ci-${CI_BUILD_KEY}"
+  nameSuffix: ""
+  shareStrategy: "targetCli"
+  shareStrategyTargetCli:
+    basename: "iqn.2003-01.org.linux-iscsi.ubuntu-19.x8664"
+    tpg:
+      attributes:
+        authentication: 0
+        generate_node_acls: 1
+        cache_dynamic_acls: 1
+        demo_mode_write_protect: 0

ci/configs/windows/smb.yaml

@@ -0,0 +1,40 @@
+driver: zfs-generic-smb
+
+sshConnection:
+  host: ${SERVER_HOST}
+  port: 22
+  username: ${SERVER_USERNAME}
+  password: ${SERVER_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  datasetProperties:
+    #aclmode: restricted
+    #aclinherit: passthrough
+    #acltype: nfsv4
+    casesensitivity: insensitive
+
+  datasetEnableQuotas: true
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: smbroot
+  datasetPermissionsGroup: smbroot
+
+smb:
+  shareHost: ${SERVER_HOST}
+  shareStrategy: "setDatasetProperties"
+  shareStrategySetDatasetProperties:
+    properties:
+      sharesmb: "on"
+
+node:
+  mount:
+    mount_flags: "username=smbroot,password=smbroot"
+
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

ci/configs/zfs-generic/iscsi.yaml

@@ -18,7 +18,7 @@ zfs:
 iscsi:
   targetPortal: ${SERVER_HOST}
   interface: ""
-  namePrefix: "csi-ci-${CI_BUILD_KEY}"
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
   nameSuffix: ""
   shareStrategy: "targetCli"
   shareStrategyTargetCli:

ci/configs/zfs-generic/nfs.yaml

@@ -11,7 +11,7 @@ zfs:
   detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
 
   datasetEnableQuotas: true
-  datasetEnableReservation: true
+  datasetEnableReservation: false
   datasetPermissionsMode: "0777"
   datasetPermissionsUser: 0
   datasetPermissionsGroup: 0
@@ -21,4 +21,5 @@ nfs:
   shareStrategy: "setDatasetProperties"
   shareStrategySetDatasetProperties:
     properties:
-      sharenfs: "on"
+      #sharenfs: "on"
+      sharenfs: "rw,no_subtree_check,no_root_squash"

ci/configs/zfs-generic/nvmeof.yaml

@@ -0,0 +1,30 @@
+driver: zfs-generic-nvmeof
+
+sshConnection:
+  host: ${SERVER_HOST}
+  port: 22
+  username: ${SERVER_USERNAME}
+  password: ${SERVER_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  zvolCompression:
+  zvolDedup:
+  zvolEnableReservation: false
+  zvolBlocksize:
+
+nvmeof:
+  transports:
+    - "tcp://${SERVER_HOST}:4420"
+  namePrefix: "csi-ci-${CI_BUILD_KEY}-"
+  nameSuffix: ""
+  shareStrategy: "nvmetCli"
+  shareStrategyNvmetCli:
+    basename: "nqn.2003-01.org.linux-nvmeof.ubuntu-19.x8664"
+    ports:
+      - "1"
+    subsystem:
+      attributes:
+        allow_any_host: 1

ci/configs/zfs-generic/smb.yaml

@@ -0,0 +1,40 @@
+driver: zfs-generic-smb
+
+sshConnection:
+  host: ${SERVER_HOST}
+  port: 22
+  username: ${SERVER_USERNAME}
+  password: ${SERVER_PASSWORD}
+
+zfs:
+  datasetParentName: tank/ci/${CI_BUILD_KEY}/v
+  detachedSnapshotsDatasetParentName: tank/ci/${CI_BUILD_KEY}/s
+
+  datasetProperties:
+    #aclmode: restricted
+    #aclinherit: passthrough
+    #acltype: nfsv4
+    casesensitivity: insensitive
+
+  datasetEnableQuotas: true
+  datasetEnableReservation: false
+  datasetPermissionsMode: "0770"
+  datasetPermissionsUser: smbroot
+  datasetPermissionsGroup: smbroot
+
+smb:
+  shareHost: ${SERVER_HOST}
+  shareStrategy: "setDatasetProperties"
+  shareStrategySetDatasetProperties:
+    properties:
+      sharesmb: "on"
+
+node:
+  mount:
+    mount_flags: "username=smbroot,password=smbroot"
+
+_private:
+  csi:
+    volume:
+      idHash:
+        strategy: crc16

contrib/pv-updater.sh

@@ -43,19 +43,19 @@ if [[ ! -f ${PV_ORIG_FILE} ]]; then
 	kubectl get pv "${PV}" -o yaml >"${PV_ORIG_FILE}"
 fi
 
-reclaimPolicy=$(yq eval '.spec.persistentVolumeReclaimPolicy' "${PV_ORIG_FILE}")
+reclaimPolicy=$(yq '.spec.persistentVolumeReclaimPolicy' "${PV_ORIG_FILE}")
 
 # copy file for editing
 cp "${PV_ORIG_FILE}" "${PV_TMP_FILE}"
 
 # pre-process before edit
-yq -i eval 'del(.metadata.resourceVersion)' "${PV_TMP_FILE}"
+yq -i -y 'del(.metadata.resourceVersion)' "${PV_TMP_FILE}"
 
 # manually edit
 ${EDITOR} "${PV_TMP_FILE}"
 
 # ask if looks good
-yq eval '.' "${PV_TMP_FILE}"
+yq '.' "${PV_TMP_FILE}"
 yes_or_no "Would you like to delete the existing PV object and recreate with the above data?"
 
 # set relaim to Retain on PV
@@ -69,7 +69,7 @@ kubectl patch pv "${PV}" -p '{"metadata":{"finalizers": null }}' &>/dev/null ||
 kubectl apply -f "${PV_TMP_FILE}"
 
 # restore original reclaim value
-kubectl patch pv "${PV}" -p "{\"spec\":{\"persistentVolumeReclaimPolicy\":\"${reclaimPolicy}\"}}"
+kubectl patch pv "${PV}" -p "{\"spec\":{\"persistentVolumeReclaimPolicy\":${reclaimPolicy}}}"
 
 # spit out any zfs properties updates
 yes_or_no "Would you like to delete the PV backup file?" && {

contrib/scale-nvmet-start.sh

@@ -0,0 +1,108 @@
+#!/bin/bash
+
+# simple script to 'start' nvmet on TrueNAS SCALE
+#
+# to reinstall nvmetcli simply rm /usr/sbin/nvmetcli
+
+# debug
+#set -x
+
+# exit non-zero
+set -e
+
+SCRIPTDIR="$(
+  cd -- "$(dirname "$0")" >/dev/null 2>&1
+  pwd -P
+)"
+cd "${SCRIPTDIR}"
+
+: "${NVMETCONFIG:="${SCRIPTDIR}/nvmet-config.json"}"
+: "${NVMETVENV:="${SCRIPTDIR}/nvmet-venv"}"
+
+export PATH=${HOME}/.local/bin:${PATH}
+
+main() {
+
+  kernel_modules
+  nvmetcli ls &>/dev/null || {
+    setup_venv
+    install_nvmetcli
+  }
+  nvmetcli_restore
+
+}
+
+kernel_modules() {
+
+  modules=()
+  modules+=("nvmet")
+  modules+=("nvmet-fc")
+  modules+=("nvmet-rdma")
+  modules+=("nvmet-tcp")
+
+  for module in "${modules[@]}"; do
+    modprobe "${module}"
+  done
+
+}
+
+setup_venv() {
+
+  rm -rf ${NVMETVENV}
+  python -m venv ${NVMETVENV} --without-pip --system-site-packages
+  activate_venv
+  curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+  python get-pip.py
+  rm get-pip.py
+  deactivate_venv
+
+}
+
+activate_venv() {
+
+  . ${NVMETVENV}/bin/activate
+
+}
+
+deactivate_venv() {
+
+  deactivate
+
+}
+
+install_nvmetcli() {
+
+  if [[ ! -d nvmetcli ]]; then
+    git clone git://git.infradead.org/users/hch/nvmetcli.git
+  fi
+
+  cd nvmetcli
+
+  activate_venv
+
+  # install to root home dir
+  python3 setup.py install --install-scripts=${HOME}/.local/bin
+
+  # install to root home dir
+  pip install configshell_fb==1.1.30
+
+  # remove source
+  cd "${SCRIPTDIR}"
+  rm -rf nvmetcli
+
+  deactivate_venv
+
+}
+
+nvmetcli_restore() {
+
+  activate_venv
+  cd "${SCRIPTDIR}"
+  nvmetcli restore "${NVMETCONFIG}"
+  deactivate_venv
+  touch /var/run/nvmet-config-loaded
+  chmod +r /var/run/nvmet-config-loaded
+
+}
+
+main

csi_proto/download-proto.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# v1.6.0
+VERSION=${1}
+
+curl -v -o "csi-${VERSION}.proto" https://raw.githubusercontent.com/container-storage-interface/spec/${VERSION}/csi.proto

csi_proxy_proto/disk/v1/api.proto

@@ -0,0 +1,111 @@
+syntax = "proto3";
+
+package v1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/disk/v1";
+
+service Disk {
+    // ListDiskLocations returns locations <Adapter, Bus, Target, LUN ID> of all
+    // disk devices enumerated by the host.
+    rpc ListDiskLocations(ListDiskLocationsRequest) returns (ListDiskLocationsResponse) {}
+
+    // PartitionDisk initializes and partitions a disk device with the GPT partition style
+    // (if the disk has not been partitioned already) and returns the resulting volume device ID.
+    rpc PartitionDisk(PartitionDiskRequest) returns (PartitionDiskResponse) {}
+
+    // Rescan refreshes the host's storage cache.
+    rpc Rescan(RescanRequest) returns (RescanResponse) {}
+
+    // ListDiskIDs returns a map of DiskID objects where the key is the disk number.
+    rpc ListDiskIDs(ListDiskIDsRequest) returns (ListDiskIDsResponse) {}
+
+    // GetDiskStats returns the stats of a disk (currently it returns the disk size).
+    rpc GetDiskStats(GetDiskStatsRequest) returns (GetDiskStatsResponse) {}
+
+    // SetDiskState sets the offline/online state of a disk.
+    rpc SetDiskState(SetDiskStateRequest) returns (SetDiskStateResponse) {}
+
+    // GetDiskState gets the offline/online state of a disk.
+    rpc GetDiskState(GetDiskStateRequest) returns (GetDiskStateResponse) {}
+}
+
+message ListDiskLocationsRequest {
+    // Intentionally empty.
+}
+
+message DiskLocation {
+    string Adapter = 1;
+    string Bus = 2;
+    string Target = 3;
+    string LUNID = 4;
+}
+
+message ListDiskLocationsResponse {
+    // Map of disk number and <adapter, bus, target, lun ID> associated with each disk device.
+    map <uint32, DiskLocation> disk_locations = 1;
+}
+
+message PartitionDiskRequest {
+    // Disk device number of the disk to partition.
+    uint32 disk_number = 1;
+}
+
+message PartitionDiskResponse {
+    // Intentionally empty.
+}
+
+message RescanRequest {
+    // Intentionally empty.
+}
+
+message RescanResponse {
+    // Intentionally empty.
+}
+
+message ListDiskIDsRequest {
+    // Intentionally empty.
+}
+
+message DiskIDs {
+    // The disk page83 id.
+    string page83 = 1;
+    // The disk serial number.
+    string serial_number = 2;
+}
+
+message ListDiskIDsResponse {
+    // Map of disk numbers and disk identifiers associated with each disk device.
+    map <uint32, DiskIDs> diskIDs = 1;  // the case is intentional for protoc to generate the field as DiskIDs
+}
+
+message GetDiskStatsRequest {
+    // Disk device number of the disk to get the stats from.
+    uint32 disk_number = 1;
+}
+
+message GetDiskStatsResponse {
+    // Total size of the volume.
+    int64 total_bytes = 1;
+}
+
+message SetDiskStateRequest {
+    // Disk device number of the disk.
+    uint32 disk_number = 1;
+
+    // Online state to set for the disk. true for online, false for offline.
+    bool is_online = 2;
+}
+
+message SetDiskStateResponse {
+    // Intentionally empty.
+}
+
+message GetDiskStateRequest {
+    // Disk device number of the disk.
+    uint32 disk_number = 1;
+}
+
+message GetDiskStateResponse {
+    // Online state of the disk. true for online, false for offline.
+    bool is_online = 1;
+}

csi_proxy_proto/disk/v1alpha1/api.proto

@@ -0,0 +1,62 @@
+syntax = "proto3";
+
+package v1alpha1;
+
+service Disk {
+    // ListDiskLocations returns locations <Adapter, Bus, Target, LUN ID> of all
+    // disk devices enumerated by the host
+    rpc ListDiskLocations(ListDiskLocationsRequest) returns (ListDiskLocationsResponse) {}
+
+    // PartitionDisk initializes and partitions a disk device (if the disk has not
+    // been partitioned already) and returns the resulting volume device ID
+    rpc PartitionDisk(PartitionDiskRequest) returns (PartitionDiskResponse) {}
+
+    // Rescan refreshes the host's storage cache
+    rpc Rescan(RescanRequest) returns (RescanResponse) {}
+
+    // GetDiskNumberByName returns disk number based on the passing disk name information
+    rpc GetDiskNumberByName(GetDiskNumberByNameRequest) returns (GetDiskNumberByNameResponse) {}
+}
+
+message ListDiskLocationsRequest {
+    // Intentionally empty
+}
+
+message DiskLocation {
+    string Adapter = 1;
+    string Bus = 2;
+    string Target = 3;
+    string LUNID = 4;
+}
+
+message ListDiskLocationsResponse {
+    // Map of disk device IDs and <adapter, bus, target, lun ID> associated with each disk device
+    map <string, DiskLocation> disk_locations = 1;
+}
+
+message PartitionDiskRequest {
+    // Disk device ID of the disk to partition
+    string diskID = 1;
+}
+
+message PartitionDiskResponse {
+    // Intentionally empty
+}
+
+message RescanRequest {
+    // Intentionally empty
+}
+
+message RescanResponse {
+    // Intentionally empty
+}
+
+message GetDiskNumberByNameRequest {
+    // Disk ID
+    string disk_name = 1;
+}
+
+message GetDiskNumberByNameResponse {
+    // Disk number
+    string disk_number = 1;
+}

csi_proxy_proto/disk/v1beta1/api.proto

@@ -0,0 +1,81 @@
+syntax = "proto3";
+
+package v1beta1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/disk/v1beta1";
+
+service Disk {
+    // ListDiskLocations returns locations <Adapter, Bus, Target, LUN ID> of all
+    // disk devices enumerated by the host
+    rpc ListDiskLocations(ListDiskLocationsRequest) returns (ListDiskLocationsResponse) {}
+
+    // PartitionDisk initializes and partitions a disk device (if the disk has not
+    // been partitioned already) and returns the resulting volume device ID
+    rpc PartitionDisk(PartitionDiskRequest) returns (PartitionDiskResponse) {}
+
+    // Rescan refreshes the host's storage cache
+    rpc Rescan(RescanRequest) returns (RescanResponse) {}
+
+    // ListDiskIDs returns a map of DiskID objects where the key is the disk number
+    rpc ListDiskIDs(ListDiskIDsRequest) returns (ListDiskIDsResponse) {}
+
+    // DiskStats returns the stats for the disk
+    rpc DiskStats(DiskStatsRequest) returns (DiskStatsResponse) {}
+}
+
+message ListDiskLocationsRequest {
+    // Intentionally empty
+}
+
+message DiskLocation {
+    string Adapter = 1;
+    string Bus = 2;
+    string Target = 3;
+    string LUNID = 4;
+}
+
+message ListDiskLocationsResponse {
+    // Map of disk device IDs and <adapter, bus, target, lun ID> associated with each disk device
+    map <string, DiskLocation> disk_locations = 1;
+}
+
+message PartitionDiskRequest {
+    // Disk device ID of the disk to partition
+    string diskID = 1;
+}
+
+message PartitionDiskResponse {
+    // Intentionally empty
+}
+
+message RescanRequest {
+    // Intentionally empty
+}
+
+message RescanResponse {
+    // Intentionally empty
+}
+
+message ListDiskIDsRequest {
+    // Intentionally empty
+}
+
+message DiskIDs {
+    // Map of Disk ID types and Disk ID values
+    map <string, string> identifiers = 1;
+}
+
+message ListDiskIDsResponse {
+    // Map of disk device numbers and IDs <page83> associated with each disk device
+    map <string, DiskIDs> diskIDs = 1;
+}
+
+message DiskStatsRequest {
+    // Disk device ID of the disk to get the size from
+    string diskID = 1;
+}
+
+message DiskStatsResponse {
+    //Total size of the volume
+    int64 diskSize = 1;
+}

csi_proxy_proto/disk/v1beta2/api.proto

@@ -0,0 +1,109 @@
+syntax = "proto3";
+
+package v1beta2;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/disk/v1beta2";
+
+service Disk {
+    // ListDiskLocations returns locations <Adapter, Bus, Target, LUN ID> of all
+    // disk devices enumerated by the host
+    rpc ListDiskLocations(ListDiskLocationsRequest) returns (ListDiskLocationsResponse) {}
+
+    // PartitionDisk initializes and partitions a disk device (if the disk has not
+    // been partitioned already) and returns the resulting volume device ID
+    rpc PartitionDisk(PartitionDiskRequest) returns (PartitionDiskResponse) {}
+
+    // Rescan refreshes the host's storage cache
+    rpc Rescan(RescanRequest) returns (RescanResponse) {}
+
+    // ListDiskIDs returns a map of DiskID objects where the key is the disk number
+    rpc ListDiskIDs(ListDiskIDsRequest) returns (ListDiskIDsResponse) {}
+
+    // DiskStats returns the stats for the disk
+    rpc DiskStats(DiskStatsRequest) returns (DiskStatsResponse) {}
+
+    // SetAttachState sets the offline/online state of a disk
+    rpc SetAttachState(SetAttachStateRequest) returns (SetAttachStateResponse) {}
+
+    // GetAttachState gets the offline/online state of a disk
+    rpc GetAttachState(GetAttachStateRequest) returns (GetAttachStateResponse) {}
+}
+
+message ListDiskLocationsRequest {
+    // Intentionally empty
+}
+
+message DiskLocation {
+    string Adapter = 1;
+    string Bus = 2;
+    string Target = 3;
+    string LUNID = 4;
+}
+
+message ListDiskLocationsResponse {
+    // Map of disk device IDs and <adapter, bus, target, lun ID> associated with each disk device
+    map <string, DiskLocation> disk_locations = 1;
+}
+
+message PartitionDiskRequest {
+    // Disk device ID of the disk to partition
+    string diskID = 1;
+}
+
+message PartitionDiskResponse {
+    // Intentionally empty
+}
+
+message RescanRequest {
+    // Intentionally empty
+}
+
+message RescanResponse {
+    // Intentionally empty
+}
+
+message ListDiskIDsRequest {
+    // Intentionally empty
+}
+
+message DiskIDs {
+    // Map of Disk ID types and Disk ID values
+    map <string, string> identifiers = 1;
+}
+
+message ListDiskIDsResponse {
+    // Map of disk device numbers and IDs <page83> associated with each disk device
+    map <string, DiskIDs> diskIDs = 1;
+}
+
+message DiskStatsRequest {
+    // Disk device ID of the disk to get the size from
+    string diskID = 1;
+}
+
+message DiskStatsResponse {
+    //Total size of the volume
+    int64 diskSize = 1;
+}
+
+message SetAttachStateRequest {
+    // Disk device ID (number) of the disk which state will change
+    string diskID = 1;
+
+    // Online state to set for the disk. true for online, false for offline
+    bool isOnline = 2;
+}
+
+message SetAttachStateResponse {
+}
+
+message GetAttachStateRequest {
+    // Disk device ID (number) of the disk
+    string diskID = 1;
+}
+
+message GetAttachStateResponse {
+    // Online state of the disk. true for online, false for offline
+    bool isOnline = 1;
+}
+

csi_proxy_proto/disk/v1beta3/api.proto

@@ -0,0 +1,111 @@
+syntax = "proto3";
+
+package v1beta3;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/disk/v1beta3";
+
+service Disk {
+    // ListDiskLocations returns locations <Adapter, Bus, Target, LUN ID> of all
+    // disk devices enumerated by the host.
+    rpc ListDiskLocations(ListDiskLocationsRequest) returns (ListDiskLocationsResponse) {}
+
+    // PartitionDisk initializes and partitions a disk device with the GPT partition style
+    // (if the disk has not been partitioned already) and returns the resulting volume device ID.
+    rpc PartitionDisk(PartitionDiskRequest) returns (PartitionDiskResponse) {}
+
+    // Rescan refreshes the host's storage cache.
+    rpc Rescan(RescanRequest) returns (RescanResponse) {}
+
+    // ListDiskIDs returns a map of DiskID objects where the key is the disk number.
+    rpc ListDiskIDs(ListDiskIDsRequest) returns (ListDiskIDsResponse) {}
+
+    // GetDiskStats returns the stats of a disk (currently it returns the disk size).
+    rpc GetDiskStats(GetDiskStatsRequest) returns (GetDiskStatsResponse) {}
+
+    // SetDiskState sets the offline/online state of a disk.
+    rpc SetDiskState(SetDiskStateRequest) returns (SetDiskStateResponse) {}
+
+    // GetDiskState gets the offline/online state of a disk.
+    rpc GetDiskState(GetDiskStateRequest) returns (GetDiskStateResponse) {}
+}
+
+message ListDiskLocationsRequest {
+    // Intentionally empty.
+}
+
+message DiskLocation {
+    string Adapter = 1;
+    string Bus = 2;
+    string Target = 3;
+    string LUNID = 4;
+}
+
+message ListDiskLocationsResponse {
+    // Map of disk number and <adapter, bus, target, lun ID> associated with each disk device.
+    map <uint32, DiskLocation> disk_locations = 1;
+}
+
+message PartitionDiskRequest {
+    // Disk device number of the disk to partition.
+    uint32 disk_number = 1;
+}
+
+message PartitionDiskResponse {
+    // Intentionally empty.
+}
+
+message RescanRequest {
+    // Intentionally empty.
+}
+
+message RescanResponse {
+    // Intentionally empty.
+}
+
+message ListDiskIDsRequest {
+    // Intentionally empty.
+}
+
+message DiskIDs {
+    // The disk page83 id.
+    string page83 = 1;
+    // The disk serial number.
+    string serial_number = 2;
+}
+
+message ListDiskIDsResponse {
+    // Map of disk numbers and disk identifiers associated with each disk device.
+    map <uint32, DiskIDs> diskIDs = 1;  // the case is intentional for protoc to generate the field as DiskIDs
+}
+
+message GetDiskStatsRequest {
+    // Disk device number of the disk to get the stats from.
+    uint32 disk_number = 1;
+}
+
+message GetDiskStatsResponse {
+    // Total size of the volume.
+    int64 total_bytes = 1;
+}
+
+message SetDiskStateRequest {
+    // Disk device number of the disk.
+    uint32 disk_number = 1;
+
+    // Online state to set for the disk. true for online, false for offline.
+    bool is_online = 2;
+}
+
+message SetDiskStateResponse {
+    // Intentionally empty.
+}
+
+message GetDiskStateRequest {
+    // Disk device number of the disk.
+    uint32 disk_number = 1;
+}
+
+message GetDiskStateResponse {
+    // Online state of the disk. true for online, false for offline.
+    bool is_online = 1;
+}

csi_proxy_proto/errors.proto

@@ -0,0 +1,17 @@
+syntax = "proto3";
+
+package api;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api";
+
+// CommandError details errors yielded by cmdlet calls.
+message CmdletError {
+    // Name of the cmdlet that errored out.
+    string cmdlet_name = 1;
+
+    // Error code that got returned.
+    uint32 code = 2;
+
+    // Human-readable error message - can be empty.
+    string message = 3;
+}

csi_proxy_proto/filesystem/v1/api.proto

@@ -0,0 +1,136 @@
+syntax = "proto3";
+
+package v1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/filesystem/v1";
+
+service Filesystem {
+    // PathExists checks if the requested path exists in the host filesystem.
+    rpc PathExists(PathExistsRequest) returns (PathExistsResponse) {}
+
+    // Mkdir creates a directory at the requested path in the host filesystem.
+    rpc Mkdir(MkdirRequest) returns (MkdirResponse) {}
+
+    // Rmdir removes the directory at the requested path in the host filesystem.
+    // This may be used for unlinking a symlink created through CreateSymlink.
+    rpc Rmdir(RmdirRequest) returns (RmdirResponse) {}
+
+    // CreateSymlink creates a symbolic link called target_path that points to source_path
+    // in the host filesystem (target_path is the name of the symbolic link created,
+    // source_path is the existing path).
+    rpc CreateSymlink(CreateSymlinkRequest) returns (CreateSymlinkResponse) {}
+
+    // IsSymlink checks if a given path is a symlink.
+    rpc IsSymlink(IsSymlinkRequest) returns (IsSymlinkResponse) {}
+}
+
+message PathExistsRequest {
+    // The path whose existence we want to check in the host's filesystem
+    string path = 1;
+}
+
+message PathExistsResponse {
+    // Indicates whether the path in PathExistsRequest exists in the host's filesystem
+    bool exists = 1;
+}
+
+message MkdirRequest {
+    // The path to create in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    // Non-existent parent directories in the path will be automatically created.
+    // Directories will be created with Read and Write privileges of the Windows
+    // User account under which csi-proxy is started (typically LocalSystem).
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // The path parameter cannot already exist in the host's filesystem.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+}
+
+message MkdirResponse {
+    // Intentionally empty.
+}
+
+message RmdirRequest {
+    // The path to remove in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Path cannot be a file of type symlink.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+
+    // Force remove all contents under path (if any).
+    bool force = 2;
+}
+
+message RmdirResponse {
+    // Intentionally empty.
+}
+
+message CreateSymlinkRequest {
+    // The path of the existing directory to be linked.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs needs to match the paths specified as
+    // kubelet-csi-plugins-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // source_path cannot already exist in the host filesystem.
+    // Maximum path length will be capped to 260 characters.
+    string source_path = 1;
+
+    // Target path is the location of the new directory entry to be created in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs to match the paths specified as
+    // kubelet-pod-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // target_path needs to exist as a directory in the host that is empty.
+    // target_path cannot be a symbolic link.
+    // Maximum path length will be capped to 260 characters.
+    string target_path = 2;
+}
+
+message CreateSymlinkResponse {
+    // Intentionally empty.
+}
+
+message IsSymlinkRequest {
+    // The path whose existence as a symlink we want to check in the host's filesystem.
+    string path = 1;
+}
+
+message IsSymlinkResponse {
+    // Indicates whether the path in IsSymlinkRequest is a symlink.
+    bool is_symlink = 1;
+}

csi_proxy_proto/filesystem/v1alpha1/api.proto

@@ -0,0 +1,168 @@
+syntax = "proto3";
+
+package v1alpha1;
+
+service Filesystem {
+    // PathExists checks if the requested path exists in the host's filesystem
+    rpc PathExists(PathExistsRequest) returns (PathExistsResponse) {}
+
+    // Mkdir creates a directory at the requested path in the host's filesystem
+    rpc Mkdir(MkdirRequest) returns (MkdirResponse) {}
+
+    // Rmdir removes the directory at the requested path in the host's filesystem.
+    // This may be used for unlinking a symlink created through LinkPath
+    rpc Rmdir(RmdirRequest) returns (RmdirResponse) {}
+
+    // LinkPath creates a local directory symbolic link between a source path 
+    // and target path in the host's filesystem
+    rpc LinkPath(LinkPathRequest) returns (LinkPathResponse) {}
+
+    //IsMountPoint checks if a given path is mount or not
+    rpc IsMountPoint(IsMountPointRequest) returns (IsMountPointResponse) {}
+}
+
+// Context of the paths used for path prefix validation
+enum PathContext {
+    // Indicates the kubelet-csi-plugins-path parameter of csi-proxy be used as
+    // the path context. This may be used while handling NodeStageVolume where
+    // a volume may need to be mounted at a plugin-specific path like:
+    // kubelet\plugins\kubernetes.io\csi\pv\<pv-name>\globalmount
+    PLUGIN = 0;
+    // Indicates the kubelet-pod-path parameter of csi-proxy be used as the path
+    // context. This may be used while handling NodePublishVolume where a staged
+    // volume may be need to be symlinked to a pod-specific path like:
+    // kubelet\pods\<pod-uuid>\volumes\kubernetes.io~csi\<pvc-name>\mount
+    POD = 1;
+}
+
+message PathExistsRequest {
+    // The path whose existence we want to check in the host's filesystem
+    string path = 1;
+
+    // Context of the path parameter.
+    // This is used to validate prefix for absolute paths passed
+    PathContext context = 2;
+}
+
+message PathExistsResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+
+    // Indicates whether the path in PathExistsRequest exists in the host's filesystem
+    bool exists = 2;
+}
+
+message MkdirRequest {
+    // The path to create in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    // Non-existent parent directories in the path will be automatically created.
+    // Directories will be created with Read and Write privileges of the Windows
+    // User account under which csi-proxy is started (typically LocalSystem).
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs 
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // The path parameter cannot already exist in the host's filesystem.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+
+    // Context of the path parameter.
+    // This is used to validate prefix for absolute paths passed
+    PathContext context = 2;
+}
+
+message MkdirResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+}
+
+message RmdirRequest {
+    // The path to remove in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs 
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Path cannot be a file of type symlink.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+
+    // Context of the path parameter.
+    // This is used to validate prefix for absolute paths passed
+    PathContext context = 2;
+
+    // Force remove all contents under path (if any).
+    bool force = 3;
+}
+
+message RmdirResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+}
+
+message LinkPathRequest {
+    // The path where the symlink is created in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs needs to match the paths specified as 
+    // kubelet-csi-plugins-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // source_path cannot already exist in the host filesystem.
+    // Maximum path length will be capped to 260 characters.
+    string source_path = 1;
+
+    // Target path in the host's filesystem used for the symlink creation.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs to match the paths specified as
+    // kubelet-pod-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // target_path needs to exist as a directory in the host that is empty.
+    // target_path cannot be a symbolic link.
+    // Maximum path length will be capped to 260 characters.
+    string target_path = 2;
+}
+
+message LinkPathResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+}
+ 
+message IsMountPointRequest {
+    // The path whose existence we want to check in the host's filesystem
+    string path = 1;
+}
+
+message IsMountPointResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+
+    // Indicates whether the path in PathExistsRequest exists in the host's filesystem
+    bool is_mount_point = 2;
+}

csi_proxy_proto/filesystem/v1beta1/api.proto

@@ -0,0 +1,168 @@
+syntax = "proto3";
+
+package v1beta1;
+
+service Filesystem {
+    // PathExists checks if the requested path exists in the host's filesystem
+    rpc PathExists(PathExistsRequest) returns (PathExistsResponse) {}
+
+    // Mkdir creates a directory at the requested path in the host's filesystem
+    rpc Mkdir(MkdirRequest) returns (MkdirResponse) {}
+
+    // Rmdir removes the directory at the requested path in the host's filesystem.
+    // This may be used for unlinking a symlink created through LinkPath
+    rpc Rmdir(RmdirRequest) returns (RmdirResponse) {}
+
+    // LinkPath creates a local directory symbolic link between a source path 
+    // and target path in the host's filesystem
+    rpc LinkPath(LinkPathRequest) returns (LinkPathResponse) {}
+
+    //IsMountPoint checks if a given path is mount or not
+    rpc IsMountPoint(IsMountPointRequest) returns (IsMountPointResponse) {}
+}
+
+// Context of the paths used for path prefix validation
+enum PathContext {
+    // Indicates the kubelet-csi-plugins-path parameter of csi-proxy be used as
+    // the path context. This may be used while handling NodeStageVolume where
+    // a volume may need to be mounted at a plugin-specific path like:
+    // kubelet\plugins\kubernetes.io\csi\pv\<pv-name>\globalmount
+    PLUGIN = 0;
+    // Indicates the kubelet-pod-path parameter of csi-proxy be used as the path
+    // context. This may be used while handling NodePublishVolume where a staged
+    // volume may be need to be symlinked to a pod-specific path like:
+    // kubelet\pods\<pod-uuid>\volumes\kubernetes.io~csi\<pvc-name>\mount
+    POD = 1;
+}
+
+message PathExistsRequest {
+    // The path whose existence we want to check in the host's filesystem
+    string path = 1;
+
+    // Context of the path parameter.
+    // This is used to validate prefix for absolute paths passed
+    PathContext context = 2;
+}
+
+message PathExistsResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+
+    // Indicates whether the path in PathExistsRequest exists in the host's filesystem
+    bool exists = 2;
+}
+
+message MkdirRequest {
+    // The path to create in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    // Non-existent parent directories in the path will be automatically created.
+    // Directories will be created with Read and Write privileges of the Windows
+    // User account under which csi-proxy is started (typically LocalSystem).
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs 
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // The path parameter cannot already exist in the host's filesystem.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+
+    // Context of the path parameter.
+    // This is used to validate prefix for absolute paths passed
+    PathContext context = 2;
+}
+
+message MkdirResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+}
+
+message RmdirRequest {
+    // The path to remove in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs 
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Path cannot be a file of type symlink.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+
+    // Context of the path parameter.
+    // This is used to validate prefix for absolute paths passed
+    PathContext context = 2;
+
+    // Force remove all contents under path (if any).
+    bool force = 3;
+}
+
+message RmdirResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+}
+
+message LinkPathRequest {
+    // The path where the symlink is created in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs needs to match the paths specified as 
+    // kubelet-csi-plugins-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // source_path cannot already exist in the host filesystem.
+    // Maximum path length will be capped to 260 characters.
+    string source_path = 1;
+
+    // Target path in the host's filesystem used for the symlink creation.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs to match the paths specified as
+    // kubelet-pod-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // target_path needs to exist as a directory in the host that is empty.
+    // target_path cannot be a symbolic link.
+    // Maximum path length will be capped to 260 characters.
+    string target_path = 2;
+}
+
+message LinkPathResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+}
+ 
+message IsMountPointRequest {
+    // The path whose existence we want to check in the host's filesystem
+    string path = 1;
+}
+
+message IsMountPointResponse {
+    // Error message if any. Empty string indicates success
+    string error = 1;
+
+    // Indicates whether the path in PathExistsRequest exists in the host's filesystem
+    bool is_mount_point = 2;
+}

csi_proxy_proto/filesystem/v1beta2/api.proto

@@ -0,0 +1,136 @@
+syntax = "proto3";
+
+package v1beta2;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/filesystem/v1beta2";
+
+service Filesystem {
+    // PathExists checks if the requested path exists in the host filesystem.
+    rpc PathExists(PathExistsRequest) returns (PathExistsResponse) {}
+
+    // Mkdir creates a directory at the requested path in the host filesystem.
+    rpc Mkdir(MkdirRequest) returns (MkdirResponse) {}
+
+    // Rmdir removes the directory at the requested path in the host filesystem.
+    // This may be used for unlinking a symlink created through CreateSymlink.
+    rpc Rmdir(RmdirRequest) returns (RmdirResponse) {}
+
+    // CreateSymlink creates a symbolic link called target_path that points to source_path
+    // in the host filesystem (target_path is the name of the symbolic link created,
+    // source_path is the existing path).
+    rpc CreateSymlink(CreateSymlinkRequest) returns (CreateSymlinkResponse) {}
+
+    // IsSymlink checks if a given path is a symlink.
+    rpc IsSymlink(IsSymlinkRequest) returns (IsSymlinkResponse) {}
+}
+
+message PathExistsRequest {
+    // The path whose existence we want to check in the host's filesystem
+    string path = 1;
+}
+
+message PathExistsResponse {
+    // Indicates whether the path in PathExistsRequest exists in the host's filesystem
+    bool exists = 1;
+}
+
+message MkdirRequest {
+    // The path to create in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    // Non-existent parent directories in the path will be automatically created.
+    // Directories will be created with Read and Write privileges of the Windows
+    // User account under which csi-proxy is started (typically LocalSystem).
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // The path parameter cannot already exist in the host's filesystem.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+}
+
+message MkdirResponse {
+    // Intentionally empty.
+}
+
+message RmdirRequest {
+    // The path to remove in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Path cannot be a file of type symlink.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+
+    // Force remove all contents under path (if any).
+    bool force = 2;
+}
+
+message RmdirResponse {
+    // Intentionally empty.
+}
+
+message CreateSymlinkRequest {
+    // The path of the existing directory to be linked.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs needs to match the paths specified as
+    // kubelet-csi-plugins-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // source_path cannot already exist in the host filesystem.
+    // Maximum path length will be capped to 260 characters.
+    string source_path = 1;
+
+    // Target path is the location of the new directory entry to be created in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs to match the paths specified as
+    // kubelet-pod-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // target_path needs to exist as a directory in the host that is empty.
+    // target_path cannot be a symbolic link.
+    // Maximum path length will be capped to 260 characters.
+    string target_path = 2;
+}
+
+message CreateSymlinkResponse {
+    // Intentionally empty.
+}
+
+message IsSymlinkRequest {
+    // The path whose existence as a symlink we want to check in the host's filesystem.
+    string path = 1;
+}
+
+message IsSymlinkResponse {
+    // Indicates whether the path in IsSymlinkRequest is a symlink.
+    bool is_symlink = 1;
+}

csi_proxy_proto/filesystem/v2alpha1/api.proto

@@ -0,0 +1,163 @@
+syntax = "proto3";
+
+package v2alpha1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/filesystem/v2alpha1";
+
+service Filesystem {
+    // PathExists checks if the requested path exists in the host filesystem.
+    rpc PathExists(PathExistsRequest) returns (PathExistsResponse) {}
+
+    // Mkdir creates a directory at the requested path in the host filesystem.
+    rpc Mkdir(MkdirRequest) returns (MkdirResponse) {}
+
+    // Rmdir removes the directory at the requested path in the host filesystem.
+    // This may be used for unlinking a symlink created through CreateSymlink.
+    rpc Rmdir(RmdirRequest) returns (RmdirResponse) {}
+
+    // RmdirContents removes the contents of a directory in the host filesystem.
+    // Unlike Rmdir it won't delete the requested path, it'll only delete its contents.
+    rpc RmdirContents(RmdirContentsRequest) returns (RmdirContentsResponse) {}
+
+    // CreateSymlink creates a symbolic link called target_path that points to source_path
+    // in the host filesystem (target_path is the name of the symbolic link created,
+    // source_path is the existing path).
+    rpc CreateSymlink(CreateSymlinkRequest) returns (CreateSymlinkResponse) {}
+
+    // IsSymlink checks if a given path is a symlink.
+    rpc IsSymlink(IsSymlinkRequest) returns (IsSymlinkResponse) {}
+}
+
+message PathExistsRequest {
+    // The path whose existence we want to check in the host's filesystem
+    string path = 1;
+}
+
+message PathExistsResponse {
+    // Indicates whether the path in PathExistsRequest exists in the host's filesystem
+    bool exists = 1;
+}
+
+message MkdirRequest {
+    // The path to create in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    // Non-existent parent directories in the path will be automatically created.
+    // Directories will be created with Read and Write privileges of the Windows
+    // User account under which csi-proxy is started (typically LocalSystem).
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // The path parameter cannot already exist in the host's filesystem.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+}
+
+message MkdirResponse {
+    // Intentionally empty.
+}
+
+message RmdirRequest {
+    // The path to remove in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Path cannot be a file of type symlink.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+
+    // Force remove all contents under path (if any).
+    bool force = 2;
+}
+
+message RmdirResponse {
+    // Intentionally empty.
+}
+
+message RmdirContentsRequest {
+    // The path whose contents will be removed in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // Depending on the context parameter of this function, the path prefix needs
+    // to match the paths specified either as kubelet-csi-plugins-path
+    // or as kubelet-pod-path parameters of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // Path cannot be a file of type symlink.
+    // Maximum path length will be capped to 260 characters.
+    string path = 1;
+}
+
+message RmdirContentsResponse {
+    // Intentionally empty.
+}
+
+message CreateSymlinkRequest {
+    // The path of the existing directory to be linked.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs needs to match the paths specified as
+    // kubelet-csi-plugins-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // source_path cannot already exist in the host filesystem.
+    // Maximum path length will be capped to 260 characters.
+    string source_path = 1;
+
+    // Target path is the location of the new directory entry to be created in the host's filesystem.
+    // All special characters allowed by Windows in path names will be allowed
+    // except for restrictions noted below. For details, please check:
+    // https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file
+    //
+    // Restrictions:
+    // Only absolute path (indicated by a drive letter prefix: e.g. "C:\") is accepted.
+    // The path prefix needs to match the paths specified as
+    // kubelet-pod-path parameter of csi-proxy.
+    // UNC paths of the form "\\server\share\path\file" are not allowed.
+    // All directory separators need to be backslash character: "\".
+    // Characters: .. / : | ? * in the path are not allowed.
+    // target_path needs to exist as a directory in the host that is empty.
+    // target_path cannot be a symbolic link.
+    // Maximum path length will be capped to 260 characters.
+    string target_path = 2;
+}
+
+message CreateSymlinkResponse {
+    // Intentionally empty.
+}
+
+message IsSymlinkRequest {
+    // The path whose existence as a symlink we want to check in the host's filesystem.
+    string path = 1;
+}
+
+message IsSymlinkResponse {
+    // Indicates whether the path in IsSymlinkRequest is a symlink.
+    bool is_symlink = 1;
+}

csi_proxy_proto/iscsi/v1alpha1/api.proto

@@ -0,0 +1,153 @@
+syntax = "proto3";
+
+package v1alpha1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/iscsi/v1alpha1";
+
+service Iscsi {
+  // AddTargetPortal registers an iSCSI target network address for later
+  // discovery.
+  // AddTargetPortal currently does not support selecting different NICs or
+  // a different iSCSI initiator (e.g a hardware initiator). This means that
+  // Windows will select the initiator NIC and instance on its own.
+  rpc AddTargetPortal(AddTargetPortalRequest)
+      returns (AddTargetPortalResponse) {}
+
+  // DiscoverTargetPortal initiates discovery on an iSCSI target network address
+  // and returns discovered IQNs.
+  rpc DiscoverTargetPortal(DiscoverTargetPortalRequest)
+      returns (DiscoverTargetPortalResponse) {}
+
+  // RemoveTargetPortal removes an iSCSI target network address registration.
+  rpc RemoveTargetPortal(RemoveTargetPortalRequest)
+      returns (RemoveTargetPortalResponse) {}
+
+  // ListTargetPortal lists all currently registered iSCSI target network
+  // addresses.
+  rpc ListTargetPortals(ListTargetPortalsRequest)
+      returns (ListTargetPortalsResponse) {}
+
+  // ConnectTarget connects to an iSCSI Target
+  rpc ConnectTarget(ConnectTargetRequest) returns (ConnectTargetResponse) {}
+
+  // DisconnectTarget disconnects from an iSCSI Target
+  rpc DisconnectTarget(DisconnectTargetRequest)
+      returns (DisconnectTargetResponse) {}
+
+  // GetTargetDisks returns the disk addresses that correspond to an iSCSI
+  // target
+  rpc GetTargetDisks(GetTargetDisksRequest) returns (GetTargetDisksResponse) {}
+}
+
+// TargetPortal is an address and port pair for a specific iSCSI storage
+// target.
+message TargetPortal {
+  // iSCSI Target (server) address
+  string target_address = 1;
+
+  // iSCSI Target port (default iSCSI port is 3260)
+  uint32 target_port = 2;
+}
+
+message AddTargetPortalRequest {
+  // iSCSI Target Portal to register in the initiator
+  TargetPortal target_portal = 1;
+}
+
+message AddTargetPortalResponse {
+  // Intentionally empty
+}
+
+message DiscoverTargetPortalRequest {
+  // iSCSI Target Portal on which to initiate discovery
+  TargetPortal target_portal = 1;
+}
+
+message DiscoverTargetPortalResponse {
+  // List of discovered IQN addresses
+  // follows IQN format: iqn.yyyy-mm.naming-authority:unique-name
+  repeated string iqns = 1;
+}
+
+message RemoveTargetPortalRequest {
+  // iSCSI Target Portal
+  TargetPortal target_portal = 1;
+}
+
+message RemoveTargetPortalResponse {
+  // Intentionally empty
+}
+
+message ListTargetPortalsRequest {
+  // Intentionally empty
+}
+
+message ListTargetPortalsResponse {
+  // A list of Target Portals currently registered in the initiator
+  repeated TargetPortal target_portals = 1;
+}
+
+enum AuthenticationType {
+  // No authentication is used
+  NONE = 0;
+
+  // One way CHAP authentication. The target authenticates the initiator.
+  ONE_WAY_CHAP = 1;
+
+  // Mutual CHAP authentication. The target and initiator authenticate each
+  // other.
+  MUTUAL_CHAP = 2;
+}
+
+message ConnectTargetRequest {
+  // Target portal to which the initiator will connect
+  TargetPortal target_portal = 1;
+
+  // IQN of the iSCSI Target
+  string iqn = 2;
+
+  // Connection authentication type, None by default
+  //
+  // One Way Chap uses the chap_username and chap_secret
+  // fields mentioned below to authenticate the initiator.
+  //
+  // Mutual Chap uses both the user/secret mentioned below
+  // and the Initiator Chap Secret to authenticate the target and initiator.
+  AuthenticationType auth_type = 3;
+
+  // CHAP Username used to authenticate the initiator
+  string chap_username = 4;
+
+  // CHAP password used to authenticate the initiator
+  string chap_secret = 5;
+}
+
+message ConnectTargetResponse {
+  // Intentionally empty
+}
+
+message GetTargetDisksRequest {
+  // Target portal whose disks will be queried
+  TargetPortal target_portal = 1;
+
+  // IQN of the iSCSI Target
+  string iqn = 2;
+}
+
+message GetTargetDisksResponse {
+  // List composed of disk ids (numbers) that are associated with the
+  // iSCSI target
+  repeated string diskIDs = 1;
+}
+
+message DisconnectTargetRequest {
+  // Target portal from which initiator will disconnect
+  TargetPortal target_portal = 1;
+
+  // IQN of the iSCSI Target
+  string iqn = 2;
+}
+
+message DisconnectTargetResponse {
+  // Intentionally empty
+}

csi_proxy_proto/iscsi/v1alpha2/api.proto

@@ -0,0 +1,175 @@
+syntax = "proto3";
+
+package v1alpha2;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/iscsi/v1alpha2";
+
+service Iscsi {
+  // AddTargetPortal registers an iSCSI target network address for later
+  // discovery.
+  // AddTargetPortal currently does not support selecting different NICs or
+  // a different iSCSI initiator (e.g a hardware initiator). This means that
+  // Windows will select the initiator NIC and instance on its own.
+  rpc AddTargetPortal(AddTargetPortalRequest)
+      returns (AddTargetPortalResponse) {}
+
+  // DiscoverTargetPortal initiates discovery on an iSCSI target network address
+  // and returns discovered IQNs.
+  rpc DiscoverTargetPortal(DiscoverTargetPortalRequest)
+      returns (DiscoverTargetPortalResponse) {}
+
+  // RemoveTargetPortal removes an iSCSI target network address registration.
+  rpc RemoveTargetPortal(RemoveTargetPortalRequest)
+      returns (RemoveTargetPortalResponse) {}
+
+  // ListTargetPortal lists all currently registered iSCSI target network
+  // addresses.
+  rpc ListTargetPortals(ListTargetPortalsRequest)
+      returns (ListTargetPortalsResponse) {}
+
+  // ConnectTarget connects to an iSCSI Target
+  rpc ConnectTarget(ConnectTargetRequest) returns (ConnectTargetResponse) {}
+
+  // DisconnectTarget disconnects from an iSCSI Target
+  rpc DisconnectTarget(DisconnectTargetRequest)
+      returns (DisconnectTargetResponse) {}
+
+  // GetTargetDisks returns the disk addresses that correspond to an iSCSI
+  // target
+  rpc GetTargetDisks(GetTargetDisksRequest) returns (GetTargetDisksResponse) {}
+
+  // SetMutualChapSecret sets the default CHAP secret that all initiators on
+  // this machine (node) use to authenticate the target on mutual CHAP
+  // authentication.
+  // NOTE: This method affects global node state and should only be used
+  //       with consideration to other CSI drivers that run concurrently.
+  rpc SetMutualChapSecret(SetMutualChapSecretRequest)
+      returns (SetMutualChapSecretResponse) {}
+}
+
+// TargetPortal is an address and port pair for a specific iSCSI storage
+// target.
+message TargetPortal {
+  // iSCSI Target (server) address
+  string target_address = 1;
+
+  // iSCSI Target port (default iSCSI port is 3260)
+  uint32 target_port = 2;
+}
+
+message AddTargetPortalRequest {
+  // iSCSI Target Portal to register in the initiator
+  TargetPortal target_portal = 1;
+}
+
+message AddTargetPortalResponse {
+  // Intentionally empty
+}
+
+message DiscoverTargetPortalRequest {
+  // iSCSI Target Portal on which to initiate discovery
+  TargetPortal target_portal = 1;
+}
+
+message DiscoverTargetPortalResponse {
+  // List of discovered IQN addresses
+  // follows IQN format: iqn.yyyy-mm.naming-authority:unique-name
+  repeated string iqns = 1;
+}
+
+message RemoveTargetPortalRequest {
+  // iSCSI Target Portal
+  TargetPortal target_portal = 1;
+}
+
+message RemoveTargetPortalResponse {
+  // Intentionally empty
+}
+
+message ListTargetPortalsRequest {
+  // Intentionally empty
+}
+
+message ListTargetPortalsResponse {
+  // A list of Target Portals currently registered in the initiator
+  repeated TargetPortal target_portals = 1;
+}
+
+// iSCSI logon authentication type
+enum AuthenticationType {
+  // No authentication is used
+  NONE = 0;
+
+  // One way CHAP authentication. The target authenticates the initiator.
+  ONE_WAY_CHAP = 1;
+
+  // Mutual CHAP authentication. The target and initiator authenticate each
+  // other.
+  MUTUAL_CHAP = 2;
+}
+
+message ConnectTargetRequest {
+  // Target portal to which the initiator will connect
+  TargetPortal target_portal = 1;
+
+  // IQN of the iSCSI Target
+  string iqn = 2;
+
+  // Connection authentication type, None by default
+  //
+  // One Way Chap uses the chap_username and chap_secret
+  // fields mentioned below to authenticate the initiator.
+  //
+  // Mutual Chap uses both the user/secret mentioned below
+  // and the Initiator Chap Secret (See `SetMutualChapSecret`)
+  // to authenticate the target and initiator.
+  AuthenticationType auth_type = 3;
+
+  // CHAP Username used to authenticate the initiator
+  string chap_username = 4;
+
+  // CHAP password used to authenticate the initiator
+  string chap_secret = 5;
+}
+
+message ConnectTargetResponse {
+  // Intentionally empty
+}
+
+message GetTargetDisksRequest {
+  // Target portal whose disks will be queried
+  TargetPortal target_portal = 1;
+
+  // IQN of the iSCSI Target
+  string iqn = 2;
+}
+
+message GetTargetDisksResponse {
+  // List composed of disk ids (numbers) that are associated with the
+  // iSCSI target
+  repeated string diskIDs = 1;
+}
+
+message DisconnectTargetRequest {
+  // Target portal from which initiator will disconnect
+  TargetPortal target_portal = 1;
+
+  // IQN of the iSCSI Target
+  string iqn = 2;
+}
+
+message DisconnectTargetResponse {
+  // Intentionally empty
+}
+
+message SetMutualChapSecretRequest {
+  // the default CHAP secret that all initiators on this machine (node) use to
+  // authenticate the target on mutual CHAP authentication.
+  // Must be at least 12 byte long for non-Ipsec connections, at least one
+  // byte long for Ipsec connections, and at most 16 bytes long.
+  string MutualChapSecret = 1;
+}
+
+message SetMutualChapSecretResponse {
+  // Intentionally empty
+}

csi_proxy_proto/smb/v1/api.proto

@@ -0,0 +1,58 @@
+syntax = "proto3";
+
+package v1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/smb/v1";
+
+service Smb {
+    // NewSmbGlobalMapping creates an SMB mapping on the SMB client to an SMB share.
+    rpc NewSmbGlobalMapping(NewSmbGlobalMappingRequest) returns (NewSmbGlobalMappingResponse) {}
+
+    // RemoveSmbGlobalMapping removes the SMB mapping to an SMB share.
+    rpc RemoveSmbGlobalMapping(RemoveSmbGlobalMappingRequest) returns (RemoveSmbGlobalMappingResponse) {}
+}
+
+
+message NewSmbGlobalMappingRequest {
+    // A remote SMB share to mount
+    // All unicode characters allowed in SMB server name specifications are
+    // permitted except for restrictions below
+    //
+    // Restrictions:
+    // SMB remote path specified in the format: \\server-name\sharename, \\server.fqdn\sharename or \\a.b.c.d\sharename
+    // If not an IP address, share name has to be a valid DNS name.
+    // UNC specifications to local paths or prefix: \\?\ is not allowed.
+    // Characters: + [ ] " / : ; | < > , ? * = $ are not allowed.
+    string remote_path = 1;
+
+    // Optional local path to mount the smb on
+    string local_path = 2;
+
+    // Username credential associated with the share
+    string username = 3;
+
+    // Password credential associated with the share
+    string password = 4;
+}
+
+message NewSmbGlobalMappingResponse {
+    // Intentionally empty.
+}
+
+
+message RemoveSmbGlobalMappingRequest {
+    // A remote SMB share mapping to remove
+    // All unicode characters allowed in SMB server name specifications are
+    // permitted except for restrictions below
+    //
+    // Restrictions:
+    // SMB share specified in the format: \\server-name\sharename, \\server.fqdn\sharename or \\a.b.c.d\sharename
+    // If not an IP address, share name has to be a valid DNS name.
+    // UNC specifications to local paths or prefix: \\?\ is not allowed.
+    // Characters: + [ ] " / : ; | < > , ? * = $ are not allowed.
+    string remote_path = 1;
+}
+
+message RemoveSmbGlobalMappingResponse {
+    // Intentionally empty.
+}

csi_proxy_proto/smb/v1alpha1/api.proto

@@ -0,0 +1,59 @@
+syntax = "proto3";
+
+package v1alpha1;
+
+service Smb {
+    // NewSmbGlobalMapping creates an SMB mapping on the SMB client to an SMB share.
+    rpc NewSmbGlobalMapping(NewSmbGlobalMappingRequest) returns (NewSmbGlobalMappingResponse) {}
+
+    // RemoveSmbGlobalMapping removes the SMB mapping to an SMB share.
+    rpc RemoveSmbGlobalMapping(RemoveSmbGlobalMappingRequest) returns (RemoveSmbGlobalMappingResponse) {}
+}
+
+
+message NewSmbGlobalMappingRequest {
+    // A remote SMB share to mount
+    // All unicode characters allowed in SMB server name specifications are
+    // permitted except for restrictions below
+    //
+    // Restrictions:
+    // SMB remote path specified in the format: \\server-name\sharename, \\server.fqdn\sharename or \\a.b.c.d\sharename
+    // If not an IP address, share name has to be a valid DNS name.
+    // UNC specifications to local paths or prefix: \\?\ is not allowed.
+    // Characters: + [ ] " / : ; | < > , ? * = $ are not allowed.
+    string remote_path = 1;
+    // Optional local path to mount the smb on
+    string local_path = 2;
+
+    // Username credential associated with the share
+    string username = 3;
+
+    // Password credential associated with the share
+    string password = 4;
+}
+
+message NewSmbGlobalMappingResponse {
+    // Windows error code
+    // Success is represented as 0
+    string error = 1;
+}
+
+
+message RemoveSmbGlobalMappingRequest {
+    // A remote SMB share mapping to remove
+    // All unicode characters allowed in SMB server name specifications are
+    // permitted except for restrictions below
+    //
+    // Restrictions:
+    // SMB share specified in the format: \\server-name\sharename, \\server.fqdn\sharename or \\a.b.c.d\sharename
+    // If not an IP address, share name has to be a valid DNS name.
+    // UNC specifications to local paths or prefix: \\?\ is not allowed.
+    // Characters: + [ ] " / : ; | < > , ? * = $ are not allowed.
+    string remote_path = 1;
+}
+
+message RemoveSmbGlobalMappingResponse {
+    // Windows error code
+    // Success is represented as 0
+    string error = 1;
+}

csi_proxy_proto/smb/v1beta1/api.proto

@@ -0,0 +1,59 @@
+syntax = "proto3";
+
+package v1beta1;
+
+service Smb {
+    // NewSmbGlobalMapping creates an SMB mapping on the SMB client to an SMB share.
+    rpc NewSmbGlobalMapping(NewSmbGlobalMappingRequest) returns (NewSmbGlobalMappingResponse) {}
+
+    // RemoveSmbGlobalMapping removes the SMB mapping to an SMB share.
+    rpc RemoveSmbGlobalMapping(RemoveSmbGlobalMappingRequest) returns (RemoveSmbGlobalMappingResponse) {}
+}
+
+
+message NewSmbGlobalMappingRequest {
+    // A remote SMB share to mount
+    // All unicode characters allowed in SMB server name specifications are
+    // permitted except for restrictions below
+    //
+    // Restrictions:
+    // SMB remote path specified in the format: \\server-name\sharename, \\server.fqdn\sharename or \\a.b.c.d\sharename
+    // If not an IP address, share name has to be a valid DNS name.
+    // UNC specifications to local paths or prefix: \\?\ is not allowed.
+    // Characters: + [ ] " / : ; | < > , ? * = $ are not allowed.
+    string remote_path = 1;
+    // Optional local path to mount the smb on
+    string local_path = 2;
+
+    // Username credential associated with the share
+    string username = 3;
+
+    // Password credential associated with the share
+    string password = 4;
+}
+
+message NewSmbGlobalMappingResponse {
+    // Windows error code
+    // Success is represented as 0
+    string error = 1;
+}
+
+
+message RemoveSmbGlobalMappingRequest {
+    // A remote SMB share mapping to remove
+    // All unicode characters allowed in SMB server name specifications are
+    // permitted except for restrictions below
+    //
+    // Restrictions:
+    // SMB share specified in the format: \\server-name\sharename, \\server.fqdn\sharename or \\a.b.c.d\sharename
+    // If not an IP address, share name has to be a valid DNS name.
+    // UNC specifications to local paths or prefix: \\?\ is not allowed.
+    // Characters: + [ ] " / : ; | < > , ? * = $ are not allowed.
+    string remote_path = 1;
+}
+
+message RemoveSmbGlobalMappingResponse {
+    // Windows error code
+    // Success is represented as 0
+    string error = 1;
+}

csi_proxy_proto/smb/v1beta2/api.proto

@@ -0,0 +1,58 @@
+syntax = "proto3";
+
+package v1beta2;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/smb/v1beta2";
+
+service Smb {
+    // NewSmbGlobalMapping creates an SMB mapping on the SMB client to an SMB share.
+    rpc NewSmbGlobalMapping(NewSmbGlobalMappingRequest) returns (NewSmbGlobalMappingResponse) {}
+
+    // RemoveSmbGlobalMapping removes the SMB mapping to an SMB share.
+    rpc RemoveSmbGlobalMapping(RemoveSmbGlobalMappingRequest) returns (RemoveSmbGlobalMappingResponse) {}
+}
+
+
+message NewSmbGlobalMappingRequest {
+    // A remote SMB share to mount
+    // All unicode characters allowed in SMB server name specifications are
+    // permitted except for restrictions below
+    //
+    // Restrictions:
+    // SMB remote path specified in the format: \\server-name\sharename, \\server.fqdn\sharename or \\a.b.c.d\sharename
+    // If not an IP address, share name has to be a valid DNS name.
+    // UNC specifications to local paths or prefix: \\?\ is not allowed.
+    // Characters: + [ ] " / : ; | < > , ? * = $ are not allowed.
+    string remote_path = 1;
+
+    // Optional local path to mount the smb on
+    string local_path = 2;
+
+    // Username credential associated with the share
+    string username = 3;
+
+    // Password credential associated with the share
+    string password = 4;
+}
+
+message NewSmbGlobalMappingResponse {
+    // Intentionally empty.
+}
+
+
+message RemoveSmbGlobalMappingRequest {
+    // A remote SMB share mapping to remove
+    // All unicode characters allowed in SMB server name specifications are
+    // permitted except for restrictions below
+    //
+    // Restrictions:
+    // SMB share specified in the format: \\server-name\sharename, \\server.fqdn\sharename or \\a.b.c.d\sharename
+    // If not an IP address, share name has to be a valid DNS name.
+    // UNC specifications to local paths or prefix: \\?\ is not allowed.
+    // Characters: + [ ] " / : ; | < > , ? * = $ are not allowed.
+    string remote_path = 1;
+}
+
+message RemoveSmbGlobalMappingResponse {
+    // Intentionally empty.
+}

csi_proxy_proto/system/v1alpha1/api.proto

@@ -0,0 +1,93 @@
+syntax = "proto3";
+
+package v1alpha1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/system/v1alpha1";
+
+service System {
+  // GetBIOSSerialNumber returns the device's serial number
+  rpc GetBIOSSerialNumber(GetBIOSSerialNumberRequest)
+      returns (GetBIOSSerialNumberResponse) {}
+
+  // StartService starts a Windows service
+  // NOTE: This method affects global node state and should only be used
+  //       with consideration to other CSI drivers that run concurrently.
+  rpc StartService(StartServiceRequest) returns (StartServiceResponse) {}
+
+  // StopService stops a Windows service
+  // NOTE: This method affects global node state and should only be used
+  //       with consideration to other CSI drivers that run concurrently.
+  rpc StopService(StopServiceRequest) returns (StopServiceResponse) {}
+
+  // GetService queries a Windows service state
+  rpc GetService(GetServiceRequest) returns (GetServiceResponse) {}
+}
+
+message GetBIOSSerialNumberRequest {
+  // Intentionally empty
+}
+
+message GetBIOSSerialNumberResponse {
+  // Serial number
+  string serial_number = 1;
+}
+
+message StartServiceRequest {
+  // Service name (as listed in System\CCS\Services keys)
+  string name = 1;
+}
+
+message StartServiceResponse {
+  // Intentionally empty
+}
+
+message StopServiceRequest {
+  // Service name (as listed in System\CCS\Services keys)
+  string name = 1;
+
+  // Forces stopping of services that has dependent services
+  bool force = 2;
+}
+
+message StopServiceResponse {
+  // Intentionally empty
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/winsvc/ns-winsvc-service_status#members
+enum ServiceStatus {
+  UNKNOWN = 0;
+  STOPPED = 1;
+  START_PENDING = 2;
+  STOP_PENDING = 3;
+  RUNNING = 4;
+  CONTINUE_PENDING = 5;
+  PAUSE_PENDING = 6;
+  PAUSED = 7;
+}
+
+// https://docs.microsoft.com/en-us/windows/win32/api/winsvc/nf-winsvc-changeserviceconfiga
+enum StartType {
+  BOOT = 0;
+  SYSTEM = 1;
+  AUTOMATIC = 2;
+  MANUAL = 3;
+  DISABLED = 4;
+}
+
+message GetServiceRequest {
+  // Service name (as listed in System\CCS\Services keys)
+  string name = 1;
+}
+
+message GetServiceResponse {
+  // Service display name
+  string display_name = 1;
+
+  // Service start type.
+  // Used to control whether a service will start on boot, and if so on which
+  // boot phase.
+  StartType start_type = 2;
+
+  // Service status, e.g. stopped, running, paused
+  ServiceStatus status = 3;
+}

csi_proxy_proto/volume/v1/api.proto

@@ -0,0 +1,143 @@
+syntax = "proto3";
+
+package v1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/volume/v1";
+
+service Volume {
+    // ListVolumesOnDisk returns the volume IDs (in \\.\Volume{GUID} format) for all volumes from a
+    // given disk number and partition number (optional)
+    rpc ListVolumesOnDisk(ListVolumesOnDiskRequest) returns (ListVolumesOnDiskResponse) {}
+
+    // MountVolume mounts the volume at the requested global staging path.
+    rpc MountVolume(MountVolumeRequest) returns (MountVolumeResponse) {}
+
+    // UnmountVolume flushes data cache to disk and removes the global staging path.
+    rpc UnmountVolume(UnmountVolumeRequest) returns (UnmountVolumeResponse) {}
+
+    // IsVolumeFormatted checks if a volume is formatted.
+    rpc IsVolumeFormatted(IsVolumeFormattedRequest) returns (IsVolumeFormattedResponse) {}
+
+    // FormatVolume formats a volume with NTFS.
+    rpc FormatVolume(FormatVolumeRequest) returns (FormatVolumeResponse) {}
+
+    // ResizeVolume performs resizing of the partition and file system for a block based volume.
+    rpc ResizeVolume(ResizeVolumeRequest) returns (ResizeVolumeResponse) {}
+
+    // GetVolumeStats gathers total bytes and used bytes for a volume.
+    rpc GetVolumeStats(GetVolumeStatsRequest) returns (GetVolumeStatsResponse) {}
+
+    // GetDiskNumberFromVolumeID gets the disk number of the disk where the volume is located.
+    rpc GetDiskNumberFromVolumeID(GetDiskNumberFromVolumeIDRequest) returns (GetDiskNumberFromVolumeIDResponse ) {}
+
+    // GetVolumeIDFromTargetPath gets the volume id for a given target path.
+    rpc GetVolumeIDFromTargetPath(GetVolumeIDFromTargetPathRequest) returns (GetVolumeIDFromTargetPathResponse) {}
+
+    // WriteVolumeCache write volume cache to disk.
+    rpc WriteVolumeCache(WriteVolumeCacheRequest) returns (WriteVolumeCacheResponse) {}
+}
+
+message ListVolumesOnDiskRequest {
+    // Disk device number of the disk to query for volumes.
+    uint32 disk_number = 1;
+    // The partition number (optional), by default it uses the first partition of the disk.
+    uint32 partition_number = 2;
+}
+
+message ListVolumesOnDiskResponse {
+    // Volume device IDs of volumes on the specified disk.
+    repeated string volume_ids = 1;
+}
+
+message MountVolumeRequest {
+    // Volume device ID of the volume to mount.
+    string volume_id = 1;
+    // Path in the host's file system where the volume needs to be mounted.
+    string target_path = 2;
+}
+
+message MountVolumeResponse {
+    // Intentionally empty.
+}
+
+message UnmountVolumeRequest {
+    // Volume device ID of the volume to dismount.
+    string volume_id = 1;
+    // Path where the volume has been mounted.
+    string target_path = 2;
+}
+
+message UnmountVolumeResponse {
+    // Intentionally empty.
+}
+
+message IsVolumeFormattedRequest {
+    // Volume device ID of the volume to check.
+    string volume_id = 1;
+}
+
+message IsVolumeFormattedResponse {
+    // Is the volume formatted with NTFS.
+    bool formatted = 1;
+}
+
+message FormatVolumeRequest {
+    // Volume device ID of the volume to format.
+    string volume_id = 1;
+}
+
+message FormatVolumeResponse {
+    // Intentionally empty.
+}
+
+message ResizeVolumeRequest {
+    // Volume device ID of the volume to resize.
+    string volume_id = 1;
+    // New size in bytes of the volume.
+    int64 size_bytes = 2;
+}
+
+message ResizeVolumeResponse {
+    // Intentionally empty.
+}
+
+message GetVolumeStatsRequest{
+    // Volume device Id of the volume to get the stats for.
+    string volume_id = 1;
+}
+
+message GetVolumeStatsResponse{
+    // Total bytes
+    int64 total_bytes = 1;
+    // Used bytes
+    int64 used_bytes = 2;
+}
+
+message GetDiskNumberFromVolumeIDRequest {
+    // Volume device ID of the volume to get the disk number for.
+    string volume_id = 1;
+}
+
+message GetDiskNumberFromVolumeIDResponse {
+    // Corresponding disk number.
+    uint32 disk_number = 1;
+}
+
+message GetVolumeIDFromTargetPathRequest {
+    // The target path.
+    string target_path = 1;
+}
+
+message GetVolumeIDFromTargetPathResponse {
+    // The volume device ID.
+    string volume_id = 1;
+}
+
+message WriteVolumeCacheRequest {
+    // Volume device ID of the volume to flush the cache.
+    string volume_id = 1;
+}
+
+message WriteVolumeCacheResponse {
+    // Intentionally empty.
+}

csi_proxy_proto/volume/v1alpha1/api.proto

@@ -0,0 +1,69 @@
+syntax = "proto3";
+
+package v1alpha1;
+
+service Volume {
+    // ListVolumesOnDisk returns the volume IDs (in \\.\Volume{GUID} format) for
+    // all volumes on a Disk device
+    rpc ListVolumesOnDisk(ListVolumesOnDiskRequest) returns (ListVolumesOnDiskResponse) {}
+    // MountVolume mounts the volume at the requested global staging path
+    rpc MountVolume(MountVolumeRequest) returns (MountVolumeResponse) {}
+    // DismountVolume gracefully dismounts a volume
+    rpc DismountVolume(DismountVolumeRequest) returns (DismountVolumeResponse) {}
+    // IsVolumeFormatted checks if a volume is formatted with NTFS
+    rpc IsVolumeFormatted(IsVolumeFormattedRequest) returns (IsVolumeFormattedResponse) {}
+    // FormatVolume formats a volume with the provided file system
+    rpc FormatVolume(FormatVolumeRequest) returns (FormatVolumeResponse) {}
+    // ResizeVolume performs resizing of the partition and file system for a block based volume
+    rpc ResizeVolume(ResizeVolumeRequest) returns (ResizeVolumeResponse) {}
+}
+message ListVolumesOnDiskRequest {
+    // Disk device ID of the disk to query for volumes
+    string disk_id = 1;
+}
+message ListVolumesOnDiskResponse {
+    // Volume device IDs of volumes on the specified disk
+    repeated string volume_ids = 1;
+}
+message MountVolumeRequest {
+    // Volume device ID of the volume to mount
+    string volume_id = 1;
+    // Path in the host's file system where the volume needs to be mounted
+    string path = 2;
+}
+message MountVolumeResponse {
+    // Intentionally empty
+}
+message DismountVolumeRequest {
+    // Volume device ID of the volume to dismount
+    string volume_id = 1;
+    // Path where the volume has been mounted.
+    string path = 2;
+}
+message DismountVolumeResponse {
+    // Intentionally empty
+}
+message IsVolumeFormattedRequest {
+    // Volume device ID of the volume to check
+    string volume_id = 1;
+}
+message IsVolumeFormattedResponse {
+    // Is the volume formatted with NTFS
+    bool formatted = 1;
+}
+message FormatVolumeRequest {
+    // Volume device ID of the volume to format
+    string volume_id = 1;
+}
+message FormatVolumeResponse {
+    // Intentionally empty
+}
+message ResizeVolumeRequest {
+    // Volume device ID of the volume to dismount
+    string volume_id = 1;
+    // New size of the volume
+    int64 size = 2;
+}
+message ResizeVolumeResponse {
+    // Intentionally empty
+}

csi_proxy_proto/volume/v1beta1/api.proto

@@ -0,0 +1,121 @@
+syntax = "proto3";
+
+package v1beta1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/volume/v1beta1";
+
+service Volume {
+    // ListVolumesOnDisk returns the volume IDs (in \\.\Volume{GUID} format) for
+    // all volumes on a Disk device
+    rpc ListVolumesOnDisk(ListVolumesOnDiskRequest) returns (ListVolumesOnDiskResponse) {}
+    // MountVolume mounts the volume at the requested global staging path
+    rpc MountVolume(MountVolumeRequest) returns (MountVolumeResponse) {}
+    // DismountVolume gracefully dismounts a volume
+    rpc DismountVolume(DismountVolumeRequest) returns (DismountVolumeResponse) {}
+    // IsVolumeFormatted checks if a volume is formatted with NTFS
+    rpc IsVolumeFormatted(IsVolumeFormattedRequest) returns (IsVolumeFormattedResponse) {}
+    // FormatVolume formats a volume with the provided file system
+    rpc FormatVolume(FormatVolumeRequest) returns (FormatVolumeResponse) {}
+    // ResizeVolume performs resizing of the partition and file system for a block based volume
+    rpc ResizeVolume(ResizeVolumeRequest) returns (ResizeVolumeResponse) {}
+    // VolumeStats gathers DiskSize, VolumeSize and VolumeUsedSize for a volume
+    rpc VolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse) {}
+    // GetVolumeDiskNumber gets the disk number of the disk where the volume is located
+    rpc GetVolumeDiskNumber(VolumeDiskNumberRequest) returns (VolumeDiskNumberResponse) {}
+    // GetVolumeIDFromMount gets the volume id for a given mount
+    rpc GetVolumeIDFromMount(VolumeIDFromMountRequest) returns (VolumeIDFromMountResponse) {}
+}
+
+message ListVolumesOnDiskRequest {
+    // Disk device ID of the disk to query for volumes
+    string disk_id = 1;
+}
+
+message ListVolumesOnDiskResponse {
+    // Volume device IDs of volumes on the specified disk
+    repeated string volume_ids = 1;
+}
+
+message MountVolumeRequest {
+    // Volume device ID of the volume to mount
+    string volume_id = 1;
+    // Path in the host's file system where the volume needs to be mounted
+    string path = 2;
+}
+
+message MountVolumeResponse {
+    // Intentionally empty
+}
+
+message DismountVolumeRequest {
+    // Volume device ID of the volume to dismount
+    string volume_id = 1;
+    // Path where the volume has been mounted.
+    string path = 2;
+}
+
+message DismountVolumeResponse {
+    // Intentionally empty
+}
+
+message IsVolumeFormattedRequest {
+    // Volume device ID of the volume to check
+    string volume_id = 1;
+}
+
+message IsVolumeFormattedResponse {
+    // Is the volume formatted with NTFS
+    bool formatted = 1;
+}
+
+message FormatVolumeRequest {
+    // Volume device ID of the volume to format
+    string volume_id = 1;
+}
+
+message FormatVolumeResponse {
+    // Intentionally empty
+}
+
+message ResizeVolumeRequest {
+    // Volume device ID of the volume to dismount
+    string volume_id = 1;
+    // New size of the volume
+    int64 size = 2;
+}
+
+message ResizeVolumeResponse {
+    // Intentionally empty
+}
+
+message VolumeStatsRequest{
+    // Volume device Id of the volume to get the stats for
+    string volume_id = 1;
+}
+
+message VolumeStatsResponse{
+    // Capacity of the volume
+    int64 volumeSize = 1;
+    // Used bytes
+    int64 volumeUsedSize = 2;
+}
+
+message VolumeDiskNumberRequest{
+    // Volume device Id of the volume to get the disk number for
+    string volume_id = 1;
+}
+
+message VolumeDiskNumberResponse{
+    // Corresponding disk number
+    int64 diskNumber = 1;
+}
+
+message VolumeIDFromMountRequest {
+    // Mount
+    string mount = 1;
+}
+
+message VolumeIDFromMountResponse {
+    // Mount
+    string volume_id = 1;
+}

csi_proxy_proto/volume/v1beta2/api.proto

@@ -0,0 +1,132 @@
+syntax = "proto3";
+
+package v1beta2;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/volume/v1beta2";
+
+service Volume {
+    // ListVolumesOnDisk returns the volume IDs (in \\.\Volume{GUID} format) for
+    // all volumes on a Disk device
+    rpc ListVolumesOnDisk(ListVolumesOnDiskRequest) returns (ListVolumesOnDiskResponse) {}
+    // MountVolume mounts the volume at the requested global staging path
+    rpc MountVolume(MountVolumeRequest) returns (MountVolumeResponse) {}
+    // DismountVolume gracefully dismounts a volume
+    rpc DismountVolume(DismountVolumeRequest) returns (DismountVolumeResponse) {}
+    // IsVolumeFormatted checks if a volume is formatted with NTFS
+    rpc IsVolumeFormatted(IsVolumeFormattedRequest) returns (IsVolumeFormattedResponse) {}
+    // FormatVolume formats a volume with the provided file system
+    rpc FormatVolume(FormatVolumeRequest) returns (FormatVolumeResponse) {}
+    // ResizeVolume performs resizing of the partition and file system for a block based volume
+    rpc ResizeVolume(ResizeVolumeRequest) returns (ResizeVolumeResponse) {}
+    // VolumeStats gathers DiskSize, VolumeSize and VolumeUsedSize for a volume
+    rpc VolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse) {}
+    // GetVolumeDiskNumber gets the disk number of the disk where the volume is located
+    rpc GetVolumeDiskNumber(VolumeDiskNumberRequest) returns (VolumeDiskNumberResponse) {}
+    // GetVolumeIDFromMount gets the volume id for a given mount
+    rpc GetVolumeIDFromMount(VolumeIDFromMountRequest) returns (VolumeIDFromMountResponse) {}
+    // WriteVolumeCache write volume cache to disk
+    rpc WriteVolumeCache(WriteVolumeCacheRequest) returns (WriteVolumeCacheResponse) {}
+}
+
+message ListVolumesOnDiskRequest {
+    // Disk device ID of the disk to query for volumes
+    string disk_id = 1;
+}
+
+message ListVolumesOnDiskResponse {
+    // Volume device IDs of volumes on the specified disk
+    repeated string volume_ids = 1;
+}
+
+message MountVolumeRequest {
+    // Volume device ID of the volume to mount
+    string volume_id = 1;
+    // Path in the host's file system where the volume needs to be mounted
+    string path = 2;
+}
+
+message MountVolumeResponse {
+    // Intentionally empty
+}
+
+message DismountVolumeRequest {
+    // Volume device ID of the volume to dismount
+    string volume_id = 1;
+    // Path where the volume has been mounted.
+    string path = 2;
+}
+
+message DismountVolumeResponse {
+    // Intentionally empty
+}
+
+message IsVolumeFormattedRequest {
+    // Volume device ID of the volume to check
+    string volume_id = 1;
+}
+
+message IsVolumeFormattedResponse {
+    // Is the volume formatted with NTFS
+    bool formatted = 1;
+}
+
+message FormatVolumeRequest {
+    // Volume device ID of the volume to format
+    string volume_id = 1;
+}
+
+message FormatVolumeResponse {
+    // Intentionally empty
+}
+
+message ResizeVolumeRequest {
+    // Volume device ID of the volume to dismount
+    string volume_id = 1;
+    // New size of the volume
+    int64 size = 2;
+}
+
+message ResizeVolumeResponse {
+    // Intentionally empty
+}
+
+message VolumeStatsRequest{
+    // Volume device Id of the volume to get the stats for
+    string volume_id = 1;
+}
+
+message VolumeStatsResponse{
+    // Capacity of the volume
+    int64 volumeSize = 1;
+    // Used bytes
+    int64 volumeUsedSize = 2;
+}
+
+message VolumeDiskNumberRequest{
+    // Volume device Id of the volume to get the disk number for
+    string volume_id = 1;
+}
+
+message VolumeDiskNumberResponse{
+    // Corresponding disk number
+    int64 diskNumber = 1;
+}
+
+message VolumeIDFromMountRequest {
+    // Mount
+    string mount = 1;
+}
+
+message VolumeIDFromMountResponse {
+    // Mount
+    string volume_id = 1;
+}
+
+message WriteVolumeCacheRequest {
+    // Volume device ID of the volume to flush the cache
+    string volume_id = 1;
+}
+
+message WriteVolumeCacheResponse {
+    // Intentionally empty
+}

csi_proxy_proto/volume/v1beta3/api.proto

@@ -0,0 +1,143 @@
+syntax = "proto3";
+
+package v1beta3;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/volume/v1beta3";
+
+service Volume {
+    // ListVolumesOnDisk returns the volume IDs (in \\.\Volume{GUID} format) for all volumes from a
+    // given disk number and partition number (optional)
+    rpc ListVolumesOnDisk(ListVolumesOnDiskRequest) returns (ListVolumesOnDiskResponse) {}
+
+    // MountVolume mounts the volume at the requested global staging path.
+    rpc MountVolume(MountVolumeRequest) returns (MountVolumeResponse) {}
+
+    // UnmountVolume flushes data cache to disk and removes the global staging path.
+    rpc UnmountVolume(UnmountVolumeRequest) returns (UnmountVolumeResponse) {}
+
+    // IsVolumeFormatted checks if a volume is formatted.
+    rpc IsVolumeFormatted(IsVolumeFormattedRequest) returns (IsVolumeFormattedResponse) {}
+
+    // FormatVolume formats a volume with NTFS.
+    rpc FormatVolume(FormatVolumeRequest) returns (FormatVolumeResponse) {}
+
+    // ResizeVolume performs resizing of the partition and file system for a block based volume.
+    rpc ResizeVolume(ResizeVolumeRequest) returns (ResizeVolumeResponse) {}
+
+    // GetVolumeStats gathers total bytes and used bytes for a volume.
+    rpc GetVolumeStats(GetVolumeStatsRequest) returns (GetVolumeStatsResponse) {}
+
+    // GetDiskNumberFromVolumeID gets the disk number of the disk where the volume is located.
+    rpc GetDiskNumberFromVolumeID(GetDiskNumberFromVolumeIDRequest) returns (GetDiskNumberFromVolumeIDResponse ) {}
+
+    // GetVolumeIDFromTargetPath gets the volume id for a given target path.
+    rpc GetVolumeIDFromTargetPath(GetVolumeIDFromTargetPathRequest) returns (GetVolumeIDFromTargetPathResponse) {}
+
+    // WriteVolumeCache write volume cache to disk.
+    rpc WriteVolumeCache(WriteVolumeCacheRequest) returns (WriteVolumeCacheResponse) {}
+}
+
+message ListVolumesOnDiskRequest {
+    // Disk device number of the disk to query for volumes.
+    uint32 disk_number = 1;
+    // The partition number (optional), by default it uses the first partition of the disk.
+    uint32 partition_number = 2;
+}
+
+message ListVolumesOnDiskResponse {
+    // Volume device IDs of volumes on the specified disk.
+    repeated string volume_ids = 1;
+}
+
+message MountVolumeRequest {
+    // Volume device ID of the volume to mount.
+    string volume_id = 1;
+    // Path in the host's file system where the volume needs to be mounted.
+    string target_path = 2;
+}
+
+message MountVolumeResponse {
+    // Intentionally empty.
+}
+
+message UnmountVolumeRequest {
+    // Volume device ID of the volume to dismount.
+    string volume_id = 1;
+    // Path where the volume has been mounted.
+    string target_path = 2;
+}
+
+message UnmountVolumeResponse {
+    // Intentionally empty.
+}
+
+message IsVolumeFormattedRequest {
+    // Volume device ID of the volume to check.
+    string volume_id = 1;
+}
+
+message IsVolumeFormattedResponse {
+    // Is the volume formatted with NTFS.
+    bool formatted = 1;
+}
+
+message FormatVolumeRequest {
+    // Volume device ID of the volume to format.
+    string volume_id = 1;
+}
+
+message FormatVolumeResponse {
+    // Intentionally empty.
+}
+
+message ResizeVolumeRequest {
+    // Volume device ID of the volume to resize.
+    string volume_id = 1;
+    // New size in bytes of the volume.
+    int64 size_bytes = 2;
+}
+
+message ResizeVolumeResponse {
+    // Intentionally empty.
+}
+
+message GetVolumeStatsRequest{
+    // Volume device Id of the volume to get the stats for.
+    string volume_id = 1;
+}
+
+message GetVolumeStatsResponse{
+    // Total bytes
+    int64 total_bytes = 1;
+    // Used bytes
+    int64 used_bytes = 2;
+}
+
+message GetDiskNumberFromVolumeIDRequest {
+    // Volume device ID of the volume to get the disk number for.
+    string volume_id = 1;
+}
+
+message GetDiskNumberFromVolumeIDResponse {
+    // Corresponding disk number.
+    uint32 disk_number = 1;
+}
+
+message GetVolumeIDFromTargetPathRequest {
+    // The target path.
+    string target_path = 1;
+}
+
+message GetVolumeIDFromTargetPathResponse {
+    // The volume device ID.
+    string volume_id = 1;
+}
+
+message WriteVolumeCacheRequest {
+    // Volume device ID of the volume to flush the cache.
+    string volume_id = 1;
+}
+
+message WriteVolumeCacheResponse {
+    // Intentionally empty.
+}

csi_proxy_proto/volume/v2alpha1/api.proto

@@ -0,0 +1,158 @@
+syntax = "proto3";
+
+package v2alpha1;
+
+option go_package = "github.com/kubernetes-csi/csi-proxy/client/api/volume/v2alpha1";
+
+service Volume {
+    // ListVolumesOnDisk returns the volume IDs (in \\.\Volume{GUID} format) for all volumes from a
+    // given disk number and partition number (optional)
+    rpc ListVolumesOnDisk(ListVolumesOnDiskRequest) returns (ListVolumesOnDiskResponse) {}
+
+    // MountVolume mounts the volume at the requested global staging path.
+    rpc MountVolume(MountVolumeRequest) returns (MountVolumeResponse) {}
+
+    // UnmountVolume flushes data cache to disk and removes the global staging path.
+    rpc UnmountVolume(UnmountVolumeRequest) returns (UnmountVolumeResponse) {}
+
+    // IsVolumeFormatted checks if a volume is formatted.
+    rpc IsVolumeFormatted(IsVolumeFormattedRequest) returns (IsVolumeFormattedResponse) {}
+
+    // FormatVolume formats a volume with NTFS.
+    rpc FormatVolume(FormatVolumeRequest) returns (FormatVolumeResponse) {}
+
+    // ResizeVolume performs resizing of the partition and file system for a block based volume.
+    rpc ResizeVolume(ResizeVolumeRequest) returns (ResizeVolumeResponse) {}
+
+    // GetVolumeStats gathers total bytes and used bytes for a volume.
+    rpc GetVolumeStats(GetVolumeStatsRequest) returns (GetVolumeStatsResponse) {}
+
+    // GetDiskNumberFromVolumeID gets the disk number of the disk where the volume is located.
+    rpc GetDiskNumberFromVolumeID(GetDiskNumberFromVolumeIDRequest) returns (GetDiskNumberFromVolumeIDResponse ) {}
+
+    // GetVolumeIDFromTargetPath gets the volume id for a given target path.
+    rpc GetVolumeIDFromTargetPath(GetVolumeIDFromTargetPathRequest) returns (GetVolumeIDFromTargetPathResponse) {}
+
+    // GetClosestVolumeIDFromTargetPath gets the closest volume id for a given target path
+    // by following symlinks and moving up in the filesystem, if after moving up in the filesystem
+    // we get to a DriveLetter then the volume corresponding to this drive letter is returned instead.
+    rpc GetClosestVolumeIDFromTargetPath(GetClosestVolumeIDFromTargetPathRequest) returns (GetClosestVolumeIDFromTargetPathResponse) {}
+
+    // WriteVolumeCache write volume cache to disk.
+    rpc WriteVolumeCache(WriteVolumeCacheRequest) returns (WriteVolumeCacheResponse) {}
+}
+
+message ListVolumesOnDiskRequest {
+    // Disk device number of the disk to query for volumes.
+    uint32 disk_number = 1;
+    // The partition number (optional), by default it uses the first partition of the disk.
+    uint32 partition_number = 2;
+}
+
+message ListVolumesOnDiskResponse {
+    // Volume device IDs of volumes on the specified disk.
+    repeated string volume_ids = 1;
+}
+
+message MountVolumeRequest {
+    // Volume device ID of the volume to mount.
+    string volume_id = 1;
+    // Path in the host's file system where the volume needs to be mounted.
+    string target_path = 2;
+}
+
+message MountVolumeResponse {
+    // Intentionally empty.
+}
+
+message UnmountVolumeRequest {
+    // Volume device ID of the volume to dismount.
+    string volume_id = 1;
+    // Path where the volume has been mounted.
+    string target_path = 2;
+}
+
+message UnmountVolumeResponse {
+    // Intentionally empty.
+}
+
+message IsVolumeFormattedRequest {
+    // Volume device ID of the volume to check.
+    string volume_id = 1;
+}
+
+message IsVolumeFormattedResponse {
+    // Is the volume formatted with NTFS.
+    bool formatted = 1;
+}
+
+message FormatVolumeRequest {
+    // Volume device ID of the volume to format.
+    string volume_id = 1;
+}
+
+message FormatVolumeResponse {
+    // Intentionally empty.
+}
+
+message ResizeVolumeRequest {
+    // Volume device ID of the volume to resize.
+    string volume_id = 1;
+    // New size in bytes of the volume.
+    int64 size_bytes = 2;
+}
+
+message ResizeVolumeResponse {
+    // Intentionally empty.
+}
+
+message GetVolumeStatsRequest{
+    // Volume device Id of the volume to get the stats for.
+    string volume_id = 1;
+}
+
+message GetVolumeStatsResponse{
+    // Total bytes
+    int64 total_bytes = 1;
+    // Used bytes
+    int64 used_bytes = 2;
+}
+
+message GetDiskNumberFromVolumeIDRequest {
+    // Volume device ID of the volume to get the disk number for.
+    string volume_id = 1;
+}
+
+message GetDiskNumberFromVolumeIDResponse {
+    // Corresponding disk number.
+    uint32 disk_number = 1;
+}
+
+message GetVolumeIDFromTargetPathRequest {
+    // The target path.
+    string target_path = 1;
+}
+
+message GetVolumeIDFromTargetPathResponse {
+    // The volume device ID.
+    string volume_id = 1;
+}
+
+message GetClosestVolumeIDFromTargetPathRequest {
+    // The target path.
+    string target_path = 1;
+}
+
+message GetClosestVolumeIDFromTargetPathResponse {
+    // The volume device ID.
+    string volume_id = 1;
+}
+
+message WriteVolumeCacheRequest {
+    // Volume device ID of the volume to flush the cache.
+    string volume_id = 1;
+}
+
+message WriteVolumeCacheResponse {
+    // Intentionally empty.
+}

docker/iscsiadm

@@ -1,5 +1,28 @@
 #!/bin/bash
 
-# https://engineering.docker.com/2019/07/road-to-containing-iscsi/
+: "${ISCSIADM_HOST_STRATEGY:=chroot}"
+: "${ISCSIADM_HOST_PATH:=iscsiadm}"
 
-chroot /host /usr/bin/env -i PATH="/sbin:/bin:/usr/bin" iscsiadm "${@:1}"
+echoerr() { printf "%s\n" "$*" >&2; }
+
+case ${ISCSIADM_HOST_STRATEGY} in
+  chroot)
+    # https://engineering.docker.com/2019/07/road-to-containing-iscsi/
+    chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" ${ISCSIADM_HOST_PATH} "${@:1}"
+    ;;
+
+  nsenter)
+    # https://github.com/siderolabs/extensions/issues/38#issuecomment-1125403043
+    iscsid_pid=$(pgrep --exact --oldest iscsid)
+    if [[ "${iscsid_pid}x" == "x" ]]; then
+      echoerr "failed to find iscsid pid for nsenter"
+      exit 1
+    fi
+    nsenter --mount="/proc/${iscsid_pid}/ns/mnt" --net="/proc/${iscsid_pid}/ns/net" -- ${ISCSIADM_HOST_PATH} "${@:1}"
+    ;;
+
+  *)
+    echoerr "invalid ISCSIADM_HOST_STRATEGY: ${ISCSIADM_HOST_STRATEGY}"
+    exit 1
+    ;;
+esac

docker/kopia-installer.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+set -e
+set -x
+
+PLATFORM_TYPE=${1}
+
+if [[ "${PLATFORM_TYPE}" == "build" ]]; then
+  PLATFORM=$BUILDPLATFORM
+else
+  PLATFORM=$TARGETPLATFORM
+fi
+
+if [[ "x${PLATFORM}" == "x" ]]; then
+  PLATFORM="linux/amd64"
+fi
+
+# these come from the --platform option of buildx, indirectly from DOCKER_BUILD_PLATFORM in main.yaml
+if [ "$PLATFORM" = "linux/amd64" ]; then
+  export PLATFORM_ARCH="amd64"
+elif [ "$PLATFORM" = "linux/arm64" ]; then
+  export PLATFORM_ARCH="arm64"
+elif [ "$PLATFORM" = "linux/arm/v7" ]; then
+  export PLATFORM_ARCH="armhf"
+else
+  echo "unsupported/unknown kopia PLATFORM ${PLATFORM}"
+  exit 0
+fi
+
+echo "I am installing kopia $KOPIA_VERSION"
+
+export DEB_FILE="kopia.deb"
+wget -O "${DEB_FILE}" "https://github.com/kopia/kopia/releases/download/v${KOPIA_VERSION}/kopia_${KOPIA_VERSION}_linux_${PLATFORM_ARCH}.deb"
+dpkg -i "${DEB_FILE}"
+
+rm "${DEB_FILE}"

docker/mount

@@ -20,16 +20,18 @@ container_supported_filesystems=(
 while getopts "t:" opt; do
   case "$opt" in
     t)
-      [[ "${OPTARG,,}" == "zfs" ]]     && USE_HOST_MOUNT_TOOLS=1
-      [[ "${OPTARG,,}" == "lustre" ]]  && USE_HOST_MOUNT_TOOLS=1
-      [[ "${OPTARG,,}" == "onedata" ]] && USE_HOST_MOUNT_TOOLS=1
-      #(printf '%s\0' "${container_supported_filesystems[@]}" | grep -Fqxz -- "${OPTARG}") || USE_HOST_MOUNT_TOOLS=1
+      if [[ "x${USE_HOST_MOUNT_TOOLS}" == "x" ]]; then
+        [[ "${OPTARG,,}" == "zfs" ]] && USE_HOST_MOUNT_TOOLS=1
+        [[ "${OPTARG,,}" == "lustre" ]] && USE_HOST_MOUNT_TOOLS=1
+        [[ "${OPTARG,,}" == "onedata" ]] && USE_HOST_MOUNT_TOOLS=1
+        #(printf '%s\0' "${container_supported_filesystems[@]}" | grep -Fqxz -- "${OPTARG}") || USE_HOST_MOUNT_TOOLS=1
+      fi
       ;;
   esac
 done
 
-if [[ ${USE_HOST_MOUNT_TOOLS} -eq 1 ]];then
-  chroot /host /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" mount "${@:1}"
+if [[ ${USE_HOST_MOUNT_TOOLS} -eq 1 ]]; then
+  chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" mount "${@:1}"
 else
-  /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" mount "${@:1}"
+  /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" mount "${@:1}"
 fi

docker/multipath

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-chroot /host /usr/bin/env -i PATH="/sbin:/bin:/usr/sbin:/usr/bin" multipath "${@:1}"
+chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" multipath "${@:1}"

docker/objectivefs-installer.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+set -e
+set -x
+
+if [[ -z "${OBJECTIVEFS_DOWNLOAD_ID}" ]]; then
+  echo 'missing OBJECTIVEFS_DOWNLOAD_ID, moving on'
+  exit 0
+fi
+
+PLATFORM_TYPE=${1}
+
+if [[ "${PLATFORM_TYPE}" == "build" ]]; then
+  PLATFORM=$BUILDPLATFORM
+else
+  PLATFORM=$TARGETPLATFORM
+fi
+
+if [[ "x${PLATFORM}" == "x" ]]; then
+  PLATFORM="linux/amd64"
+fi
+
+# these come from the --platform option of buildx, indirectly from DOCKER_BUILD_PLATFORM in main.yaml
+if [ "$PLATFORM" = "linux/amd64" ]; then
+  export OBJECTIVEFS_ARCH="amd64"
+elif [ "$PLATFORM" = "linux/arm64" ]; then
+  export OBJECTIVEFS_ARCH="arm64"
+else
+  echo "unsupported/unknown PLATFORM ${PLATFORM}"
+  exit 0
+fi
+
+export DEB_FILE="objectivefs_${OBJECTIVEFS_VERSION}_${OBJECTIVEFS_ARCH}.deb"
+
+echo "I am installing objectivefs $OBJECTIVEFS_VERSION"
+
+wget "https://objectivefs.com/user/download/${OBJECTIVEFS_DOWNLOAD_ID}/${DEB_FILE}"
+dpkg -i "${DEB_FILE}"
+
+rm "${DEB_FILE}"

docker/oneclient

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-chroot /host /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" oneclient "${@:1}"
+chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" oneclient "${@:1}"

docker/rclone-installer.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+set -e
+set -x
+
+PLATFORM_TYPE=${1}
+
+if [[ "${PLATFORM_TYPE}" == "build" ]]; then
+  PLATFORM=$BUILDPLATFORM
+else
+  PLATFORM=$TARGETPLATFORM
+fi
+
+# linux/amd64,linux/arm64,linux/arm/v7,linux/s390x,linux/ppc64le
+if [[ "x${PLATFORM}" == "x" ]]; then
+  PLATFORM="linux/amd64"
+fi
+
+# these come from the --platform option of buildx, indirectly from DOCKER_BUILD_PLATFORM in main.yaml
+# linux/amd64,linux/arm64,linux/arm/v7,linux/s390x,linux/ppc64le
+if [ "$PLATFORM" = "linux/amd64" ]; then
+  export PLATFORM_ARCH="amd64"
+elif [ "$PLATFORM" = "linux/arm64" ]; then
+  export PLATFORM_ARCH="arm"
+elif [ "$PLATFORM" = "linux/arm/v7" ]; then
+  export PLATFORM_ARCH="arm-v7"
+else
+  echo "unsupported/unknown restic PLATFORM ${PLATFORM}"
+  exit 0
+fi
+
+echo "I am installing rclone $RCLONE_VERSION"
+
+export ZIP_FILE="rclone.zip"
+wget -O "${ZIP_FILE}" "https://github.com/rclone/rclone/releases/download/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-${PLATFORM_ARCH}.zip"
+unzip "${ZIP_FILE}"
+
+mv rclone-*-linux-*/rclone /usr/local/bin/rclone
+rm -rf rclone-*-linux-*
+chown root:root /usr/local/bin/rclone
+chmod +x /usr/local/bin/rclone

docker/restic-installer.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+set -x
+
+PLATFORM_TYPE=${1}
+
+if [[ "${PLATFORM_TYPE}" == "build" ]]; then
+  PLATFORM=$BUILDPLATFORM
+else
+  PLATFORM=$TARGETPLATFORM
+fi
+
+if [[ "x${PLATFORM}" == "x" ]]; then
+  PLATFORM="linux/amd64"
+fi
+
+# these come from the --platform option of buildx, indirectly from DOCKER_BUILD_PLATFORM in main.yaml
+# linux/amd64,linux/arm64,linux/arm/v7,linux/s390x,linux/ppc64le
+if [ "$PLATFORM" = "linux/amd64" ]; then
+  export PLATFORM_ARCH="amd64"
+elif [ "$PLATFORM" = "linux/arm64" ]; then
+  export PLATFORM_ARCH="arm64"
+elif [ "$PLATFORM" = "linux/arm/v7" ]; then
+  export PLATFORM_ARCH="arm"
+elif [ "$PLATFORM" = "linux/s390x" ]; then
+  export PLATFORM_ARCH="s390x"
+elif [ "$PLATFORM" = "linux/ppc64le" ]; then
+  export PLATFORM_ARCH="ppc64le"
+else
+  echo "unsupported/unknown restic PLATFORM ${PLATFORM}"
+  exit 0
+fi
+
+echo "I am installing restic $RESTIC_VERSION"
+
+export TAR_FILE="restic.bz2"
+wget -O "${TAR_FILE}" "https://github.com/restic/restic/releases/download/v${RESTIC_VERSION}/restic_${RESTIC_VERSION}_linux_${PLATFORM_ARCH}.bz2"
+bunzip2 "${TAR_FILE}"
+mv restic /usr/local/bin
+chown root:root /usr/local/bin/restic
+chmod +x /usr/local/bin/restic

docker/umount

@@ -20,16 +20,18 @@ container_supported_filesystems=(
 while getopts "t:" opt; do
   case "$opt" in
     t)
-      [[ "${OPTARG,,}" == "zfs" ]]     && USE_HOST_MOUNT_TOOLS=1
-      [[ "${OPTARG,,}" == "lustre" ]]  && USE_HOST_MOUNT_TOOLS=1
-      [[ "${OPTARG,,}" == "onedata" ]] && USE_HOST_MOUNT_TOOLS=1
-      #(printf '%s\0' "${container_supported_filesystems[@]}" | grep -Fqxz -- "${OPTARG}") || USE_HOST_MOUNT_TOOLS=1
+      if [[ "x${USE_HOST_MOUNT_TOOLS}" == "x" ]]; then
+        [[ "${OPTARG,,}" == "zfs" ]] && USE_HOST_MOUNT_TOOLS=1
+        [[ "${OPTARG,,}" == "lustre" ]] && USE_HOST_MOUNT_TOOLS=1
+        [[ "${OPTARG,,}" == "onedata" ]] && USE_HOST_MOUNT_TOOLS=1
+        #(printf '%s\0' "${container_supported_filesystems[@]}" | grep -Fqxz -- "${OPTARG}") || USE_HOST_MOUNT_TOOLS=1
+      fi
       ;;
   esac
 done
 
-if [[ ${USE_HOST_MOUNT_TOOLS} -eq 1 ]];then
-  chroot /host /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" umount "${@:1}"
+if [[ ${USE_HOST_MOUNT_TOOLS} -eq 1 ]]; then
+  chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" umount "${@:1}"
 else
-  /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" umount "${@:1}"
+  /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" umount "${@:1}"
 fi

docker/zfs

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-chroot /host /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" zfs "${@:1}"
+chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" zfs "${@:1}"

docker/zpool

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-chroot /host /usr/bin/env -i PATH="/sbin:/bin:/usr/bin:/usr/sbin" zpool "${@:1}"
+chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" zpool "${@:1}"

docs/Nomad/examples/democratic-csi-iscsi-node.hcl

@@ -11,6 +11,10 @@ job "democratic-csi-iscsi-node" {
 
       env {
         CSI_NODE_ID = "${attr.unique.hostname}"
+        
+        # if you run into a scenario where your iscsi volumes are zeroed each time they are mounted,
+        # you can configure the fs detection system used with the following envvar:
+        #FILESYSTEM_TYPE_DETECTION_STRATEGY = "blkid"
       }
 
       config {
@@ -38,6 +42,15 @@ job "democratic-csi-iscsi-node" {
           source = "/"
           readonly=false
         }
+        
+        # if you run into a scenario where your iscsi volumes are zeroed each time they are mounted,
+        # you can try uncommenting the following additional mount block:
+        #mount {
+        #  type     = "bind"
+        #  target   = "/run/udev"
+        #  source   = "/run/udev"
+        #  readonly = true
+        #}
       }
 
       template {

docs/Nomad/examples/democratic-csi-nfs-controller.hcl

@@ -6,17 +6,44 @@ job "democratic-csi-nfs-controller" {
       driver = "docker"
 
       config {
-        image = "docker.io/democraticcsi/democratic-csi:latest"
+        image = "docker.io/democraticcsi/democratic-csi:${var.version}"
 
-        args = [
-          "--csi-version=1.5.0",
-          # must match the csi_plugin.id attribute below
-          "--csi-name=org.democratic-csi.nfs",
-          "--driver-config-file=${NOMAD_TASK_DIR}/driver-config-file.yaml",
-          "--log-level=info",
-          "--csi-mode=controller",
-          "--server-socket=/csi/csi.sock",
+        entrypoint = [
+          "${NOMAD_TASK_DIR}/init.sh"
         ]
+
+        network_mode = "host"
+        privileged = true
+      }
+
+      env {
+        NFS_SERVER = "<nfs server>"
+        NFS_SHARE  = "<nfs share>"
+      }
+
+      # The nfs share is mounted in the controller so it can create the volumes
+      # sub directories inside the nfs share
+      template {
+        destination = "${NOMAD_TASK_DIR}/init.sh"
+        perms = "755"
+
+        data = <<-EOT
+          #!/bin/sh
+
+          if [ ! -d /storage ]; then
+            mkdir -p /storage
+          fi
+
+          mount "{{ env "NFS_SERVER" }}:{{ env "NFS_SHARE" }}" /storage
+
+          exec ./bin/democratic-csi \
+            --csi-version=1.5.0 \
+            --csi-name=org.democratic-csi.nfs \
+            --driver-config-file={{ env "NOMAD_TASK_DIR" }}/driver-config-file.yaml \
+            --log-level=info \
+            --csi-mode=controller \
+            --server-socket=/csi/csi.sock
+        EOT
       }
 
       template {

docs/storage-class-parameters.md

@@ -0,0 +1,138 @@
+# Storage Class Parameters
+
+Some drivers support different settings for volumes. These can be configured via the driver configuration and/or storage
+classes.
+
+## `synology-iscsi`
+The `synology-iscsi` driver supports several storage class parameters. Note however that not all parameters/values are
+supported for all backing file systems and LUN type. The following options are available:
+
+### Configure Storage Classes
+```yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: synology-iscsi
+parameters:
+    fsType: ext4
+    # The following options affect the LUN representing the volume. These options are passed directly to the Synology API.
+    # The following options are known.
+    lunTemplate: |
+      type: BLUN       # Btrfs thin provisioning
+      type: BLUN_THICK # Btrfs thick provisioning
+      type: THIN       # Ext4 thin provisioning
+      type: ADV        # Ext4 thin provisioning with legacy advanced feature set
+      type: FILE       # Ext4 thick provisioning
+      description: Some Description
+      
+      # Only for thick provisioned volumes. Known values:
+      # 0: Buffered Writes
+      # 3: Direct Write
+      direct_io_pattern: 0
+      
+      # Device Attributes. See below for more info
+      dev_attribs:
+      - dev_attrib: emulate_tpws
+        enable: 1
+      - ...
+
+    # The following options affect the iSCSI target. These options will be passed directly to the Synology API.
+    # The following options are known.
+    targetTemplate: |
+      has_header_checksum: false
+      has_data_checksum: false
+      
+      # Note that this option requires a compatible filesystem. Use 0 for unlimited sessions.
+      max_sessions: 0
+      multi_sessions: true
+      max_recv_seg_bytes: 262144
+      max_send_seg_bytes: 262144
+
+      # Use this to disable authentication. To configure authentication see below
+      auth_type: 0
+```
+
+#### About LUN Types
+The availability of the different types of LUNs depends on the filesystem used on your Synology volume. For Btrfs volumes
+you can use `BLUN` and `BLUN_THICK` volumes. For Ext4 volumes you can use `THIN`, `ADV` or `FILE` volumes. These
+correspond to the options available in the UI.
+
+#### About `dev_attribs`
+Most of the LUN options are configured via the `dev_attribs` list. This list can be specified both in the `lunTemplate`
+of the global configuration and in the `lunTemplate` of the `StorageClass`. If both lists are present they will be merged
+(with the `StorageClass` taking precedence). The following  `dev_attribs` are known to work:
+
+- `emulate_tpws`: Hardware-assisted zeroing
+- `emulate_caw`: Hardware-assisted locking
+- `emulate_3pc`: Hardware-assisted data transfer
+- `emulate_tpu`: Space Reclamation
+- `emulate_fua_write`: Enable the FUA iSCSI command (DSM 7+)
+- `emulate_sync_cache`: Enable the Sync Cache iSCSI command (DSM 7+)
+- `can_snapshot`: Enable snapshots for this volume. Only works for thin provisioned volumes.
+
+### Configure Snapshot Classes
+`synology-iscsi` can also configure different parameters on snapshot classes:
+
+```yaml
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: synology-iscsi-snapshot
+parameters:
+  # This inline yaml object will be passed to the Synology API when creating the snapshot.
+  lunSnapshotTemplate: |
+    is_locked: true
+    
+    # https://kb.synology.com/en-me/DSM/tutorial/What_is_file_system_consistent_snapshot
+    # Note that app consistent snapshots require a working Synology Storage Console. Otherwise both values will have
+    # equivalent behavior.
+    is_app_consistent: true
+...
+```
+
+Note that it is currently not supported by Synology devices to restore a snapshot onto a different volume. You can
+create volumes from snapshots, but you should use the same `StorageClass` as the original volume of the snapshot did. 
+
+### Enabling CHAP Authentication
+You can enable CHAP Authentication for `StorageClass`es by supplying an appropriate `StorageClass` secret (see the
+[documentation](https://kubernetes-csi.github.io/docs/secrets-and-credentials-storage-class.html) for more details). You
+can use the same password for alle volumes of a `StorageClass` or use different passwords per volume.
+
+```yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: synology-iscsi-chap
+parameters:
+  fsType: ext4
+  lunTemplate: |
+    type: BLUN
+    description: iSCSI volumes with CHAP Authentication
+secrets:
+  # Use this to configure a single set of credentials for all volumes of this StorageClass
+  csi.storage.k8s.io/provisioner-secret-name: chap-secret
+  csi.storage.k8s.io/provisioner-secret-namespace: default
+  # Use substitutions to use different credentials for volumes based on the PVC
+  csi.storage.k8s.io/provisioner-secret-name: "${pvc.name}-chap-secret"
+  csi.storage.k8s.io/provisioner-secret-namespace: "${pvc.namespace}"
+...
+---
+# Use a secret like this to supply CHAP credentials.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: chap-secret
+stringData:
+  # Client Credentials
+  user: client
+  password: MySecretPassword
+  # Mutual CHAP Credentials. If these are specified mutual CHAP will be enabled.
+  mutualUser: server
+  mutualPassword: MyOtherPassword
+```
+
+Note that CHAP authentication will only be enabled if the secret contains a username and password. If e.g. a password is
+missing CHAP authentication will not be enabled (but the volume will still be created). You cannot automatically
+enable/disable CHAP or change the password after the volume has been created.
+
+If the secret itself is referenced but not present, the volume will not be created.

examples/controller-common.yaml

@@ -0,0 +1,6 @@
+# common options for the controller service
+
+csi:
+  # manual override of the available access modes for the deployment
+  # generally highly uncessary to alter so only use in advanced scenarios
+  #access_modes: []

examples/freenas-api-iscsi.yaml

@@ -33,7 +33,7 @@ zfs:
   #  "org.freenas:description": "{{ parameters.[csi.storage.k8s.io/pvc/namespace] }}/{{ parameters.[csi.storage.k8s.io/pvc/name] }}"
   #  "org.freenas:test": "{{ parameters.foo }}"
   #  "org.freenas:test2": "some value"
-  
+
   # total volume name (zvol/<datasetParentName>/<pvc name>) length cannot exceed 63 chars
   # https://www.ixsystems.com/documentation/freenas/11.2-U5/storage.html#zfs-zvol-config-opts-tab
   # standard volume naming overhead is 46 chars
@@ -41,7 +41,8 @@ zfs:
   # for work-arounds see https://github.com/democratic-csi/democratic-csi/issues/54
   datasetParentName: tank/k8s/b/vols
   # do NOT make datasetParentName and detachedSnapshotsDatasetParentName overlap
-  # they may be siblings, but neither should be nested in the other 
+  # they may be siblings, but neither should be nested in the other
+  # do NOT comment this option out even if you don't plan to use snapshots, just leave it with dummy value
   detachedSnapshotsDatasetParentName: tanks/k8s/b/snaps
   # "" (inherit), lz4, gzip-9, etc
   zvolCompression:
@@ -67,6 +68,8 @@ iscsi:
   # add as many as needed
   targetGroups:
     # get the correct ID from the "portal" section in the UI
+    # https://github.com/democratic-csi/democratic-csi/issues/302
+    # NOTE: the ID in the UI does NOT always match the ID in the DB, you must use the DB value
     - targetGroupPortalGroup: 1
       # get the correct ID from the "initiators" section in the UI
       targetGroupInitiatorGroup: 1

examples/freenas-api-nfs.yaml

@@ -37,12 +37,15 @@ zfs:
   datasetParentName: tank/k8s/a/vols
   # do NOT make datasetParentName and detachedSnapshotsDatasetParentName overlap
   # they may be siblings, but neither should be nested in the other
+  # do NOT comment this option out even if you don't plan to use snapshots, just leave it with dummy value
   detachedSnapshotsDatasetParentName: tank/k8s/a/snaps
   datasetEnableQuotas: true
   datasetEnableReservation: false
   datasetPermissionsMode: "0777"
   datasetPermissionsUser: 0
   datasetPermissionsGroup: 0
+
+  # not supported yet
   #datasetPermissionsAcls:
   #- "-m everyone@:full_set:allow"
   #- "-m u:kube:full_set:allow"

examples/freenas-api-smb.yaml

@@ -34,21 +34,25 @@ zfs:
   #  "org.freenas:test": "{{ parameters.foo }}"
   #  "org.freenas:test2": "some value"
 
-  datasetProperties:
-    aclmode: restricted
-    casesensitivity: mixed
+  # these are managed automatically via the volume creation process when flagged as an smb volume
+  #datasetProperties:
+  #  aclmode: restricted
+  #  casesensitivity: mixed
 
   datasetParentName: tank/k8s/a/vols
   # do NOT make datasetParentName and detachedSnapshotsDatasetParentName overlap
   # they may be siblings, but neither should be nested in the other
+  # do NOT comment this option out even if you don't plan to use snapshots, just leave it with dummy value
   detachedSnapshotsDatasetParentName: tank/k8s/a/snaps
   datasetEnableQuotas: true
   datasetEnableReservation: false
   datasetPermissionsMode: "0777"
   datasetPermissionsUser: 0
   datasetPermissionsGroup: 0
-  datasetPermissionsAcls:
-  - "-m everyone@:full_set:allow"
+  
+  # not supported yet in api
+  #datasetPermissionsAcls:
+  #- "-m everyone@:full_set:allow"
   #- "-m u:kube:full_set:allow"
 
 smb:

examples/freenas-iscsi.yaml

@@ -43,14 +43,15 @@ zfs:
   #  "org.freenas:description": "{{ parameters.[csi.storage.k8s.io/pvc/namespace] }}/{{ parameters.[csi.storage.k8s.io/pvc/name] }}"
   #  "org.freenas:test": "{{ parameters.foo }}"
   #  "org.freenas:test2": "some value"
-  
+
   # total volume name (zvol/<datasetParentName>/<pvc name>) length cannot exceed 63 chars
   # https://www.ixsystems.com/documentation/freenas/11.2-U5/storage.html#zfs-zvol-config-opts-tab
   # standard volume naming overhead is 46 chars
   # datasetParentName should therefore be 17 chars or less when using TrueNAS 12 or below
   datasetParentName: tank/k8s/b/vols
   # do NOT make datasetParentName and detachedSnapshotsDatasetParentName overlap
-  # they may be siblings, but neither should be nested in the other 
+  # they may be siblings, but neither should be nested in the other
+  # do NOT comment this option out even if you don't plan to use snapshots, just leave it with dummy value
   detachedSnapshotsDatasetParentName: tanks/k8s/b/snaps
   # "" (inherit), lz4, gzip-9, etc
   zvolCompression:
@@ -76,6 +77,8 @@ iscsi:
   # add as many as needed
   targetGroups:
     # get the correct ID from the "portal" section in the UI
+    # https://github.com/democratic-csi/democratic-csi/issues/302
+    # NOTE: the ID in the UI does NOT always match the ID in the DB, you must use the DB value
     - targetGroupPortalGroup: 1
       # get the correct ID from the "initiators" section in the UI
       targetGroupInitiatorGroup: 1