docs, prep for release

Signed-off-by: Travis Glenn Hansen <travisghansen@yahoo.com>
This commit is contained in:
Travis Glenn Hansen
2026-01-06 12:07:19 -07:00
parent 91ad61e98a
commit 333ff2b30b
14 changed files with 415 additions and 20 deletions
+35 -5
View File
@@ -184,7 +184,7 @@ jobs:
TRUENAS_PASSWORD: ${{ secrets.SANITY_TRUENAS_PASSWORD }}
# ssh-based drivers
csi-sanity-zfs-generic:
csi-sanity-zfs-generic-targetcli:
needs:
- build-npm-linux-amd64
strategy:
@@ -192,7 +192,6 @@ jobs:
max-parallel: 1
matrix:
config:
#- zfs-generic/iscsi-pcs.yaml # TODO: enable this once the server is setup
- zfs-generic/iscsi-targetcli.yaml
- zfs-generic/nfs.yaml
- zfs-generic/smb.yaml
@@ -213,7 +212,36 @@ jobs:
ci/bin/run.sh
env:
TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
SERVER_HOST: ${{ secrets.SANITY_ZFS_GENERIC_HOST }}
SERVER_HOST: ${{ secrets.SANITY_ZFS_GENERIC_TARGETCLI_HOST }}
SERVER_USERNAME: ${{ secrets.SANITY_ZFS_GENERIC_USERNAME }}
SERVER_PASSWORD: ${{ secrets.SANITY_ZFS_GENERIC_PASSWORD }}
csi-sanity-zfs-generic-pcs:
needs:
- build-npm-linux-amd64
strategy:
fail-fast: false
max-parallel: 1
matrix:
config:
- zfs-generic/iscsi-pcs.yaml
runs-on:
- self-hosted
- Linux
- X64
- csi-sanity-zfs-generic
steps:
- uses: actions/checkout@v4
- uses: actions/download-artifact@v4
with:
name: node-modules-linux-amd64
- name: csi-sanity
run: |
# run tests
ci/bin/run.sh
env:
TEMPLATE_CONFIG_FILE: "./ci/configs/${{ matrix.config }}"
SERVER_HOST: ${{ secrets.SANITY_ZFS_GENERIC_PCS_HOST }}
SERVER_USERNAME: ${{ secrets.SANITY_ZFS_GENERIC_USERNAME }}
SERVER_PASSWORD: ${{ secrets.SANITY_ZFS_GENERIC_PASSWORD }}
@@ -439,7 +467,8 @@ jobs:
- csi-sanity-synology-dsm7
- csi-sanity-truenas-scale-25_10
- csi-sanity-truenas-core-13_0
- csi-sanity-zfs-generic
- csi-sanity-zfs-generic-targetcli
- csi-sanity-zfs-generic-pcs
- csi-sanity-objectivefs
- csi-sanity-client
- csi-sanity-client-windows
@@ -480,7 +509,8 @@ jobs:
- csi-sanity-synology-dsm7
- csi-sanity-truenas-scale-25_10
- csi-sanity-truenas-core-13_0
- csi-sanity-zfs-generic
- csi-sanity-zfs-generic-targetcli
- csi-sanity-zfs-generic-pcs
- csi-sanity-objectivefs
- csi-sanity-client
- csi-sanity-client-windows
+5 -4
View File
@@ -76,14 +76,15 @@ COPY . .
######################
# actual image
######################
FROM mcr.microsoft.com/windows/nanoserver:${NANO_BASE_TAG}
#FROM mcr.microsoft.com/windows/nanoserver:${NANO_BASE_TAG}
FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0
SHELL ["cmd.exe", "/s" , "/c"]
#SHELL ["cmd.exe", "/s" , "/c"]
#https://github.com/PowerShell/PowerShell-Docker/issues/236
# NOTE: this works for non-host process containers, but host process containers will have specials PATH requirements
# C:\Windows\System32\WindowsPowerShell\v1.0\
ENV PATH="C:\Windows\system32;C:\Windows;C:\PowerShell;C:\app\bin;"
#ENV PATH="C:\Windows\system32;C:\Windows;C:\PowerShell;C:\app\bin;"
ENV DEMOCRATIC_CSI_IS_CONTAINER=true
ENV NODE_ENV=production
@@ -93,7 +94,7 @@ LABEL org.opencontainers.image.url https://github.com/democratic-csi/democratic-
LABEL org.opencontainers.image.licenses MIT
# install powershell
COPY --from=build /PowerShell /PowerShell
#COPY --from=build /PowerShell /PowerShell
# install app
COPY --from=build /app /app
+10 -1
View File
@@ -19,9 +19,11 @@ have access to resizing, snapshots, clones, etc functionality.
- `freenas-nfs` (manages zfs datasets to share over nfs)
- `freenas-iscsi` (manages zfs zvols to share over iscsi)
- `freenas-smb` (manages zfs datasets to share over smb)
- `freenas-nvmeof` (manages zfs zvols to share over nvmeof)
- `freenas-api-nfs` experimental use with SCALE only (manages zfs datasets to share over nfs)
- `freenas-api-iscsi` experimental use with SCALE only (manages zfs zvols to share over iscsi)
- `freenas-api-smb` experimental use with SCALE only (manages zfs datasets to share over smb)
- `freenas-api-nvmeof` experimental use with SCALE only (manages zfs zvols to share over nvmeof)
- `zfs-generic-nfs` (works with any ZoL installation...ie: Ubuntu)
- `zfs-generic-iscsi` (works with any ZoL installation...ie: Ubuntu)
- `zfs-generic-smb` (works with any ZoL installation...ie: Ubuntu)
@@ -41,6 +43,8 @@ have access to resizing, snapshots, clones, etc functionality.
- `node-manual` (allows connecting to manually created smb, nfs, lustre,
oneclient, nvmeof, and iscsi volumes, see sample PVs in the `examples`
directory)
- `containerd-oci-ephemeral-inline` (provisions ephemeral rw node-local storage using oci images as a base)
- `vhd-ephemeral-inline` (provisions ephemeral rw node-local storage using vhd images as a base)
- framework for developing `csi` drivers
If you have any interest in providing a `csi` driver, simply open an issue to
@@ -58,6 +62,8 @@ Predominantly 3 things are needed:
## Community Guides
Join us in the Home Operations discord server in #democratic-csi
- https://jonathangazeley.com/2021/01/05/using-truenas-to-provide-persistent-storage-for-kubernetes/
- https://www.lisenet.com/2021/moving-to-truenas-and-democratic-csi-for-kubernetes-persistent-storage/
- https://gist.github.com/admun/4372899f20421a947b7544e5fc9f9117 (migrating
@@ -65,6 +71,7 @@ Predominantly 3 things are needed:
- https://gist.github.com/deefdragon/d58a4210622ff64088bd62a5d8a4e8cc
(migrating between storage classes using `velero`)
- https://github.com/fenio/k8s-truenas (NFS/iSCSI over API with TrueNAS Scale)
- https://wazaari.dev/blog/truenas-talos-democratic-csi
## Node Prep
@@ -328,7 +335,7 @@ Set-MSDSMGlobalLoadBalancePolicy -Policy RR
Server preparation depends slightly on which `driver` you are using.
### FreeNAS (freenas-nfs, freenas-iscsi, freenas-smb, freenas-api-nfs, freenas-api-iscsi, freenas-api-smb)
### FreeNAS (freenas-nfs, freenas-iscsi, freenas-smb, freenas-nvmeof, freenas-api-nfs, freenas-api-iscsi, freenas-api-smb, freenas-api-nvmeof)
The recommended version of FreeNAS is 12.0-U2+, however the driver should work
with much older versions as well.
@@ -371,6 +378,8 @@ Ensure the following services are configurged and running:
Be sure to properly adjust both [tunables](https://www.freebsd.org/cgi/man.cgi?query=ctl&sektion=4#end) `kern.cam.ctl.max_ports` and `kern.cam.ctl.max_luns` to avoid running out of resources when dynamically provisioning iSCSI volumes on FreeNAS or TrueNAS Core.
- smb
- nvmeof
- ensure you have at least 1 listener/port configured (typcially TCP port 4420)
If you would prefer you can configure `democratic-csi` to use a
non-`root` user when connecting to the FreeNAS server:
@@ -0,0 +1,37 @@
kind: Pod
apiVersion: v1
metadata:
name: some-oci-pod-windows
spec:
nodeSelector:
kubernetes.io/os: windows
containers:
- name: hello
image: mcr.microsoft.com/windows/servercore:ltsc2022
command:
- powershell.exe
- -command
- while ($true) { Start-Sleep -Seconds 1 }
resources:
requests:
memory: "128Mi"
cpu: "500m"
limits:
memory: "128Mi"
cpu: "500m"
volumeMounts:
- name: oci
mountPath: /mnt/oci
volumes:
- name: oci
csi:
driver: org.democratic-csi.containerd-oci-inline-ephemeral
volumeAttributes:
# "image.reference": "ubuntu:24.04"
"image.reference": "democraticcsi/csi-grpc-proxy"
# NOTE: windows is incapable of using linux-based platform images
# windows/amd64
# NOTE: linux seemingly can mount windows-based images however
# "image.platform": "linux/amd64"
# "image.pullPolicy": "Always",
"snapshot.label.containerd.io/snapshot/windows/rootfs.sizebytes": "107374182400"
@@ -0,0 +1,29 @@
kind: Pod
apiVersion: v1
metadata:
name: some-oci-pod
spec:
nodeName: node01
containers:
- name: hello
image: busybox:1.37
command: ["sh", "-c", 'echo "Hello, Kubernetes!" && sleep Infinity']
resources:
requests:
memory: "128Mi"
cpu: "500m"
limits:
memory: "128Mi"
cpu: "500m"
volumeMounts:
- name: oci
mountPath: /mnt/oci
volumes:
- name: oci
csi:
driver: org.democratic-csi.containerd-oci-inline-ephemeral
volumeAttributes:
"image.reference": "ubuntu:24.04"
# "image.platform": ""
# "image.pullPolicy": "Always",
# "snapshot.label.containerd.io/snapshot/windows/rootfs.sizebytes": "107374182400"
@@ -2,5 +2,7 @@ driver: containerd-oci-ephemeral-inline
containerd:
#address: /run/containerd/containerd.sock
#windowsAddress: \\\\.\\pipe\\containerd-containerd
# use k8s.io to use the k8s ns
#namespace: default
#creds encryption key
+94
View File
@@ -0,0 +1,94 @@
driver: freenas-api-nvmeof
instance_id:
httpConnection:
protocol: http
host: server address
port: 80
# use only 1 of apiKey or username/password
# if both are present, apiKey is preferred
# apiKey is only available starting in TrueNAS-12
#apiKey:
username: root
password:
allowInsecure: true
# use apiVersion 2 for TrueNAS-12 and up (will work on 11.x in some scenarios as well)
# leave unset for auto-detection
#apiVersion: 2
zfs:
# can be used to override defaults if necessary
# the example below is useful for TrueNAS 12
#cli:
# sudoEnabled: true
#
# leave paths unset for auto-detection
# paths:
# zfs: /usr/local/sbin/zfs
# zpool: /usr/local/sbin/zpool
# sudo: /usr/local/bin/sudo
# chroot: /usr/sbin/chroot
# can be used to set arbitrary values on the dataset/zvol
# can use handlebars templates with the parameters from the storage class/CO
#datasetProperties:
# "org.freenas:description": "{{ parameters.[csi.storage.k8s.io/pvc/namespace] }}/{{ parameters.[csi.storage.k8s.io/pvc/name] }}"
# "org.freenas:test": "{{ parameters.foo }}"
# "org.freenas:test2": "some value"
# total volume name (zvol/<datasetParentName>/<pvc name>) length cannot exceed 63 chars
# https://www.ixsystems.com/documentation/freenas/11.2-U5/storage.html#zfs-zvol-config-opts-tab
# standard volume naming overhead is 46 chars
# datasetParentName should therefore be 17 chars or less when using TrueNAS 12 or below
datasetParentName: tank/k8s/b/vols
# do NOT make datasetParentName and detachedSnapshotsDatasetParentName overlap
# they may be siblings, but neither should be nested in the other
# do NOT comment this option out even if you don't plan to use snapshots, just leave it with dummy value
detachedSnapshotsDatasetParentName: tanks/k8s/b/snaps
# "" (inherit), lz4, gzip-9, etc
zvolCompression:
# "" (inherit), on, off, verify
zvolDedup:
zvolEnableReservation: false
# 512, 1K, 2K, 4K, 8K, 16K, 64K, 128K default is 16K
zvolBlocksize:
nvmeof:
# these are for the node/client aspect
transports:
- tcp://server:port
#- "tcp://127.0.0.1:4420?host-iface=eth0"
#- "tcp://[2001:123:456::1]:4420"
#- "rdma://127.0.0.1:4420"
#- "fc://[nn-0x203b00a098cbcac6:pn-0x203d00a098cbcac6]"
# MUST ensure uniqueness
# full iqn limit is 223 bytes, plan accordingly
# default is "{{ name }}"
#nameTemplate: "{{ parameters.[csi.storage.k8s.io/pvc/namespace] }}-{{ parameters.[csi.storage.k8s.io/pvc/name] }}"
namePrefix: csi-
nameSuffix: "-clustera"
# port IDs to associate to the newly created subsystem
# the ports should be created as a pre-req to using the driver, democratic-csi does NOT manage the ports
#
# http://<IP>/api/docs/current/api_methods_nvmet.port.create.html
#
# curl -v 'http://username:password@IP/api/v2.0/nvmet/port'
#
# curl -v 'http://username:password@IP/api/v2.0/nvmet/port' \
# --header "Content-Type: application/json" \
# --request POST \
# --data '{"addr_trtype": "TCP","addr_trsvcid": 4420,"addr_traddr": "<YOUR NAS IP HERE>","addr_adrfam": "IPV4"}'
ports:
- <your port ID here>
# http://<ip>/api/docs/current/api_methods_nvmet.subsys.create.html
subsystemTemplate:
pi_enable: true
qid_max:
ieee_oui:
ana:
# http://<ip>/api/docs/current/api_methods_nvmet.namespace.create.html
# currently none of the fields can be tweaked so leave empty for now
namespaceTemplate:
+104
View File
@@ -0,0 +1,104 @@
driver: freenas-nvmeof
instance_id:
httpConnection:
protocol: http
host: server address
port: 80
# use only 1 of apiKey or username/password
# if both are present, apiKey is preferred
# apiKey is only available starting in TrueNAS-12
#apiKey:
username: root
password:
allowInsecure: true
# use apiVersion 2 for TrueNAS-12 and up (will work on 11.x in some scenarios as well)
# leave unset for auto-detection
#apiVersion: 2
sshConnection:
host: server address
port: 22
username: root
# use either password or key
password: ""
privateKey: |
-----BEGIN RSA PRIVATE KEY-----
...
-----END RSA PRIVATE KEY-----
zfs:
# can be used to override defaults if necessary
# the example below is useful for TrueNAS 12
#cli:
# sudoEnabled: true
#
# leave paths unset for auto-detection
# paths:
# zfs: /usr/local/sbin/zfs
# zpool: /usr/local/sbin/zpool
# sudo: /usr/local/bin/sudo
# chroot: /usr/sbin/chroot
# can be used to set arbitrary values on the dataset/zvol
# can use handlebars templates with the parameters from the storage class/CO
#datasetProperties:
# "org.freenas:description": "{{ parameters.[csi.storage.k8s.io/pvc/namespace] }}/{{ parameters.[csi.storage.k8s.io/pvc/name] }}"
# "org.freenas:test": "{{ parameters.foo }}"
# "org.freenas:test2": "some value"
# total volume name (zvol/<datasetParentName>/<pvc name>) length cannot exceed 63 chars
# https://www.ixsystems.com/documentation/freenas/11.2-U5/storage.html#zfs-zvol-config-opts-tab
# standard volume naming overhead is 46 chars
# datasetParentName should therefore be 17 chars or less when using TrueNAS 12 or below
datasetParentName: tank/k8s/b/vols
# do NOT make datasetParentName and detachedSnapshotsDatasetParentName overlap
# they may be siblings, but neither should be nested in the other
# do NOT comment this option out even if you don't plan to use snapshots, just leave it with dummy value
detachedSnapshotsDatasetParentName: tanks/k8s/b/snaps
# "" (inherit), lz4, gzip-9, etc
zvolCompression:
# "" (inherit), on, off, verify
zvolDedup:
zvolEnableReservation: false
# 512, 1K, 2K, 4K, 8K, 16K, 64K, 128K default is 16K
zvolBlocksize:
nvmeof:
# these are for the node/client aspect
transports:
- tcp://server:port
#- "tcp://127.0.0.1:4420?host-iface=eth0"
#- "tcp://[2001:123:456::1]:4420"
#- "rdma://127.0.0.1:4420"
#- "fc://[nn-0x203b00a098cbcac6:pn-0x203d00a098cbcac6]"
# MUST ensure uniqueness
# full iqn limit is 223 bytes, plan accordingly
# default is "{{ name }}"
#nameTemplate: "{{ parameters.[csi.storage.k8s.io/pvc/namespace] }}-{{ parameters.[csi.storage.k8s.io/pvc/name] }}"
namePrefix: csi-
nameSuffix: "-clustera"
# port IDs to associate to the newly created subsystem
# the ports should be created as a pre-req to using the driver, democratic-csi does NOT manage the ports
#
# http://<IP>/api/docs/current/api_methods_nvmet.port.create.html
#
# curl -v 'http://username:password@IP/api/v2.0/nvmet/port'
#
# curl -v 'http://username:password@IP/api/v2.0/nvmet/port' \
# --header "Content-Type: application/json" \
# --request POST \
# --data '{"addr_trtype": "TCP","addr_trsvcid": 4420,"addr_traddr": "<YOUR NAS IP HERE>","addr_adrfam": "IPV4"}'
ports:
- <your port ID here>
# http://<ip>/api/docs/current/api_methods_nvmet.subsys.create.html
subsystemTemplate:
pi_enable: true
qid_max:
ieee_oui:
ana:
# http://<ip>/api/docs/current/api_methods_nvmet.namespace.create.html
# currently none of the fields can be tweaked so leave empty for now
namespaceTemplate:
@@ -0,0 +1,30 @@
kind: Pod
apiVersion: v1
metadata:
name: some-vhd-pod-windows
spec:
nodeSelector:
kubernetes.io/os: windows
containers:
- name: hello
image: mcr.microsoft.com/windows/servercore:ltsc2022
command:
- powershell.exe
- -command
- while ($true) { Start-Sleep -Seconds 1 }
resources:
requests:
memory: "1Gi"
cpu: "500m"
limits:
memory: "1Gi"
cpu: "500m"
volumeMounts:
- name: vhd
mountPath: /mnt/vhd
volumes:
- name: vhd
csi:
driver: org.democratic-csi.vhd-ephemeral-inline
volumeAttributes:
vhd.parentPath: "C:\\some\\host\\path\\to\\SampleDisk.vhdx"
+3
View File
@@ -0,0 +1,3 @@
driver: vhd-ephemeral-inline
vhd:
nameTemplate: "csi-ephemeral-inline-{{ volume_id }}"
+15 -6
View File
@@ -3,6 +3,7 @@ const { CsiBaseDriver } = require("../index");
const { GrpcError, grpc } = require("../../utils/grpc");
const GeneralUtils = require("../../utils/general");
const getLargestNumber = require("../../utils/general").getLargestNumber;
const Mount = require("../../utils/mount").Mount;
const Handlebars = require("handlebars");
const uuidv4 = require("uuid").v4;
@@ -1193,10 +1194,21 @@ class ControllerZfsBaseDriver extends CsiBaseDriver {
properties = properties[datasetName];
driver.ctx.logger.debug("zfs props data: %j", properties);
// get mountpoint
let mountpoint = properties.mountpoint.value;
if (mountpoint == "legacy") {
let mount = new Mount();
let mounts = await mount.getDeviceMounts(datasetName);
if (mounts.filesystems[0]) {
mountpoint = mounts.filesystems[0].target;
}
}
// set mode
if (driverOptions.zfs.datasetPermissionsMode) {
await driver.setFilesystemMode(
properties.mountpoint.value,
mountpoint,
driverOptions.zfs.datasetPermissionsMode
);
}
@@ -1209,7 +1221,7 @@ class ControllerZfsBaseDriver extends CsiBaseDriver {
.length > 0
) {
await driver.setFilesystemOwnership(
properties.mountpoint.value,
mountpoint,
driverOptions.zfs.datasetPermissionsUser,
driverOptions.zfs.datasetPermissionsGroup
);
@@ -1225,10 +1237,7 @@ class ControllerZfsBaseDriver extends CsiBaseDriver {
"setfacl"
);
for (const acl of driverOptions.zfs.datasetPermissionsAcls) {
command = execClient.buildCommand(aclBinary, [
acl,
properties.mountpoint.value,
]);
command = execClient.buildCommand(aclBinary, [acl, mountpoint]);
if ((await this.getWhoAmI()) != "root") {
command = (await this.getSudoPath()) + " " + command;
}
@@ -115,6 +115,10 @@ class EphemeralInlineContainerDOciDriver extends CsiBaseDriver {
}
}
/**
* TODO: add Probe here with ctr check to ensure socket is alive
*/
/**
*
* @returns CTR
@@ -273,7 +277,7 @@ class EphemeralInlineContainerDOciDriver extends CsiBaseDriver {
// create publish directory
if (!fs.existsSync(target_path)) {
await fs.mkdirSync(target_path, { recursive: true });
fs.mkdirSync(target_path, { recursive: true });
}
if (process.platform != "win32") {
+15 -3
View File
@@ -3800,10 +3800,14 @@ class CsiBaseDriver {
if (!volume_path) {
throw new GrpcError(grpc.status.INVALID_ARGUMENT, `missing volume_path`);
}
const block_path = volume_path + "/block_device";
const capacity_range = call.request.capacity_range;
const volume_capability = call.request.volume_capability;
// placeholder
let capacity_bytes;
switch (driver.__getNodeOsDriver()) {
case NODE_OS_DRIVER_POSIX:
if (
@@ -3864,6 +3868,7 @@ class CsiBaseDriver {
await GeneralUtils.sleep(2000);
}
// is_formatted = false;
if (is_formatted && access_type == "mount") {
fs_info = await filesystem.getDeviceFilesystemInfo(device);
fs_type = fs_info.type;
@@ -3898,13 +3903,20 @@ class CsiBaseDriver {
);
}
}
result = await mount.getMountDetails(device_path, ["size"]);
capacity_bytes = result.size;
} else {
//block device unformatted
return {};
result = await filesystem.getBlockDevice(device);
capacity_bytes = result.size;
return { capacity_bytes };
}
} else {
// not block device
return {};
result = await mount.getMountDetails(device_path, ["size"]);
capacity_bytes = result.size;
return { capacity_bytes };
}
break;
@@ -4069,7 +4081,7 @@ class CsiBaseDriver {
);
}
return {};
return { capacity_bytes };
}
}
+31
View File
@@ -94,6 +94,37 @@ class Mount {
return true;
}
/**
* findmnt --source <device> --output source,target,fstype,label,options,avail,size,used -b -J
*
* @param {*} device
*/
async getDeviceMounts(device) {
const mount = this;
const filesystem = await mount.getFilesystemInstance();
if (device.startsWith("/")) {
device = await filesystem.realpath(device);
}
let args = [];
args = args.concat(["--source", device]);
args = args.concat(FINDMNT_COMMON_OPTIONS);
let result;
try {
result = await mount.exec(mount.options.paths.findmnt, args);
} catch (err) {
// no results
if (err.code == 1) {
return { filesystems: [] };
} else {
throw err;
}
}
return JSON.parse(result.stdout);
}
/**
* findmnt --mountpoint / --output source,target,fstype,label,options,avail,size,used -b -J
*