From cf38cb5d3dcff81dc3c60f4ad7b452b660c66729 Mon Sep 17 00:00:00 2001 From: 5cat Date: Fri, 5 Aug 2022 22:04:55 +0800 Subject: [PATCH 1/5] adding support for talos.dev clusters iscsi --- README.md | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 62254a7..8a85ee7 100644 --- a/README.md +++ b/README.md @@ -94,9 +94,9 @@ If you are running Kubernetes with rancher/rke please see the following: - https://github.com/rancher/rke/issues/1846 -``` -RHEL / CentOS +#### RHEL / CentOS +``` # Install the following system packages sudo yum install -y lsscsi iscsi-initiator-utils sg3_utils device-mapper-multipath @@ -110,10 +110,11 @@ sudo systemctl start iscsid multipathd # Start and enable iscsi sudo systemctl enable iscsi sudo systemctl start iscsi +``` +#### Ubuntu / Debian -Ubuntu / Debian - +``` # Install the following system packages sudo apt-get install -y open-iscsi lsscsi sg3-utils multipath-tools scsitools @@ -134,6 +135,62 @@ sudo systemctl enable open-iscsi.service sudo service open-iscsi start sudo systemctl status open-iscsi ``` +#### [Talos](https://www.talos.dev/) +To use iscsi storage in kubernetes cluster in talos these steps are needed which are similar to the ones explained in https://www.talos.dev/v1.1/kubernetes-guides/configuration/replicated-local-storage-with-openebs-jiva/#patching-the-jiva-installation + +##### Patch nodes +since talos does not have iscsi support by default, the iscsi extension is needed +create a `patch.yaml` file with +```yaml +- op: add + path: /machine/install/extensions + value: + - image: ghcr.io/siderolabs/iscsi-tools:v0.1.1 +``` +and apply the patch across all of your nodes +```bash +talosctl -e -n patch mc -p @patch.yaml +``` +the extension will not activate until you "upgrade" the nodes, even if there is no update, use the latest version of talos installer. +VERIFY THE TALOS VERSION IN THIS COMMAND BEFORE RUNNING IT AND READ THE [OpenEBS Jiva](https://www.talos.dev/v1.1/kubernetes-guides/configuration/replicated-local-storage-with-openebs-jiva/#patching-the-jiva-installation). +upgrade all of the nodes in the cluster to get the extension +```bash +talosctl -e -n upgrade --image=ghcr.io/siderolabs/installer:v1.1.1 +``` + +since the default [iscsi](https://github.com/democratic-csi/democratic-csi/blob/master/docker/iscsiadm) does not work with talos, this config map is needed to be applied in the same namespace as the democratic-csi installation +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: talos-iscsiadm +data: + iscsiadm: | + #!/bin/bash + iscsid_pid=$(for proc in /proc/*/cmdline; do grep -q "iscsid -f" <<< $(cat $proc 2>/dev/null | tr "\0" " ") && echo $(basename $(dirname $proc)) && break; done) + nsenter --mount="/proc/${iscsid_pid}/ns/mnt" --net="/proc/${iscsid_pid}/ns/net" -- /usr/local/sbin/iscsiadm "${@:1}" +``` + +in your `values.yaml` file make sure to enable these settings +```yaml + +node: + hostPID: true + extraVolumes: + - name: talos-iscsiadm + configMap: + name: talos-iscsiadm + defaultMode: 0777 + driver: + extraVolumeMounts: + - name: talos-iscsiadm + mountPath: /usr/local/sbin/iscsiadm + subPath: iscsiadm + iscsiDirHostPath: /usr/local/etc/iscsi + iscsiDirHostPathCheckDirectory: false +``` +and continue your democratic installation as usuall with other iscsi drivers. + ### freenas-smb From c0b8590e1bd664b3f05de3d79404bb63d49ba8c6 Mon Sep 17 00:00:00 2001 From: 5cat Date: Sat, 6 Aug 2022 03:48:56 +0800 Subject: [PATCH 2/5] adding nsenter and ISCSIADM_HOST_STRATEGY variable --- README.md | 25 +++---------------------- docker/iscsiadm | 27 +++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 8a85ee7..dfd74d3 100644 --- a/README.md +++ b/README.md @@ -158,34 +158,15 @@ upgrade all of the nodes in the cluster to get the extension talosctl -e -n upgrade --image=ghcr.io/siderolabs/installer:v1.1.1 ``` -since the default [iscsi](https://github.com/democratic-csi/democratic-csi/blob/master/docker/iscsiadm) does not work with talos, this config map is needed to be applied in the same namespace as the democratic-csi installation -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: talos-iscsiadm -data: - iscsiadm: | - #!/bin/bash - iscsid_pid=$(for proc in /proc/*/cmdline; do grep -q "iscsid -f" <<< $(cat $proc 2>/dev/null | tr "\0" " ") && echo $(basename $(dirname $proc)) && break; done) - nsenter --mount="/proc/${iscsid_pid}/ns/mnt" --net="/proc/${iscsid_pid}/ns/net" -- /usr/local/sbin/iscsiadm "${@:1}" -``` - in your `values.yaml` file make sure to enable these settings ```yaml node: hostPID: true - extraVolumes: - - name: talos-iscsiadm - configMap: - name: talos-iscsiadm - defaultMode: 0777 driver: - extraVolumeMounts: - - name: talos-iscsiadm - mountPath: /usr/local/sbin/iscsiadm - subPath: iscsiadm + extraEnv: + - name: ISCSIADM_HOST_STRATEGY + value: nsenter iscsiDirHostPath: /usr/local/etc/iscsi iscsiDirHostPathCheckDirectory: false ``` diff --git a/docker/iscsiadm b/docker/iscsiadm index 56623d7..2fb2505 100755 --- a/docker/iscsiadm +++ b/docker/iscsiadm @@ -1,5 +1,28 @@ #!/bin/bash -# https://engineering.docker.com/2019/07/road-to-containing-iscsi/ -chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" iscsiadm "${@:1}" +iscsiadm_host_strategy=$([ $ISCSIADM_HOST_STRATEGY ] && echo $ISCSIADM_HOST_STRATEGY || echo "chroot") + +echo "using $iscsiadm_host_strategy strategy" + +case $iscsiadm_host_strategy in + chroot) + # https://engineering.docker.com/2019/07/road-to-containing-iscsi/ + chroot /host /usr/bin/env -i PATH="/usr/sbin:/usr/bin:/sbin:/bin" iscsiadm "${@:1}" + ;; + + nsenter) + # https://github.com/siderolabs/extensions/issues/38#issuecomment-1125403043 + iscsid_pid=$(for proc in /proc/*/cmdline; do grep -q "iscsid -f" <<< $(cat $proc 2>/dev/null | tr "\0" " ") && echo $(basename $(dirname $proc)) && break; done) + if [ "$iscsid_pid" = "" ]; then + echo "could not find the iscsid process" + exit 1 + fi + nsenter --mount="/proc/${iscsid_pid}/ns/mnt" --net="/proc/${iscsid_pid}/ns/net" -- /usr/local/sbin/iscsiadm "${@:1}" + ;; + + *) + echo "$iscsiadm_host_strategy is not a valid strategy, choose either 'chroot' or 'nsenter'" + exit 1 + ;; +esac From f4a8e14f334a5c97f7221cfc151982025c3e079b Mon Sep 17 00:00:00 2001 From: 5cat Date: Sat, 6 Aug 2022 04:06:58 +0800 Subject: [PATCH 3/5] adding ISCSIADM_HOST_PATH to iscisadm --- README.md | 2 ++ docker/iscsiadm | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index dfd74d3..e2ff7b4 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,8 @@ node: extraEnv: - name: ISCSIADM_HOST_STRATEGY value: nsenter + - name: ISCSIADM_HOST_PATH + value: /usr/local/sbin/iscsiadm iscsiDirHostPath: /usr/local/etc/iscsi iscsiDirHostPathCheckDirectory: false ``` diff --git a/docker/iscsiadm b/docker/iscsiadm index 2fb2505..d1c988e 100755 --- a/docker/iscsiadm +++ b/docker/iscsiadm @@ -1,9 +1,12 @@ #!/bin/bash +set -e +set -x iscsiadm_host_strategy=$([ $ISCSIADM_HOST_STRATEGY ] && echo $ISCSIADM_HOST_STRATEGY || echo "chroot") - -echo "using $iscsiadm_host_strategy strategy" +iscsiadm_host_path=$( [ $ISCSIADM_HOST_PATH ] && echo $ISCSIADM_HOST_PATH || echo "/sbin/iscsiadm") +echo "using iscsiadm_host_strategy=$iscsiadm_host_strategy" +echo "using iscsiadm_host_path=$iscsiadm_host_path" case $iscsiadm_host_strategy in chroot) @@ -18,7 +21,7 @@ case $iscsiadm_host_strategy in echo "could not find the iscsid process" exit 1 fi - nsenter --mount="/proc/${iscsid_pid}/ns/mnt" --net="/proc/${iscsid_pid}/ns/net" -- /usr/local/sbin/iscsiadm "${@:1}" + nsenter --mount="/proc/${iscsid_pid}/ns/mnt" --net="/proc/${iscsid_pid}/ns/net" -- $iscsiadm_host_path "${@:1}" ;; *) From 716df8fdd09dc792c918a0fc3ee1a650bc633734 Mon Sep 17 00:00:00 2001 From: 5cat Date: Sat, 6 Aug 2022 22:27:49 +0800 Subject: [PATCH 4/5] remove iscsiDirHostPathCheckDirectory and replace it with iscsiDirHostType --- README.md | 2 +- docker/iscsiadm | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/README.md b/README.md index e2ff7b4..78ea8d3 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ node: - name: ISCSIADM_HOST_PATH value: /usr/local/sbin/iscsiadm iscsiDirHostPath: /usr/local/etc/iscsi - iscsiDirHostPathCheckDirectory: false + iscsiDirHostType: "" ``` and continue your democratic installation as usuall with other iscsi drivers. diff --git a/docker/iscsiadm b/docker/iscsiadm index d1c988e..1642aaa 100755 --- a/docker/iscsiadm +++ b/docker/iscsiadm @@ -1,12 +1,7 @@ #!/bin/bash -set -e -set -x - iscsiadm_host_strategy=$([ $ISCSIADM_HOST_STRATEGY ] && echo $ISCSIADM_HOST_STRATEGY || echo "chroot") iscsiadm_host_path=$( [ $ISCSIADM_HOST_PATH ] && echo $ISCSIADM_HOST_PATH || echo "/sbin/iscsiadm") -echo "using iscsiadm_host_strategy=$iscsiadm_host_strategy" -echo "using iscsiadm_host_path=$iscsiadm_host_path" case $iscsiadm_host_strategy in chroot) @@ -18,14 +13,12 @@ case $iscsiadm_host_strategy in # https://github.com/siderolabs/extensions/issues/38#issuecomment-1125403043 iscsid_pid=$(for proc in /proc/*/cmdline; do grep -q "iscsid -f" <<< $(cat $proc 2>/dev/null | tr "\0" " ") && echo $(basename $(dirname $proc)) && break; done) if [ "$iscsid_pid" = "" ]; then - echo "could not find the iscsid process" exit 1 fi nsenter --mount="/proc/${iscsid_pid}/ns/mnt" --net="/proc/${iscsid_pid}/ns/net" -- $iscsiadm_host_path "${@:1}" ;; *) - echo "$iscsiadm_host_strategy is not a valid strategy, choose either 'chroot' or 'nsenter'" exit 1 ;; esac From 3464929a31164a168a73d4976652fc4cd82aa610 Mon Sep 17 00:00:00 2001 From: 5cat Date: Sat, 6 Aug 2022 22:34:44 +0800 Subject: [PATCH 5/5] changing from iscsiDirHostType to iscsiDirHostPathType --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78ea8d3..613a9b6 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ node: - name: ISCSIADM_HOST_PATH value: /usr/local/sbin/iscsiadm iscsiDirHostPath: /usr/local/etc/iscsi - iscsiDirHostType: "" + iscsiDirHostPathType: "" ``` and continue your democratic installation as usuall with other iscsi drivers.