democratic-csi/contrib/freenas-provisioner-to-demo...

406 lines
17 KiB
Bash
Executable File

#!/bin/bash
set -e
#set -x
######## LINKS ########
# https://www.truenas.com/community/threads/moving-zfs-dataset-into-an-other.17720/
# https://www.truenas.com/community/threads/moving-a-zvol.76574/
# https://github.com/kubernetes/kubernetes/issues/77086
######## REQUIRMENTS #########
# kubectl
# curl
# jq
######### NOTES ############
# This script is meant to be downloaded and modified to your specific needs.
# The process is relatively intricate and the matrix of various options is
# quite large.
#
# It is highly recommended to create a test PV/PVC with the old provisioner to
# use as a playground to ensure you have things configured correctly and that
# the transition is smooth.
#
# From a high level the intent of this script is to:
# - update *existing* PVs/PVCs created with the non-csi provisioners to be managed by democratic-csi
# - ultimately after all PVs/PVCs have been migrated remove the non-csi provisioners from your cluster(s)
#
# To achieve the above goals the following happens:
# - each execution of the script is meant to migrate 1 PV/PVC to democratic-csi
# - original PV/PVC api object data is stored in ${PWD/tmp/*.json
# - PVC api object is deleted and recreated with proper values
# - PV api object is deleted and recreated with proper values
# - you will be required to `zfs rename` your old zfs zvols/dataset to place in the new democratic-csi structure
# - you will run several `curl` commands to update various share assets in TrueNAS via the API
# - you will run several `zfs` commands to set zfs properties on the datasets
# - you will be required to *stop* individual workloads using the volumes for a short period of time while migrating each PV/PVC
# - you should incure no data loss, after migration workloads should come up in exactly the same state they were before migration
#
# Several assumptions are made in this script
# - your intent is to use the *same* pool you used previously
# - you have already created/deployed democratic-csi
# - you have already deployed freenas-{nfs,iscsi}-provisioner(s)
# - you have direct access to the storage nas to run zfs commands
# - you can execute curl commands to manipulate shares/etc with the freenas api
# - where you execute the script should be setup with administrative kubectl/KUBECONFIG access
# currently only support v2 api so unused
# API_VERSION=2
FREENAS_USERNAME="root"
FREENAS_PASSWORD="secret"
FREENAS_URI="http://<ip>:port"
# where your pools get mounted
POOL_MNT_DIR="/mnt"
## nfs
#SHARE_STRATEGY="nfs" # hard-coded if migrating nfs-based volumes
#OLD_STORAGE_CLASS_NAME="freenas-nfs"
#NEW_STORAGE_CLASS_NAME="zfs-nfs"
#NEW_PARENT_DATASET="" # datasetParentName option from democratic-csi config
#PROVISIONER_IDENTITY="" # find this by looking at a PV created by the new storage class .spec.csi.volumeAttributes."storage.kubernetes.io/csiProvisionerIdentity"
#PROVISIONER_DRIVER="freenas-nfs"
#PROVISIONER_INSTANCE_ID="" # optional, should match the driver.instance_id attribute in your democratic-csi config
# should be the mountpoint, not the zfs path to the dataset
# zfs create tank/tmpnfs
# zfs destroy -r tank/tmpnfs
#TMP_ASSET="/mnt/tank/tmpnfs"
## end nfs
## iscsi
#SHARE_STRATEGY="iscsi" # hard-coded if migrating iscsi-based volumes
#OLD_STORAGE_CLASS_NAME="freenas-iscsi"
#NEW_STORAGE_CLASS_NAME="zfs-iscsi"
#NEW_PARENT_DATASET="" # datasetParentName option from democratic-csi config
#PROVISIONER_IDENTITY="" # find this by looking at a PV created by the new storage class .spec.csi.volumeAttributes."storage.kubernetes.io/csiProvisionerIdentity"
#PROVISIONER_DRIVER="freenas-iscsi"
#PROVISIONER_INSTANCE_ID="" # optional, should match the driver.instance_id attribute in your democratic-csi config
# should be the path to the zfs asset *not* a mountpath
# zfs create -V 1MB tank/tmpiscsi
# zfs destroy -r tank/tmpiscsi
#TMP_ASSET="tank/tmpiscsi"
# should match your iscsi.namePrefix/nameSuffix/template syntax in the democratic-csi config
# %s is replaced by the pvc-<id> string
#ISCSI_ASSET_NAME_TEMPLATE="csi.%s.primary"
## end iscsi
###### make sure you uncomment appropriate variable above in either the nfs or
###### iscsi block (just pick one of the blocks at a time), every thing below
###### here is script logic and should not need to be tampered with unless
###### special circumstances/configuration requires it
# get secret details
nscJSON=$(kubectl get sc "${NEW_STORAGE_CLASS_NAME}" -o json)
CONTROLLER_NAMESPACE=$(echo "${nscJSON}" | jq -crM '.parameters."csi.storage.k8s.io/controller-expand-secret-namespace"')
CE_SECRET_NAME=$(echo "${nscJSON}" | jq -crM '.parameters."csi.storage.k8s.io/controller-expand-secret-name"')
CP_SECRET_NAME=$(echo "${nscJSON}" | jq -crM '.parameters."csi.storage.k8s.io/controller-publish-secret-name"')
NP_SECRET_NAME=$(echo "${nscJSON}" | jq -crM '.parameters."csi.storage.k8s.io/node-publish-secret-name"')
NS_SECRET_NAME=$(echo "${nscJSON}" | jq -crM '.parameters."csi.storage.k8s.io/node-stage-secret-name"')
NEW_CSI_DRIVER_NAME=$(echo "${nscJSON}" | jq -crM ".provisioner")
function yes_or_no {
while true; do
read -p "$* [y/n]: " yn
case $yn in
[Yy]*) return 0 ;;
[Nn]*)
echo "Aborted"
return 1
;;
esac
done
}
for ipv in $(kubectl get pv -o json | jq -crM ".items[] | select(.spec.storageClassName|test(\"${OLD_STORAGE_CLASS_NAME}\")) | (.metadata.name,.spec.claimRef.namespace,.spec.claimRef.name)"); do
:
echo "${ipv}"
done
read -p "Which PV would you like to migrate? " TO_UPDATE
export pv="${TO_UPDATE}"
echo "migrating ${pv} to new provisioner"
# create temporary directory to store all original PV and PVC json
mkdir -p tmp
if [[ ! -f "tmp/${pv}-pv.json" ]]; then
pvJSON=$(kubectl get pv "${pv}" -o json)
echo "${pvJSON}" >>"tmp/${pv}-pv.json"
else
pvJSON=$(cat "tmp/${pv}-pv.json")
fi
npvJSON="${pvJSON}"
name=$(echo "${pvJSON}" | jq -crM ".metadata.name")
status=$(echo "${pvJSON}" | jq -crM ".status.phase")
reclaimPolicy=$(echo "${pvJSON}" | jq -crM ".spec.persistentVolumeReclaimPolicy")
if [[ ${SHARE_STRATEGY} == "nfs" ]]; then
:
pool=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.pool")
dataset=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.dataset")
shareId=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.shareId")
server=$(echo "${pvJSON}" | jq -crM ".spec.nfs.server")
path=$(echo "${pvJSON}" | jq -crM ".spec.nfs.path")
fsType="nfs"
npath="${POOL_MNT_DIR}/${NEW_PARENT_DATASET}/${pv}"
# only need to remove these from the new json
for annotation in shareId dataset datasetEnableQuotas datasetEnableReservation datasetParent datasetPreExisted freenasNFSProvisionerIdentity pool sharePreExisted; do
:
echo "removing annotation: ${annotation}"
npvJSON=$(echo "${npvJSON}" | jq "del(.metadata.annotations.${annotation})")
done
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.volumeAttributes.server = \"${server}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.volumeAttributes.share = \"${npath}\"")
src="${dataset}"
fi
if [[ ${SHARE_STRATEGY} == "iscsi" ]]; then
:
pool=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.pool")
zvol=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.zvol")
targetId=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.targetId")
extentId=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.extentId")
targetGroupId=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.targetGroupId")
targetToExtentId=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.targetToExtentId")
zvol=$(echo "${pvJSON}" | jq -crM ".metadata.annotations.zvol")
fsType=$(echo "${pvJSON}" | jq -crM ".spec.iscsi.fsType")
lun=$(echo "${pvJSON}" | jq -crM ".spec.iscsi.lun")
iqn=$(echo "${pvJSON}" | jq -crM ".spec.iscsi.iqn")
targetPortal=$(echo "${pvJSON}" | jq -crM ".spec.iscsi.targetPortal")
# only need to remove these from the new json
for annotation in datasetParent extentId freenasISCSIProvisionerIdentity iscsiName pool targetGroupId targetId targetToExtentId zvol; do
:
echo "removing annotation: ${annotation}"
npvJSON=$(echo "${npvJSON}" | jq "del(.metadata.annotations.${annotation})")
done
ISCSI_BASE_NAME="$(echo "${iqn}" | cut -d ":" -f1)"
ISCSI_ASSET_NAME=$(printf "${ISCSI_ASSET_NAME_TEMPLATE}" "${pv}")
niqn="${ISCSI_BASE_NAME}:${ISCSI_ASSET_NAME}"
npvJSON=$(echo "${npvJSON}" | jq '.spec.csi.volumeAttributes.interface = ""')
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.volumeAttributes.iqn = \"${niqn}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.volumeAttributes.lun = \"${lun}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.volumeAttributes.portal = \"${targetPortal}\"")
npvJSON=$(echo "${npvJSON}" | jq '.spec.csi.volumeAttributes.portals = ""')
src="${pool}/${zvol}"
fi
dst="${NEW_PARENT_DATASET}/${name}"
npvJSON=$(echo "${npvJSON}" | jq ".metadata.annotations.\"pv.kubernetes.io/provisioned-by\" = \"${NEW_CSI_DRIVER_NAME}\"")
# remove old, update old
npvJSON=$(echo "${npvJSON}" | jq "del(.metadata.resourceVersion)")
npvJSON=$(echo "${npvJSON}" | jq "del(.spec.nfs)")
npvJSON=$(echo "${npvJSON}" | jq "del(.spec.iscsi)")
npvJSON=$(echo "${npvJSON}" | jq ".spec.storageClassName = \"${NEW_STORAGE_CLASS_NAME}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.driver = \"${NEW_CSI_DRIVER_NAME}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.volumeHandle = \"${name}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.fsType = \"${fsType}\"")
npvJSON=$(echo "${npvJSON}" | jq '.spec.persistentVolumeReclaimPolicy = "Retain"')
# secrets
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.controllerExpandSecretRef.name = \"${CE_SECRET_NAME}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.controllerExpandSecretRef.namespace = \"${CONTROLLER_NAMESPACE}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.controllerPublishSecretRef.name = \"${CP_SECRET_NAME}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.controllerPublishSecretRef.namespace = \"${CONTROLLER_NAMESPACE}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.nodePublishSecretRef.name = \"${NP_SECRET_NAME}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.nodePublishSecretRef.namespace = \"${CONTROLLER_NAMESPACE}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.nodeStageSecretRef.name = \"${NS_SECRET_NAME}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.nodeStageSecretRef.namespace = \"${CONTROLLER_NAMESPACE}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.volumeAttributes.node_attach_driver = \"${SHARE_STRATEGY}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.volumeAttributes.provisioner_driver = \"${PROVISIONER_DRIVER}\"")
npvJSON=$(echo "${npvJSON}" | jq ".spec.csi.volumeAttributes.\"storage.kubernetes.io/csiProvisionerIdentity\" = \"${PROVISIONER_IDENTITY}\"")
if [[ ${status} == "Bound" ]]; then
:
# ensure any workloads are shutdown
yes_or_no "Please type y when all workloads using the PV/PVC have been scaled to 0"
yes_or_no "Are you certain nothing is using the share?"
claimName=$(echo "${pvJSON}" | jq -crM ".spec.claimRef.name")
claimNamespace=$(echo "${pvJSON}" | jq -crM ".spec.claimRef.namespace")
echo "${claimNamespace}/${claimName}"
if [[ ! -f "tmp/${pv}-pvc.json" ]]; then
pvcJSON=$(kubectl -n "${claimNamespace}" get pvc "${claimName}" -o json)
echo "${pvcJSON}" >>"tmp/${pv}-pvc.json"
else
pvcJSON=$(cat "tmp/${pv}-pvc.json")
fi
npvcJSON="${pvcJSON}"
kubectl patch pv "${name}" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
kubectl -n "${claimNamespace}" delete pvc "${claimName}" --wait=false || true
sleep 3
kubectl -n "${claimNamespace}" patch pvc "${claimName}" -p '{"metadata":{"finalizers": null }}' || true
sleep 3
# update pvc
npvcJSON=$(echo "${npvcJSON}" | jq "del(.metadata.resourceVersion)")
npvcJSON=$(echo "${npvcJSON}" | jq ".metadata.annotations.\"volume.beta.kubernetes.io/storage-provisioner\" = \"${NEW_CSI_DRIVER_NAME}\"")
npvcJSON=$(echo "${npvcJSON}" | jq ".spec.storageClassName = \"${NEW_STORAGE_CLASS_NAME}\"")
# recreate pvc
echo "${npvcJSON}" | jq .
yes_or_no "Would you like to contiue with the update to the PVC with the above details? "
echo "${npvcJSON}" | kubectl apply -f -
# get pvc .metadata.uid
uid=$(kubectl -n "${claimNamespace}" get pvc "${claimName}" -o jsonpath='{.metadata.uid}')
# set pv .spec.claimRef.uid
#npvJSON="${pvJSON}"
npvJSON=$(echo "${npvJSON}" | jq "del(.metadata.resourceVersion)")
npvJSON=$(echo "${npvJSON}" | jq ".spec.claimRef.uid = \"${uid}\"")
# wait for things to settle and all should be well
sleep 3
fi
if [[ ${status} == "Released" ]]; then
yes_or_no "PV status is Released, not updating PVC details, is this OK?"
fi
echo "${npvJSON}" | jq .
yes_or_no "Would you like to contiue with the update to the PV with the above details? " && {
:
echo "starting PV update PV ${pv}"
kubectl patch pv "${name}" -p '{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}'
kubectl delete pv "${name}"
echo "${npvJSON}" | kubectl apply -f -
echo "successfully updated PV ${pv}"
} || {
:
echo "you decided no"
}
if [[ -z ${src} || ${src} == "null" || ${src} == "null" ]]; then
read -p "Prompt for src zvol/dataset (share path: ${path}): " src
fi
if [[ ${SHARE_STRATEGY} == "nfs" ]]; then
if [[ -z ${shareId} || ${shareId} == "null" ]]; then
echo "Edit the share in the FreeNAS UI and observe the id in the URL address bar"
read -p "shareId: " shareId
fi
fi
echo ""
echo ""
yes_or_no "Do you understand that you *must* execute all the commands shown after this message in the *exact* order shown? You cannot skip any of them, they all must succeed (including 200s from the curl commands)." && {
echo "OK then, moving on :)"
} || {
echo "It's best you stop here"
exit 1
}
echo ""
echo ""
echo "################## commands to run on TrueNAS cli #############################"
echo ""
echo "# set properties"
# common
if [[ -n ${PROVISIONER_INSTANCE_ID} ]]; then
echo "zfs set democratic-csi:volume_context_provisioner_instance_id=${PROVISIONER_INSTANCE_ID} ${src}"
fi
echo "zfs set democratic-csi:csi_volume_name=${pv} ${src}"
echo "zfs set democratic-csi:provision_success=true ${src}"
echo "zfs set democratic-csi:managed_resource=true ${src}"
if [[ ${SHARE_STRATEGY} == "nfs" ]]; then
# nfs
volume_context="{\"node_attach_driver\":\"nfs\",\"server\":\"${server}\",\"share\":\"${npath}\"}"
echo "zfs set democratic-csi:csi_share_volume_context='${volume_context}' ${src}"
echo "zfs set democratic-csi:freenas_nfs_share_id=${shareId} ${src}"
echo "zfs set democratic-csi:volume_context_provisioner_driver=freenas-nfs ${src}"
fi
if [[ ${SHARE_STRATEGY} == "iscsi" ]]; then
# iscsi
echo "zfs set democratic-csi:freenas_iscsi_assets_name=${ISCSI_ASSET_NAME} ${src}"
volume_context="{\"node_attach_driver\":\"iscsi\",\"portal\":\"${targetPortal}\",\"portals\":\"\",\"interface\":\"\",\"iqn\":\"${niqn}\",\"lun\":${lun}}"
echo "zfs set democratic-csi:csi_share_volume_context='${volume_context}' ${src}"
echo "zfs set democratic-csi:freenas_iscsi_target_id=${targetId} ${src}"
echo "zfs set democratic-csi:freenas_iscsi_extent_id=${extentId} ${src}"
echo "zfs set democratic-csi:freenas_iscsi_targettoextent_id=${targetToExtentId} ${src}"
echo "zfs set democratic-csi:volume_context_provisioner_driver=freenas-iscsi ${src}"
fi
echo ""
echo ""
echo "################## end commands to run on FreeNAS cli #############################"
echo ""
echo "#################### API curl command to update share #########################"
echo ""
# update shares to point to new location of vol/dataset
# rename dataset/zvol
if [[ ${SHARE_STRATEGY} == "nfs" ]]; then
# nfs
:
echo "# temporarily assign share to different path to free up dataset for rename"
echo "curl -v -u\"${FREENAS_USERNAME}:${FREENAS_PASSWORD}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\" -XPUT \"${FREENAS_URI}/api/v2.0/sharing/nfs/id/${shareId}\" -d '{\"paths\":[\"${TMP_ASSET}\"]}'"
echo ""
echo "# rename asset"
echo "zfs rename -p -f ${src} ${dst}"
echo ""
echo "# re-associate the share with the dataset"
echo "curl -v -u\"${FREENAS_USERNAME}:${FREENAS_PASSWORD}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\" -XPUT \"${FREENAS_URI}/api/v2.0/sharing/nfs/id/${shareId}\" -d '{\"paths\":[\"${npath}\"]}'"
fi
if [[ ${SHARE_STRATEGY} == "iscsi" ]]; then
# iscsi
:
echo "# temporarily assign extent to different asset to free up zvol for rename"
echo "curl -v -u\"${FREENAS_USERNAME}:${FREENAS_PASSWORD}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\" -XPUT \"${FREENAS_URI}/api/v2.0/iscsi/extent/id/${extentId}\" -d '{\"path\":\"zvol/${TMP_ASSET}\", \"disk\":\"zvol/${TMP_ASSET}\"}'"
echo ""
echo "# rename asset"
echo "zfs rename -p -f ${src} ${dst}"
echo ""
echo "curl -v -u\"${FREENAS_USERNAME}:${FREENAS_PASSWORD}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\" -XPUT \"${FREENAS_URI}/api/v2.0/iscsi/target/id/${targetId}\" -d '{\"name\":\"${ISCSI_ASSET_NAME}\"}'"
echo "curl -v -u\"${FREENAS_USERNAME}:${FREENAS_PASSWORD}\" -H \"Content-Type: application/json\" -H \"Accept: application/json\" -XPUT \"${FREENAS_URI}/api/v2.0/iscsi/extent/id/${extentId}\" -d '{\"name\":\"${ISCSI_ASSET_NAME}\", \"path\":\"zvol/${dst}\", \"disk\":\"zvol/${dst}\"}'"
fi
echo ""
echo "################## end API curl command to update share #############################"
echo "################## final cleanup ######################"
echo ""
echo "# ensure volumes are bound/etc as appropriate and restart your workloads here and ensure all is well"
echo ""
echo "# restore original reclaim policy"
echo "kubectl patch pv \"${name}\" -p '{\"spec\":{\"persistentVolumeReclaimPolicy\":\"${reclaimPolicy}\"}}'"
echo ""
echo "################## end final cleanup ######################"