Add optional pre-upgrade hook delay (#82434)

* Add optional pre-upgrade hook delay

On clusters with slow network initialization the pre-upgrade hook
may fail when etcdctl tries to contact etcd members before pod network
is actually initialized. Optional customizable delay can be injected
before running etcdctl to give kubelet enough time to initialize
network.

fixes #82322

Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>

* Update preupgrade.sh

Signed-off-by: Alvaro Neira Ayuso <alvaro.neira@broadcom.com>

---------

Signed-off-by: Robert Moucha <robert.moucha@gooddata.com>
Signed-off-by: Alvaro Neira Ayuso <alvaro.neira@broadcom.com>
Co-authored-by: Alvaro Neira Ayuso <alvaro.neira@broadcom.com>
This commit is contained in:
Robert Moucha 2025-07-01 11:37:19 +02:00 committed by GitHub
parent 3bcea4bed0
commit 95534844ec
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 8 additions and 0 deletions

View File

@ -34,6 +34,7 @@ etcd_env_vars=(
ETCD_DISASTER_RECOVERY
ETCD_ON_K8S
ETCD_INIT_SNAPSHOT_FILENAME
ETCD_PREUPGRADE_START_DELAY
ETCD_NAME
ETCD_LOG_LEVEL
ETCD_LISTEN_CLIENT_URLS
@ -90,6 +91,7 @@ export ETCD_START_FROM_SNAPSHOT="${ETCD_START_FROM_SNAPSHOT:-no}"
export ETCD_DISASTER_RECOVERY="${ETCD_DISASTER_RECOVERY:-no}"
export ETCD_ON_K8S="${ETCD_ON_K8S:-no}"
export ETCD_INIT_SNAPSHOT_FILENAME="${ETCD_INIT_SNAPSHOT_FILENAME:-}"
export ETCD_PREUPGRADE_START_DELAY="${ETCD_PREUPGRADE_START_DELAY:-}"
# etcd native environment variables (see https://etcd.io/docs/current/op-guide/configuration)
export ETCD_NAME="${ETCD_NAME:-}"

View File

@ -37,6 +37,11 @@ endpoints_as_host_port() {
read -r -a extra_flags <<<"$(etcdctl_auth_flags)"
is_boolean_yes "$ETCD_ON_K8S" && extra_flags+=("--endpoints=$(endpoints_as_host_port)")
if [[ -n "$ETCD_PREUPGRADE_START_DELAY" ]]; then
info "Waiting for $ETCD_PREUPGRADE_START_DELAY seconds before starting pre-upgrade checks"
sleep "$ETCD_PREUPGRADE_START_DELAY"
fi
debug "Listing members"
if ! current="$(etcdctl member list "${extra_flags[@]}" --write-out simple | awk -F ", " '{print $3 ":" $1}')"; then
error "Unable to list members, are all members healthy?"

View File

@ -195,6 +195,7 @@ Apart from providing your custom configuration file, you can also modify the ser
| `ETCD_START_FROM_SNAPSHOT` | Whether etcd should start from an existing snapshot or not. | `no` |
| `ETCD_DISASTER_RECOVERY` | Whether etcd should try or not to recover from snapshots when the cluste disastrously fails. | `no` |
| `ETCD_ON_K8S` | Whether etcd is running on a K8s environment or not. | `no` |
| `ETCD_PREUPGRADE_START_DELAY` | Optional delay before starting the pre-upgrade hook (in seconds). | `nil` |
| `ETCD_INIT_SNAPSHOT_FILENAME` | Existing snapshot filename to start the etcd cluster from. | `nil` |
| `ETCD_NAME` | etcd member name. | `nil` |
| `ETCD_LOG_LEVEL` | etcd log level. | `info` |