fix(backup): find race and disable search in subdir, always test latest backup image in e2e, bump backup to 0.3.x (#1011)
Co-authored-by: brokenpip3 <brokenpip3@gmail.com>
This commit is contained in:
Evgenii
GitHub
parent
e40032497b
commit
f98b0bc5ee
15
Makefile
15
Makefile
|
|
@ -89,11 +89,18 @@ test: ## Runs the go tests
|
|||
@RUNNING_TESTS=1 go test -tags "$(BUILDTAGS) cgo" $(PACKAGES_FOR_UNIT_TESTS)
|
||||
|
||||
.PHONY: e2e
|
||||
e2e: deepcopy-gen manifests ## Runs e2e tests, you can use EXTRA_ARGS
|
||||
e2e: deepcopy-gen manifests backup-kind-load ## Runs e2e tests, you can use EXTRA_ARGS
|
||||
@echo "+ $@"
|
||||
RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -ginkgo.v -tags "$(BUILDTAGS) cgo" -v -timeout 60m -run "$(E2E_TEST_SELECTOR)" \
|
||||
-jenkins-api-hostname=$(JENKINS_API_HOSTNAME) -jenkins-api-port=$(JENKINS_API_PORT) -jenkins-api-use-nodeport=$(JENKINS_API_USE_NODEPORT) $(E2E_TEST_ARGS)
|
||||
|
||||
## Backup Section
|
||||
|
||||
.PHONY: backup-kind-load
|
||||
backup-kind-load: ## Load latest backup image in the cluster
|
||||
@echo "+ $@"
|
||||
make -C backup/pvc backup-kind-load
|
||||
|
||||
## HELM Section
|
||||
|
||||
.PHONY: helm
|
||||
|
|
@ -370,6 +377,12 @@ kind-clean: ## Delete kind cluster
|
|||
@echo "+ $@"
|
||||
kind delete cluster --name $(KIND_CLUSTER_NAME)
|
||||
|
||||
.PHONY: kind-load-backup
|
||||
kind-load-backup:
|
||||
@echo "+ $@"
|
||||
make -C
|
||||
kind delete cluster --name $(KIND_CLUSTER_NAME)
|
||||
|
||||
.PHONY: bats-tests
|
||||
IMAGE_NAME := quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY):$(GITCOMMIT)-amd64
|
||||
BUILD_PRESENT := $(shell docker images |grep -q ${IMAGE_NAME})
|
||||
|
|
|
|||
|
|
@ -118,6 +118,12 @@ docker-release-latest: docker-build ## Release image with latest tags (in additi
|
|||
docker-release: docker-release-version docker-release-latest ## Release image with version and latest tags (in addition to build tag)
|
||||
@echo "+ $@"
|
||||
|
||||
.PHONY: backup-kind-load
|
||||
backup-kind-load: docker-build ## Build and load backup img in kind with e2e-test tag
|
||||
@echo "+ $@"
|
||||
docker tag quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):$(GITCOMMIT) quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):e2e-test
|
||||
kind load docker-image quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):e2e-test --name $(KIND_CLUSTER_NAME)
|
||||
|
||||
# if this session isn't interactive, then we don't want to allocate a
|
||||
# TTY, which would fail, but if it is interactive, we do want to attach
|
||||
# so that the user can send e.g. ^C through.
|
||||
|
|
@ -127,7 +133,7 @@ ifeq ($(INTERACTIVE), 1)
|
|||
endif
|
||||
|
||||
.PHONY: docker-run
|
||||
docker-run: docker-build ## Run the container in docker, you can use EXTRA_ARGS
|
||||
docker-run: docker-build
|
||||
@echo "+ $@"
|
||||
docker run --rm -i $(DOCKER_FLAGS) \
|
||||
quay.io/$(QUAY_ORGANIZATION)/$(QUAY_REGISTRY)-$(NAME):$(GITCOMMIT) $(ARGS)
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
v0.2.5
|
||||
v0.3.0
|
||||
|
|
|
|||
|
|
@ -2,9 +2,38 @@
|
|||
|
||||
set -eo pipefail
|
||||
|
||||
[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1
|
||||
is_backup_not_exist() {
|
||||
local backup_dir="$1"
|
||||
# Save the current value of 'set -e'
|
||||
local previous_e
|
||||
previous_e=$(set +e; :; echo $?)
|
||||
|
||||
# Temporarily turn off 'set -e'
|
||||
set +e
|
||||
|
||||
# Run ls command to check if any files matching the pattern exist
|
||||
ls "${backup_dir}"/*.tar.* 1> /dev/null 2>&1
|
||||
|
||||
# Store the exit status of the ls command
|
||||
local ls_exit_status=$?
|
||||
|
||||
# Restore the previous value of 'set -e'
|
||||
[ "$previous_e" = "0" ] && set -e
|
||||
|
||||
# Return true if ls command succeeded (no files found), otherwise return false
|
||||
[ $ls_exit_status -ne 0 ]
|
||||
}
|
||||
|
||||
[[ -z "${BACKUP_DIR}" ]] && { echo "Required 'BACKUP_DIR' env not set"; exit 1; }
|
||||
|
||||
# Check if we have any backup
|
||||
if is_backup_not_exist "${BACKUP_DIR}"; then
|
||||
echo "-1"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Search for all the tar.* inside the backup dir to support the migration between gzip vs zstd
|
||||
latest=$(find ${BACKUP_DIR} -name '*.tar.*' -exec basename {} \; | sort -g | tail -n 1)
|
||||
latest=$(find "${BACKUP_DIR}"/*.tar.* -maxdepth 0 -exec basename {} \; | sort -g | tail -n 1)
|
||||
|
||||
if [[ "${latest}" == "" ]]; then
|
||||
echo "-1"
|
||||
|
|
|
|||
|
|
@ -2,15 +2,70 @@
|
|||
|
||||
set -eo pipefail
|
||||
|
||||
[[ -z "${BACKUP_DIR}" ]] && echo "Required 'BACKUP_DIR' env not set" && exit 1;
|
||||
[[ -z "${JENKINS_HOME}" ]] && echo "Required 'JENKINS_HOME' env not set" && exit 1;
|
||||
# Use 60 as default in case BACKUP_CLEANUP_INTERVAL did not set
|
||||
BACKUP_CLEANUP_INTERVAL=${BACKUP_CLEANUP_INTERVAL:=60}
|
||||
|
||||
# Ensure required environment variables are set
|
||||
check_env_var() {
|
||||
if [[ -z "${!1}" ]]; then
|
||||
echo "Required '$1' environment variable is not set"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
is_backup_not_exist() {
|
||||
local backup_dir="$1"
|
||||
# Save the current value of 'set -e'
|
||||
local previous_e
|
||||
previous_e=$(set +e; :; echo $?)
|
||||
|
||||
# Temporarily turn off 'set -e'
|
||||
set +e
|
||||
|
||||
# Run ls command to check if any files matching the pattern exist
|
||||
ls "${backup_dir}"/*.tar.* 1> /dev/null 2>&1
|
||||
|
||||
# Store the exit status of the ls command
|
||||
local ls_exit_status=$?
|
||||
|
||||
# Restore the previous value of 'set -e'
|
||||
[ "$previous_e" = "0" ] && set -e
|
||||
|
||||
# Return true if ls command succeeded (no files found), otherwise return false
|
||||
[ $ls_exit_status -ne 0 ]
|
||||
}
|
||||
|
||||
# Function to find exceeding backups
|
||||
find_exceeding_backups() {
|
||||
local backup_dir="$1"
|
||||
local backup_count="$2"
|
||||
# Check if we have any backup
|
||||
if is_backup_not_exist "${backup_dir}"; then
|
||||
echo "backups not found in ${backup_dir}" >&2
|
||||
return
|
||||
fi
|
||||
find "${backup_dir}"/*.tar.zstd -maxdepth 0 -exec basename {} \; | sort -gr | tail -n +$((backup_count +1))
|
||||
}
|
||||
|
||||
check_env_var "BACKUP_DIR"
|
||||
check_env_var "JENKINS_HOME"
|
||||
|
||||
if [[ -z "${BACKUP_COUNT}" ]]; then
|
||||
echo "ATTENTION! No BACKUP_COUNT set, it means you MUST delete old backups manually or by custom script"
|
||||
else
|
||||
echo "Retaining only the ${BACKUP_COUNT} most recent backups, cleanup occurs every ${BACKUP_CLEANUP_INTERVAL} seconds"
|
||||
fi
|
||||
|
||||
while true;
|
||||
do
|
||||
sleep 10
|
||||
if [[ ! -z "${BACKUP_COUNT}" ]]; then
|
||||
echo "Trimming to only ${BACKUP_COUNT} recent backups in preparation for new backup"
|
||||
#TODO: add the list of exceeding backup before delete
|
||||
find ${BACKUP_DIR} -maxdepth 1 -name '*.tar.zstd' -exec basename {} \; | sort -gr | tail -n +$((BACKUP_COUNT +1)) | xargs -I '{}' rm ${BACKUP_DIR}/'{}'
|
||||
sleep "$BACKUP_CLEANUP_INTERVAL"
|
||||
if [[ -n "${BACKUP_COUNT}" ]]; then
|
||||
exceeding_backups=$(find_exceeding_backups "${BACKUP_DIR}" "${BACKUP_COUNT}")
|
||||
if [[ -n "$exceeding_backups" ]]; then
|
||||
echo "Removing backups: $(echo "$exceeding_backups" | tr '\n' ', ' | sed 's/,$//')"
|
||||
echo "$exceeding_backups" | while read -r file; do
|
||||
rm "${BACKUP_DIR}/${file}"
|
||||
done
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
|
|
|||
|
|
@ -4,3 +4,4 @@ QUAY_ORGANIZATION=jenkins-kubernetes-operator
|
|||
QUAY_REGISTRY=backup
|
||||
UID=1000
|
||||
GID=1000
|
||||
KIND_CLUSTER_NAME=jenkins
|
||||
|
|
|
|||
|
|
@ -30,6 +30,9 @@ touch ${BACKUP_DIR}/8.tar.zstd
|
|||
touch ${BACKUP_DIR}/9.tar.zstd
|
||||
touch ${BACKUP_DIR}/10.tar.zstd
|
||||
touch ${BACKUP_DIR}/11.tar.zstd
|
||||
# Emulate backup creation
|
||||
BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}")
|
||||
touch ${BACKUP_TMP_DIR}/12.tar.zstd
|
||||
|
||||
# Create an instance of the container under testing
|
||||
cid="$(docker run -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
|
||||
|
|
@ -38,10 +41,20 @@ echo "Docker container ID '${cid}'"
|
|||
# Remove test directory and container afterwards
|
||||
trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT
|
||||
|
||||
echo "Try to get latest against 11 backups and one in progress"
|
||||
latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
|
||||
|
||||
rm ${BACKUP_DIR}/*.tar.zstd
|
||||
echo "Try to get latest against one in progress"
|
||||
empty_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
|
||||
|
||||
rmdir ${BACKUP_DIR}/lost+found
|
||||
rm ${BACKUP_TMP_DIR}/*.tar.zstd
|
||||
rmdir ${BACKUP_TMP_DIR}
|
||||
echo "Try to get latest against empty dir"
|
||||
empty_dir_latest=$(docker exec ${cid} /bin/bash -c "JENKINS_HOME=${RESTORE_FOLDER};/home/user/bin/get-latest.sh")
|
||||
|
||||
|
||||
if [[ "${DEBUG}" ]]; then
|
||||
docker logs ${cid}
|
||||
ls -la ${BACKUP_DIR}
|
||||
|
|
@ -55,5 +68,9 @@ if [[ ! "${empty_latest}" == "-1" ]]; then
|
|||
echo "Latest backup number should be '-1' but is '${empty_latest}'"
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! "${empty_dir_latest}" == "-1" ]]; then
|
||||
echo "Latest backup number should be '-1' but is '${empty_dir_latest}'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo PASS
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
echo "Running limit_backup_count e2e test..."
|
||||
|
||||
[[ "${DEBUG}" ]] && set -x
|
||||
|
||||
# set current working directory to the directory of the script
|
||||
|
|
@ -30,17 +32,20 @@ touch ${BACKUP_DIR}/8.tar.zstd
|
|||
touch ${BACKUP_DIR}/9.tar.zstd
|
||||
touch ${BACKUP_DIR}/10.tar.zstd
|
||||
touch ${BACKUP_DIR}/11.tar.zstd
|
||||
# Emulate backup creation
|
||||
BACKUP_TMP_DIR=$(mktemp -d --tmpdir="${BACKUP_DIR}")
|
||||
touch ${BACKUP_TMP_DIR}/12.tar.zstd
|
||||
|
||||
# Create an instance of the container under testing
|
||||
cid="$(docker run -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
|
||||
cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
|
||||
echo "Docker container ID '${cid}'"
|
||||
|
||||
# Remove test directory and container afterwards
|
||||
trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}" EXIT
|
||||
|
||||
sleep 11
|
||||
touch ${BACKUP_DIR}/12.tar.zstd
|
||||
sleep 11
|
||||
sleep 2
|
||||
mv ${BACKUP_TMP_DIR}/12.tar.zstd ${BACKUP_DIR}/
|
||||
sleep 2
|
||||
|
||||
if [[ "${DEBUG}" ]]; then
|
||||
docker logs ${cid}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
echo "Running limit_backup_count_no_backups e2e test..."
|
||||
|
||||
[[ "${DEBUG}" ]] && set -x
|
||||
|
||||
# set current working directory to the directory of the script
|
||||
|
|
@ -19,7 +21,7 @@ mkdir -p ${BACKUP_DIR}
|
|||
mkdir -p ${JENKINS_HOME}
|
||||
|
||||
# Create an instance of the container under testing
|
||||
cid="$(docker run -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
|
||||
cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e BACKUP_COUNT=2 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
|
||||
echo "Docker container ID '${cid}'"
|
||||
|
||||
# Remove test directory and container afterwards
|
||||
|
|
@ -27,8 +29,16 @@ trap "docker rm -vf $cid > /dev/null;rm -rf ${BACKUP_DIR};rm -rf ${JENKINS_HOME}
|
|||
|
||||
# container should be running
|
||||
echo 'Checking if container is running'
|
||||
sleep 11
|
||||
sleep 3
|
||||
set +e
|
||||
docker exec ${cid} echo
|
||||
exit_code=$?
|
||||
set -e
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
echo "container terminated with following logs:"
|
||||
docker logs "${cid}"
|
||||
exit 1
|
||||
fi
|
||||
echo 'Container is running'
|
||||
|
||||
echo PASS
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#!/bin/bash
|
||||
set -eo pipefail
|
||||
|
||||
echo "Running tmp_dir_clean_after_backup_creation e2e test..."
|
||||
|
||||
[[ "${DEBUG}" ]] && set -x
|
||||
|
||||
# set current working directory to the directory of the script
|
||||
|
|
@ -18,7 +20,7 @@ BACKUP_DIR="$(pwd)/backup"
|
|||
mkdir -p ${BACKUP_DIR}
|
||||
|
||||
# Create an instance of the container under testing
|
||||
cid="$(docker run -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
|
||||
cid="$(docker run -e BACKUP_CLEANUP_INTERVAL=1 -e JENKINS_HOME=${JENKINS_HOME} -v ${JENKINS_HOME}:${JENKINS_HOME}:ro -e BACKUP_DIR=${BACKUP_DIR} -v ${BACKUP_DIR}:${BACKUP_DIR}:rw -d ${docker_image})"
|
||||
echo "Docker container ID '${cid}'"
|
||||
|
||||
# Remove test directory and container afterwards
|
||||
|
|
@ -28,6 +30,8 @@ backup_number=1
|
|||
docker exec ${cid} /home/user/bin/backup.sh ${backup_number}
|
||||
|
||||
[ "$(docker exec ${cid} ls /tmp | grep 'tmp')" ] && echo "tmp directory not empty" && exit 1;
|
||||
# We should also check backup directory, since after #1000 we create temp directory at backup filesystem
|
||||
[ "$(docker exec ${cid} ls ${BACKUP_DIR} | grep 'tmp')" ] && echo "backup dir consists temp directory" && exit 1;
|
||||
|
||||
backup_file="${BACKUP_DIR}/${backup_number}.tar.zstd"
|
||||
[[ ! -f ${backup_file} ]] && echo "Backup file ${backup_file} not found" && exit 1;
|
||||
|
|
|
|||
|
|
@ -157,7 +157,7 @@ func createJenkinsWithBackupAndRestoreConfigured(name, namespace string) *v1alph
|
|||
},
|
||||
{
|
||||
Name: containerName,
|
||||
Image: "virtuslab/jenkins-operator-backup-pvc:v0.1.1",
|
||||
Image: "quay.io/jenkins-kubernetes-operator/backup-pvc:e2e-test",
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
Env: []corev1.EnvVar{
|
||||
{
|
||||
|
|
|
|||
Loading…
Reference in New Issue