[bitnami/*] Change workflow to fix issues retrieving tag information (#3331)

* Change pipelines to fix issues retrieving tag information

Signed-off-by: Fran Mulero <fmulero@vmware.com>

* Add 'verify-all' job to report if whole matrix finished OK in case of several containers are tested

Signed-off-by: Fran Mulero <fmulero@vmware.com>

* Amend verify-all conditions

Signed-off-by: Fran Mulero <fmulero@vmware.com>

* Set 'build-maintainers' as team for failures

Signed-off-by: Fran Mulero <fmulero@vmware.com>

* Apply suggestions

Signed-off-by: Fran Mulero <fmulero@vmware.com>

* Apply suggestions: simplify get-containers output

Signed-off-by: Fran Mulero <fmulero@vmware.com>

Signed-off-by: Fran Mulero <fmulero@vmware.com>
This commit is contained in:
Fran Mulero 2022-08-24 13:54:19 +02:00 committed by GitHub
parent 38785b125c
commit 73cda2097f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 221 additions and 182 deletions

View File

@ -13,6 +13,9 @@ jobs:
get-containers:
runs-on: ubuntu-latest
name: Get modified containers path
if: |
github.event.pusher.name == 'bitnami-bot' &&
github.event.forced == false
outputs:
result: ${{ steps.get-containers.outputs.result }}
containers: ${{ steps.get-containers.outputs.containers }}
@ -22,70 +25,54 @@ jobs:
env:
GITHUB_COMMITS: ${{ toJson(github.event.commits) }}
run: |
if [[ "${{ github.event.forced }}" == "false" ]]; then
# Get all commits associated to the push
commits=($(echo "${GITHUB_COMMITS}" | jq -r '.[] | .id'))
containers=()
for commit in "${commits[@]}"; do
containers_in_commit=()
# Using the Github API to detect the files changed as git merge-base stops working when the branch is behind
URL="https://api.github.com/repos/${{ github.repository }}/commits/${commit}"
files_changed_data=$(curl -s --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -X GET -G "$URL")
files_changed="$(echo $files_changed_data | jq -r '.files[] | .filename')"
# Adding || true to avoid "Process exited with code 1" errors
containers_in_commit+=($(echo "$files_changed" | xargs dirname | grep -o "^bitnami/[^/]*/[^/]*/[^/]*" | sort | uniq || true))
# Changes done in readme files?
readme_object="$(echo $files_changed_data | jq -r '[ .files[] | select(.filename | test("^bitnami/[^/]+/README.md"))] | first')"
if [[ "${readme_object}" != "null" ]]; then
# Look for modified branch in Readme content
container_name="$(echo ${readme_object} | jq -r '.filename' | sed -nr "s|bitnami/([^/]+)/README.md|\1|p")"
branch="$(echo ${readme_object} | jq -r '.patch' | grep -o "([^/]*/[^/]*/Dockerfile)" | sort -u | head -n 1 | sed -nr "s|\((.+)/Dockerfile\)|\1|p")"
if [[ ! -z "${branch}" ]] && [[ ! -z "${container_name}" ]]; then
# readme file changed and the diff contains a change about the latest version generated by EAM
containers_in_commit+=("bitnami/${container_name}/${branch}")
fi
# Get all commits associated to the push
commits=($(echo "${GITHUB_COMMITS}" | jq -r '.[] | .id'))
containers=()
for commit in "${commits[@]}"; do
containers_in_commit=()
# Using the Github API to detect the files changed as git merge-base stops working when the branch is behind
URL="https://api.github.com/repos/${{ github.repository }}/commits/${commit}"
files_changed_data=$(curl -s --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -X GET -G "$URL")
files_changed="$(echo $files_changed_data | jq -r '.files[] | .filename')"
# Adding || true to avoid "Process exited with code 1" errors
containers_in_commit+=($(echo "$files_changed" | xargs dirname | grep -o "^bitnami/[^/]*/[^/]*/[^/]*" | sort | uniq || true))
for container in "${containers_in_commit[@]}"; do
if [[ ! $containers =~ (^|[[:space:]])$container($|[[:space:]]) ]]; then
# Avoid duplicates
containers+=("${container}")
fi
for container in "${containers_in_commit[@]}"; do
if [[ ! $containers =~ (^|[[:space:]])$container($|[[:space:]]) ]]; then
# Avoid duplicates
containers+=("${container}")
fi
done
done
done
if [[ "${#containers[@]}" -le "0" ]]; then
echo "No changes detected in containers. The rest of the steps will be skipped."
echo "::set-output name=result::skip"
else
containers_json=$(printf "%s\n" "${containers[@]}" | jq -R . | jq -cs .)
echo "::set-output name=result::ok"
echo "::set-output name=containers::${containers_json}"
fi
else
echo "Forced push detected. CD pipeline will be skipped"
if [[ "${#containers[@]}" -le "0" ]]; then
echo "No changes detected in containers. The rest of the steps will be skipped."
echo "::set-output name=result::skip"
else
containers_json=$(printf "%s\n" "${containers[@]}" | jq -R . | jq -cs .)
echo "::set-output name=result::ok"
echo "::set-output name=containers::${containers_json}"
fi
vib-publish:
runs-on: ubuntu-latest
needs: get-containers
if: ${{ needs.get-containers.outputs.result == 'ok' }}
name: publish
name: Publish
strategy:
fail-fast: false
max-parallel: 2
matrix:
container: ${{ fromJSON(needs.get-containers.outputs.containers) }}
steps:
- uses: actions/checkout@v3
name: Checkout Repository
# Required to search the latest commit with the tag
# Full history is not required anymore
with:
fetch-depth: 0
fetch-depth: 1
- id: get-container-metadata
name: Get image tag and container name
run: |
if [[ -d "${{ matrix.container }}" ]]; then
# Now EAM commits has the following format "[bitnami/<container>] Release <tag>"
tag="$(git log --pretty=tformat:"%s" -n 1 --grep="[R|r]elease" --author bitnami-bot@vmware.com --author containers@bitnami.com --author containers-bot@bitnami.com -- ${{ matrix.container }} | sed "s|\[.*\]||" | sed "s|[R|r]elease||" | awk '{print $1}')"
tag="$(grep -oE "org.opencontainers.image.ref.name=\".+\"" ${{ matrix.container }}/Dockerfile | sed -nr "s|org.opencontainers.image.ref.name=\"(.+)\"|\1|p")"
if [[ -z "${tag}" ]]; then
echo "No tag found for: ${{ matrix.container }}"
exit 1

View File

@ -0,0 +1,143 @@
name: '[CI/CD] CI Pipeline for automated PRs'
on: # Triggered only on open PR events done by bitnami-bot on containers folders
pull_request_target:
types:
- opened
branches:
- main
- bitnami:main
paths:
- 'bitnami/**'
env:
CSP_API_URL: https://console.cloud.vmware.com
CSP_API_TOKEN: ${{ secrets.CSP_API_TOKEN }}
VIB_PUBLIC_URL: https://cp.bromelia.vmware.com
jobs:
auto-pr-triage:
runs-on: ubuntu-latest
name: Triage for automated PRs
if: |
contains(github.event.pull_request.title, 'Release') &&
github.actor == 'bitnami-bot'
steps:
# Enables auto-merge and adds necessary labels for automated releases' PRs
- id: labeling
name: Label PR
uses: andymckay/labeler@1.0.4
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
add-labels: "verify, auto-merge"
- id: auto-merge
name: Enable auto-merge
run: |
curl --request POST \
--url https://api.github.com/graphql \
--header 'authorization: Bearer ${{ secrets.BITNAMI_BOT_TOKEN }}' \
--data '{
"query": "mutation { enablePullRequestAutoMerge(input: {pullRequestId: \"${{ github.event.pull_request.node_id }}\", mergeMethod: SQUASH}) { clientMutationId }}"
}' \
--fail
get-container:
runs-on: ubuntu-latest
name: Get modified containers
needs: auto-pr-triage
outputs:
container: ${{ steps.get-container.outputs.container }}
path: ${{ steps.get-container.outputs.path }}
steps:
- id: get-container
name: Get modified containers
run: |
# Using the Github API to detect the files changed as git merge-base stops working when the branch is behind
# and jitterbit/get-changed-files does not support pull_request_target
URL="https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files"
files_changed_data=$(curl -s --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -X GET -G "$URL")
files_changed="$(echo $files_changed_data | jq -r '.[] | .filename')"
# Adding || true to avoid "Process exited with code 1" errors
containers_dirs_changed="$(echo "$files_changed" | xargs dirname | grep -o "^bitnami/[^/]*" | sort | uniq || true)"
flavors=($(echo "$files_changed" | xargs dirname | grep -o "^bitnami/[^/]*/[^/]*/[^/]*" | sort | uniq || true))
container_name=$(echo "$containers_dirs_changed" | sed "s|bitnami/||g")
if [[ "${#flavors[@]}" -eq "1" ]]; then
# Changes done in only one container -> OK
echo "::set-output name=container::${container_name}"
echo "::set-output name=path::${flavors[0]}"
else
# No changes detected or there are changes done in more than one container -> FAIL. It doesn't make sense in automated PRs.
echo -e "No changes detected or there are changes done in more than one container:\n${containers_dirs_changed}\n. The rest of the tests will be skipped."
exit 1
fi
vib-verify:
runs-on: ubuntu-latest
needs: get-container
name: Verify
steps:
- uses: actions/checkout@v3
name: Checkout Repository
with:
# Required to search the latest commit with the tag
fetch-depth: 1
# labeled events trigger the event with the latest commit in main
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- id: get-tag
name: Get latest image tag
run: |
# Now EAM commits has the following format "[bitnami/<container>] Release <tag>"
tag="$(grep -oE "org.opencontainers.image.ref.name=\".+\"" ${{ needs.get-container.outputs.path }}/Dockerfile | sed -nr "s|org.opencontainers.image.ref.name=\"(.+)\"|\1|p")"
if [[ -z "${tag}" ]]; then
echo "No tag found for: ${{ needs.get-container.outputs.path }}"
exit 1
else
echo "::set-output name=tag::${tag}"
fi
- uses: vmware-labs/vmware-image-builder-action@main
name: Verify
with:
pipeline: vib-verify.json
env:
# Path with docker resources
VIB_ENV_PATH: ${{ needs.get-container.outputs.path }}
# Container name
VIB_ENV_CONTAINER: ${{ needs.get-container.outputs.container }}
VIB_ENV_TAG: ${{ steps.get-tag.outputs.tag }}
auto-pr-review:
runs-on: ubuntu-latest
needs: vib-verify
name: Reviewal for automated PRs
if: |
always() &&
github.actor == 'bitnami-bot'
steps:
# Approves the CI's PR if the 'VIB Verify' job succeeded
# Approved by the 'github-actions' user. A PR can't be approved by its author
- name: Approval
if: ${{ needs.vib-verify.result == 'success' }}
run: |
curl --request POST \
--url https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews \
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
--header 'content-type: application/json' \
--data '{
"event": "APPROVE"
}' \
--fail
- name: Manual review required
if: ${{ needs.vib-verify.result != 'success' }}
run: |
curl --request POST \
--url https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
--header 'content-type: application/json' \
--data '{
"body": "There has been an error during the automated release process. Manual revision is now required."
}' \
--fail
- name: Assign to a person to work on it
if: ${{ needs.vib-verify.result != 'success' }}
uses: pozil/auto-assign-issue@v1.9.0
with:
numOfAssignee: 1
removePreviousAssignees: true
teams: build-maintainers
repo-token: "${{ secrets.BITNAMI_BOT_TOKEN }}"

View File

@ -2,7 +2,6 @@ name: '[CI/CD] CI Pipeline'
on: # rebuild any PRs and main branch changes
pull_request_target:
types:
- opened
- reopened
- synchronize
- labeled
@ -14,33 +13,7 @@ env:
CSP_API_TOKEN: ${{ secrets.CSP_API_TOKEN }}
VIB_PUBLIC_URL: https://cp.bromelia.vmware.com
jobs:
auto-pr-triage:
runs-on: ubuntu-latest
name: Triage for automated PRs
if: |
contains(github.event.pull_request.title, 'Release') &&
github.event.action == 'opened' &&
github.actor == 'bitnami-bot'
steps:
# Enables auto-merge and adds necessary labels for automated releases' PRs
- id: labeling
name: Label PR
uses: andymckay/labeler@1.0.4
with:
# We can't use GITHUB_TOKEN because the labeling needs to trigger a new CI pipeline
repo-token: "${{ secrets.BITNAMI_BOT_TOKEN }}"
add-labels: "verify, auto-merge"
- id: auto-merge
name: Enable auto-merge
run: |
curl --request POST \
--url https://api.github.com/graphql \
--header 'authorization: Bearer ${{ secrets.BITNAMI_BOT_TOKEN }}' \
--data '{
"query": "mutation { enablePullRequestAutoMerge(input: {pullRequestId: \"${{ github.event.pull_request.node_id }}\", mergeMethod: SQUASH}) { clientMutationId }}"
}' \
--fail
get-container:
get-containers:
runs-on: ubuntu-latest
name: Get modified containers
if: |
@ -50,11 +23,10 @@ jobs:
(github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'verify'))
)
outputs:
container: ${{ steps.get-container.outputs.container }}
result: ${{ steps.get-container.outputs.result }}
path: ${{ steps.get-container.outputs.path }}
result: ${{ steps.get-containers.outputs.result }}
containers: ${{ steps.get-containers.outputs.containers }}
steps:
- id: get-container
- id: get-containers
name: Get modified containers
run: |
# Using the Github API to detect the files changed as git merge-base stops working when the branch is behind
@ -63,136 +35,73 @@ jobs:
files_changed_data=$(curl -s --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -X GET -G "$URL")
files_changed="$(echo $files_changed_data | jq -r '.[] | .filename')"
# Adding || true to avoid "Process exited with code 1" errors
containers_dirs_changed="$(echo "$files_changed" | xargs dirname | grep -o "^bitnami/[^/]*" | sort | uniq || true)"
flavors=($(echo "$files_changed" | xargs dirname | grep -o "^bitnami/[^/]*/[^/]*/[^/]*" | sort | uniq || true))
container_name=$(echo "$containers_dirs_changed" | sed "s|bitnami/||g")
assets=($(echo "$files_changed" | xargs dirname | sed -nr "s|bitnami/([^/]*)/.*|\1|p" | sort | uniq || true))
non_readme_files=$(echo "$files_changed" | grep -vc "\.md" || true)
if [[ "${#flavors[@]}" -eq "1" ]]; then
# Changes done in only one container -> OK
echo "::set-output name=result::ok"
echo "::set-output name=container::${container_name}"
echo "::set-output name=path::${flavors[0]}"
elif [[ "${#flavors[@]}" -le "0" ]]; then
# Changes done in the containers/ folder but not inside a container subfolder -> Look for readme changes
readme_object="$(echo $files_changed_data | jq -r '[ .[] | select(.filename | test("^bitnami/[^/]+/README.md"))] | first')"
if [[ "${readme_object}" == "null" ]]; then
echo "::set-output name=error::No changes detected in containers. The rest of the tests will be skipped."
echo "::set-output name=result::skip"
else
# Look for the modified branch in Readme content
branch="$(echo ${readme_object} | jq -r '.patch' | grep -o "([^/]*/[^/]*/Dockerfile)" | sort -u | head -n 1 | sed -nr "s|\((.+)/Dockerfile\)|\1|p")"
if [[ ! -z "${branch}" ]]; then
echo "::set-output name=result::ok"
echo "::set-output name=container::${container_name}"
echo "::set-output name=path::bitnami/${container_name}/${branch}"
else
echo "::set-output name=error::No changes detected in containers. The rest of the tests will be skipped."
echo "::set-output name=result::skip"
fi
fi
else
# Changes done in more than container -> SKIP
echo -e "::set-output name=error::Changes detected in more than one container directory:\n${containers_dirs_changed}\nIt is strongly advised to change only one container in a PR. The rest of the tests will be skipped."
if [[ "$non_readme_files" -le "0" ]]; then
# The only changes are .md files -> SKIP
echo "::set-output name=result::skip"
elif [[ "${#assets[@]}" -ne "1" ]]; then
echo "Changes should affect to only one asset. You are currently modifying: ${assets[@]}"
echo "::set-output name=result::skip"
else
containers_json=$(printf "%s\n" "${flavors[@]}" | jq -R . | jq -cs .)
echo "::set-output name=result::ok"
echo "::set-output name=containers::${containers_json}"
fi
# Using actions/github-scripts because using exit 1 in the script above would not provide any output
# Source: https://github.community/t/no-output-on-process-completed-with-exit-code-1/123821/3
- id: show-error
name: Show error
if: ${{ steps.get-container.outputs.result == 'fail' }}
uses: actions/github-script@v3
with:
script: |
core.setFailed('${{ steps.get-container.outputs.error }}')
vib-verify:
runs-on: ubuntu-latest
needs: get-container
if: ${{ needs.get-container.outputs.result == 'ok' && contains(github.event.pull_request.labels.*.name, 'verify') }}
name: Verify
needs: get-containers
if: ${{ needs.get-containers.outputs.result == 'ok' && contains(github.event.pull_request.labels.*.name, 'verify') }}
name: Verify container
continue-on-error: false
strategy:
fail-fast: false
max-parallel: 2
matrix:
container: ${{ fromJSON(needs.get-containers.outputs.containers) }}
steps:
- uses: actions/checkout@v3
name: Checkout Repository
with:
# Required to search the latest commit with the tag
fetch-depth: 0
# Full history is not required anymore
fetch-depth: 1
# labeled events trigger the event with the latest commit in main
ref: ${{ github.event.pull_request.head.ref }}
repository: ${{ github.event.pull_request.head.repo.full_name }}
- id: get-tag
name: Get latest image tag
- id: get-container-metadata
name: Get image tag and container name
run: |
# Now EAM commits has the following format "[bitnami/<container>] Release <tag>"
tag="$(git log --pretty=tformat:"%s" -n 1 --grep="[R|r]elease" --author bitnami-bot@vmware.com --author containers@bitnami.com --author containers-bot@bitnami.com -- ${{ needs.get-container.outputs.path }} | sed "s|\[.*\]||" | sed "s|[R|r]elease||" | awk '{print $1}')"
if [[ -z "${tag}" ]]; then
echo "No tag found for: ${{ needs.get-container.outputs.path }}"
exit 1
else
if [[ -d "${{ matrix.container }}" ]]; then
name="$(echo "${{ matrix.container }}" | awk -F '/' '{print $2}')"
# Build a tag based on current SHA
tag="$(echo "${{ matrix.container }}" | awk -F '/' -v run_number="${GITHUB_RUN_NUMBER}" '{printf "%s-rc.%s", $3, run_number}')"
echo "::set-output name=tag::${tag}"
echo "::set-output name=name::${name}"
echo "::set-output name=result::ok"
else
# Container folder doesn't exists we are assuming a deprecation
echo "::set-output name=result::skip"
fi
- uses: vmware-labs/vmware-image-builder-action@main
name: Verify
if: ${{ steps.get-container-metadata.outputs.result == 'ok' }}
with:
pipeline: vib-verify.json
env:
# Path with docker resources
VIB_ENV_PATH: ${{ needs.get-container.outputs.path }}
VIB_ENV_PATH: ${{ matrix.container }}
# Container name
VIB_ENV_CONTAINER: ${{ needs.get-container.outputs.container }}
VIB_ENV_TAG: ${{ steps.get-tag.outputs.tag }}
auto-pr-review:
VIB_ENV_CONTAINER: ${{ steps.get-container-metadata.outputs.name }}
VIB_ENV_TAG: ${{ steps.get-container-metadata.outputs.tag }}
verify-all:
# Ensure all containers passed the verification
runs-on: ubuntu-latest
name: Verify
needs: vib-verify
name: Reviewal for automated PRs
if: |
always() &&
github.actor == 'bitnami-bot' &&
contains(github.event.pull_request.labels.*.name, 'auto-merge')
if: ${{ always() }}
steps:
# Approves the CI's PR if the 'VIB Verify' job succeeded
# Approved by the 'github-actions' user. A PR can't be approved by its author
- name: Approval
if: ${{ needs.vib-verify.result == 'success' }}
run: |
curl --request POST \
--url https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/reviews \
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
--header 'content-type: application/json' \
--data '{
"event": "APPROVE"
}' \
--fail
- name: Remove auto-merge label
if: ${{ needs.vib-verify.result == 'failure' }}
uses: andymckay/labeler@1.0.4
with:
repo-token: "${{ secrets.GITHUB_TOKEN }}"
remove-labels: "auto-merge"
- name: Disable auto-merge
if: ${{ needs.vib-verify.result == 'failure' }}
run: |
curl --request POST \
--url https://api.github.com/graphql \
--header 'authorization: Bearer ${{ secrets.BITNAMI_BOT_TOKEN }}' \
--data '{
"query": "mutation { disablePullRequestAutoMerge(input: {pullRequestId: \"${{ github.event.pull_request.node_id }}\"}) { clientMutationId }}"
}' \
--fail
- name: Manual review required
if: ${{ needs.vib-verify.result == 'failure' }}
run: |
curl --request POST \
--url https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/comments \
--header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
--header 'content-type: application/json' \
--data '{
"body": "There has been an error during the automated release process. Manual revision is now required."
}' \
--fail
curl --request POST \
--url https://api.github.com/repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/requested_reviewers \
--header 'authorization: Bearer ${{ secrets.BITNAMI_BOT_TOKEN }}' \
--header 'content-type: application/json' \
--data '{
"reviewers": ["fmulero"]
}' \
--fail
- name: Verify
if: ${{ needs.vib-verify.result != 'success' && needs.vib-verify.result != 'skipped'}}
run: exit 1