From bef796af8accefb21f1f8186871ab48ad6b74c94 Mon Sep 17 00:00:00 2001 From: Bartosz Surma Date: Tue, 28 Jun 2022 19:14:31 +0200 Subject: [PATCH] Update and use install-plugins.sh (#756) * Download install-plugins.sh manually from remote * Reformat * Goimports * Use install-plugins.sh bundled with the operator * Fix compile error * Update install-plugins.sh * Remove redundant param to new install-plugins.sh Co-authored-by: Piotr Ryba --- api/v1alpha2/zz_generated.deepcopy.go | 4 +- .../base/resources/scripts_configmap.go | 149 +++++++++++++----- 2 files changed, 114 insertions(+), 39 deletions(-) diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go index 35547563..cfd1b305 100644 --- a/api/v1alpha2/zz_generated.deepcopy.go +++ b/api/v1alpha2/zz_generated.deepcopy.go @@ -23,7 +23,7 @@ package v1alpha2 import ( corev1 "k8s.io/api/core/v1" - "k8s.io/api/rbac/v1" + rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -400,7 +400,7 @@ func (in *JenkinsSpec) DeepCopyInto(out *JenkinsSpec) { in.ConfigurationAsCode.DeepCopyInto(&out.ConfigurationAsCode) if in.Roles != nil { in, out := &in.Roles, &out.Roles - *out = make([]v1.RoleRef, len(*in)) + *out = make([]rbacv1.RoleRef, len(*in)) copy(*out, *in) } in.ServiceAccount.DeepCopyInto(&out.ServiceAccount) diff --git a/pkg/configuration/base/resources/scripts_configmap.go b/pkg/configuration/base/resources/scripts_configmap.go index 2014a42e..ef2bc3b6 100644 --- a/pkg/configuration/base/resources/scripts_configmap.go +++ b/pkg/configuration/base/resources/scripts_configmap.go @@ -15,42 +15,60 @@ import ( const installPluginsCommand = "install-plugins.sh" // bash scripts installs single jenkins plugin with specific version -const installPluginsBashFmt = `#!/bin/bash -eu +const installPluginsBashScript = `#!/bin/bash -eu # Resolve dependencies and download plugins given on the command line # # FROM jenkins # RUN install-plugins.sh docker-slaves github-branch-source +# +# Environment variables: +# REF: directory with preinstalled plugins. Default: /usr/share/jenkins/ref/plugins +# JENKINS_WAR: full path to the jenkins.war. Default: /usr/share/jenkins/jenkins.war +# JENKINS_UC: url of the Update Center. Default: "" +# JENKINS_UC_EXPERIMENTAL: url of the Experimental Update Center for experimental versions of plugins. Default: "" +# JENKINS_INCREMENTALS_REPO_MIRROR: url of the incrementals repo mirror. Default: "" +# JENKINS_UC_DOWNLOAD: download url of the Update Center. Default: JENKINS_UC/download +# CURL_OPTIONS When downloading the plugins with curl. Curl options. Default: -sSfL +# CURL_CONNECTION_TIMEOUT When downloading the plugins with curl. Maximum time allowed for connection. Default: 20 +# CURL_RETRY When downloading the plugins with curl. Retry request if transient problems occur. Default: 3 +# CURL_RETRY_DELAY When downloading the plugins with curl. Wait time between retries. Default: 0 +# CURL_RETRY_MAX_TIME When downloading the plugins with curl. Retry only within this period. Default: 60 set -o pipefail -REF_DIR=${REF:-%s/plugins} -FAILED="$REF_DIR/failed-plugins.txt" +echo "WARN: install-plugins.sh is deprecated, please switch to jenkins-plugin-cli" + +JENKINS_WAR=${JENKINS_WAR:-/usr/share/jenkins/jenkins.war} . /usr/local/bin/jenkins-support +REF_DIR="${REF}/plugins" +FAILED="$REF_DIR/failed-plugins.txt" + getLockFile() { - printf '%%s' "$REF_DIR/${1}.lock" + printf '%s' "$REF_DIR/${1}.lock" } getArchiveFilename() { - printf '%%s' "$REF_DIR/${1}.jpi" + printf '%s' "$REF_DIR/${1}.jpi" } download() { - local plugin originalPlugin version lock ignoreLockFile + local plugin originalPlugin version lock ignoreLockFile url plugin="$1" version="${2:-latest}" ignoreLockFile="${3:-}" + url="${4:-}" lock="$(getLockFile "$plugin")" if [[ $ignoreLockFile ]] || mkdir "$lock" &>/dev/null; then - if ! doDownload "$plugin" "$version"; then + if ! doDownload "$plugin" "$version" "$url"; then # some plugin don't follow the rules about artifact ID # typically: docker-plugin originalPlugin="$plugin" plugin="${plugin}-plugin" - if ! doDownload "$plugin" "$version"; then + if ! doDownload "$plugin" "$version" "$url"; then echo "Failed to download plugin: $originalPlugin or $plugin" >&2 echo "Not downloaded: ${originalPlugin}" >> "$FAILED" return 1 @@ -60,9 +78,11 @@ download() { if ! checkIntegrity "$plugin"; then echo "Downloaded file is not a valid ZIP: $(getArchiveFilename "$plugin")" >&2 echo "Download integrity: ${plugin}" >> "$FAILED" + rm $(getArchiveFilename "$plugin") return 1 fi + resolveDependencies "$plugin" fi } @@ -70,6 +90,7 @@ doDownload() { local plugin version url jpi plugin="$1" version="$2" + url="$3" jpi="$(getArchiveFilename "$plugin")" # If plugin already exists and is the same version do not download @@ -78,7 +99,9 @@ doDownload() { return 0 fi - if [[ "$version" == "latest" && -n "$JENKINS_UC_LATEST" ]]; then + if [[ -n $url ]] ; then + echo "Will use url=$url" + elif [[ "$version" == "latest" && -n "$JENKINS_UC_LATEST" ]]; then # If version-specific Update Center is available, which is the case for LTS versions, # use it to resolve latest versions. url="$JENKINS_UC_LATEST/latest/${plugin}.hpi" @@ -89,7 +112,9 @@ doDownload() { # Download from Incrementals repo: https://jenkins.io/blog/2018/05/15/incremental-deployment/ # Example URL: https://repo.jenkins-ci.org/incrementals/org/jenkins-ci/plugins/workflow/workflow-support/2.19-rc289.d09828a05a74/workflow-support-2.19-rc289.d09828a05a74.hpi local groupId incrementalsVersion - arrIN=(${version//;/ }) + # add a trailing ; so the \n gets added to the end + readarray -t "-d;" arrIN <<<"${version};"; + unset 'arrIN[-1]'; groupId=${arrIN[1]} incrementalsVersion=${arrIN[2]} url="${JENKINS_INCREMENTALS_REPO_MIRROR}/$(echo "${groupId}" | tr '.' '/')/${plugin}/${incrementalsVersion}/${plugin}-${incrementalsVersion}.hpi" @@ -99,7 +124,10 @@ doDownload() { fi echo "Downloading plugin: $plugin from $url" - retry_command curl "${CURL_OPTIONS:--sSfL}" --connect-timeout "${CURL_CONNECTION_TIMEOUT:-20}" --retry "${CURL_RETRY:-5}" --retry-delay "${CURL_RETRY_DELAY:-0}" --retry-max-time "${CURL_RETRY_MAX_TIME:-60}" "$url" -o "$jpi" + # We actually want to allow variable value to be split into multiple options passed to curl. + # This is needed to allow long options and any options that take value. + # shellcheck disable=SC2086 + retry_command curl ${CURL_OPTIONS:--sSfL} --connect-timeout "${CURL_CONNECTION_TIMEOUT:-20}" --retry "${CURL_RETRY:-3}" --retry-delay "${CURL_RETRY_DELAY:-0}" --retry-max-time "${CURL_RETRY_MAX_TIME:-60}" "$url" -o "$jpi" return $? } @@ -112,12 +140,52 @@ checkIntegrity() { return $? } +resolveDependencies() { + local plugin jpi dependencies + plugin="$1" + jpi="$(getArchiveFilename "$plugin")" + + dependencies="$(unzip -p "$jpi" META-INF/MANIFEST.MF | tr -d '\r' | tr '\n' '|' | sed -e 's#| ##g' | tr '|' '\n' | grep "^Plugin-Dependencies: " | sed -e 's#^Plugin-Dependencies: ##')" + + if [[ ! $dependencies ]]; then + echo " > $plugin has no dependencies" + return + fi + + echo " > $plugin depends on $dependencies" + + IFS=',' read -r -a array <<< "$dependencies" + + for d in "${array[@]}" + do + plugin="$(cut -d':' -f1 - <<< "$d")" + if [[ $d == *"resolution:=optional"* ]]; then + echo "Skipping optional dependency $plugin" + else + local pluginInstalled + if pluginInstalled="$(echo -e "${bundledPlugins}\n${installedPlugins}" | grep "^${plugin}:")"; then + pluginInstalled="${pluginInstalled//[$'\r']}" + local versionInstalled; versionInstalled=$(versionFromPlugin "${pluginInstalled}") + local minVersion; minVersion=$(versionFromPlugin "${d}") + if versionLT "${versionInstalled}" "${minVersion}"; then + echo "Upgrading bundled dependency $d ($minVersion > $versionInstalled)" + download "$plugin" & + else + echo "Skipping already installed dependency $d ($minVersion <= $versionInstalled)" + fi + else + download "$plugin" & + fi + fi + done + wait +} + bundledPlugins() { - local JENKINS_WAR=/usr/share/jenkins/jenkins.war - if [ -f $JENKINS_WAR ] + if [ -f "$JENKINS_WAR" ] then TEMP_PLUGIN_DIR=/tmp/plugintemp.$$ - for i in $(jar tf $JENKINS_WAR | grep -E '[^detached-]plugins.*\..pi' | sort) + for i in $(jar tf "$JENKINS_WAR" | grep -E '[^detached-]plugins.*\..pi' | sort) do rm -fr $TEMP_PLUGIN_DIR mkdir -p $TEMP_PLUGIN_DIR @@ -128,9 +196,7 @@ bundledPlugins() { done rm -fr $TEMP_PLUGIN_DIR else - rm -f "$TEMP_ALREADY_INSTALLED" - echo "ERROR file not found: $JENKINS_WAR" - exit 1 + echo "war not found, installing all plugins: $JENKINS_WAR" fi } @@ -151,27 +217,29 @@ installedPlugins() { } jenkinsMajorMinorVersion() { - local JENKINS_WAR - JENKINS_WAR=/usr/share/jenkins/jenkins.war if [[ -f "$JENKINS_WAR" ]]; then local version major minor - version="$(java -jar /usr/share/jenkins/jenkins.war --version)" + version="$(java -jar "$JENKINS_WAR" --version)" major="$(echo "$version" | cut -d '.' -f 1)" minor="$(echo "$version" | cut -d '.' -f 2)" echo "$major.$minor" else - echo "ERROR file not found: $JENKINS_WAR" - return 1 + echo "" fi } main() { - local plugin pluginVersion jenkinsVersion + local plugin jenkinsVersion local plugins=() mkdir -p "$REF_DIR" || exit 1 rm -f "$FAILED" + echo "Cleaning up locks" + find "$REF_DIR" -regex ".*.lock" | while read -r filepath; do + rm -r "$filepath" + done + # Read plugins from stdin or from the command line arguments if [[ ($# -eq 0) ]]; then while read -r line || [ "$line" != "" ]; do @@ -193,16 +261,18 @@ main() { mkdir "$(getLockFile "${plugin%%:*}")" done - echo "Analyzing war..." + echo "Analyzing war $JENKINS_WAR..." bundledPlugins="$(bundledPlugins)" echo "Registering preinstalled plugins..." installedPlugins="$(installedPlugins)" - # Check if there's a version-specific update center, which is the case for LTS versions + # Get the update center URL based on the jenkins version jenkinsVersion="$(jenkinsMajorMinorVersion)" - if curl -fsL -o /dev/null "$JENKINS_UC/$jenkinsVersion"; then - JENKINS_UC_LATEST="$JENKINS_UC/$jenkinsVersion" + # shellcheck disable=SC2086 + jenkinsUcJson=$(curl ${CURL_OPTIONS:--sSfL} -o /dev/null -w "%{url_effective}" "${JENKINS_UC}/update-center.json?version=${jenkinsVersion}") + if [ -n "${jenkinsUcJson}" ]; then + JENKINS_UC_LATEST=${jenkinsUcJson//update-center.json/} echo "Using version-specific update center: $JENKINS_UC_LATEST..." else JENKINS_UC_LATEST= @@ -210,14 +280,16 @@ main() { echo "Downloading plugins..." for plugin in "${plugins[@]}"; do - pluginVersion="" - - if [[ $plugin =~ .*:.* ]]; then - pluginVersion=$(versionFromPlugin "${plugin}") - plugin="${plugin%%:*}" + local reg='^([^:]+):?([^:]+)?:?([^:]+)?:?(http.+)?' + if [[ $plugin =~ $reg ]]; then + local pluginId="${BASH_REMATCH[1]}" + local version="${BASH_REMATCH[2]}" + local lock="${BASH_REMATCH[3]}" + local url="${BASH_REMATCH[4]}" + download "$pluginId" "$version" "${lock:-true}" "${url}" & + else + echo "Skipping the line '${plugin}' as it does not look like a reference to a plugin" fi - - download "$plugin" "$pluginVersion" "true" & done wait @@ -234,7 +306,10 @@ main() { fi echo "Cleaning up locks" - rm -r "$REF_DIR"/*.lock + find "$REF_DIR" -regex ".*.lock" | while read -r filepath; do + rm -r "$filepath" + done + } main "$@" @@ -313,7 +388,7 @@ func buildInitBashScript(jenkins *v1alpha2.Jenkins) (*string, error) { InitConfigurationPath: jenkinsInitConfigurationVolumePath, BasePlugins: jenkins.Spec.Master.BasePlugins, UserPlugins: jenkins.Spec.Master.Plugins, - InstallPluginsCommand: installPluginsCommand, + InstallPluginsCommand: JenkinsScriptsVolumePath + "/" + installPluginsCommand, JenkinsScriptsVolumePath: JenkinsScriptsVolumePath, } @@ -343,7 +418,7 @@ func NewScriptsConfigMap(meta metav1.ObjectMeta, jenkins *v1alpha2.Jenkins) (*co ObjectMeta: meta, Data: map[string]string{ InitScriptName: *initBashScript, - installPluginsCommand: fmt.Sprintf(installPluginsBashFmt, getJenkinsHomePath(jenkins)), + installPluginsCommand: installPluginsBashScript, }, }, nil }