Update and use install-plugins.sh (#756)

* Download install-plugins.sh manually from remote

* Reformat

* Goimports

* Use install-plugins.sh bundled with the operator

* Fix compile error

* Update install-plugins.sh

* Remove redundant param to new install-plugins.sh

Co-authored-by: Piotr Ryba <pryba@virtuslab.com>

api/v1alpha2/zz_generated.deepcopy.go

@@ -23,7 +23,7 @@ package v1alpha2
 
 import (
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/api/rbac/v1"
+	rbacv1 "k8s.io/api/rbac/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 )
 
@@ -400,7 +400,7 @@ func (in *JenkinsSpec) DeepCopyInto(out *JenkinsSpec) {
 	in.ConfigurationAsCode.DeepCopyInto(&out.ConfigurationAsCode)
 	if in.Roles != nil {
 		in, out := &in.Roles, &out.Roles
-		*out = make([]v1.RoleRef, len(*in))
+		*out = make([]rbacv1.RoleRef, len(*in))
 		copy(*out, *in)
 	}
 	in.ServiceAccount.DeepCopyInto(&out.ServiceAccount)

pkg/configuration/base/resources/scripts_configmap.go

@@ -15,42 +15,60 @@ import (
 const installPluginsCommand = "install-plugins.sh"
 
 // bash scripts installs single jenkins plugin with specific version
-const installPluginsBashFmt = `#!/bin/bash -eu
+const installPluginsBashScript = `#!/bin/bash -eu
 
 # Resolve dependencies and download plugins given on the command line
 #
 # FROM jenkins
 # RUN install-plugins.sh docker-slaves github-branch-source
+#
+# Environment variables:
+# REF: directory with preinstalled plugins. Default: /usr/share/jenkins/ref/plugins
+# JENKINS_WAR: full path to the jenkins.war. Default: /usr/share/jenkins/jenkins.war
+# JENKINS_UC: url of the Update Center. Default: ""
+# JENKINS_UC_EXPERIMENTAL: url of the Experimental Update Center for experimental versions of plugins. Default: ""
+# JENKINS_INCREMENTALS_REPO_MIRROR: url of the incrementals repo mirror. Default: ""
+# JENKINS_UC_DOWNLOAD: download url of the Update Center. Default: JENKINS_UC/download
+# CURL_OPTIONS When downloading the plugins with curl. Curl options. Default: -sSfL
+# CURL_CONNECTION_TIMEOUT When downloading the plugins with curl. <seconds> Maximum time allowed for connection. Default: 20
+# CURL_RETRY When downloading the plugins with curl. Retry request if transient problems occur. Default: 3
+# CURL_RETRY_DELAY When downloading the plugins with curl. <seconds> Wait time between retries. Default: 0
+# CURL_RETRY_MAX_TIME When downloading the plugins with curl. <seconds> Retry only within this period. Default: 60
 
 set -o pipefail
 
-REF_DIR=${REF:-%s/plugins}
+echo "WARN: install-plugins.sh is deprecated, please switch to jenkins-plugin-cli"
-FAILED="$REF_DIR/failed-plugins.txt"
+
+JENKINS_WAR=${JENKINS_WAR:-/usr/share/jenkins/jenkins.war}
 
 . /usr/local/bin/jenkins-support
 
+REF_DIR="${REF}/plugins"
+FAILED="$REF_DIR/failed-plugins.txt"
+
 getLockFile() {
-    printf '%%s' "$REF_DIR/${1}.lock"
+    printf '%s' "$REF_DIR/${1}.lock"
 }
 
 getArchiveFilename() {
-    printf '%%s' "$REF_DIR/${1}.jpi"
+    printf '%s' "$REF_DIR/${1}.jpi"
 }
 
 download() {
-    local plugin originalPlugin version lock ignoreLockFile
+    local plugin originalPlugin version lock ignoreLockFile url
     plugin="$1"
     version="${2:-latest}"
     ignoreLockFile="${3:-}"
+    url="${4:-}"
     lock="$(getLockFile "$plugin")"
 
     if [[ $ignoreLockFile ]] || mkdir "$lock" &>/dev/null; then
-        if ! doDownload "$plugin" "$version"; then
+        if ! doDownload "$plugin" "$version" "$url"; then
             # some plugin don't follow the rules about artifact ID
             # typically: docker-plugin
             originalPlugin="$plugin"
             plugin="${plugin}-plugin"
-            if ! doDownload "$plugin" "$version"; then
+            if ! doDownload "$plugin" "$version" "$url"; then
                 echo "Failed to download plugin: $originalPlugin or $plugin" >&2
                 echo "Not downloaded: ${originalPlugin}" >> "$FAILED"
                 return 1
@@ -60,9 +78,11 @@ download() {
         if ! checkIntegrity "$plugin"; then
             echo "Downloaded file is not a valid ZIP: $(getArchiveFilename "$plugin")" >&2
             echo "Download integrity: ${plugin}" >> "$FAILED"
+            rm $(getArchiveFilename "$plugin")
             return 1
         fi
 
+        resolveDependencies "$plugin"
     fi
 }
 
@@ -70,6 +90,7 @@ doDownload() {
     local plugin version url jpi
     plugin="$1"
     version="$2"
+    url="$3"
     jpi="$(getArchiveFilename "$plugin")"
 
     # If plugin already exists and is the same version do not download
@@ -78,7 +99,9 @@ doDownload() {
         return 0
     fi
 
-    if [[ "$version" == "latest" && -n "$JENKINS_UC_LATEST" ]]; then
+    if [[ -n $url ]] ; then
+        echo "Will use url=$url"
+    elif [[ "$version" == "latest" && -n "$JENKINS_UC_LATEST" ]]; then
         # If version-specific Update Center is available, which is the case for LTS versions,
         # use it to resolve latest versions.
         url="$JENKINS_UC_LATEST/latest/${plugin}.hpi"
@@ -89,7 +112,9 @@ doDownload() {
         # Download from Incrementals repo: https://jenkins.io/blog/2018/05/15/incremental-deployment/
         # Example URL: https://repo.jenkins-ci.org/incrementals/org/jenkins-ci/plugins/workflow/workflow-support/2.19-rc289.d09828a05a74/workflow-support-2.19-rc289.d09828a05a74.hpi
         local groupId incrementalsVersion
-        arrIN=(${version//;/ })
+        # add a trailing ; so the \n gets added to the end
+        readarray -t "-d;" arrIN <<<"${version};";
+        unset 'arrIN[-1]';
         groupId=${arrIN[1]}
         incrementalsVersion=${arrIN[2]}
         url="${JENKINS_INCREMENTALS_REPO_MIRROR}/$(echo "${groupId}" | tr '.' '/')/${plugin}/${incrementalsVersion}/${plugin}-${incrementalsVersion}.hpi"
@@ -99,7 +124,10 @@ doDownload() {
     fi
 
     echo "Downloading plugin: $plugin from $url"
-    retry_command curl "${CURL_OPTIONS:--sSfL}" --connect-timeout "${CURL_CONNECTION_TIMEOUT:-20}" --retry "${CURL_RETRY:-5}" --retry-delay "${CURL_RETRY_DELAY:-0}" --retry-max-time "${CURL_RETRY_MAX_TIME:-60}" "$url" -o "$jpi"
+    # We actually want to allow variable value to be split into multiple options passed to curl.
+    # This is needed to allow long options and any options that take value.
+    # shellcheck disable=SC2086
+    retry_command curl ${CURL_OPTIONS:--sSfL} --connect-timeout "${CURL_CONNECTION_TIMEOUT:-20}" --retry "${CURL_RETRY:-3}" --retry-delay "${CURL_RETRY_DELAY:-0}" --retry-max-time "${CURL_RETRY_MAX_TIME:-60}" "$url" -o "$jpi"
     return $?
 }
 
@@ -112,12 +140,52 @@ checkIntegrity() {
     return $?
 }
 
+resolveDependencies() {
+    local plugin jpi dependencies
+    plugin="$1"
+    jpi="$(getArchiveFilename "$plugin")"
+
+    dependencies="$(unzip -p "$jpi" META-INF/MANIFEST.MF | tr -d '\r' | tr '\n' '|' | sed -e 's#| ##g' | tr '|' '\n' | grep "^Plugin-Dependencies: " | sed -e 's#^Plugin-Dependencies: ##')"
+
+    if [[ ! $dependencies ]]; then
+        echo " > $plugin has no dependencies"
+        return
+    fi
+
+    echo " > $plugin depends on $dependencies"
+
+    IFS=',' read -r -a array <<< "$dependencies"
+
+    for d in "${array[@]}"
+    do
+        plugin="$(cut -d':' -f1 - <<< "$d")"
+        if [[ $d == *"resolution:=optional"* ]]; then
+            echo "Skipping optional dependency $plugin"
+        else
+            local pluginInstalled
+            if pluginInstalled="$(echo -e "${bundledPlugins}\n${installedPlugins}" | grep "^${plugin}:")"; then
+                pluginInstalled="${pluginInstalled//[$'\r']}"
+                local versionInstalled; versionInstalled=$(versionFromPlugin "${pluginInstalled}")
+                local minVersion; minVersion=$(versionFromPlugin "${d}")
+                if versionLT "${versionInstalled}" "${minVersion}"; then
+                    echo "Upgrading bundled dependency $d ($minVersion > $versionInstalled)"
+                    download "$plugin" &
+                else
+                    echo "Skipping already installed dependency $d ($minVersion <= $versionInstalled)"
+                fi
+            else
+                download "$plugin" &
+            fi
+        fi
+    done
+    wait
+}
+
 bundledPlugins() {
-    local JENKINS_WAR=/usr/share/jenkins/jenkins.war
+    if [ -f "$JENKINS_WAR" ]
-    if [ -f $JENKINS_WAR ]
     then
         TEMP_PLUGIN_DIR=/tmp/plugintemp.$$
-        for i in $(jar tf $JENKINS_WAR | grep -E '[^detached-]plugins.*\..pi' | sort)
+        for i in $(jar tf "$JENKINS_WAR" | grep -E '[^detached-]plugins.*\..pi' | sort)
         do
             rm -fr $TEMP_PLUGIN_DIR
             mkdir -p $TEMP_PLUGIN_DIR
@@ -128,9 +196,7 @@ bundledPlugins() {
         done
         rm -fr $TEMP_PLUGIN_DIR
     else
-        rm -f "$TEMP_ALREADY_INSTALLED"
+        echo "war not found, installing all plugins: $JENKINS_WAR"
-        echo "ERROR file not found: $JENKINS_WAR"
-        exit 1
     fi
 }
 
@@ -151,27 +217,29 @@ installedPlugins() {
 }
 
 jenkinsMajorMinorVersion() {
-    local JENKINS_WAR
-    JENKINS_WAR=/usr/share/jenkins/jenkins.war
     if [[ -f "$JENKINS_WAR" ]]; then
         local version major minor
-        version="$(java -jar /usr/share/jenkins/jenkins.war --version)"
+        version="$(java -jar "$JENKINS_WAR" --version)"
         major="$(echo "$version" | cut -d '.' -f 1)"
         minor="$(echo "$version" | cut -d '.' -f 2)"
         echo "$major.$minor"
     else
-        echo "ERROR file not found: $JENKINS_WAR"
+        echo ""
-        return 1
     fi
 }
 
 main() {
-    local plugin pluginVersion jenkinsVersion
+    local plugin jenkinsVersion
     local plugins=()
 
     mkdir -p "$REF_DIR" || exit 1
     rm -f "$FAILED"
 
+	echo "Cleaning up locks"
+	find "$REF_DIR" -regex ".*.lock" | while read -r filepath; do
+		rm -r "$filepath"
+	done
+
     # Read plugins from stdin or from the command line arguments
     if [[ ($# -eq 0) ]]; then
         while read -r line || [ "$line" != "" ]; do
@@ -193,16 +261,18 @@ main() {
         mkdir "$(getLockFile "${plugin%%:*}")"
     done
 
-    echo "Analyzing war..."
+    echo "Analyzing war $JENKINS_WAR..."
     bundledPlugins="$(bundledPlugins)"
 
     echo "Registering preinstalled plugins..."
     installedPlugins="$(installedPlugins)"
 
-    # Check if there's a version-specific update center, which is the case for LTS versions
+    # Get the update center URL based on the jenkins version
     jenkinsVersion="$(jenkinsMajorMinorVersion)"
-    if curl -fsL -o /dev/null "$JENKINS_UC/$jenkinsVersion"; then
+    # shellcheck disable=SC2086
-        JENKINS_UC_LATEST="$JENKINS_UC/$jenkinsVersion"
+    jenkinsUcJson=$(curl ${CURL_OPTIONS:--sSfL} -o /dev/null -w "%{url_effective}" "${JENKINS_UC}/update-center.json?version=${jenkinsVersion}")
+    if [ -n "${jenkinsUcJson}" ]; then
+        JENKINS_UC_LATEST=${jenkinsUcJson//update-center.json/}
         echo "Using version-specific update center: $JENKINS_UC_LATEST..."
     else
         JENKINS_UC_LATEST=
@@ -210,14 +280,16 @@ main() {
 
     echo "Downloading plugins..."
     for plugin in "${plugins[@]}"; do
-        pluginVersion=""
+        local reg='^([^:]+):?([^:]+)?:?([^:]+)?:?(http.+)?'
-
+        if [[ $plugin =~ $reg ]]; then
-        if [[ $plugin =~ .*:.* ]]; then
+            local pluginId="${BASH_REMATCH[1]}"
-            pluginVersion=$(versionFromPlugin "${plugin}")
+            local version="${BASH_REMATCH[2]}"
-            plugin="${plugin%%:*}"
+            local lock="${BASH_REMATCH[3]}"
+            local url="${BASH_REMATCH[4]}"
+            download "$pluginId" "$version" "${lock:-true}" "${url}" &
+        else
+          echo "Skipping the line '${plugin}' as it does not look like a reference to a plugin"
         fi
-
-        download "$plugin" "$pluginVersion" "true" &
     done
     wait
 
@@ -234,7 +306,10 @@ main() {
     fi
 
     echo "Cleaning up locks"
-    rm -r "$REF_DIR"/*.lock
+    find "$REF_DIR" -regex ".*.lock" | while read -r filepath; do
+        rm -r "$filepath"
+    done
+
 }
 
 main "$@"
@@ -313,7 +388,7 @@ func buildInitBashScript(jenkins *v1alpha2.Jenkins) (*string, error) {
 		InitConfigurationPath:    jenkinsInitConfigurationVolumePath,
 		BasePlugins:              jenkins.Spec.Master.BasePlugins,
 		UserPlugins:              jenkins.Spec.Master.Plugins,
-		InstallPluginsCommand:    installPluginsCommand,
+		InstallPluginsCommand:    JenkinsScriptsVolumePath + "/" + installPluginsCommand,
 		JenkinsScriptsVolumePath: JenkinsScriptsVolumePath,
 	}
 
@@ -343,7 +418,7 @@ func NewScriptsConfigMap(meta metav1.ObjectMeta, jenkins *v1alpha2.Jenkins) (*co
 		ObjectMeta: meta,
 		Data: map[string]string{
 			InitScriptName:        *initBashScript,
-			installPluginsCommand: fmt.Sprintf(installPluginsBashFmt, getJenkinsHomePath(jenkins)),
+			installPluginsCommand: installPluginsBashScript,
 		},
 	}, nil
 }