Reimplemented the validation logic with caching the security warnings

- Reimplemented the validator interface - Updated manifests to allocate more resources
2026-10-08 09:01:15 +02:00 · 2021-08-05 04:39:27 +05:30
parent 52fe5fe95e
commit 37d0eac4e3
7 changed files with 286 additions and 105 deletions
@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

-    http://www.apache.org/licenses/LICENSE-2.0
+    http://www.apache.org/lictenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,10 +17,13 @@ limitations under the License.
 package v1alpha2

 import (
+	"compress/gzip"
 	"encoding/json"
 	"errors"
+	"io"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"time"

 	"github.com/jenkinsci/kubernetes-operator/pkg/plugins"
@@ -32,8 +35,16 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 )

-// log is for logging in this package.
-var jenkinslog = logf.Log.WithName("jenkins-resource")
+var (
+	jenkinslog       = logf.Log.WithName("jenkins-resource") // log is for logging in this package.
+	isRetrieved bool = false                                 // For checking whether the data file is downloaded and extracted or not
+)
+
+const (
+	hosturl        string = "https://ci.jenkins.io/job/Infra/job/plugin-site-api/job/generate-data/lastSuccessfulBuild/artifact/plugins.json.gzip" // Url for downloading the plugins file
+	compressedFile string = "/tmp/plugins.json.gzip"                                                                                               // location where the gzip file will be downloaded
+	pluginDataFile string = "/tmp/plugins.json"                                                                                                    // location where the file will be extracted
+)

 func (in *Jenkins) SetupWebhookWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewWebhookManagedBy(mgr).
@@ -72,8 +83,13 @@ func (in *Jenkins) ValidateDelete() error {
 	return nil
 }

-type Warnings struct {
-	Warnings []Warning `json:"securityWarnings"`
+type PluginsInfo struct {
+	Plugins []PluginInfo `json:"plugins"`
+}
+
+type PluginInfo struct {
+	Name             string    `json:"name"`
+	SecurityWarnings []Warning `json:"securityWarnings"`
 }

 type Warning struct {
@@ -83,18 +99,197 @@ type Warning struct {
 	URL      string    `json:"url"`
 	Active   bool      `json:"active"`
 }
+
 type Version struct {
 	FirstVersion string `json:"firstVersion"`
 	LastVersion  string `json:"lastVersion"`
 }

-const APIURL string = "https://plugins.jenkins.io/api/plugin/"
+type PluginData struct {
+	Version string
+	Kind    string
+}

+// Validates security warnings for both updating and creating a Jenkins CR
+func Validate(r Jenkins) error {
+
+	pluginset := make(map[string]PluginData)
+	var warningmsg string
+	basePlugins := plugins.BasePlugins()
+	temp, err := NewPluginsInfo()
+	AllPluginData := *temp
+	if err != nil {
+		return err
+	}
+
+	for _, plugin := range basePlugins {
+		// Only Update the map if the plugin is not present or a lower version is being used
+		if pluginData, ispresent := pluginset[plugin.Name]; !ispresent || semver.Compare(MakeSemanticVersion(plugin.Version), pluginData.Version) == 1 {
+			jenkinslog.Info("Validate", plugin.Name, plugin.Version)
+			pluginset[plugin.Name] = PluginData{Version: plugin.Version, Kind: "base"}
+		}
+	}
+
+	for _, plugin := range r.Spec.Master.Plugins {
+		if pluginData, ispresent := pluginset[plugin.Name]; !ispresent || semver.Compare(MakeSemanticVersion(plugin.Version), pluginData.Version) == 1 {
+			jenkinslog.Info("Validate", plugin.Name, plugin.Version)
+			pluginset[plugin.Name] = PluginData{Version: plugin.Version, Kind: "user-defined"}
+		}
+	}
+
+	jenkinslog.Info("Checking through all the warnings")
+	for _, plugin := range AllPluginData.Plugins {
+
+		if pluginData, ispresent := pluginset[plugin.Name]; ispresent {
+			jenkinslog.Info("Checking for plugin", "name", plugin.Name)
+			for _, warning := range plugin.SecurityWarnings {
+				for _, version := range warning.Versions {
+					firstVersion := version.FirstVersion
+					lastVersion := version.LastVersion
+					if len(firstVersion) == 0 {
+						firstVersion = "0" // setting default value in case of empty string
+					}
+					if len(lastVersion) == 0 {
+						lastVersion = pluginData.Version // setting default value in case of empty string
+					}
+
+					if CompareVersions(firstVersion, lastVersion, pluginData.Version) {
+						jenkinslog.Info("security Vulnerabilities detected", "message", warning.Message, "Check security Advisory", warning.URL)
+						warningmsg += "Security Vulnerabilities detected in " + pluginData.Kind + " plugin " + plugin.Name + "\n"
+
+					}
+
+				}
+			}
+
+		}
+
+	}
+
+	if len(warningmsg) > 0 {
+		return errors.New(warningmsg)
+	}
+
+	return nil
+
+}
+
+// Returns an object containing information of all the plugins present in the security center
+func NewPluginsInfo() (*PluginsInfo, error) {
+	var AllPluginData PluginsInfo
+	for i := 0; i < 28; i++ {
+		if isRetrieved {
+			break
+		}
+		time.Sleep(1 * time.Second)
+	}
+	if !isRetrieved {
+		jenkinslog.Info("Plugins Data file hasn't been downloaded and extracted")
+		return &AllPluginData, errors.New("plugins data file not found")
+	}
+
+	jsonFile, err := os.Open(pluginDataFile)
+	if err != nil {
+		jenkinslog.Info("Failed to open the Plugins Data File")
+		return &AllPluginData, err
+	}
+	defer jsonFile.Close()
+
+	byteValue, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		jenkinslog.Info("Failed to convert the JSON file into a byte array")
+		return &AllPluginData, err
+	}
+	err = json.Unmarshal(byteValue, &AllPluginData)
+	if err != nil {
+		jenkinslog.Info("Failed to decode the Plugin JSON data file")
+		return &AllPluginData, err
+	}
+
+	return &AllPluginData, nil
+}
+
+// Downloads and extracts the JSON file in every 12 hours
+func RetrieveDataFile() {
+	for {
+		jenkinslog.Info("Retreiving file", "Host Url", hosturl)
+		err := Download()
+		if err != nil {
+			jenkinslog.Info("Retrieving File", "Error while downloading", err)
+			continue
+		}
+
+		jenkinslog.Info("Retrieve File", "Successfully downloaded", compressedFile)
+		err = Extract()
+		if err != nil {
+			jenkinslog.Info("Retreive File", "Error while extracting", err)
+			continue
+		}
+		jenkinslog.Info("Retreive File", "Successfully extracted", pluginDataFile)
+		isRetrieved = true
+		time.Sleep(12 * time.Hour)
+
+	}
+}
+
+func Download() error {
+
+	out, err := os.Create(compressedFile)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	client := http.Client{
+		Timeout: 2000 * time.Second,
+	}
+
+	resp, err := client.Get(hosturl)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	jenkinslog.Info("Successfully Downloaded")
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+func Extract() error {
+	reader, err := os.Open(compressedFile)
+
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+	archive, err := gzip.NewReader(reader)
+	if err != nil {
+		return err
+	}
+
+	defer archive.Close()
+	writer, err := os.Create(pluginDataFile)
+	if err != nil {
+		return err
+	}
+	defer writer.Close()
+
+	_, err = io.Copy(writer, archive)
+	return err
+
+}
+
+// returns a semantic version that can be used for comparision
 func MakeSemanticVersion(version string) string {
 	version = "v" + version
 	return semver.Canonical(version)
 }

+// Compare if the current version lies between first version and last version
 func CompareVersions(firstVersion string, lastVersion string, pluginVersion string) bool {
 	firstSemVer := MakeSemanticVersion(firstVersion)
 	lastSemVer := MakeSemanticVersion(lastVersion)
@@ -104,88 +299,3 @@ func CompareVersions(firstVersion string, lastVersion string, pluginVersion stri
 	}
 	return true
 }
-
-func CheckSecurityWarnings(pluginName string, pluginVersion string) (bool, error) {
-	jenkinslog.Info("checking security warnings", "plugin: ", pluginName)
-	pluginURL := APIURL + pluginName
-	client := &http.Client{
-		Timeout: time.Second * 30,
-	}
-	request, err := http.NewRequest("GET", pluginURL, nil)
-	if err != nil {
-		return false, err
-	}
-	request.Header.Add("Accept", "application/json")
-	request.Header.Add("Content-Type", "application/json")
-	response, err := client.Do(request)
-	if err != nil {
-		return false, err
-	}
-	defer response.Body.Close()
-	bodyBytes, err := ioutil.ReadAll(response.Body)
-	if err != nil {
-		return false, err
-	}
-	securityWarnings := Warnings{}
-
-	jsonErr := json.Unmarshal(bodyBytes, &securityWarnings)
-	if jsonErr != nil {
-		return false, err
-	}
-
-	jenkinslog.Info("Validate()", "warnings", securityWarnings)
-
-	for _, warning := range securityWarnings.Warnings {
-		for _, version := range warning.Versions {
-			firstVersion := version.FirstVersion
-			lastVersion := version.LastVersion
-			if len(firstVersion) == 0 {
-				firstVersion = "0" // setting default value in case of empty string
-			}
-			if len(lastVersion) == 0 {
-				lastVersion = pluginVersion // setting default value in case of empty string
-			}
-
-			if CompareVersions(firstVersion, lastVersion, pluginVersion) {
-				jenkinslog.Info("security Vulnerabilities detected", "message", warning.Message, "Check security Advisory", warning.URL)
-				return true, nil
-			}
-		}
-	}
-
-	return false, nil
-}
-
-func Validate(r Jenkins) error {
-	basePlugins := plugins.BasePlugins()
-	var warnings string = ""
-
-	for _, plugin := range basePlugins {
-		name := plugin.Name
-		version := plugin.Version
-		hasWarnings, err := CheckSecurityWarnings(name, version)
-		if err != nil {
-			return err
-		}
-		if hasWarnings {
-			warnings += "Security Vulnerabilities detected in base plugin:" + name
-		}
-	}
-
-	for _, plugin := range r.Spec.Master.Plugins {
-		name := plugin.Name
-		version := plugin.Version
-		hasWarnings, err := CheckSecurityWarnings(name, version)
-		if err != nil {
-			return err
-		}
-		if hasWarnings {
-			warnings += "Security Vulnerabilities detected in the user defined plugin: " + name
-		}
-	}
-	if len(warnings) > 0 {
-		return errors.New(warnings)
-	}
-
-	return nil
-}