Prepare for Security Validator release (#680)

* Tidy up k8s events logging
* Increase memory and decrease CPU in resources
* Standarize API fields
* Refactor tests
* Change image and resources for e2e tests
* Increase readability

api/v1alpha2/jenkins_types.go

@@ -20,7 +20,7 @@ type JenkinsSpec struct {
 
 	// ValidateSecurityWarnings enables or disables validating potential security warnings in Jenkins plugins via admission webhooks.
 	//+optional
-	ValidateSecurityWarnings bool `json:"ValidateSecurityWarnings,omitempty"`
+	ValidateSecurityWarnings bool `json:"validateSecurityWarnings,omitempty"`
 
 	// Notifications defines list of a services which are used to inform about Jenkins status
 	// Can be used to integrate chat services like Slack, Microsoft Teams or Mailgun

api/v1alpha2/jenkins_webhook.go

@@ -37,12 +37,19 @@ import (
 )
 
 var (
-	jenkinslog                   = logf.Log.WithName("jenkins-resource") // log is for logging in this package.
-	PluginsMgr PluginDataManager = *NewPluginsDataManager("/tmp/plugins.json.gzip", "/tmp/plugins.json", false, time.Duration(1000)*time.Second)
-	_          webhook.Validator = &Jenkins{}
+	jenkinslog                                                = logf.Log.WithName("jenkins-resource") // log is for logging in this package.
+	SecValidator                                              = *NewSecurityValidator()
+	_                                       webhook.Validator = &Jenkins{}
+	initialSecurityWarningsDownloadSucceded                   = false
 )
 
-const Hosturl = "https://ci.jenkins.io/job/Infra/job/plugin-site-api/job/generate-data/lastSuccessfulBuild/artifact/plugins.json.gzip"
+const (
+	Hosturl                 = "https://ci.jenkins.io/job/Infra/job/plugin-site-api/job/generate-data/lastSuccessfulBuild/artifact/plugins.json.gzip"
+	CompressedFilePath      = "/tmp/plugins.json.gzip"
+	PluginDataFile          = "/tmp/plugins.json"
+	shortenedCheckingPeriod = 1 * time.Hour
+	defaultCheckingPeriod   = 12 * time.Minute
+)
 
 func (in *Jenkins) SetupWebhookWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewWebhookManagedBy(mgr).
@@ -50,8 +57,6 @@ func (in *Jenkins) SetupWebhookWithManager(mgr ctrl.Manager) error {
 		Complete()
 }
 
-// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
-
 // TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation.
 // +kubebuilder:webhook:path=/validate-jenkins-io-jenkins-io-v1alpha2-jenkins,mutating=false,failurePolicy=fail,sideEffects=None,groups=jenkins.io.jenkins.io,resources=jenkins,verbs=create;update,versions=v1alpha2,name=vjenkins.kb.io,admissionReviewVersions={v1,v1beta1}
 
@@ -79,14 +84,11 @@ func (in *Jenkins) ValidateDelete() error {
 	return nil
 }
 
-type PluginDataManager struct {
-	PluginDataCache    PluginsInfo
-	Timeout            time.Duration
-	CompressedFilePath string
-	PluginDataFile     string
-	IsCached           bool
-	Attempts           int
-	SleepTime          time.Duration
+type SecurityValidator struct {
+	PluginDataCache PluginsInfo
+	isCached        bool
+	Attempts        int
+	checkingPeriod  time.Duration
 }
 
 type PluginsInfo struct {
@@ -118,7 +120,7 @@ type PluginData struct {
 
 // Validates security warnings for both updating and creating a Jenkins CR
 func Validate(r Jenkins) error {
-	if !PluginsMgr.IsCached {
+	if !SecValidator.isCached {
 		return errors.New("plugins data has not been fetched")
 	}
 
@@ -140,7 +142,7 @@ func Validate(r Jenkins) error {
 		}
 	}
 
-	for _, plugin := range PluginsMgr.PluginDataCache.Plugins {
+	for _, plugin := range SecValidator.PluginDataCache.Plugins {
 		if pluginData, ispresent := pluginSet[plugin.Name]; ispresent {
 			var hasVulnerabilities bool
 			for _, warning := range plugin.SecurityWarnings {
@@ -184,44 +186,42 @@ func Validate(r Jenkins) error {
 	return nil
 }
 
-func NewPluginsDataManager(compressedFilePath string, pluginDataFile string, isCached bool, timeout time.Duration) *PluginDataManager {
-	return &PluginDataManager{
-		CompressedFilePath: compressedFilePath,
-		PluginDataFile:     pluginDataFile,
-		IsCached:           isCached,
-		Timeout:            timeout,
+// NewMonitor creates a new worker and instantiates all the data structures required
+func NewSecurityValidator() *SecurityValidator {
+	return &SecurityValidator{
+		isCached:       false,
+		Attempts:       0,
+		checkingPeriod: shortenedCheckingPeriod,
 	}
 }
 
-func (in *PluginDataManager) ManagePluginData(sig chan bool) {
-	var isInit bool
-	var retryInterval time.Duration
+func (in *SecurityValidator) MonitorSecurityWarnings(securityWarningsFetched chan bool) {
+	jenkinslog.Info("Security warnings check: enabled\n")
 	for {
-		var isCached bool
-		err := in.fetchPluginData()
-		if err == nil {
-			isCached = true
-		} else {
-			jenkinslog.Info("Cache plugin data", "failed to fetch plugin data", err)
-		}
-		// should only be executed once when the operator starts
-		if !isInit {
-			sig <- isCached // sending signal to main to continue
-			isInit = true
-		}
+		in.checkForSecurityVulnerabilities(securityWarningsFetched)
+		<-time.After(in.checkingPeriod)
+	}
+}
 
-		in.IsCached = in.IsCached || isCached
-		if !isCached {
-			retryInterval = time.Duration(1) * time.Hour
-		} else {
-			retryInterval = time.Duration(12) * time.Hour
-		}
-		time.Sleep(retryInterval)
+func (in *SecurityValidator) checkForSecurityVulnerabilities(securityWarningsFetched chan bool) {
+	err := in.fetchPluginData()
+	if err != nil {
+		jenkinslog.Info("Cache plugin data", "failed to fetch plugin data", err)
+		in.checkingPeriod = shortenedCheckingPeriod
+		return
+	}
+	in.isCached = true
+	in.checkingPeriod = defaultCheckingPeriod
+
+	// should only be executed once when the operator starts
+	if !initialSecurityWarningsDownloadSucceded {
+		securityWarningsFetched <- in.isCached
+		initialSecurityWarningsDownloadSucceded = true
 	}
 }
 
 // Downloads extracts and reads the JSON data in every 12 hours
-func (in *PluginDataManager) fetchPluginData() error {
+func (in *SecurityValidator) fetchPluginData() error {
 	jenkinslog.Info("Initializing/Updating the plugin data cache")
 	var err error
 	for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
@@ -262,29 +262,36 @@ func (in *PluginDataManager) fetchPluginData() error {
 	return err
 }
 
-func (in *PluginDataManager) download() error {
-	out, err := os.Create(in.CompressedFilePath)
+func (in *SecurityValidator) download() error {
+	out, err := os.Create(CompressedFilePath)
 	if err != nil {
 		return err
 	}
 	defer out.Close()
 
-	client := http.Client{
-		Timeout: in.Timeout,
-	}
-
-	resp, err := client.Get(Hosturl)
+	req, err := http.NewRequest(http.MethodGet, Hosturl, nil)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+	req.Header.Set("Content-Type", "application/json")
 
-	_, err = io.Copy(out, resp.Body)
+	Client := http.Client{
+		Timeout: 1 * time.Minute,
+	}
+
+	response, err := Client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer response.Body.Close()
+
+	_, err = io.Copy(out, response.Body)
 	return err
 }
 
-func (in *PluginDataManager) extract() error {
-	reader, err := os.Open(in.CompressedFilePath)
+func (in *SecurityValidator) extract() error {
+	reader, err := os.Open(CompressedFilePath)
 
 	if err != nil {
 		return err
@@ -296,7 +303,7 @@ func (in *PluginDataManager) extract() error {
 	}
 
 	defer archive.Close()
-	writer, err := os.Create(in.PluginDataFile)
+	writer, err := os.Create(PluginDataFile)
 	if err != nil {
 		return err
 	}
@@ -307,8 +314,8 @@ func (in *PluginDataManager) extract() error {
 }
 
 // Loads the JSON data into memory and stores it
-func (in *PluginDataManager) cache() error {
-	jsonFile, err := os.Open(in.PluginDataFile)
+func (in *SecurityValidator) cache() error {
+	jsonFile, err := os.Open(PluginDataFile)
 	if err != nil {
 		return err
 	}

api/v1alpha2/jenkins_webhook_test.go

@@ -83,9 +83,9 @@ func TestValidate(t *testing.T) {
 		assert.Equal(t, got, errors.New("plugins data has not been fetched"))
 	})
 
-	PluginsMgr.IsCached = true
+	SecValidator.isCached = true
 	t.Run("Validating a Jenkins CR with plugins not having security warnings and validation is turned on", func(t *testing.T) {
-		PluginsMgr.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
+		SecValidator.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
 			{Name: "security-script"},
 			{Name: "git-client"},
 			{Name: "git"},
@@ -100,7 +100,7 @@ func TestValidate(t *testing.T) {
 	})
 
 	t.Run("Validating a Jenkins CR with some of the plugins having security warnings and validation is turned on", func(t *testing.T) {
-		PluginsMgr.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
+		SecValidator.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
 			{Name: "security-script", SecurityWarnings: createSecurityWarnings("1.2", "2.2")},
 			{Name: "workflow-cps", SecurityWarnings: createSecurityWarnings("2.59", "")},
 			{Name: "git-client"},
@@ -118,7 +118,7 @@ func TestValidate(t *testing.T) {
 	})
 
 	t.Run("Updating a Jenkins CR with some of the plugins having security warnings and validation is turned on", func(t *testing.T) {
-		PluginsMgr.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
+		SecValidator.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
 			{Name: "handy-uri-templates-2-api", SecurityWarnings: createSecurityWarnings("2.1.8-1.0", "2.2.8-1.0")},
 			{Name: "workflow-cps", SecurityWarnings: createSecurityWarnings("2.59", "")},
 			{Name: "resource-disposer", SecurityWarnings: createSecurityWarnings("0.7", "1.2")},

api/v1alpha2/zz_generated.deepcopy.go

@@ -23,7 +23,7 @@ package v1alpha2
 import (
 	corev1 "k8s.io/api/core/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
-	runtime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime"
 )
 
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
@@ -356,6 +356,13 @@ func (in *JenkinsMaster) DeepCopyInto(out *JenkinsMaster) {
 		*out = make([]Plugin, len(*in))
 		copy(*out, *in)
 	}
+	if in.HostAliases != nil {
+		in, out := &in.HostAliases, &out.HostAliases
+		*out = make([]corev1.HostAlias, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JenkinsMaster.
@@ -528,6 +535,65 @@ func (in *Plugin) DeepCopy() *Plugin {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PluginData) DeepCopyInto(out *PluginData) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PluginData.
+func (in *PluginData) DeepCopy() *PluginData {
+	if in == nil {
+		return nil
+	}
+	out := new(PluginData)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PluginInfo) DeepCopyInto(out *PluginInfo) {
+	*out = *in
+	if in.SecurityWarnings != nil {
+		in, out := &in.SecurityWarnings, &out.SecurityWarnings
+		*out = make([]Warning, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PluginInfo.
+func (in *PluginInfo) DeepCopy() *PluginInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(PluginInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *PluginsInfo) DeepCopyInto(out *PluginsInfo) {
+	*out = *in
+	if in.Plugins != nil {
+		in, out := &in.Plugins, &out.Plugins
+		*out = make([]PluginInfo, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PluginsInfo.
+func (in *PluginsInfo) DeepCopy() *PluginsInfo {
+	if in == nil {
+		return nil
+	}
+	out := new(PluginsInfo)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Restore) DeepCopyInto(out *Restore) {
 	*out = *in
@@ -593,6 +659,22 @@ func (in *SecretRef) DeepCopy() *SecretRef {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *SecurityValidator) DeepCopyInto(out *SecurityValidator) {
+	*out = *in
+	in.PluginDataCache.DeepCopyInto(&out.PluginDataCache)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecurityValidator.
+func (in *SecurityValidator) DeepCopy() *SecurityValidator {
+	if in == nil {
+		return nil
+	}
+	out := new(SecurityValidator)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SeedJob) DeepCopyInto(out *SeedJob) {
 	*out = *in
@@ -679,3 +761,38 @@ func (in *Slack) DeepCopy() *Slack {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Version) DeepCopyInto(out *Version) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Version.
+func (in *Version) DeepCopy() *Version {
+	if in == nil {
+		return nil
+	}
+	out := new(Version)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Warning) DeepCopyInto(out *Warning) {
+	*out = *in
+	if in.Versions != nil {
+		in, out := &in.Versions, &out.Versions
+		*out = make([]Version, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Warning.
+func (in *Warning) DeepCopy() *Warning {
+	if in == nil {
+		return nil
+	}
+	out := new(Warning)
+	in.DeepCopyInto(out)
+	return out
+}