public_git
/
kubernetes-operator
mirror of https://github.com/jenkinsci/kubernetes-operator.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Code optimization and cleanup

This commit is contained in:
sharmapulkit04 2021-08-21 20:36:39 +05:30
parent f527a8c5cb
commit 9594c8e7cd
3 changed files with 114 additions and 127 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

171
api/v1alpha2/jenkins_webhook.go
View File

@ -27,12 +27,10 @@ import (
"time"
//"github.com/jenkinsci/kubernetes-operator/pkg/configuration/base/resources"
"github.com/jenkinsci/kubernetes-operator/pkg/constants"
"github.com/jenkinsci/kubernetes-operator/pkg/plugins"
"golang.org/x/mod/semver"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
logf "sigs.k8s.io/controller-runtime/pkg/log"
@ -41,10 +39,12 @@ import (
var (
jenkinslog = logf.Log.WithName("jenkins-resource") // log is for logging in this package.
PluginsMgr PluginDataManager = *NewPluginsDataManager("https://ci.jenkins.io/job/Infra/job/plugin-site-api/job/generate-data/lastSuccessfulBuild/artifact/plugins.json.gzip", "/tmp/plugins.json.gzip", "/tmp/plugins.json", false, time.Duration(1000)*time.Second)
PluginsMgr PluginDataManager = *NewPluginsDataManager("/tmp/plugins.json.gzip", "/tmp/plugins.json", false, time.Duration(1000)*time.Second)
_ webhook.Validator = &Jenkins{}
)
const Hosturl = "https://ci.jenkins.io/job/Infra/job/plugin-site-api/job/generate-data/lastSuccessfulBuild/artifact/plugins.json.gzip"
func (in *Jenkins) SetupWebhookWithManager(mgr ctrl.Manager) error {
return ctrl.NewWebhookManagedBy(mgr).
For(in).
@ -82,13 +82,12 @@ func (in *Jenkins) ValidateDelete() error {
type PluginDataManager struct {
PluginDataCache PluginsInfo
Hosturl string
Timeout time.Duration
CompressedFilePath string
PluginDataFile string
IsCached bool
Attempts int
SleepTime int
SleepTime time.Duration
}
type PluginsInfo struct {
@ -155,7 +154,7 @@ func Validate(r Jenkins) error {
if len(lastVersion) == 0 {
lastVersion = pluginData.Version // setting default value in case of empty string
}
// checking if this warning applies to our version as well
if compareVersions(firstVersion, lastVersion, pluginData.Version) {
jenkinslog.Info("Security Vulnerability detected in "+pluginData.Kind+" "+plugin.Name+":"+pluginData.Version, "Warning message", warning.Message, "For more details,check security advisory", warning.URL)
hasVulnerabilities = true
@ -186,9 +185,8 @@ func Validate(r Jenkins) error {
return nil
}
func NewPluginsDataManager(hosturl string, compressedFilePath string, pluginDataFile string, isCached bool, timeout time.Duration) *PluginDataManager {
func NewPluginsDataManager(compressedFilePath string, pluginDataFile string, isCached bool, timeout time.Duration) *PluginDataManager {
return &PluginDataManager{
Hosturl: hosturl,
CompressedFilePath: compressedFilePath,
PluginDataFile: pluginDataFile,
IsCached: isCached,
@ -196,63 +194,72 @@ func NewPluginsDataManager(hosturl string, compressedFilePath string, pluginData
}
}
// Downloads extracts and reads the JSON data in every 12 hours
func (in *PluginDataManager) FetchPluginData(isInitialized chan bool) {
func (in *PluginDataManager) ManagePluginData(sig chan bool) {
var isInit bool // checks if the operator
var retryInterval time.Duration
for {
jenkinslog.Info("Initializing/Updating the plugin data cache")
var isDownloaded, isExtracted, isCached bool
var err error
for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
err = in.download()
if err == nil {
isDownloaded = true
break
}
isCached := in.fetchPluginData()
if !isInit {
sig <- isCached // sending signal to main to continue
isInit = false
}
if isDownloaded {
for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
err = in.extract()
if err == nil {
isExtracted = true
break
}
}
in.IsCached = in.IsCached || isCached
if !isCached {
retryInterval = time.Duration(1) * time.Hour
} else {
jenkinslog.Info("Cache Plugin Data", "failed to download file", err)
retryInterval = time.Duration(12) * time.Hour
}
if isExtracted {
for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
err = in.cache()
if err == nil {
isCached = true
break
}
}
if !isCached {
jenkinslog.Info("Cache Plugin Data", "failed to read plugin data file", err)
}
} else {
jenkinslog.Info("Cache Plugin Data", "failed to extract file", err)
}
// Checks for the first time
if !in.IsCached {
isInitialized <- isCached
in.IsCached = isCached
}
if isCached {
in.SleepTime = 12
} else {
in.SleepTime = 1
}
time.Sleep(time.Duration(in.SleepTime) * time.Hour)
time.Sleep(retryInterval)
}
}
// Downloads extracts and reads the JSON data in every 12 hours
func (in *PluginDataManager) fetchPluginData() bool {
jenkinslog.Info("Initializing/Updating the plugin data cache")
var err error
for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
err = in.download()
if err == nil {
break
}
jenkinslog.V(1).Info("Cache Plugin Data", "failed to download file", err)
}
if err != nil {
jenkinslog.Info("Cache Plugin Data", "failed to download file", err)
return false
}
for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
err = in.extract()
if err == nil {
break
}
jenkinslog.V(1).Info("Cache Plugin Data", "failed to extract file", err)
}
if err != nil {
jenkinslog.Info("Cache Plugin Data", "failed to extract file", err)
return false
}
for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
err = in.cache()
if err == nil {
break
}
jenkinslog.V(1).Info("Cache Plugin Data", "failed to read plugin data file", err)
}
if err != nil {
jenkinslog.Info("Cache Plugin Data", "failed to read plugin data file", err)
return false
}
return true
}
func (in *PluginDataManager) download() error {
out, err := os.Create(in.CompressedFilePath)
if err != nil {
@ -264,11 +271,12 @@ func (in *PluginDataManager) download() error {
Timeout: in.Timeout,
}
resp, err := client.Get(in.Hosturl)
resp, err := client.Get(Hosturl)
if err != nil {
return err
}
defer resp.Body.Close()
_, err = io.Copy(out, resp.Body)
if err != nil {
return err
@ -307,7 +315,6 @@ func (in *PluginDataManager) cache() error {
return err
}
defer jsonFile.Close()
byteValue, err := ioutil.ReadAll(jsonFile)
if err != nil {
return err
@ -337,47 +344,3 @@ func compareVersions(firstVersion string, lastVersion string, pluginVersion stri
}
return true
}
func CreateJenkinsCR(name string, namespace string, userPlugins []Plugin, validateSecurityWarnings bool) *Jenkins {
jenkins := &Jenkins{
TypeMeta: JenkinsTypeMeta(),
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: namespace,
},
Spec: JenkinsSpec{
GroovyScripts: GroovyScripts{
Customization: Customization{
Configurations: []ConfigMapRef{},
Secret: SecretRef{
Name: "",
},
},
},
ConfigurationAsCode: ConfigurationAsCode{
Customization: Customization{
Configurations: []ConfigMapRef{},
Secret: SecretRef{
Name: "",
},
},
},
Master: JenkinsMaster{
Plugins: userPlugins,
DisableCSRFProtection: false,
},
ValidateSecurityWarnings: validateSecurityWarnings,
Service: Service{
Type: corev1.ServiceTypeNodePort,
Port: constants.DefaultHTTPPortInt32,
},
JenkinsAPISettings: JenkinsAPISettings{AuthorizationStrategy: CreateUserAuthorizationStrategy},
},
}
return jenkins
}
func CreateSecurityWarnings(firstVersion string, lastVersion string) []Warning {
return []Warning{{Versions: []Version{{FirstVersion: firstVersion, LastVersion: lastVersion}}, ID: "null", Message: "unit testing", URL: "null", Active: false}}
}

68
api/v1alpha2/jenkins_webhook_test.go
View File

@ -5,6 +5,7 @@ import (
"testing"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestMakeSemanticVersion(t *testing.T) {
@ -77,7 +78,7 @@ func TestCompareVersions(t *testing.T) {
func TestValidate(t *testing.T) {
t.Run("Validating when plugins data file is not fetched", func(t *testing.T) {
userplugins := []Plugin{{Name: "script-security", Version: "1.77"}, {Name: "git-client", Version: "3.9"}, {Name: "git", Version: "4.8.1"}, {Name: "plain-credentials", Version: "1.7"}}
jenkinscr := *CreateJenkinsCR("Jenkins", "test", userplugins, true)
jenkinscr := *createJenkinsCR(userplugins, true)
got := jenkinscr.ValidateCreate()
assert.Equal(t, got, errors.New("plugins data has not been fetched"))
})
@ -88,66 +89,89 @@ func TestValidate(t *testing.T) {
{Name: "security-script"},
{Name: "git-client"},
{Name: "git"},
{Name: "google-login", SecurityWarnings: CreateSecurityWarnings("", "1.2")},
{Name: "sample-plugin", SecurityWarnings: CreateSecurityWarnings("", "0.8")},
{Name: "google-login", SecurityWarnings: createSecurityWarnings("", "1.2")},
{Name: "sample-plugin", SecurityWarnings: createSecurityWarnings("", "0.8")},
{Name: "mailer"},
{Name: "plain-credentials"}}}
userplugins := []Plugin{{Name: "script-security", Version: "1.77"}, {Name: "git-client", Version: "3.9"}, {Name: "git", Version: "4.8.1"}, {Name: "plain-credentials", Version: "1.7"}}
jenkinscr := *CreateJenkinsCR("Jenkins", "test", userplugins, true)
jenkinscr := *createJenkinsCR(userplugins, true)
got := jenkinscr.ValidateCreate()
assert.Nil(t, got)
})
t.Run("Validating a Jenkins CR with some of the plugins having security warnings and validation is turned on", func(t *testing.T) {
PluginsMgr.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
{Name: "security-script", SecurityWarnings: CreateSecurityWarnings("1.2", "2.2")},
{Name: "workflow-cps", SecurityWarnings: CreateSecurityWarnings("2.59", "")},
{Name: "security-script", SecurityWarnings: createSecurityWarnings("1.2", "2.2")},
{Name: "workflow-cps", SecurityWarnings: createSecurityWarnings("2.59", "")},
{Name: "git-client"},
{Name: "git"},
{Name: "sample-plugin", SecurityWarnings: CreateSecurityWarnings("0.8", "")},
{Name: "command-launcher", SecurityWarnings: CreateSecurityWarnings("1.2", "1.4")},
{Name: "sample-plugin", SecurityWarnings: createSecurityWarnings("0.8", "")},
{Name: "command-launcher", SecurityWarnings: createSecurityWarnings("1.2", "1.4")},
{Name: "plain-credentials"},
{Name: "google-login", SecurityWarnings: CreateSecurityWarnings("1.1", "1.3")},
{Name: "mailer", SecurityWarnings: CreateSecurityWarnings("1.0.3", "1.1.4")},
{Name: "google-login", SecurityWarnings: createSecurityWarnings("1.1", "1.3")},
{Name: "mailer", SecurityWarnings: createSecurityWarnings("1.0.3", "1.1.4")},
}}
userplugins := []Plugin{{Name: "google-login", Version: "1.2"}, {Name: "mailer", Version: "1.1"}, {Name: "git", Version: "4.8.1"}, {Name: "command-launcher", Version: "1.6"}, {Name: "workflow-cps", Version: "2.59"}}
jenkinscr := *CreateJenkinsCR("Jenkins", "test", userplugins, true)
jenkinscr := *createJenkinsCR(userplugins, true)
got := jenkinscr.ValidateCreate()
assert.Equal(t, got, errors.New("security vulnerabilities detected in the following user-defined plugins: \nworkflow-cps:2.59\ngoogle-login:1.2\nmailer:1.1"))
})
t.Run("Updating a Jenkins CR with some of the plugins having security warnings and validation is turned on", func(t *testing.T) {
PluginsMgr.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
{Name: "handy-uri-templates-2-api", SecurityWarnings: CreateSecurityWarnings("2.1.8-1.0", "2.2.8-1.0")},
{Name: "workflow-cps", SecurityWarnings: CreateSecurityWarnings("2.59", "")},
{Name: "resource-disposer", SecurityWarnings: CreateSecurityWarnings("0.7", "1.2")},
{Name: "handy-uri-templates-2-api", SecurityWarnings: createSecurityWarnings("2.1.8-1.0", "2.2.8-1.0")},
{Name: "workflow-cps", SecurityWarnings: createSecurityWarnings("2.59", "")},
{Name: "resource-disposer", SecurityWarnings: createSecurityWarnings("0.7", "1.2")},
{Name: "git"},
{Name: "jjwt-api"},
{Name: "blueocean-github-pipeline", SecurityWarnings: CreateSecurityWarnings("1.2.0-alpha-2", "1.2.0-beta-5")},
{Name: "command-launcher", SecurityWarnings: CreateSecurityWarnings("1.2", "1.4")},
{Name: "blueocean-github-pipeline", SecurityWarnings: createSecurityWarnings("1.2.0-alpha-2", "1.2.0-beta-5")},
{Name: "command-launcher", SecurityWarnings: createSecurityWarnings("1.2", "1.4")},
{Name: "plain-credentials"},
{Name: "ghprb", SecurityWarnings: CreateSecurityWarnings("1.1", "1.43")},
{Name: "mailer", SecurityWarnings: CreateSecurityWarnings("1.0.3", "1.1.4")},
{Name: "ghprb", SecurityWarnings: createSecurityWarnings("1.1", "1.43")},
{Name: "mailer", SecurityWarnings: createSecurityWarnings("1.0.3", "1.1.4")},
}}
userplugins := []Plugin{{Name: "google-login", Version: "1.2"}, {Name: "mailer", Version: "1.1"}, {Name: "git", Version: "4.8.1"}, {Name: "command-launcher", Version: "1.6"}, {Name: "workflow-cps", Version: "2.59"}}
oldjenkinscr := *CreateJenkinsCR("Jenkins", "test", userplugins, true)
oldjenkinscr := *createJenkinsCR(userplugins, true)
userplugins = []Plugin{{Name: "handy-uri-templates-2-api", Version: "2.1.8-1.0"}, {Name: "resource-disposer", Version: "0.8"}, {Name: "jjwt-api", Version: "0.11.2-9.c8b45b8bb173"}, {Name: "blueocean-github-pipeline", Version: "1.2.0-beta-3"}, {Name: "ghprb", Version: "1.39"}}
newjenkinscr := *CreateJenkinsCR("Jenkins", "test", userplugins, true)
newjenkinscr := *createJenkinsCR(userplugins, true)
got := newjenkinscr.ValidateUpdate(&oldjenkinscr)
assert.Equal(t, got, errors.New("security vulnerabilities detected in the following user-defined plugins: \nhandy-uri-templates-2-api:2.1.8-1.0\nresource-disposer:0.8\nblueocean-github-pipeline:1.2.0-beta-3\nghprb:1.39"))
})
t.Run("Validation is turned off", func(t *testing.T) {
userplugins := []Plugin{{Name: "google-login", Version: "1.2"}, {Name: "mailer", Version: "1.1"}, {Name: "git", Version: "4.8.1"}, {Name: "command-launcher", Version: "1.6"}, {Name: "workflow-cps", Version: "2.59"}}
jenkinscr := *CreateJenkinsCR("Jenkins", "test", userplugins, false)
jenkinscr := *createJenkinsCR(userplugins, false)
got := jenkinscr.ValidateCreate()
assert.Nil(t, got)
userplugins = []Plugin{{Name: "google-login", Version: "1.2"}, {Name: "mailer", Version: "1.1"}, {Name: "git", Version: "4.8.1"}, {Name: "command-launcher", Version: "1.6"}, {Name: "workflow-cps", Version: "2.59"}}
newjenkinscr := *CreateJenkinsCR("jenkins", "test", userplugins, false)
newjenkinscr := *createJenkinsCR(userplugins, false)
got = newjenkinscr.ValidateUpdate(&jenkinscr)
assert.Nil(t, got)
})
}
func createJenkinsCR(userPlugins []Plugin, validateSecurityWarnings bool) *Jenkins {
jenkins := &Jenkins{
TypeMeta: JenkinsTypeMeta(),
ObjectMeta: metav1.ObjectMeta{
Name: "jenkins",
Namespace: "test",
},
Spec: JenkinsSpec{
Master: JenkinsMaster{
Plugins: userPlugins,
DisableCSRFProtection: false,
},
ValidateSecurityWarnings: validateSecurityWarnings,
},
}
return jenkins
}
func createSecurityWarnings(firstVersion string, lastVersion string) []Warning {
return []Warning{{Versions: []Version{{FirstVersion: firstVersion, LastVersion: lastVersion}}, ID: "null", Message: "unit testing", URL: "null", Active: false}}
}

2
main.go
View File

@ -113,7 +113,7 @@ func main() {
if ValidateSecurityWarnings {
isInitialized := make(chan bool)
go v1alpha2.PluginsMgr.FetchPluginData(isInitialized)
go v1alpha2.PluginsMgr.ManagePluginData(isInitialized)
if !<-isInitialized {
logger.Info("Unable to get the plugins data")