Implemented validation logic for the webhook (#593)

* Fix workflow for autogenerating docs (#592)

* Use grep -c flag in check for changes step to fix case when more than 1 website file was modified

* Implemented validation logic for the webhook
- Created a single Validate() function to validate both updating and creating Jenkins CR.
- Implemented the Validate function to fetch warnings from the API and do security check if
  being enabled.
- Updated the helm charts and helm-e2e target to run the helm tests.

* Configure bot for labelling new issues as needing triage (#597)

* Configure bot for managing stale issues (#598)

* Docs: explanation what is backed up and why (#599)

* Explanation what's backed up and why

* Auto-updated docs (#600)

Co-authored-by: prryb <prryb@users.noreply.github.com>

* Docs: clarification of description of get latest command in backup (#601)

* Auto-updated docs (#602)

Co-authored-by: Sig00rd <Sig00rd@users.noreply.github.com>

* Bump seedjobs agent image version to 4.9-1 (#604)

* Add GitLFS pull after checkout behaviour to SeedJob GroovyScript Template (#483)

Add GitLFS pull after checkout behaviour to support also repositories which are relying on Git LFS

Close #482

* Docs: minor fixes (#608)

* Link to project's DockerHub in README's section on nightly builds, add paragraph about nightly builds in installation docs

* Fix repositoryURL in sample seedJob configuration with SSH auth

* Slightly expand on #348

* Fix formatting in docs on Jenkins' customization, update plugin versions

* Add notes on Jenkins home Volume in Helm chart values.yaml and docs (#589)

* Auto-updated docs (#610)

Co-authored-by: Sig00rd <Sig00rd@users.noreply.github.com>

* Reimplemented the validation logic with caching the security warnings
- Reimplemented the validator interface
- Updated manifests to allocate more resources

* Add an issue template for documentation (#613)

* Docs: add info on restricted volumeMounts other than jenkins-home(#612)

* Update note in installation docs

* Update Helm chart default values.yaml

* Update schema

* Auto-updated docs (#616)

Co-authored-by: Sig00rd <Sig00rd@users.noreply.github.com>

* Auto-updated docs (#617)

Co-authored-by: Sig00rd <Sig00rd@users.noreply.github.com>

* Updated Validation logic
- Defined a security manager struct to cache all the plugin data
- Added flag to make validating security warnings optional while deploying the operator

* Helm Chart: Remove empty priorityClassName from Jenkins template (#618)

Also bump Helm Chart version to v0.5.2

* Added unit test cases for webhook

* Updated Helm Charts
- Optimized the charts
- Made the webhook optional
- Added cert manager as dependency to be installed while running webhook

* Updated unit tests, helm charts and validation logic

* Completed helm e2e tests and updated helm charts
- Completed helm tests for various scenarios
- Disabled startupapi check for cert manager webhook, defined a secret and updated templates
- Made the webhook completely optional

* Code optimization and cleanup

* Modified helm tests

* code cleanup and optimization
This commit is contained in:
sharmapulkit04
2021-08-30 08:21:52 +02:00
committed by Sylwia Brant
parent 4aa34157c3
commit 51f7ec8248
23 changed files with 19807 additions and 2909 deletions
+289 -9
View File
@@ -17,14 +17,32 @@ limitations under the License.
package v1alpha2
import (
"compress/gzip"
"encoding/json"
"errors"
"io"
"io/ioutil"
"net/http"
"os"
"time"
"github.com/jenkinsci/kubernetes-operator/pkg/log"
"github.com/jenkinsci/kubernetes-operator/pkg/plugins"
"golang.org/x/mod/semver"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/webhook"
)
// log is for logging in this package.
var jenkinslog = logf.Log.WithName("jenkins-resource")
var (
jenkinslog = logf.Log.WithName("jenkins-resource") // log is for logging in this package.
PluginsMgr PluginDataManager = *NewPluginsDataManager("/tmp/plugins.json.gzip", "/tmp/plugins.json", false, time.Duration(1000)*time.Second)
_ webhook.Validator = &Jenkins{}
)
const Hosturl = "https://ci.jenkins.io/job/Infra/job/plugin-site-api/job/generate-data/lastSuccessfulBuild/artifact/plugins.json.gzip"
func (in *Jenkins) SetupWebhookWithManager(mgr ctrl.Manager) error {
return ctrl.NewWebhookManagedBy(mgr).
@@ -37,25 +55,287 @@ func (in *Jenkins) SetupWebhookWithManager(mgr ctrl.Manager) error {
// TODO(user): change verbs to "verbs=create;update;delete" if you want to enable deletion validation.
// +kubebuilder:webhook:path=/validate-jenkins-io-jenkins-io-v1alpha2-jenkins,mutating=false,failurePolicy=fail,sideEffects=None,groups=jenkins.io.jenkins.io,resources=jenkins,verbs=create;update,versions=v1alpha2,name=vjenkins.kb.io,admissionReviewVersions={v1,v1beta1}
var _ webhook.Validator = &Jenkins{}
// ValidateCreate implements webhook.Validator so a webhook will be registered for the type
func (in *Jenkins) ValidateCreate() error {
jenkinslog.Info("validate create", "name", in.Name)
if in.Spec.ValidateSecurityWarnings {
jenkinslog.Info("validate create", "name", in.Name)
return Validate(*in)
}
// TODO(user): fill in your validation logic upon object creation.
return nil
}
// ValidateUpdate implements webhook.Validator so a webhook will be registered for the type
func (in *Jenkins) ValidateUpdate(old runtime.Object) error {
jenkinslog.Info("validate update", "name", in.Name)
if in.Spec.ValidateSecurityWarnings {
jenkinslog.Info("validate update", "name", in.Name)
return Validate(*in)
}
// TODO(user): fill in your validation logic upon object update.
return nil
}
func (in *Jenkins) ValidateDelete() error {
// TODO(user): fill in your validation logic upon object deletion.
return nil
}
type PluginDataManager struct {
PluginDataCache PluginsInfo
Timeout time.Duration
CompressedFilePath string
PluginDataFile string
IsCached bool
Attempts int
SleepTime time.Duration
}
type PluginsInfo struct {
Plugins []PluginInfo `json:"plugins"`
}
type PluginInfo struct {
Name string `json:"name"`
SecurityWarnings []Warning `json:"securityWarnings"`
}
type Warning struct {
Versions []Version `json:"versions"`
ID string `json:"id"`
Message string `json:"message"`
URL string `json:"url"`
Active bool `json:"active"`
}
type Version struct {
FirstVersion string `json:"firstVersion"`
LastVersion string `json:"lastVersion"`
}
type PluginData struct {
Version string
Kind string
}
// Validates security warnings for both updating and creating a Jenkins CR
func Validate(r Jenkins) error {
if !PluginsMgr.IsCached {
return errors.New("plugins data has not been fetched")
}
pluginSet := make(map[string]PluginData)
var faultyBasePlugins string
var faultyUserPlugins string
basePlugins := plugins.BasePlugins()
for _, plugin := range basePlugins {
// Only Update the map if the plugin is not present or a lower version is being used
if pluginData, ispresent := pluginSet[plugin.Name]; !ispresent || semver.Compare(makeSemanticVersion(plugin.Version), pluginData.Version) == 1 {
pluginSet[plugin.Name] = PluginData{Version: plugin.Version, Kind: "base"}
}
}
for _, plugin := range r.Spec.Master.Plugins {
if pluginData, ispresent := pluginSet[plugin.Name]; !ispresent || semver.Compare(makeSemanticVersion(plugin.Version), pluginData.Version) == 1 {
pluginSet[plugin.Name] = PluginData{Version: plugin.Version, Kind: "user-defined"}
}
}
for _, plugin := range PluginsMgr.PluginDataCache.Plugins {
if pluginData, ispresent := pluginSet[plugin.Name]; ispresent {
var hasVulnerabilities bool
for _, warning := range plugin.SecurityWarnings {
for _, version := range warning.Versions {
firstVersion := version.FirstVersion
lastVersion := version.LastVersion
if len(firstVersion) == 0 {
firstVersion = "0" // setting default value in case of empty string
}
if len(lastVersion) == 0 {
lastVersion = pluginData.Version // setting default value in case of empty string
}
// checking if this warning applies to our version as well
if compareVersions(firstVersion, lastVersion, pluginData.Version) {
jenkinslog.Info("Security Vulnerability detected in "+pluginData.Kind+" "+plugin.Name+":"+pluginData.Version, "Warning message", warning.Message, "For more details,check security advisory", warning.URL)
hasVulnerabilities = true
}
}
}
if hasVulnerabilities {
if pluginData.Kind == "base" {
faultyBasePlugins += "\n" + plugin.Name + ":" + pluginData.Version
} else {
faultyUserPlugins += "\n" + plugin.Name + ":" + pluginData.Version
}
}
}
}
if len(faultyBasePlugins) > 0 || len(faultyUserPlugins) > 0 {
var errormsg string
if len(faultyBasePlugins) > 0 {
errormsg += "security vulnerabilities detected in the following base plugins: " + faultyBasePlugins
}
if len(faultyUserPlugins) > 0 {
errormsg += "security vulnerabilities detected in the following user-defined plugins: " + faultyUserPlugins
}
return errors.New(errormsg)
}
return nil
}
func NewPluginsDataManager(compressedFilePath string, pluginDataFile string, isCached bool, timeout time.Duration) *PluginDataManager {
return &PluginDataManager{
CompressedFilePath: compressedFilePath,
PluginDataFile: pluginDataFile,
IsCached: isCached,
Timeout: timeout,
}
}
func (in *PluginDataManager) ManagePluginData(sig chan bool) {
var isInit bool
var retryInterval time.Duration
for {
var isCached bool
err := in.fetchPluginData()
if err == nil {
isCached = true
} else {
jenkinslog.Info("Cache plugin data", "failed to fetch plugin data", err)
}
// should only be executed once when the operator starts
if !isInit {
sig <- isCached // sending signal to main to continue
isInit = true
}
in.IsCached = in.IsCached || isCached
if !isCached {
retryInterval = time.Duration(1) * time.Hour
} else {
retryInterval = time.Duration(12) * time.Hour
}
time.Sleep(retryInterval)
}
}
// Downloads extracts and reads the JSON data in every 12 hours
func (in *PluginDataManager) fetchPluginData() error {
jenkinslog.Info("Initializing/Updating the plugin data cache")
var err error
for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
err = in.download()
if err != nil {
jenkinslog.V(log.VDebug).Info("Cache Plugin Data", "failed to download file", err)
continue
}
break
}
if err != nil {
return err
}
for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
err = in.extract()
if err != nil {
jenkinslog.V(log.VDebug).Info("Cache Plugin Data", "failed to extract file", err)
continue
}
break
}
if err != nil {
return err
}
for in.Attempts = 0; in.Attempts < 5; in.Attempts++ {
err = in.cache()
if err != nil {
jenkinslog.V(log.VDebug).Info("Cache Plugin Data", "failed to read plugin data file", err)
continue
}
break
}
return err
}
func (in *PluginDataManager) download() error {
out, err := os.Create(in.CompressedFilePath)
if err != nil {
return err
}
defer out.Close()
client := http.Client{
Timeout: in.Timeout,
}
resp, err := client.Get(Hosturl)
if err != nil {
return err
}
defer resp.Body.Close()
_, err = io.Copy(out, resp.Body)
return err
}
func (in *PluginDataManager) extract() error {
reader, err := os.Open(in.CompressedFilePath)
if err != nil {
return err
}
defer reader.Close()
archive, err := gzip.NewReader(reader)
if err != nil {
return err
}
defer archive.Close()
writer, err := os.Create(in.PluginDataFile)
if err != nil {
return err
}
defer writer.Close()
_, err = io.Copy(writer, archive)
return err
}
// Loads the JSON data into memory and stores it
func (in *PluginDataManager) cache() error {
jsonFile, err := os.Open(in.PluginDataFile)
if err != nil {
return err
}
defer jsonFile.Close()
byteValue, err := ioutil.ReadAll(jsonFile)
if err != nil {
return err
}
err = json.Unmarshal(byteValue, &in.PluginDataCache)
return err
}
// returns a semantic version that can be used for comparison, allowed versioning format vMAJOR.MINOR.PATCH or MAJOR.MINOR.PATCH
func makeSemanticVersion(version string) string {
if version[0] != 'v' {
version = "v" + version
}
return semver.Canonical(version)
}
// Compare if the current version lies between first version and last version
func compareVersions(firstVersion string, lastVersion string, pluginVersion string) bool {
firstSemVer := makeSemanticVersion(firstVersion)
lastSemVer := makeSemanticVersion(lastVersion)
pluginSemVer := makeSemanticVersion(pluginVersion)
if semver.Compare(pluginSemVer, firstSemVer) == -1 || semver.Compare(pluginSemVer, lastSemVer) == 1 {
return false
}
return true
}
+177
View File
@@ -0,0 +1,177 @@
package v1alpha2
import (
"errors"
"testing"
"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestMakeSemanticVersion(t *testing.T) {
t.Run("only major version specified", func(t *testing.T) {
got := makeSemanticVersion("1")
assert.Equal(t, got, "v1.0.0")
})
t.Run("major and minor version specified", func(t *testing.T) {
got := makeSemanticVersion("1.2")
assert.Equal(t, got, "v1.2.0")
})
t.Run("major,minor and patch version specified", func(t *testing.T) {
got := makeSemanticVersion("1.2.3")
assert.Equal(t, got, "v1.2.3")
})
t.Run("semantic versions begin with a leading v and no patch version", func(t *testing.T) {
got := makeSemanticVersion("v2.5")
assert.Equal(t, got, "v2.5.0")
})
t.Run("semantic versions with prerelease versions", func(t *testing.T) {
got := makeSemanticVersion("2.1.2-alpha.1")
assert.Equal(t, got, "v2.1.2-alpha.1")
})
t.Run("semantic versions with prerelease versions", func(t *testing.T) {
got := makeSemanticVersion("0.11.2-9.c8b45b8bb173")
assert.Equal(t, got, "v0.11.2-9.c8b45b8bb173")
})
t.Run("semantic versions with build suffix", func(t *testing.T) {
got := makeSemanticVersion("1.7.9+meta")
assert.Equal(t, got, "v1.7.9")
})
t.Run("invalid semantic version", func(t *testing.T) {
got := makeSemanticVersion("google-login-1.2")
assert.Equal(t, got, "")
})
}
func TestCompareVersions(t *testing.T) {
t.Run("Plugin Version lies between first and last version", func(t *testing.T) {
got := compareVersions("1.2", "1.6", "1.4")
assert.Equal(t, got, true)
})
t.Run("Plugin Version is greater than the last version", func(t *testing.T) {
got := compareVersions("1", "2", "3")
assert.Equal(t, got, false)
})
t.Run("Plugin Version is less than the first version", func(t *testing.T) {
got := compareVersions("1.4", "2.5", "1.1")
assert.Equal(t, got, false)
})
t.Run("Plugins Versions have prerelease version and it lies between first and last version", func(t *testing.T) {
got := compareVersions("1.2.1-alpha", "1.2.1", "1.2.1-beta")
assert.Equal(t, got, true)
})
t.Run("Plugins Versions have prerelease version and it is greater than the last version", func(t *testing.T) {
got := compareVersions("v2.2.1-alpha", "v2.5.1-beta.1", "v2.5.1-beta.2")
assert.Equal(t, got, false)
})
}
func TestValidate(t *testing.T) {
t.Run("Validating when plugins data file is not fetched", func(t *testing.T) {
userplugins := []Plugin{{Name: "script-security", Version: "1.77"}, {Name: "git-client", Version: "3.9"}, {Name: "git", Version: "4.8.1"}, {Name: "plain-credentials", Version: "1.7"}}
jenkinscr := *createJenkinsCR(userplugins, true)
got := jenkinscr.ValidateCreate()
assert.Equal(t, got, errors.New("plugins data has not been fetched"))
})
PluginsMgr.IsCached = true
t.Run("Validating a Jenkins CR with plugins not having security warnings and validation is turned on", func(t *testing.T) {
PluginsMgr.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
{Name: "security-script"},
{Name: "git-client"},
{Name: "git"},
{Name: "google-login", SecurityWarnings: createSecurityWarnings("", "1.2")},
{Name: "sample-plugin", SecurityWarnings: createSecurityWarnings("", "0.8")},
{Name: "mailer"},
{Name: "plain-credentials"}}}
userplugins := []Plugin{{Name: "script-security", Version: "1.77"}, {Name: "git-client", Version: "3.9"}, {Name: "git", Version: "4.8.1"}, {Name: "plain-credentials", Version: "1.7"}}
jenkinscr := *createJenkinsCR(userplugins, true)
got := jenkinscr.ValidateCreate()
assert.Nil(t, got)
})
t.Run("Validating a Jenkins CR with some of the plugins having security warnings and validation is turned on", func(t *testing.T) {
PluginsMgr.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
{Name: "security-script", SecurityWarnings: createSecurityWarnings("1.2", "2.2")},
{Name: "workflow-cps", SecurityWarnings: createSecurityWarnings("2.59", "")},
{Name: "git-client"},
{Name: "git"},
{Name: "sample-plugin", SecurityWarnings: createSecurityWarnings("0.8", "")},
{Name: "command-launcher", SecurityWarnings: createSecurityWarnings("1.2", "1.4")},
{Name: "plain-credentials"},
{Name: "google-login", SecurityWarnings: createSecurityWarnings("1.1", "1.3")},
{Name: "mailer", SecurityWarnings: createSecurityWarnings("1.0.3", "1.1.4")},
}}
userplugins := []Plugin{{Name: "google-login", Version: "1.2"}, {Name: "mailer", Version: "1.1"}, {Name: "git", Version: "4.8.1"}, {Name: "command-launcher", Version: "1.6"}, {Name: "workflow-cps", Version: "2.59"}}
jenkinscr := *createJenkinsCR(userplugins, true)
got := jenkinscr.ValidateCreate()
assert.Equal(t, got, errors.New("security vulnerabilities detected in the following user-defined plugins: \nworkflow-cps:2.59\ngoogle-login:1.2\nmailer:1.1"))
})
t.Run("Updating a Jenkins CR with some of the plugins having security warnings and validation is turned on", func(t *testing.T) {
PluginsMgr.PluginDataCache = PluginsInfo{Plugins: []PluginInfo{
{Name: "handy-uri-templates-2-api", SecurityWarnings: createSecurityWarnings("2.1.8-1.0", "2.2.8-1.0")},
{Name: "workflow-cps", SecurityWarnings: createSecurityWarnings("2.59", "")},
{Name: "resource-disposer", SecurityWarnings: createSecurityWarnings("0.7", "1.2")},
{Name: "git"},
{Name: "jjwt-api"},
{Name: "blueocean-github-pipeline", SecurityWarnings: createSecurityWarnings("1.2.0-alpha-2", "1.2.0-beta-5")},
{Name: "command-launcher", SecurityWarnings: createSecurityWarnings("1.2", "1.4")},
{Name: "plain-credentials"},
{Name: "ghprb", SecurityWarnings: createSecurityWarnings("1.1", "1.43")},
{Name: "mailer", SecurityWarnings: createSecurityWarnings("1.0.3", "1.1.4")},
}}
userplugins := []Plugin{{Name: "google-login", Version: "1.2"}, {Name: "mailer", Version: "1.1"}, {Name: "git", Version: "4.8.1"}, {Name: "command-launcher", Version: "1.6"}, {Name: "workflow-cps", Version: "2.59"}}
oldjenkinscr := *createJenkinsCR(userplugins, true)
userplugins = []Plugin{{Name: "handy-uri-templates-2-api", Version: "2.1.8-1.0"}, {Name: "resource-disposer", Version: "0.8"}, {Name: "jjwt-api", Version: "0.11.2-9.c8b45b8bb173"}, {Name: "blueocean-github-pipeline", Version: "1.2.0-beta-3"}, {Name: "ghprb", Version: "1.39"}}
newjenkinscr := *createJenkinsCR(userplugins, true)
got := newjenkinscr.ValidateUpdate(&oldjenkinscr)
assert.Equal(t, got, errors.New("security vulnerabilities detected in the following user-defined plugins: \nhandy-uri-templates-2-api:2.1.8-1.0\nresource-disposer:0.8\nblueocean-github-pipeline:1.2.0-beta-3\nghprb:1.39"))
})
t.Run("Validation is turned off", func(t *testing.T) {
userplugins := []Plugin{{Name: "google-login", Version: "1.2"}, {Name: "mailer", Version: "1.1"}, {Name: "git", Version: "4.8.1"}, {Name: "command-launcher", Version: "1.6"}, {Name: "workflow-cps", Version: "2.59"}}
jenkinscr := *createJenkinsCR(userplugins, false)
got := jenkinscr.ValidateCreate()
assert.Nil(t, got)
userplugins = []Plugin{{Name: "google-login", Version: "1.2"}, {Name: "mailer", Version: "1.1"}, {Name: "git", Version: "4.8.1"}, {Name: "command-launcher", Version: "1.6"}, {Name: "workflow-cps", Version: "2.59"}}
newjenkinscr := *createJenkinsCR(userplugins, false)
got = newjenkinscr.ValidateUpdate(&jenkinscr)
assert.Nil(t, got)
})
}
func createJenkinsCR(userPlugins []Plugin, validateSecurityWarnings bool) *Jenkins {
jenkins := &Jenkins{
TypeMeta: JenkinsTypeMeta(),
ObjectMeta: metav1.ObjectMeta{
Name: "jenkins",
Namespace: "test",
},
Spec: JenkinsSpec{
Master: JenkinsMaster{
Plugins: userPlugins,
DisableCSRFProtection: false,
},
ValidateSecurityWarnings: validateSecurityWarnings,
},
}
return jenkins
}
func createSecurityWarnings(firstVersion string, lastVersion string) []Warning {
return []Warning{{Versions: []Version{{FirstVersion: firstVersion, LastVersion: lastVersion}}, ID: "null", Message: "unit testing", URL: "null", Active: false}}
}