diff --git a/Gopkg.lock b/Gopkg.lock index a13a4167..a6b767aa 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -33,6 +33,49 @@ pruneopts = "NT" revision = "de5bf2ad457846296e2031421a34e2568e304e35" +[[projects]] + digest = "1:d6d8c5e82a2ac7f2d44a9cd37412f72c4c0342b61ed5fa57e0c6c53bf3f1a4cc" + name = "github.com/aws/aws-sdk-go" + packages = [ + "aws", + "aws/awserr", + "aws/awsutil", + "aws/client", + "aws/client/metadata", + "aws/corehandlers", + "aws/credentials", + "aws/credentials/ec2rolecreds", + "aws/credentials/endpointcreds", + "aws/credentials/processcreds", + "aws/credentials/stscreds", + "aws/csm", + "aws/defaults", + "aws/ec2metadata", + "aws/endpoints", + "aws/request", + "aws/session", + "aws/signer/v4", + "internal/ini", + "internal/s3err", + "internal/sdkio", + "internal/sdkrand", + "internal/sdkuri", + "internal/shareddefaults", + "private/protocol", + "private/protocol/eventstream", + "private/protocol/eventstream/eventstreamapi", + "private/protocol/query", + "private/protocol/query/queryutil", + "private/protocol/rest", + "private/protocol/restxml", + "private/protocol/xml/xmlutil", + "service/s3", + "service/sts", + ] + pruneopts = "NT" + revision = "a2484497433aa110263c045df2f1d49336b38cfd" + version = "v1.16.22" + [[projects]] digest = "1:8d13c70d5898b091728540686c696baee0d64013b8e43089da80621a49410391" name = "github.com/bndr/gojenkins" @@ -249,6 +292,13 @@ revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4" version = "v0.3.6" +[[projects]] + digest = "1:1234e31f3de67447e344dabcdf72c4588d31b8eed2d28f1889377ec006a086a9" + name = "github.com/jmespath/go-jmespath" + packages = ["."] + pruneopts = "NT" + revision = "c2b33e84" + [[projects]] digest = "1:f5b9328966ccea0970b1d15075698eff0ddb3e75889560aad2e9f76b289b536a" name = "github.com/joho/godotenv" @@ -426,7 +476,10 @@ [[projects]] digest = "1:4af061277c04a7660e082acc2020f4c66d2c21dfc62e0242ffa1d2120cdfb4ec" name = "github.com/stretchr/testify" - packages = ["assert"] + packages = [ + "assert", + "require", + ] pruneopts = "NT" revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" version = "v1.2.2" @@ -869,6 +922,10 @@ analyzer-name = "dep" analyzer-version = 1 input-imports = [ + "github.com/aws/aws-sdk-go/aws", + "github.com/aws/aws-sdk-go/aws/credentials", + "github.com/aws/aws-sdk-go/aws/session", + "github.com/aws/aws-sdk-go/service/s3", "github.com/bndr/gojenkins", "github.com/docker/distribution/reference", "github.com/go-logr/logr", @@ -882,6 +939,7 @@ "github.com/operator-framework/operator-sdk/version", "github.com/pkg/errors", "github.com/stretchr/testify/assert", + "github.com/stretchr/testify/require", "k8s.io/api/core/v1", "k8s.io/api/rbac/v1", "k8s.io/apimachinery/pkg/api/errors", diff --git a/Makefile b/Makefile index 2595ccd4..955a140f 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ test: ## Runs the go tests .PHONY: e2e CURRENT_DIRECTORY := $(shell pwd) -e2e: build docker-build ## Runs e2e tests +e2e: build docker-build ## Runs e2e tests, you can use EXTRA_ARGS @echo "+ $@" @echo "Docker image: $(REPO):$(GITCOMMIT)" cp deploy/service_account.yaml deploy/namespace-init.yaml @@ -156,7 +156,7 @@ ifeq ($(ENVIRONMENT),minikube) endif @RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -tags "$(BUILDTAGS) cgo" -v -timeout 30m \ - -root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/virtuslab_v1alpha1_jenkins_crd.yaml -namespacedMan deploy/namespace-init.yaml + -root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/virtuslab_v1alpha1_jenkins_crd.yaml -namespacedMan deploy/namespace-init.yaml $(EXTRA_ARGS) .PHONY: vet vet: ## Verifies `go vet` passes diff --git a/pkg/controller/jenkins/configuration/user/reconcile.go b/pkg/controller/jenkins/configuration/user/reconcile.go index 10a0f545..fa7bcf1b 100644 --- a/pkg/controller/jenkins/configuration/user/reconcile.go +++ b/pkg/controller/jenkins/configuration/user/reconcile.go @@ -2,7 +2,6 @@ package user import ( "context" - "fmt" "time" virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" @@ -103,7 +102,7 @@ func (r *ReconcileUserConfiguration) ensureSeedJobs() (reconcile.Result, error) } func (r *ReconcileUserConfiguration) ensureUserConfiguration(jenkinsClient jenkinsclient.Jenkins) (reconcile.Result, error) { - groovyClient := groovy.New(jenkinsClient, r.k8sClient, r.logger, fmt.Sprintf("%s-user-configuration", constants.OperatorName), resources.JenkinsUserConfigurationVolumePath) + groovyClient := groovy.New(jenkinsClient, r.k8sClient, r.logger, constants.UserConfigurationJobName, resources.JenkinsUserConfigurationVolumePath) err := groovyClient.ConfigureGroovyJob() if err != nil { diff --git a/pkg/controller/jenkins/constants/constants.go b/pkg/controller/jenkins/constants/constants.go index b968b842..3f6526ae 100644 --- a/pkg/controller/jenkins/constants/constants.go +++ b/pkg/controller/jenkins/constants/constants.go @@ -15,6 +15,8 @@ const ( BackupAmazonS3SecretSecretKey = "secret-key" // BackupJobName is the Jenkins job name used to backup jobs history BackupJobName = OperatorName + "-backup" + // UserConfigurationJobName is the Jenkins job name used to configure Jenkins by groovy scripts provided by user + UserConfigurationJobName = OperatorName + "-user-configuration" // BackupLatestFileName is the latest backup file name BackupLatestFileName = "build-history-latest.tar.gz" ) diff --git a/test/e2e/aws_s3_backup_test.go b/test/e2e/aws_s3_backup_test.go new file mode 100644 index 00000000..1fab28d1 --- /dev/null +++ b/test/e2e/aws_s3_backup_test.go @@ -0,0 +1,157 @@ +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "testing" + + virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/base/resources" + "github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/constants" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/bndr/gojenkins" + framework "github.com/operator-framework/operator-sdk/pkg/test" + assert "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type amazonS3BackupConfiguration struct { + BucketName string `json:"bucketName,omitempty"` + BucketPath string `json:"bucketPath,omitempty"` + Region string `json:"region,omitempty"` + AccessKey string `json:"accessKey,omitempty"` + SecretKey string `json:"secretKey,omitempty"` +} + +func TestAmazonS3Backup(t *testing.T) { + t.Parallel() + if amazonS3BackupConfigurationFile == nil || len(*amazonS3BackupConfigurationFile) == 0 { + t.Skipf("Skipping testing because flag '%s' is not set", amazonS3BackupConfigurationParameterName) + } + backupConfig := loadAmazonS3BackupConfig(t) + + s3Client := createS3Client(t, backupConfig) + deleteAllBackupsInS3(t, backupConfig, s3Client) + namespace, ctx := setupTest(t) + defer ctx.Cleanup() // Deletes test namespace + + jenkins := createJenkinsCRWithAmazonS3Backup(t, namespace, backupConfig) + waitForJenkinsBaseConfigurationToComplete(t, jenkins) + waitForJenkinsUserConfigurationToComplete(t, jenkins) + + restartJenkinsMasterPod(t, jenkins) + waitForRecreateJenkinsMasterPod(t, jenkins) + + waitForJenkinsBaseConfigurationToComplete(t, jenkins) + waitForJenkinsUserConfigurationToComplete(t, jenkins) + jenkinsClient := verifyJenkinsAPIConnection(t, jenkins) + verifyIfBackupAndRestoreWasSuccessfull(t, jenkinsClient, backupConfig, s3Client) +} + +func createS3Client(t *testing.T, backupConfig amazonS3BackupConfiguration) *s3.S3 { + sess, err := session.NewSession(&aws.Config{ + Region: aws.String(backupConfig.Region), + Credentials: credentials.NewStaticCredentials(backupConfig.AccessKey, backupConfig.SecretKey, ""), + }) + assert.NoError(t, err) + + return s3.New(sess) +} + +func deleteAllBackupsInS3(t *testing.T, backupConfig amazonS3BackupConfiguration, s3Client *s3.S3) { + input := &s3.DeleteObjectInput{ + Bucket: aws.String(backupConfig.BucketName), + Key: aws.String(backupConfig.BucketPath), + } + + _, err := s3Client.DeleteObject(input) + assert.NoError(t, err) +} + +func verifyIfBackupAndRestoreWasSuccessfull(t *testing.T, jenkinsClient *gojenkins.Jenkins, backupConfig amazonS3BackupConfiguration, s3Client *s3.S3) { + job, err := jenkinsClient.GetJob(constants.UserConfigurationJobName) + assert.NoError(t, err) + // jenkins runs twice(2) + 1 as next build number + assert.Equal(t, int64(3), job.Raw.NextBuildNumber) + + listObjects, err := s3Client.ListObjects(&s3.ListObjectsInput{ + Bucket: aws.String(backupConfig.BucketName), + Marker: aws.String(backupConfig.BucketPath), + }) + assert.NoError(t, err) + t.Logf("Backups in S3:%+v", listObjects.Contents) + assert.Equal(t, len(listObjects.Contents), 2) + latestBackupFound := false + for _, backup := range listObjects.Contents { + if *backup.Key == fmt.Sprintf("%s/%s", backupConfig.BucketPath, constants.BackupLatestFileName) { + latestBackupFound = true + } + } + assert.True(t, latestBackupFound) +} + +func createJenkinsCRWithAmazonS3Backup(t *testing.T, namespace string, backupConfig amazonS3BackupConfiguration) *virtuslabv1alpha1.Jenkins { + jenkins := &virtuslabv1alpha1.Jenkins{ + ObjectMeta: metav1.ObjectMeta{ + Name: "e2e", + Namespace: namespace, + }, + Spec: virtuslabv1alpha1.JenkinsSpec{ + Backup: virtuslabv1alpha1.JenkinsBackupTypeAmazonS3, + BackupAmazonS3: virtuslabv1alpha1.JenkinsBackupAmazonS3{ + Region: backupConfig.Region, + BucketPath: backupConfig.BucketPath, + BucketName: backupConfig.BucketName, + }, + Master: virtuslabv1alpha1.JenkinsMaster{ + Image: "jenkins/jenkins", + }, + }, + } + + t.Logf("Jenkins CR %+v", *jenkins) + err := framework.Global.Client.Create(context.TODO(), jenkins, nil) + assert.NoError(t, err) + + backupCredentialsSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: resources.GetBackupCredentialsSecretName(jenkins), + Namespace: namespace, + }, + Data: map[string][]byte{ + constants.BackupAmazonS3SecretAccessKey: []byte(backupConfig.AccessKey), + constants.BackupAmazonS3SecretSecretKey: []byte(backupConfig.SecretKey), + }, + } + err = framework.Global.Client.Create(context.TODO(), backupCredentialsSecret, nil) + assert.NoError(t, err) + + return jenkins +} + +func loadAmazonS3BackupConfig(t *testing.T) amazonS3BackupConfiguration { + jsonFile, err := os.Open(*amazonS3BackupConfigurationFile) + assert.NoError(t, err) + defer func() { _ = jsonFile.Close() }() + + byteValue, err := ioutil.ReadAll(jsonFile) + assert.NoError(t, err) + + var result amazonS3BackupConfiguration + err = json.Unmarshal([]byte(byteValue), &result) + assert.NoError(t, err) + assert.NotEmpty(t, result.AccessKey) + assert.NotEmpty(t, result.BucketName) + assert.NotEmpty(t, result.Region) + assert.NotEmpty(t, result.SecretKey) + result.BucketPath = t.Name() + return result +} diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 7a194634..361ffcfd 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -1,6 +1,7 @@ package e2e import ( + "flag" "testing" "github.com/VirtusLab/jenkins-operator/pkg/apis" @@ -14,10 +15,16 @@ import ( ) const ( - jenkinsOperatorDeploymentName = constants.OperatorName + jenkinsOperatorDeploymentName = constants.OperatorName + amazonS3BackupConfigurationParameterName = "s3BackupConfig" +) + +var ( + amazonS3BackupConfigurationFile *string ) func TestMain(m *testing.M) { + amazonS3BackupConfigurationFile = flag.String(amazonS3BackupConfigurationParameterName, "", "path to AWS S3 backup config") f.MainEntry(m) }