Add AWS S3 backup e2e test

Gopkg.lock

@@ -33,6 +33,49 @@
   pruneopts = "NT"
   revision = "de5bf2ad457846296e2031421a34e2568e304e35"
 
+[[projects]]
+  digest = "1:d6d8c5e82a2ac7f2d44a9cd37412f72c4c0342b61ed5fa57e0c6c53bf3f1a4cc"
+  name = "github.com/aws/aws-sdk-go"
+  packages = [
+    "aws",
+    "aws/awserr",
+    "aws/awsutil",
+    "aws/client",
+    "aws/client/metadata",
+    "aws/corehandlers",
+    "aws/credentials",
+    "aws/credentials/ec2rolecreds",
+    "aws/credentials/endpointcreds",
+    "aws/credentials/processcreds",
+    "aws/credentials/stscreds",
+    "aws/csm",
+    "aws/defaults",
+    "aws/ec2metadata",
+    "aws/endpoints",
+    "aws/request",
+    "aws/session",
+    "aws/signer/v4",
+    "internal/ini",
+    "internal/s3err",
+    "internal/sdkio",
+    "internal/sdkrand",
+    "internal/sdkuri",
+    "internal/shareddefaults",
+    "private/protocol",
+    "private/protocol/eventstream",
+    "private/protocol/eventstream/eventstreamapi",
+    "private/protocol/query",
+    "private/protocol/query/queryutil",
+    "private/protocol/rest",
+    "private/protocol/restxml",
+    "private/protocol/xml/xmlutil",
+    "service/s3",
+    "service/sts",
+  ]
+  pruneopts = "NT"
+  revision = "a2484497433aa110263c045df2f1d49336b38cfd"
+  version = "v1.16.22"
+
 [[projects]]
   digest = "1:8d13c70d5898b091728540686c696baee0d64013b8e43089da80621a49410391"
   name = "github.com/bndr/gojenkins"
@@ -249,6 +292,13 @@
   revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4"
   version = "v0.3.6"
 
+[[projects]]
+  digest = "1:1234e31f3de67447e344dabcdf72c4588d31b8eed2d28f1889377ec006a086a9"
+  name = "github.com/jmespath/go-jmespath"
+  packages = ["."]
+  pruneopts = "NT"
+  revision = "c2b33e84"
+
 [[projects]]
   digest = "1:f5b9328966ccea0970b1d15075698eff0ddb3e75889560aad2e9f76b289b536a"
   name = "github.com/joho/godotenv"
@@ -426,7 +476,10 @@
 [[projects]]
   digest = "1:4af061277c04a7660e082acc2020f4c66d2c21dfc62e0242ffa1d2120cdfb4ec"
   name = "github.com/stretchr/testify"
-  packages = ["assert"]
+  packages = [
+    "assert",
+    "require",
+  ]
   pruneopts = "NT"
   revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
   version = "v1.2.2"
@@ -869,6 +922,10 @@
   analyzer-name = "dep"
   analyzer-version = 1
   input-imports = [
+    "github.com/aws/aws-sdk-go/aws",
+    "github.com/aws/aws-sdk-go/aws/credentials",
+    "github.com/aws/aws-sdk-go/aws/session",
+    "github.com/aws/aws-sdk-go/service/s3",
     "github.com/bndr/gojenkins",
     "github.com/docker/distribution/reference",
     "github.com/go-logr/logr",
@@ -882,6 +939,7 @@
     "github.com/operator-framework/operator-sdk/version",
     "github.com/pkg/errors",
     "github.com/stretchr/testify/assert",
+    "github.com/stretchr/testify/require",
     "k8s.io/api/core/v1",
     "k8s.io/api/rbac/v1",
     "k8s.io/apimachinery/pkg/api/errors",

Makefile

@@ -142,7 +142,7 @@ test: ## Runs the go tests
 
 .PHONY: e2e
 CURRENT_DIRECTORY := $(shell pwd)
-e2e: build docker-build ## Runs e2e tests
+e2e: build docker-build ## Runs e2e tests, you can use EXTRA_ARGS
 	@echo "+ $@"
 	@echo "Docker image: $(REPO):$(GITCOMMIT)"
 	cp deploy/service_account.yaml deploy/namespace-init.yaml
@@ -156,7 +156,7 @@ ifeq ($(ENVIRONMENT),minikube)
 endif
 
 	@RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -tags "$(BUILDTAGS) cgo" -v -timeout 30m \
-		-root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/virtuslab_v1alpha1_jenkins_crd.yaml -namespacedMan deploy/namespace-init.yaml
+		-root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/virtuslab_v1alpha1_jenkins_crd.yaml -namespacedMan deploy/namespace-init.yaml $(EXTRA_ARGS)
 
 .PHONY: vet
 vet: ## Verifies `go vet` passes

pkg/controller/jenkins/configuration/user/reconcile.go

@@ -2,7 +2,6 @@ package user
 
 import (
 	"context"
-	"fmt"
 	"time"
 
 	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
@@ -103,7 +102,7 @@ func (r *ReconcileUserConfiguration) ensureSeedJobs() (reconcile.Result, error)
 }
 
 func (r *ReconcileUserConfiguration) ensureUserConfiguration(jenkinsClient jenkinsclient.Jenkins) (reconcile.Result, error) {
-	groovyClient := groovy.New(jenkinsClient, r.k8sClient, r.logger, fmt.Sprintf("%s-user-configuration", constants.OperatorName), resources.JenkinsUserConfigurationVolumePath)
+	groovyClient := groovy.New(jenkinsClient, r.k8sClient, r.logger, constants.UserConfigurationJobName, resources.JenkinsUserConfigurationVolumePath)
 
 	err := groovyClient.ConfigureGroovyJob()
 	if err != nil {

pkg/controller/jenkins/constants/constants.go

@@ -15,6 +15,8 @@ const (
 	BackupAmazonS3SecretSecretKey = "secret-key"
 	// BackupJobName is the Jenkins job name used to backup jobs history
 	BackupJobName = OperatorName + "-backup"
+	// UserConfigurationJobName is the Jenkins job name used to configure Jenkins by groovy scripts provided by user
+	UserConfigurationJobName = OperatorName + "-user-configuration"
 	// BackupLatestFileName is the latest backup file name
 	BackupLatestFileName = "build-history-latest.tar.gz"
 )

test/e2e/aws_s3_backup_test.go

@@ -0,0 +1,157 @@
+package e2e
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/base/resources"
+	"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/constants"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/bndr/gojenkins"
+	framework "github.com/operator-framework/operator-sdk/pkg/test"
+	assert "github.com/stretchr/testify/require"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type amazonS3BackupConfiguration struct {
+	BucketName string `json:"bucketName,omitempty"`
+	BucketPath string `json:"bucketPath,omitempty"`
+	Region     string `json:"region,omitempty"`
+	AccessKey  string `json:"accessKey,omitempty"`
+	SecretKey  string `json:"secretKey,omitempty"`
+}
+
+func TestAmazonS3Backup(t *testing.T) {
+	t.Parallel()
+	if amazonS3BackupConfigurationFile == nil || len(*amazonS3BackupConfigurationFile) == 0 {
+		t.Skipf("Skipping testing because flag '%s' is not set", amazonS3BackupConfigurationParameterName)
+	}
+	backupConfig := loadAmazonS3BackupConfig(t)
+
+	s3Client := createS3Client(t, backupConfig)
+	deleteAllBackupsInS3(t, backupConfig, s3Client)
+	namespace, ctx := setupTest(t)
+	defer ctx.Cleanup() // Deletes test namespace
+
+	jenkins := createJenkinsCRWithAmazonS3Backup(t, namespace, backupConfig)
+	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
+	waitForJenkinsUserConfigurationToComplete(t, jenkins)
+
+	restartJenkinsMasterPod(t, jenkins)
+	waitForRecreateJenkinsMasterPod(t, jenkins)
+
+	waitForJenkinsBaseConfigurationToComplete(t, jenkins)
+	waitForJenkinsUserConfigurationToComplete(t, jenkins)
+	jenkinsClient := verifyJenkinsAPIConnection(t, jenkins)
+	verifyIfBackupAndRestoreWasSuccessfull(t, jenkinsClient, backupConfig, s3Client)
+}
+
+func createS3Client(t *testing.T, backupConfig amazonS3BackupConfiguration) *s3.S3 {
+	sess, err := session.NewSession(&aws.Config{
+		Region:      aws.String(backupConfig.Region),
+		Credentials: credentials.NewStaticCredentials(backupConfig.AccessKey, backupConfig.SecretKey, ""),
+	})
+	assert.NoError(t, err)
+
+	return s3.New(sess)
+}
+
+func deleteAllBackupsInS3(t *testing.T, backupConfig amazonS3BackupConfiguration, s3Client *s3.S3) {
+	input := &s3.DeleteObjectInput{
+		Bucket: aws.String(backupConfig.BucketName),
+		Key:    aws.String(backupConfig.BucketPath),
+	}
+
+	_, err := s3Client.DeleteObject(input)
+	assert.NoError(t, err)
+}
+
+func verifyIfBackupAndRestoreWasSuccessfull(t *testing.T, jenkinsClient *gojenkins.Jenkins, backupConfig amazonS3BackupConfiguration, s3Client *s3.S3) {
+	job, err := jenkinsClient.GetJob(constants.UserConfigurationJobName)
+	assert.NoError(t, err)
+	// jenkins runs twice(2) + 1 as next build number
+	assert.Equal(t, int64(3), job.Raw.NextBuildNumber)
+
+	listObjects, err := s3Client.ListObjects(&s3.ListObjectsInput{
+		Bucket: aws.String(backupConfig.BucketName),
+		Marker: aws.String(backupConfig.BucketPath),
+	})
+	assert.NoError(t, err)
+	t.Logf("Backups in S3:%+v", listObjects.Contents)
+	assert.Equal(t, len(listObjects.Contents), 2)
+	latestBackupFound := false
+	for _, backup := range listObjects.Contents {
+		if *backup.Key == fmt.Sprintf("%s/%s", backupConfig.BucketPath, constants.BackupLatestFileName) {
+			latestBackupFound = true
+		}
+	}
+	assert.True(t, latestBackupFound)
+}
+
+func createJenkinsCRWithAmazonS3Backup(t *testing.T, namespace string, backupConfig amazonS3BackupConfiguration) *virtuslabv1alpha1.Jenkins {
+	jenkins := &virtuslabv1alpha1.Jenkins{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "e2e",
+			Namespace: namespace,
+		},
+		Spec: virtuslabv1alpha1.JenkinsSpec{
+			Backup: virtuslabv1alpha1.JenkinsBackupTypeAmazonS3,
+			BackupAmazonS3: virtuslabv1alpha1.JenkinsBackupAmazonS3{
+				Region:     backupConfig.Region,
+				BucketPath: backupConfig.BucketPath,
+				BucketName: backupConfig.BucketName,
+			},
+			Master: virtuslabv1alpha1.JenkinsMaster{
+				Image: "jenkins/jenkins",
+			},
+		},
+	}
+
+	t.Logf("Jenkins CR %+v", *jenkins)
+	err := framework.Global.Client.Create(context.TODO(), jenkins, nil)
+	assert.NoError(t, err)
+
+	backupCredentialsSecret := &corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      resources.GetBackupCredentialsSecretName(jenkins),
+			Namespace: namespace,
+		},
+		Data: map[string][]byte{
+			constants.BackupAmazonS3SecretAccessKey: []byte(backupConfig.AccessKey),
+			constants.BackupAmazonS3SecretSecretKey: []byte(backupConfig.SecretKey),
+		},
+	}
+	err = framework.Global.Client.Create(context.TODO(), backupCredentialsSecret, nil)
+	assert.NoError(t, err)
+
+	return jenkins
+}
+
+func loadAmazonS3BackupConfig(t *testing.T) amazonS3BackupConfiguration {
+	jsonFile, err := os.Open(*amazonS3BackupConfigurationFile)
+	assert.NoError(t, err)
+	defer func() { _ = jsonFile.Close() }()
+
+	byteValue, err := ioutil.ReadAll(jsonFile)
+	assert.NoError(t, err)
+
+	var result amazonS3BackupConfiguration
+	err = json.Unmarshal([]byte(byteValue), &result)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, result.AccessKey)
+	assert.NotEmpty(t, result.BucketName)
+	assert.NotEmpty(t, result.Region)
+	assert.NotEmpty(t, result.SecretKey)
+	result.BucketPath = t.Name()
+	return result
+}

test/e2e/main_test.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"flag"
 	"testing"
 
 	"github.com/VirtusLab/jenkins-operator/pkg/apis"
@@ -15,9 +16,15 @@ import (
 
 const (
 	jenkinsOperatorDeploymentName            = constants.OperatorName
+	amazonS3BackupConfigurationParameterName = "s3BackupConfig"
+)
+
+var (
+	amazonS3BackupConfigurationFile *string
 )
 
 func TestMain(m *testing.M) {
+	amazonS3BackupConfigurationFile = flag.String(amazonS3BackupConfigurationParameterName, "", "path to AWS S3 backup config")
 	f.MainEntry(m)
 }