public_git
/
kubernetes-operator
mirror of https://github.com/jenkinsci/kubernetes-operator.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Add AWS S3 backup e2e test

This commit is contained in:
Tomasz Sęk 2019-01-20 22:29:39 +01:00
parent 060a0bddda
commit a9dcd61a07
No known key found for this signature in database
GPG Key ID: DC356D23F6A644D0
6 changed files with 229 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
Gopkg.lock generated
View File

@ -33,6 +33,49 @@
pruneopts = "NT"
revision = "de5bf2ad457846296e2031421a34e2568e304e35"
[[projects]]
digest = "1:d6d8c5e82a2ac7f2d44a9cd37412f72c4c0342b61ed5fa57e0c6c53bf3f1a4cc"
name = "github.com/aws/aws-sdk-go"
packages = [
"aws",
"aws/awserr",
"aws/awsutil",
"aws/client",
"aws/client/metadata",
"aws/corehandlers",
"aws/credentials",
"aws/credentials/ec2rolecreds",
"aws/credentials/endpointcreds",
"aws/credentials/processcreds",
"aws/credentials/stscreds",
"aws/csm",
"aws/defaults",
"aws/ec2metadata",
"aws/endpoints",
"aws/request",
"aws/session",
"aws/signer/v4",
"internal/ini",
"internal/s3err",
"internal/sdkio",
"internal/sdkrand",
"internal/sdkuri",
"internal/shareddefaults",
"private/protocol",
"private/protocol/eventstream",
"private/protocol/eventstream/eventstreamapi",
"private/protocol/query",
"private/protocol/query/queryutil",
"private/protocol/rest",
"private/protocol/restxml",
"private/protocol/xml/xmlutil",
"service/s3",
"service/sts",
]
pruneopts = "NT"
revision = "a2484497433aa110263c045df2f1d49336b38cfd"
version = "v1.16.22"
[[projects]]
digest = "1:8d13c70d5898b091728540686c696baee0d64013b8e43089da80621a49410391"
name = "github.com/bndr/gojenkins"
@ -249,6 +292,13 @@
revision = "9f23e2d6bd2a77f959b2bf6acdbefd708a83a4a4"
version = "v0.3.6"
[[projects]]
digest = "1:1234e31f3de67447e344dabcdf72c4588d31b8eed2d28f1889377ec006a086a9"
name = "github.com/jmespath/go-jmespath"
packages = ["."]
pruneopts = "NT"
revision = "c2b33e84"
[[projects]]
digest = "1:f5b9328966ccea0970b1d15075698eff0ddb3e75889560aad2e9f76b289b536a"
name = "github.com/joho/godotenv"
@ -426,7 +476,10 @@
[[projects]]
digest = "1:4af061277c04a7660e082acc2020f4c66d2c21dfc62e0242ffa1d2120cdfb4ec"
name = "github.com/stretchr/testify"
packages = ["assert"]
packages = [
"assert",
"require",
]
pruneopts = "NT"
revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686"
version = "v1.2.2"
@ -869,6 +922,10 @@
analyzer-name = "dep"
analyzer-version = 1
input-imports = [
"github.com/aws/aws-sdk-go/aws",
"github.com/aws/aws-sdk-go/aws/credentials",
"github.com/aws/aws-sdk-go/aws/session",
"github.com/aws/aws-sdk-go/service/s3",
"github.com/bndr/gojenkins",
"github.com/docker/distribution/reference",
"github.com/go-logr/logr",
@ -882,6 +939,7 @@
"github.com/operator-framework/operator-sdk/version",
"github.com/pkg/errors",
"github.com/stretchr/testify/assert",
"github.com/stretchr/testify/require",
"k8s.io/api/core/v1",
"k8s.io/api/rbac/v1",
"k8s.io/apimachinery/pkg/api/errors",

4
Makefile
View File

@ -142,7 +142,7 @@ test: ## Runs the go tests
.PHONY: e2e
CURRENT_DIRECTORY := $(shell pwd)
e2e: build docker-build ## Runs e2e tests
e2e: build docker-build ## Runs e2e tests, you can use EXTRA_ARGS
@echo "+ $@"
@echo "Docker image: $(REPO):$(GITCOMMIT)"
cp deploy/service_account.yaml deploy/namespace-init.yaml
@ -156,7 +156,7 @@ ifeq ($(ENVIRONMENT),minikube)
endif
@RUNNING_TESTS=1 go test -parallel=1 "./test/e2e/" -tags "$(BUILDTAGS) cgo" -v -timeout 30m \
-root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/virtuslab_v1alpha1_jenkins_crd.yaml -namespacedMan deploy/namespace-init.yaml
-root=$(CURRENT_DIRECTORY) -kubeconfig=$(HOME)/.kube/config -globalMan deploy/crds/virtuslab_v1alpha1_jenkins_crd.yaml -namespacedMan deploy/namespace-init.yaml $(EXTRA_ARGS)
.PHONY: vet
vet: ## Verifies `go vet` passes

3
pkg/controller/jenkins/configuration/user/reconcile.go
View File

@ -2,7 +2,6 @@ package user
import (
"context"
"fmt"
"time"
virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
@ -103,7 +102,7 @@ func (r *ReconcileUserConfiguration) ensureSeedJobs() (reconcile.Result, error)
}
func (r *ReconcileUserConfiguration) ensureUserConfiguration(jenkinsClient jenkinsclient.Jenkins) (reconcile.Result, error) {
groovyClient := groovy.New(jenkinsClient, r.k8sClient, r.logger, fmt.Sprintf("%s-user-configuration", constants.OperatorName), resources.JenkinsUserConfigurationVolumePath)
groovyClient := groovy.New(jenkinsClient, r.k8sClient, r.logger, constants.UserConfigurationJobName, resources.JenkinsUserConfigurationVolumePath)
err := groovyClient.ConfigureGroovyJob()
if err != nil {

2
pkg/controller/jenkins/constants/constants.go
View File

@ -15,6 +15,8 @@ const (
BackupAmazonS3SecretSecretKey = "secret-key"
// BackupJobName is the Jenkins job name used to backup jobs history
BackupJobName = OperatorName + "-backup"
// UserConfigurationJobName is the Jenkins job name used to configure Jenkins by groovy scripts provided by user
UserConfigurationJobName = OperatorName + "-user-configuration"
// BackupLatestFileName is the latest backup file name
BackupLatestFileName = "build-history-latest.tar.gz"
)

157
test/e2e/aws_s3_backup_test.go Normal file
View File

@ -0,0 +1,157 @@
package e2e
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"testing"
virtuslabv1alpha1 "github.com/VirtusLab/jenkins-operator/pkg/apis/virtuslab/v1alpha1"
"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/configuration/base/resources"
"github.com/VirtusLab/jenkins-operator/pkg/controller/jenkins/constants"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/bndr/gojenkins"
framework "github.com/operator-framework/operator-sdk/pkg/test"
assert "github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type amazonS3BackupConfiguration struct {
BucketName string `json:"bucketName,omitempty"`
BucketPath string `json:"bucketPath,omitempty"`
Region string `json:"region,omitempty"`
AccessKey string `json:"accessKey,omitempty"`
SecretKey string `json:"secretKey,omitempty"`
}
func TestAmazonS3Backup(t *testing.T) {
t.Parallel()
if amazonS3BackupConfigurationFile == nil || len(*amazonS3BackupConfigurationFile) == 0 {
t.Skipf("Skipping testing because flag '%s' is not set", amazonS3BackupConfigurationParameterName)
}
backupConfig := loadAmazonS3BackupConfig(t)
s3Client := createS3Client(t, backupConfig)
deleteAllBackupsInS3(t, backupConfig, s3Client)
namespace, ctx := setupTest(t)
defer ctx.Cleanup() // Deletes test namespace
jenkins := createJenkinsCRWithAmazonS3Backup(t, namespace, backupConfig)
waitForJenkinsBaseConfigurationToComplete(t, jenkins)
waitForJenkinsUserConfigurationToComplete(t, jenkins)
restartJenkinsMasterPod(t, jenkins)
waitForRecreateJenkinsMasterPod(t, jenkins)
waitForJenkinsBaseConfigurationToComplete(t, jenkins)
waitForJenkinsUserConfigurationToComplete(t, jenkins)
jenkinsClient := verifyJenkinsAPIConnection(t, jenkins)
verifyIfBackupAndRestoreWasSuccessfull(t, jenkinsClient, backupConfig, s3Client)
}
func createS3Client(t *testing.T, backupConfig amazonS3BackupConfiguration) *s3.S3 {
sess, err := session.NewSession(&aws.Config{
Region: aws.String(backupConfig.Region),
Credentials: credentials.NewStaticCredentials(backupConfig.AccessKey, backupConfig.SecretKey, ""),
})
assert.NoError(t, err)
return s3.New(sess)
}
func deleteAllBackupsInS3(t *testing.T, backupConfig amazonS3BackupConfiguration, s3Client *s3.S3) {
input := &s3.DeleteObjectInput{
Bucket: aws.String(backupConfig.BucketName),
Key: aws.String(backupConfig.BucketPath),
}
_, err := s3Client.DeleteObject(input)
assert.NoError(t, err)
}
func verifyIfBackupAndRestoreWasSuccessfull(t *testing.T, jenkinsClient *gojenkins.Jenkins, backupConfig amazonS3BackupConfiguration, s3Client *s3.S3) {
job, err := jenkinsClient.GetJob(constants.UserConfigurationJobName)
assert.NoError(t, err)
// jenkins runs twice(2) + 1 as next build number
assert.Equal(t, int64(3), job.Raw.NextBuildNumber)
listObjects, err := s3Client.ListObjects(&s3.ListObjectsInput{
Bucket: aws.String(backupConfig.BucketName),
Marker: aws.String(backupConfig.BucketPath),
})
assert.NoError(t, err)
t.Logf("Backups in S3:%+v", listObjects.Contents)
assert.Equal(t, len(listObjects.Contents), 2)
latestBackupFound := false
for _, backup := range listObjects.Contents {
if *backup.Key == fmt.Sprintf("%s/%s", backupConfig.BucketPath, constants.BackupLatestFileName) {
latestBackupFound = true
}
}
assert.True(t, latestBackupFound)
}
func createJenkinsCRWithAmazonS3Backup(t *testing.T, namespace string, backupConfig amazonS3BackupConfiguration) *virtuslabv1alpha1.Jenkins {
jenkins := &virtuslabv1alpha1.Jenkins{
ObjectMeta: metav1.ObjectMeta{
Name: "e2e",
Namespace: namespace,
},
Spec: virtuslabv1alpha1.JenkinsSpec{
Backup: virtuslabv1alpha1.JenkinsBackupTypeAmazonS3,
BackupAmazonS3: virtuslabv1alpha1.JenkinsBackupAmazonS3{
Region: backupConfig.Region,
BucketPath: backupConfig.BucketPath,
BucketName: backupConfig.BucketName,
},
Master: virtuslabv1alpha1.JenkinsMaster{
Image: "jenkins/jenkins",
},
},
}
t.Logf("Jenkins CR %+v", *jenkins)
err := framework.Global.Client.Create(context.TODO(), jenkins, nil)
assert.NoError(t, err)
backupCredentialsSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: resources.GetBackupCredentialsSecretName(jenkins),
Namespace: namespace,
},
Data: map[string][]byte{
constants.BackupAmazonS3SecretAccessKey: []byte(backupConfig.AccessKey),
constants.BackupAmazonS3SecretSecretKey: []byte(backupConfig.SecretKey),
},
}
err = framework.Global.Client.Create(context.TODO(), backupCredentialsSecret, nil)
assert.NoError(t, err)
return jenkins
}
func loadAmazonS3BackupConfig(t *testing.T) amazonS3BackupConfiguration {
jsonFile, err := os.Open(*amazonS3BackupConfigurationFile)
assert.NoError(t, err)
defer func() { _ = jsonFile.Close() }()
byteValue, err := ioutil.ReadAll(jsonFile)
assert.NoError(t, err)
var result amazonS3BackupConfiguration
err = json.Unmarshal([]byte(byteValue), &result)
assert.NoError(t, err)
assert.NotEmpty(t, result.AccessKey)
assert.NotEmpty(t, result.BucketName)
assert.NotEmpty(t, result.Region)
assert.NotEmpty(t, result.SecretKey)
result.BucketPath = t.Name()
return result
}

9
test/e2e/main_test.go
View File

@ -1,6 +1,7 @@
package e2e
import (
"flag"
"testing"
"github.com/VirtusLab/jenkins-operator/pkg/apis"
@ -14,10 +15,16 @@ import (
)
const (
jenkinsOperatorDeploymentName = constants.OperatorName
jenkinsOperatorDeploymentName = constants.OperatorName
amazonS3BackupConfigurationParameterName = "s3BackupConfig"
)
var (
amazonS3BackupConfigurationFile *string
)
func TestMain(m *testing.M) {
amazonS3BackupConfigurationFile = flag.String(amazonS3BackupConfigurationParameterName, "", "path to AWS S3 backup config")
f.MainEntry(m)
}