Scalyr agent sidecar for log shipping (#190)

* Scalyr agent sidecar for log shipping

* Remove the default for the Scalyr image

Now the image needs to be specified explicitly to enable log shipping to
Scalyr.  This removes the problem of having to generate the config file
or publish our agent image repository.

* Add configuration variable for Scalyr server URL

Defaults to the EU address.

* Alter style

Newlines are cheap and make code easier to edit/refactor, but ok.

* Fix StatefulSet comparison logic

I broke it when I made the comparison consider all containers in the
PostgreSQL pod.

pkg/cluster/cluster.go

@@ -307,6 +307,30 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) *comp
 	if len(c.Statefulset.Spec.Template.Spec.Containers) != len(statefulSet.Spec.Template.Spec.Containers) {
 		needsRollUpdate = true
 		reasons = append(reasons, "new statefulset's container specification doesn't match the current one")
+	} else {
+		for index, container1 := range c.Statefulset.Spec.Template.Spec.Containers {
+			container2 := statefulSet.Spec.Template.Spec.Containers[index]
+			if container1.Image != container2.Image {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d image doesn't match the current one", index))
+			}
+			if !reflect.DeepEqual(container1.Ports, container2.Ports) {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d ports don't match the current one", index))
+			}
+			if !compareResources(&container1.Resources, &container2.Resources) {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d resources don't match the current ones", index))
+			}
+			if !reflect.DeepEqual(container1.Env, container2.Env) {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d environment doesn't match the current one", index))
+			}
+			if !reflect.DeepEqual(container1.EnvFrom, container2.EnvFrom) {
+				needsRollUpdate = true
+				reasons = append(reasons, fmt.Sprintf("new statefulset's container %d environment sources don't match the current one", index))
+			}
+		}
 	}
 	if len(c.Statefulset.Spec.Template.Spec.Containers) == 0 {
 		c.logger.Warningf("statefulset %q has no container", util.NameFromMeta(c.Statefulset.ObjectMeta))
@@ -367,31 +391,6 @@ func (c *Cluster) compareStatefulSetWith(statefulSet *v1beta1.StatefulSet) *comp
 		}
 	}
 
-	container1 := c.Statefulset.Spec.Template.Spec.Containers[0]
-	container2 := statefulSet.Spec.Template.Spec.Containers[0]
-	if container1.Image != container2.Image {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container image doesn't match the current one")
-	}
-
-	if !reflect.DeepEqual(container1.Ports, container2.Ports) {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container ports don't match the current one")
-	}
-
-	if !compareResources(&container1.Resources, &container2.Resources) {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container resources don't match the current ones")
-	}
-	if !reflect.DeepEqual(container1.Env, container2.Env) {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container environment doesn't match the current one")
-	}
-	if !reflect.DeepEqual(container1.EnvFrom, container2.EnvFrom) {
-		needsRollUpdate = true
-		reasons = append(reasons, "new statefulset's container environment sources don't match the current one")
-	}
-
 	if needsRollUpdate || needsReplace {
 		match = false
 	}

pkg/cluster/k8sres.go

@@ -276,13 +276,16 @@ func (c *Cluster) tolerations(tolerationsSpec *[]v1.Toleration) []v1.Toleration
 	}
 }
 
-func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequirements,
+func (c *Cluster) generatePodTemplate(
+	resourceRequirements *v1.ResourceRequirements,
+	resourceRequirementsScalyrSidecar *v1.ResourceRequirements,
 	tolerationsSpec *[]v1.Toleration,
 	pgParameters *spec.PostgresqlParam,
 	patroniParameters *spec.Patroni,
 	cloneDescription *spec.CloneDescription,
 	dockerImage *string,
-	customPodEnvVars map[string]string) *v1.PodTemplateSpec {
+	customPodEnvVars map[string]string,
+) *v1.PodTemplateSpec {
 	spiloConfiguration := c.generateSpiloJSONConfiguration(pgParameters, patroniParameters)
 
 	envVars := []v1.EnvVar{
@@ -391,6 +394,12 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 	if dockerImage != nil && *dockerImage != "" {
 		containerImage = *dockerImage
 	}
+	volumeMounts := []v1.VolumeMount{
+		{
+			Name:      constants.DataVolumeName,
+			MountPath: constants.PostgresDataMount, //TODO: fetch from manifest
+		},
+	}
 	container := v1.Container{
 		Name:            c.containerName(),
 		Image:           containerImage,
@@ -410,13 +419,8 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 				Protocol:      v1.ProtocolTCP,
 			},
 		},
-		VolumeMounts: []v1.VolumeMount{
-			{
-				Name:      constants.DataVolumeName,
-				MountPath: constants.PostgresDataMount, //TODO: fetch from manifesto
-			},
-		},
-		Env: envVars,
+		VolumeMounts: volumeMounts,
+		Env:          envVars,
 		SecurityContext: &v1.SecurityContext{
 			Privileged: &privilegedMode,
 		},
@@ -431,6 +435,33 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 		Tolerations:                   c.tolerations(tolerationsSpec),
 	}
 
+	if c.OpConfig.ScalyrAPIKey != "" && c.OpConfig.ScalyrImage != "" {
+		podSpec.Containers = append(
+			podSpec.Containers,
+			v1.Container{
+				Name:            "scalyr-sidecar",
+				Image:           c.OpConfig.ScalyrImage,
+				ImagePullPolicy: v1.PullIfNotPresent,
+				Resources:       *resourceRequirementsScalyrSidecar,
+				VolumeMounts:    volumeMounts,
+				Env: []v1.EnvVar{
+					{
+						Name:  "SCALYR_API_KEY",
+						Value: c.OpConfig.ScalyrAPIKey,
+					},
+					{
+						Name:  "SCALYR_SERVER_HOST",
+						Value: c.Name,
+					},
+					{
+						Name:  "SCALYR_SERVER_URL",
+						Value: c.OpConfig.ScalyrServerURL,
+					},
+				},
+			},
+		)
+	}
+
 	template := v1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels:    c.labelsSet(),
@@ -445,11 +476,35 @@ func (c *Cluster) generatePodTemplate(resourceRequirements *v1.ResourceRequireme
 	return &template
 }
 
+func makeResources(cpuRequest, memoryRequest, cpuLimit, memoryLimit string) spec.Resources {
+	return spec.Resources{
+		ResourceRequest: spec.ResourceDescription{
+			CPU:    cpuRequest,
+			Memory: memoryRequest,
+		},
+		ResourceLimits: spec.ResourceDescription{
+			CPU:    cpuLimit,
+			Memory: memoryLimit,
+		},
+	}
+}
+
 func (c *Cluster) generateStatefulSet(spec *spec.PostgresSpec) (*v1beta1.StatefulSet, error) {
 	resourceRequirements, err := c.resourceRequirements(spec.Resources)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate resource requirements: %v", err)
 	}
+	resourceRequirementsScalyrSidecar, err := c.resourceRequirements(
+		makeResources(
+			c.OpConfig.ScalyrCPURequest,
+			c.OpConfig.ScalyrMemoryRequest,
+			c.OpConfig.ScalyrCPULimit,
+			c.OpConfig.ScalyrMemoryLimit,
+		),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("could not generate Scalyr sidecar resource requirements: %v", err)
+	}
 	var customPodEnvVars map[string]string
 	if c.OpConfig.PodEnvironmentConfigMap != "" {
 		if cm, err := c.KubeClient.ConfigMaps(c.Namespace).Get(c.OpConfig.PodEnvironmentConfigMap, metav1.GetOptions{}); err != nil {
@@ -458,7 +513,7 @@ func (c *Cluster) generateStatefulSet(spec *spec.PostgresSpec) (*v1beta1.Statefu
 			customPodEnvVars = cm.Data
 		}
 	}
-	podTemplate := c.generatePodTemplate(resourceRequirements, &spec.Tolerations, &spec.PostgresqlParam, &spec.Patroni, &spec.Clone, &spec.DockerImage, customPodEnvVars)
+	podTemplate := c.generatePodTemplate(resourceRequirements, resourceRequirementsScalyrSidecar, &spec.Tolerations, &spec.PostgresqlParam, &spec.Patroni, &spec.Clone, &spec.DockerImage, customPodEnvVars)
 	volumeClaimTemplate, err := generatePersistentVolumeClaimTemplate(spec.Volume.Size, spec.Volume.StorageClass)
 	if err != nil {
 		return nil, fmt.Errorf("could not generate volume claim template: %v", err)

pkg/cluster/sync.go

@@ -320,7 +320,7 @@ func (c *Cluster) syncSecrets() error {
 			if err2 != nil {
 				return fmt.Errorf("could not get current secret: %v", err2)
 			}
-			c.logger.Debugf("secret %q already exists, fetching it's password", util.NameFromMeta(curSecret.ObjectMeta))
+			c.logger.Debugf("secret %q already exists, fetching its password", util.NameFromMeta(curSecret.ObjectMeta))
 			if secretUsername == c.systemUsers[constants.SuperuserKeyName].Name {
 				secretUsername = constants.SuperuserKeyName
 				userMap = c.systemUsers

pkg/util/config/config.go

@@ -50,11 +50,23 @@ type Auth struct {
 	ReplicationUsername           string              `name:"replication_username" default:"standby"`
 }
 
+// Scalyr holds the configuration for the Scalyr Agent sidecar for log shipping:
+type Scalyr struct {
+	ScalyrAPIKey        string `name:"scalyr_api_key" default:""`
+	ScalyrImage         string `name:"scalyr_image" default:""`
+	ScalyrServerURL     string `name:"scalyr_server_url" default:"https://upload.eu.scalyr.com"`
+	ScalyrCPURequest    string `name:"scalyr_cpu_request" default:"100m"`
+	ScalyrMemoryRequest string `name:"scalyr_memory_request" default:"50Mi"`
+	ScalyrCPULimit      string `name:"scalyr_cpu_limit" default:"1"`
+	ScalyrMemoryLimit   string `name:"scalyr_memory_limit" default:"1Gi"`
+}
+
 // Config describes operator config
 type Config struct {
 	CRD
 	Resources
 	Auth
+	Scalyr
 	Namespace                string            `name:"namespace"`
 	EtcdHost                 string            `name:"etcd_host" default:"etcd-client.default.svc.cluster.local:2379"`
 	DockerImage              string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spiloprivate-9.6:1.2-p4"`