diff --git a/acceptance/testdata/runnerset.envsubst.yaml b/acceptance/testdata/runnerset.envsubst.yaml index 1a47a521..220cf4e3 100644 --- a/acceptance/testdata/runnerset.envsubst.yaml +++ b/acceptance/testdata/runnerset.envsubst.yaml @@ -1,3 +1,48 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${NAME} +# In kind environments, the provider writes: +# /var/lib/docker/volumes/KIND_NODE_CONTAINER_VOL_ID/_data/local-path-provisioner/PV_NAME +# It can be hundreds of gigabytes depending on what you cache in the test workflow. Beware to not encounter `no space left on device` errors! +# If you did encounter no space errorrs try: +# docker system prune +# docker buildx prune #=> frees up /var/lib/docker/volumes/buildx_buildkit_container-builder0_state +# sudo rm -rf /var/lib/docker/volumes/KIND_NODE_CONTAINER_VOL_ID/_data/local-path-provisioner #=> frees up local-path-provisioner's data +provisioner: rancher.io/local-path +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${NAME}-var-lib-docker + labels: + content: ${NAME}-var-lib-docker +provisioner: rancher.io/local-path +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${NAME}-cache + labels: + content: ${NAME}-cache +provisioner: rancher.io/local-path +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: ${NAME}-runner-tool-cache + labels: + content: ${NAME}-runner-tool-cache +provisioner: rancher.io/local-path +reclaimPolicy: Retain +volumeBindingMode: WaitForFirstConsumer +--- apiVersion: actions.summerwind.dev/v1alpha1 kind: RunnerSet metadata: @@ -59,8 +104,108 @@ spec: containers: - name: runner imagePullPolicy: IfNotPresent - #- name: docker - # #image: mumoshu/actions-runner-dind:dev + env: + - name: RUNNER_FEATURE_FLAG_EPHEMERAL + value: "${RUNNER_FEATURE_FLAG_EPHEMERAL}" + - name: GOMODCACHE + value: "/home/runner/.cache/go-mod" + volumeMounts: + # Cache docker image layers, in case dockerdWithinRunnerContainer=true + - name: var-lib-docker + mountPath: /var/lib/docker + # Cache go modules and builds + # - name: gocache + # # Run `goenv | grep GOCACHE` to verify the path is correct for your env + # mountPath: /home/runner/.cache/go-build + # - name: gomodcache + # # Run `goenv | grep GOMODCACHE` to verify the path is correct for your env + # # mountPath: /home/runner/go/pkg/mod + - name: cache + # go: could not create module cache: stat /home/runner/.cache/go-mod: permission denied + mountPath: "/home/runner/.cache" + - name: runner-tool-cache + # This corresponds to our runner image's default setting of RUNNER_TOOL_CACHE=/opt/hostedtoolcache. + # + # In case you customize the envvar in both runner and docker containers of the runner pod spec, + # You'd need to change this mountPath accordingly. + # + # The tool cache directory is defined in actions/toolkit's tool-cache module: + # https://github.com/actions/toolkit/blob/2f164000dcd42fb08287824a3bc3030dbed33687/packages/tool-cache/src/tool-cache.ts#L621-L638 + # + # Many setup-* actions like setup-go utilizes the tool-cache module to download and cache installed binaries: + # https://github.com/actions/setup-go/blob/56a61c9834b4a4950dbbf4740af0b8a98c73b768/src/installer.ts#L144 + mountPath: "/opt/hostedtoolcache" + # Valid only when dockerdWithinRunnerContainer=false + - name: docker + volumeMounts: + # Cache docker image layers, in case dockerdWithinRunnerContainer=false + - name: var-lib-docker + mountPath: /var/lib/docker + # image: mumoshu/actions-runner-dind:dev + + # For buildx cache + - name: cache + mountPath: "/home/runner/.cache" + volumeClaimTemplates: + - metadata: + name: vol1 + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Mi + storageClassName: ${NAME} + ## Dunno which provider supports auto-provisioning with selector. + ## At least the rancher local path provider stopped with: + ## waiting for a volume to be created, either by external provisioner "rancher.io/local-path" or manually created by system administrator + # selector: + # matchLabels: + # runnerset-volume-id: ${NAME}-vol1 + - metadata: + name: vol2 + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Mi + storageClassName: ${NAME} + # selector: + # matchLabels: + # runnerset-volume-id: ${NAME}-vol2 + - metadata: + name: var-lib-docker + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Mi + storageClassName: ${NAME}-var-lib-docker + - metadata: + name: cache + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Mi + storageClassName: ${NAME}-cache + - metadata: + name: runner-tool-cache + # It turns out labels doesn't distinguish PVs across PVCs and the + # end result is PVs are reused by wrong PVCs. + # The correct way seems to be to differentiate storage class per pvc template. + # labels: + # id: runner-tool-cache + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Mi + storageClassName: ${NAME}-runner-tool-cache --- apiVersion: actions.summerwind.dev/v1alpha1 kind: HorizontalRunnerAutoscaler diff --git a/charts/actions-runner-controller/templates/manager_role.yaml b/charts/actions-runner-controller/templates/manager_role.yaml index 2d189e1c..9b6a8395 100644 --- a/charts/actions-runner-controller/templates/manager_role.yaml +++ b/charts/actions-runner-controller/templates/manager_role.yaml @@ -195,6 +195,28 @@ rules: verbs: - create - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - delete + - get + - list + - patch + - update + - watch - apiGroups: - coordination.k8s.io resources: diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 2cba72e3..e0fda3a1 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -202,6 +202,29 @@ rules: verbs: - create - patch +- apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/controllers/persistent_volume_claim_controller.go b/controllers/persistent_volume_claim_controller.go new file mode 100644 index 00000000..461908ab --- /dev/null +++ b/controllers/persistent_volume_claim_controller.go @@ -0,0 +1,76 @@ +/* +Copyright 2022 The actions-runner-controller authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + + "github.com/go-logr/logr" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" +) + +// RunnerPersistentVolumeClaimReconciler reconciles a PersistentVolume object +type RunnerPersistentVolumeClaimReconciler struct { + client.Client + Log logr.Logger + Recorder record.EventRecorder + Scheme *runtime.Scheme + Name string +} + +// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch + +func (r *RunnerPersistentVolumeClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("pvc", req.NamespacedName) + + var pvc corev1.PersistentVolumeClaim + if err := r.Get(ctx, req.NamespacedName, &pvc); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + log.Info("Reconciling runner pvc") + + res, err := syncPVC(ctx, r.Client, log, req.Namespace, &pvc) + + if res == nil { + res = &ctrl.Result{} + } + + return *res, err +} + +func (r *RunnerPersistentVolumeClaimReconciler) SetupWithManager(mgr ctrl.Manager) error { + name := "runnerpersistentvolumeclaim-controller" + if r.Name != "" { + name = r.Name + } + + r.Recorder = mgr.GetEventRecorderFor(name) + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.PersistentVolumeClaim{}). + Named(name). + Complete(r) +} diff --git a/controllers/persistent_volume_controller.go b/controllers/persistent_volume_controller.go new file mode 100644 index 00000000..dff8aa8c --- /dev/null +++ b/controllers/persistent_volume_controller.go @@ -0,0 +1,72 @@ +/* +Copyright 2022 The actions-runner-controller authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + + "github.com/go-logr/logr" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + corev1 "k8s.io/api/core/v1" +) + +// RunnerPersistentVolumeReconciler reconciles a PersistentVolume object +type RunnerPersistentVolumeReconciler struct { + client.Client + Log logr.Logger + Recorder record.EventRecorder + Scheme *runtime.Scheme + Name string +} + +// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch;update;patch;delete +// +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch + +func (r *RunnerPersistentVolumeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := r.Log.WithValues("pv", req.NamespacedName) + + var pv corev1.PersistentVolume + if err := r.Get(ctx, req.NamespacedName, &pv); err != nil { + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + res, err := syncPV(ctx, r.Client, log, req.Namespace, &pv) + if res == nil { + res = &ctrl.Result{} + } + + return *res, err +} + +func (r *RunnerPersistentVolumeReconciler) SetupWithManager(mgr ctrl.Manager) error { + name := "runnerpersistentvolume-controller" + if r.Name != "" { + name = r.Name + } + + r.Recorder = mgr.GetEventRecorderFor(name) + + return ctrl.NewControllerManagedBy(mgr). + For(&corev1.PersistentVolume{}). + Named(name). + Complete(r) +} diff --git a/controllers/runnerset_controller.go b/controllers/runnerset_controller.go index c5e2a9d6..01d37c2c 100644 --- a/controllers/runnerset_controller.go +++ b/controllers/runnerset_controller.go @@ -58,6 +58,7 @@ type RunnerSetReconciler struct { // +kubebuilder:rbac:groups=actions.summerwind.dev,resources=runnersets/status,verbs=get;update;patch // +kubebuilder:rbac:groups=apps,resources=statefulsets,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=apps,resources=statefulsets/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;create;update @@ -129,6 +130,12 @@ func (r *RunnerSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( owners = append(owners, &ss) } + if res, err := syncVolumes(ctx, r.Client, log, req.Namespace, runnerSet, statefulsets); err != nil { + return ctrl.Result{}, err + } else if res != nil { + return *res, nil + } + res, err := syncRunnerPodsOwners(ctx, r.Client, log, effectiveTime, newDesiredReplicas, func() client.Object { return create.DeepCopy() }, ephemeral, owners) if err != nil || res == nil { return ctrl.Result{}, err diff --git a/controllers/sync_volumes.go b/controllers/sync_volumes.go new file mode 100644 index 00000000..8e83a3d3 --- /dev/null +++ b/controllers/sync_volumes.go @@ -0,0 +1,175 @@ +package controllers + +import ( + "context" + "fmt" + "time" + + "github.com/actions-runner-controller/actions-runner-controller/api/v1alpha1" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + labelKeyCleanup = "pending-cleanup" + labelKeyRunnerStatefulSetName = "runner-statefulset-name" +) + +func syncVolumes(ctx context.Context, c client.Client, log logr.Logger, ns string, runnerSet *v1alpha1.RunnerSet, statefulsets []appsv1.StatefulSet) (*ctrl.Result, error) { + log = log.WithValues("ns", ns) + + for _, t := range runnerSet.Spec.StatefulSetSpec.VolumeClaimTemplates { + for _, sts := range statefulsets { + pvcName := fmt.Sprintf("%s-%s-0", t.Name, sts.Name) + + var pvc corev1.PersistentVolumeClaim + if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvcName}, &pvc); err != nil { + if !kerrors.IsNotFound(err) { + return nil, err + } + continue + } + + // TODO move this to statefulset reconciler so that we spam this less, + // by starting the loop only after the statefulset got deletionTimestamp set. + // Perhaps you can just wrap this in a finalizer here. + if pvc.Labels[labelKeyRunnerStatefulSetName] == "" { + updated := pvc.DeepCopy() + updated.Labels[labelKeyRunnerStatefulSetName] = sts.Name + if err := c.Update(ctx, updated); err != nil { + return nil, err + } + log.V(1).Info("Added runner-statefulset-name label to PVC", "sts", sts.Name, "pvc", pvcName) + } + } + } + + // PVs are not namespaced hence we don't need client.InNamespace(ns). + // If we added that, c.List will silently return zero items. + // + // This `List` needs to be done in a dedicated reconciler that is registered to the manager via the `For` func. + // Otherwise the List func might return outdated contents(I saw status.phase being Bound even after K8s updated it to Released, and it lasted minutes). + // + // cleanupLabels := map[string]string{ + // labelKeyCleanup: runnerSet.Name, + // } + // pvList := &corev1.PersistentVolumeList{} + // if err := c.List(ctx, pvList, client.MatchingLabels(cleanupLabels)); err != nil { + // log.Info("retrying pv listing", "ns", ns, "err", err) + // return nil, err + // } + + return nil, nil +} + +func syncPVC(ctx context.Context, c client.Client, log logr.Logger, ns string, pvc *corev1.PersistentVolumeClaim) (*ctrl.Result, error) { + stsName := pvc.Labels[labelKeyRunnerStatefulSetName] + if stsName == "" { + return nil, nil + } + + var sts appsv1.StatefulSet + if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: stsName}, &sts); err != nil { + if !kerrors.IsNotFound(err) { + return nil, err + } + } else { + // We assume that the statefulset is shortly terminated, hence retry forever until it gets removed. + retry := 10 * time.Second + log.V(1).Info("Retrying sync until statefulset gets removed", "requeueAfter", retry) + return &ctrl.Result{RequeueAfter: retry}, nil + } + + log = log.WithValues("pvc", pvc.Name, "sts", stsName) + + pvName := pvc.Spec.VolumeName + + if pvName != "" { + // If we deleted PVC before unsetting pv.spec.claimRef, + // K8s seems to revive the claimRef :thinking: + // So we need to mark PV for claimRef unset first, and delete PVC, and finally unset claimRef on PV. + + var pv corev1.PersistentVolume + if err := c.Get(ctx, types.NamespacedName{Namespace: ns, Name: pvName}, &pv); err != nil { + if !kerrors.IsNotFound(err) { + return nil, err + } + return nil, nil + } + + pvCopy := pv.DeepCopy() + if pvCopy.Labels == nil { + pvCopy.Labels = map[string]string{} + } + pvCopy.Labels[labelKeyCleanup] = stsName + + log.Info("Scheduling to unset PV's claimRef", "pv", pv.Name) + + // Apparently K8s doesn't reconcile PV immediately after PVC deletion. + // So we start a relatively busy loop of PV reconcilation slightly before the PVC deletion, + // so that PV can be unbound as soon as possible after the PVC got deleted. + if err := c.Update(ctx, pvCopy); err != nil { + return nil, err + } + + // At this point, the PV is still Bound + + log.Info("Deleting unused pvc") + + if err := c.Delete(ctx, pvc); err != nil { + return nil, err + } + + // At this point, the PV is still "Bound", but we are ready to unset pv.spec.claimRef in pv controller. + // Once the pv controller unsets claimRef, the PV becomes "Released", hence available for reuse by another eligible PVC. + } + + return nil, nil +} + +func syncPV(ctx context.Context, c client.Client, log logr.Logger, ns string, pv *corev1.PersistentVolume) (*ctrl.Result, error) { + log.V(2).Info("checking pv claimRef") + + if pv.Spec.ClaimRef == nil { + return nil, nil + } + + log.V(2).Info("checking labels") + + if pv.Labels[labelKeyCleanup] == "" { + // We assume that the pvc is shortly terminated, hence retry forever until it gets removed. + retry := 10 * time.Second + log.V(1).Info("Retrying sync until pvc gets removed", "requeueAfter", retry) + return &ctrl.Result{RequeueAfter: retry}, nil + } + + log.V(2).Info("checking pv phase", "phase", pv.Status.Phase) + + if pv.Status.Phase != corev1.VolumeReleased { + // We assume that the pvc is shortly terminated, hence retry forever until it gets removed. + retry := 10 * time.Second + log.V(1).Info("Retrying sync until pvc gets released", "requeueAfter", retry) + return &ctrl.Result{RequeueAfter: retry}, nil + } + + // At this point, the PV is still Released + + pvCopy := pv.DeepCopy() + delete(pvCopy.Labels, labelKeyCleanup) + pvCopy.Spec.ClaimRef = nil + log.Info("Unsetting PV's claimRef", "pv", pv.Name) + if err := c.Update(ctx, pvCopy); err != nil { + return nil, err + } + + // At this point, the PV becomes Available, if it's reclaim policy is "Retain". + // I have not yet tested it with "Delete" but perhaps it's deleted automatically after the update? + // https://kubernetes.io/docs/concepts/storage/persistent-volumes/#retain + + return nil, nil +} diff --git a/main.go b/main.go index b28c6cfd..9bd31304 100644 --- a/main.go +++ b/main.go @@ -240,6 +240,18 @@ func main() { GitHubClient: ghClient, } + runnerPersistentVolumeReconciler := &controllers.RunnerPersistentVolumeReconciler{ + Client: mgr.GetClient(), + Log: log.WithName("runnerpersistentvolume"), + Scheme: mgr.GetScheme(), + } + + runnerPersistentVolumeClaimReconciler := &controllers.RunnerPersistentVolumeClaimReconciler{ + Client: mgr.GetClient(), + Log: log.WithName("runnerpersistentvolumeclaim"), + Scheme: mgr.GetScheme(), + } + if err = runnerPodReconciler.SetupWithManager(mgr); err != nil { log.Error(err, "unable to create controller", "controller", "RunnerPod") os.Exit(1) @@ -250,6 +262,16 @@ func main() { os.Exit(1) } + if err = runnerPersistentVolumeReconciler.SetupWithManager(mgr); err != nil { + log.Error(err, "unable to create controller", "controller", "RunnerPersistentVolume") + os.Exit(1) + } + + if err = runnerPersistentVolumeClaimReconciler.SetupWithManager(mgr); err != nil { + log.Error(err, "unable to create controller", "controller", "RunnerPersistentVolumeClaim") + os.Exit(1) + } + if err = (&actionsv1alpha1.Runner{}).SetupWebhookWithManager(mgr); err != nil { log.Error(err, "unable to create webhook", "webhook", "Runner") os.Exit(1) diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 897a73a5..0dae7713 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -231,7 +231,7 @@ func initTestEnv(t *testing.T) *env { e.testOrgRepo = testing.Getenv(t, "TEST_ORG_REPO", "") e.testEnterprise = testing.Getenv(t, "TEST_ENTERPRISE", "") e.testEphemeral = testing.Getenv(t, "TEST_EPHEMERAL", "") - e.testJobs = createTestJobs(id, testResultCMNamePrefix, 20) + e.testJobs = createTestJobs(id, testResultCMNamePrefix, 6) e.scaleDownDelaySecondsAfterScaleOut, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_SCALE_DOWN_DELAY_SECONDS_AFTER_SCALE_OUT", "10"), 10, 32) e.minReplicas, _ = strconv.ParseInt(testing.Getenv(t, "TEST_RUNNER_MIN_REPLICAS", "1"), 10, 32) @@ -398,6 +398,62 @@ func installActionsWorkflow(t *testing.T, testName, runnerLabel, testResultCMNam { Uses: testing.ActionsCheckoutV2, }, + { + // This might be the easiest way to handle permissions without use of securityContext + // https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320 + Run: "sudo chmod 777 -R \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"", + }, + { + // This might be the easiest way to handle permissions without use of securityContext + // https://stackoverflow.com/questions/50156124/kubernetes-nfs-persistent-volumes-permission-denied#comment107483717_53186320 + Run: "ls -lah \"${RUNNER_TOOL_CACHE}\" \"${HOME}/.cache\" \"/var/lib/docker\"", + }, + { + Uses: "actions/setup-go@v3", + With: &testing.With{ + GoVersion: ">=1.18.0", + }, + }, + { + Run: "go version", + }, + { + Run: "go build .", + }, + { + // https://github.com/docker/buildx/issues/413#issuecomment-710660155 + // To prevent setup-buildx-action from failing with: + // error: could not create a builder instance with TLS data loaded from environment. Please use `docker context create ` to create a context for current environment and then create a builder instance with `docker buildx create ` + Run: "docker context create mycontext", + }, + { + Run: "docker context use mycontext", + }, + { + Name: "Set up Docker Buildx", + Uses: "docker/setup-buildx-action@v1", + With: &testing.With{ + BuildkitdFlags: "--debug", + Endpoint: "mycontext", + // As the consequence of setting `install: false`, it doesn't install buildx as an alias to `docker build` + // so we need to use `docker buildx build` in the next step + Install: false, + }, + }, + { + Run: "docker buildx build --platform=linux/amd64 " + + "--cache-from=type=local,src=/home/runner/.cache/buildx " + + "--cache-to=type=local,dest=/home/runner/.cache/buildx-new,mode=max " + + ".", + }, + { + // https://github.com/docker/build-push-action/blob/master/docs/advanced/cache.md#local-cache + // See https://github.com/moby/buildkit/issues/1896 for why this is needed + Run: "rm -rf /home/runner/.cache/buildx && mv /home/runner/.cache/buildx-new /home/runner/.cache/buildx", + }, + { + Run: "ls -lah /home/runner/.cache/*", + }, { Uses: "azure/setup-kubectl@v1", With: &testing.With{ diff --git a/testing/workflow.go b/testing/workflow.go index 8150bee1..7da9b7a7 100644 --- a/testing/workflow.go +++ b/testing/workflow.go @@ -42,5 +42,13 @@ type Step struct { } type With struct { - Version string `json:"version,omitempty"` + Version string `json:"version,omitempty"` + GoVersion string `json:"go-version,omitempty"` + + // https://github.com/docker/setup-buildx-action#inputs + BuildkitdFlags string `json:"buildkitd-flags,omitempty"` + Install bool `json:"install,omitempty"` + // This can be either the address or the context name + // https://github.com/docker/buildx/blob/master/docs/reference/buildx_create.md#description + Endpoint string `json:"endpoint,omitempty"` }