diff --git a/test/platforms/aws-eks/.gitignore b/test/platforms/aws-eks/.gitignore new file mode 100644 index 00000000..dc0ad0e5 --- /dev/null +++ b/test/platforms/aws-eks/.gitignore @@ -0,0 +1,16 @@ +**/.terraform/* +.terraformrc +terraform.rc + +*.tfstate +*.tfstate.* +*.tfvars +*.tfvars.json + +crash.log +crash.*.log + +override.tf +override.tf.json +*_override.tf +*_override.tf.json \ No newline at end of file diff --git a/test/platforms/aws-eks/.terraform.lock.hcl b/test/platforms/aws-eks/.terraform.lock.hcl new file mode 100644 index 00000000..9e1de941 --- /dev/null +++ b/test/platforms/aws-eks/.terraform.lock.hcl @@ -0,0 +1,104 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/aws" { + version = "4.54.0" + constraints = ">= 3.72.0, >= 3.73.0, >= 4.47.0, ~> 4.54.0" + hashes = [ + "h1:j/L01+hlHVM2X2VrkQC2WtMZyu4ZLhDMw+HDJ7k0Y2Q=", + "zh:24358aefc06b3f38878680fe606dab2570cb58ab952750c47e90b81d3b05e606", + "zh:3fc0ef459d6bb4fbb0e4eb7b8adadddd636efa6d975be6e70de7327d83e15729", + "zh:67e765119726f47b1916316ac95c3cd32ac074b454f2a67b6127120b476bc483", + "zh:71aed1300debac24f11263a6f8a231c6432497b25e623e8f34e27121af65f523", + "zh:722043077e63713d4e458f3228be30c21fcff5b6660c6de8b96967337cdc604a", + "zh:76d67be4220b93cfaca0882f46db9a42b4ca48285a64fe304f108dde85f4d611", + "zh:81534c18d9f02648b1644a7937e7bea56e91caef13b41de121ee51168faad680", + "zh:89983ab2596846d5f3413ff1b5b9b21424c3c757a54dcc5a4604d3ac34fea1a6", + "zh:8a603ac6884de5dc51c372f641f9613aefd87059ff6e6a74b671f6864226e06f", + "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425", + "zh:b6fae6c1cda6d842406066dac7803d24a597b62da5fae33bcd50c5dae70140c2", + "zh:bc4c3b4bfb715beecf5186dfeb91173ef1a9c0b68e8c45cbeee180195bbfa37f", + "zh:c741a3fe7d085593a160e79596bd237afc9503c836abcc95fd627554cdf16ec0", + "zh:f6763e96485e1ea5b67a33bbd04042e412508b2b06946acf957fb68a314d893e", + "zh:fc7144577ea7d6e05c276b54a9f8f8609be7b4d0a128aa45f233a4b0e5cbf090", + ] +} + +provider "registry.terraform.io/hashicorp/cloudinit" { + version = "2.2.0" + constraints = ">= 2.0.0, ~> 2.2.0" + hashes = [ + "h1:tQLNREqesrdCQ/bIJnl0+yUK+XfdWzAG0wo4lp10LvM=", + "zh:76825122171f9ea2287fd27e23e80a7eb482f6491a4f41a096d77b666896ee96", + "zh:795a36dee548e30ca9c9d474af9ad6d29290e0a9816154ad38d55381cd0ab12d", + "zh:9200f02cb917fb99e44b40a68936fd60d338e4d30a718b7e2e48024a795a61b9", + "zh:a33cf255dc670c20678063aa84218e2c1b7a67d557f480d8ec0f68bc428ed472", + "zh:ba3c1b2cd0879286c1f531862c027ec04783ece81de67c9a3b97076f1ce7f58f", + "zh:bd575456394428a1a02191d2e46af0c00e41fd4f28cfe117d57b6aeb5154a0fb", + "zh:c68dd1db83d8437c36c92dc3fc11d71ced9def3483dd28c45f8640cfcd59de9a", + "zh:cbfe34a90852ed03cc074601527bb580a648127255c08589bc3ef4bf4f2e7e0c", + "zh:d6ffd7398c6d1f359b96f5b757e77b99b339fbb91df1b96ac974fe71bc87695c", + "zh:d9c15285f847d7a52df59e044184fb3ba1b7679fd0386291ed183782683d9517", + "zh:f7dd02f6d36844da23c9a27bb084503812c29c1aec4aba97237fec16860fdc8c", + ] +} + +provider "registry.terraform.io/hashicorp/kubernetes" { + version = "2.17.0" + constraints = ">= 2.10.0" + hashes = [ + "h1:I1L2R+OPgGSh+P6uBSycvvoyRIey/FqMwSvlJ9ccw0o=", + "zh:1cbafea8c404195d8ad2490d75dbeebef131563d3e38dec87231ceb3923a3012", + "zh:26d9584423ee77e607999b082de7d9dc3e937934aa83341e0832e7253caf4f51", + "zh:333527fc15fb43bbf1898a2f058598c596468a01d88c415627bb617878dc4d4d", + "zh:391b8c80e3115af485977d6e949d7260b7fc0b641089b884256bfd36a7077db2", + "zh:4d18ba55247486181759d60195777945bcd68e17ccd980820ca18e8a8b94aeb5", + "zh:607ae94d85d1c1ed3845bd71095daadea4b2468e16f57fa05c98eab0de6b14ae", + "zh:95c6cf22f8ef14e7a4f85e33cff5d6f11056c7880041b71d425d1b5ebbe246e7", + "zh:b077edcedb46a313b461ac1e49317872063b3871f2acbe1a50498612cefff387", + "zh:c6a7891683e44148b0c928fd4748b7abac727266ab551d679015f5fe8b72d1e6", + "zh:e5cebfdf873770c37a4304362003d3fea8d6c2fd819663ad121bc65bb81e4738", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + "zh:feb19269e7c0de473ad412b37818b48da0cc91e5c93dd4c77a72676ca97a16b1", + ] +} + +provider "registry.terraform.io/hashicorp/random" { + version = "3.4.3" + constraints = "~> 3.4.3" + hashes = [ + "h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=", + "zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752", + "zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b", + "zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3", + "zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5", + "zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda", + "zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6", + "zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1", + "zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d", + "zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8", + "zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93", + ] +} + +provider "registry.terraform.io/hashicorp/tls" { + version = "4.0.4" + constraints = ">= 3.0.0, ~> 4.0.4" + hashes = [ + "h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=", + "zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55", + "zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848", + "zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be", + "zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5", + "zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe", + "zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e", + "zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48", + "zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8", + "zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60", + "zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e", + "zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/test/platforms/aws-eks/README.md b/test/platforms/aws-eks/README.md new file mode 100644 index 00000000..354d1df8 --- /dev/null +++ b/test/platforms/aws-eks/README.md @@ -0,0 +1,81 @@ + +# Context + +Terraform templates to quickly create an EKS cluster with a managed node group. This is not a reference setup! It's a vanilla setup to be used when attempting to replicate issues and/or to test new features. + +⚠️ Do not use this setup in production. + +## Pre-requisites + +- Terraform v1.3+ installed locally. +- an AWS account +- the AWS CLI v2.7.0/v1.24.0 or newer, installed and configured +- AWS IAM Authenticator +- kubectl v1.24.0 or newer + +
+ Download & Authenticate + +```bash +brew install awscli aws-iam-authenticator terraform +``` + +Configure & authenticate AWS CLI. This will vary based on your AWS account and IAM setup + +
+ +## Setup + +```bash +# Export AWS region & profile env variables +export AWS_REGION="eu-west-2" # Replace with your region +export AWS_PROFILE="actions-compute" # Replace with your profile +``` + +```bash +# You're free to use terraform cloud but you need to update main.tf first +terraform init +``` + +```bash +# Run terraform plan +terraform plan +``` + +```bash +# Verify the plan output from the previous step +# Run terraform apply +terraform apply +``` + +```bash +# Retrieve access credentials for the cluster and configure kubectl +aws eks --region "${AWS_REGION}" update-kubeconfig \ + --name "$(terraform output -raw cluster_name)" \ + --profile "${AWS_PROFILE}" + +# If you get this error: 'NoneType' object is not iterable +# Remove the ~/.kube/config file and try again +# https://github.com/aws/aws-cli/issues/4843 +``` + +```bash +# Verify your installation +kubectl cluster-info +``` + +Setup ARC by following [this quick-start guide](https://github.com/actions/actions-runner-controller/tree/master/docs/preview/actions-runner-controller-2). + +### Troubleshooting + +#### dial tcp: lookup api.github.com: i/o timeout + +If you see this error in the controller pod logs: + +```log +ERROR AutoscalingRunnerSet Failed to initialize Actions service client for creating a new runner scale set {"autoscalingrunnerset": "arc-runners/arc-runner-set", "error": "failed to get runner registration token: Post \"https://api.github.com/app/installations/33454774/access_tokens\": POST https://api.github.com/app/installations/33454774/access_tokens giving up after 5 attempt(s): Post \"https://api.github.com/app/installations/33454774/access_tokens\": dial tcp: lookup api.github.com: i/o timeout"} +``` + +This is because the controller pod is not able to resolve the `api.github.com` domain name. This is a good guide for [troubleshooting DNS failures in EKS](https://aws.amazon.com/premiumsupport/knowledge-center/eks-dns-failure/). For a fresh setup this is most likely **a security group configuration problem.** + +The controller could have allocated to a node that cannot reach coredns. You need to allow the DNS (TCP / UDP) traffic to flow between the worker nodes' security groups. \ No newline at end of file diff --git a/test/platforms/aws-eks/main.tf b/test/platforms/aws-eks/main.tf new file mode 100644 index 00000000..7bcc8597 --- /dev/null +++ b/test/platforms/aws-eks/main.tf @@ -0,0 +1,82 @@ +provider "aws" {} + +data "aws_availability_zones" "available" {} + +locals { + cluster_name = "arc-e2etests-eks-${random_string.suffix.result}" +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "3.19.0" + + name = "arc-e2etests-vpc" + + cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + + enable_nat_gateway = true + single_nat_gateway = true + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/elb" = 1 + } + + private_subnet_tags = { + "kubernetes.io/cluster/${local.cluster_name}" = "shared" + "kubernetes.io/role/internal-elb" = 1 + } + + tags = { + # Critical: GitHub specific tag + "catalog_service" = "actions-runner-controller" + } +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" + version = "19.5.1" + + cluster_name = local.cluster_name + cluster_version = "1.24" + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.private_subnets + cluster_endpoint_public_access = true + + tags = { + # Critical: GitHub specific tag + # If removed, EC2 instance creation will fail + "catalog_service" = "actions-runner-controller" + } + + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + } + + eks_managed_node_groups = { + default = { + use_custom_launch_template = false + } + + primary = { + name = "primary-node-group" + + instance_types = ["t3.small"] + + min_size = 1 + max_size = 3 + desired_size = 2 + } + } +} \ No newline at end of file diff --git a/test/platforms/aws-eks/outputs.tf b/test/platforms/aws-eks/outputs.tf new file mode 100644 index 00000000..61e54272 --- /dev/null +++ b/test/platforms/aws-eks/outputs.tf @@ -0,0 +1,14 @@ +output "cluster_endpoint" { + description = "Endpoint for EKS control plane" + value = module.eks.cluster_endpoint +} + +output "cluster_security_group_id" { + description = "Security group ids" + value = module.eks.cluster_security_group_id +} + +output "cluster_name" { + description = "Cluster Name" + value = module.eks.cluster_name +} \ No newline at end of file diff --git a/test/platforms/aws-eks/terraform.tf b/test/platforms/aws-eks/terraform.tf new file mode 100644 index 00000000..0328621a --- /dev/null +++ b/test/platforms/aws-eks/terraform.tf @@ -0,0 +1,26 @@ +terraform { + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.54.0" + } + + random = { + source = "hashicorp/random" + version = "~> 3.4.3" + } + + tls = { + source = "hashicorp/tls" + version = "~> 4.0.4" + } + + cloudinit = { + source = "hashicorp/cloudinit" + version = "~> 2.2.0" + } + } + + required_version = "~> 1.3" +} diff --git a/test/platforms/azure-aks/.keep b/test/platforms/azure-aks/.keep new file mode 100644 index 00000000..e69de29b diff --git a/test/platforms/gcp-gks/.keep b/test/platforms/gcp-gks/.keep new file mode 100644 index 00000000..e69de29b