Add EKS test environment Terraform templates (#2290)
Co-authored-by: Francesco Renzi <rentziass@gmail.com>
This commit is contained in:
Bassem Dghaidi
GitHub
parent
dab900462b
commit
3886f285f8
|
|
@ -0,0 +1,16 @@
|
||||||
|
**/.terraform/*
|
||||||
|
.terraformrc
|
||||||
|
terraform.rc
|
||||||
|
|
||||||
|
*.tfstate
|
||||||
|
*.tfstate.*
|
||||||
|
*.tfvars
|
||||||
|
*.tfvars.json
|
||||||
|
|
||||||
|
crash.log
|
||||||
|
crash.*.log
|
||||||
|
|
||||||
|
override.tf
|
||||||
|
override.tf.json
|
||||||
|
*_override.tf
|
||||||
|
*_override.tf.json
|
||||||
|
|
@ -0,0 +1,104 @@
|
||||||
|
# This file is maintained automatically by "terraform init".
|
||||||
|
# Manual edits may be lost in future updates.
|
||||||
|
|
||||||
|
provider "registry.terraform.io/hashicorp/aws" {
|
||||||
|
version = "4.54.0"
|
||||||
|
constraints = ">= 3.72.0, >= 3.73.0, >= 4.47.0, ~> 4.54.0"
|
||||||
|
hashes = [
|
||||||
|
"h1:j/L01+hlHVM2X2VrkQC2WtMZyu4ZLhDMw+HDJ7k0Y2Q=",
|
||||||
|
"zh:24358aefc06b3f38878680fe606dab2570cb58ab952750c47e90b81d3b05e606",
|
||||||
|
"zh:3fc0ef459d6bb4fbb0e4eb7b8adadddd636efa6d975be6e70de7327d83e15729",
|
||||||
|
"zh:67e765119726f47b1916316ac95c3cd32ac074b454f2a67b6127120b476bc483",
|
||||||
|
"zh:71aed1300debac24f11263a6f8a231c6432497b25e623e8f34e27121af65f523",
|
||||||
|
"zh:722043077e63713d4e458f3228be30c21fcff5b6660c6de8b96967337cdc604a",
|
||||||
|
"zh:76d67be4220b93cfaca0882f46db9a42b4ca48285a64fe304f108dde85f4d611",
|
||||||
|
"zh:81534c18d9f02648b1644a7937e7bea56e91caef13b41de121ee51168faad680",
|
||||||
|
"zh:89983ab2596846d5f3413ff1b5b9b21424c3c757a54dcc5a4604d3ac34fea1a6",
|
||||||
|
"zh:8a603ac6884de5dc51c372f641f9613aefd87059ff6e6a74b671f6864226e06f",
|
||||||
|
"zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
|
||||||
|
"zh:b6fae6c1cda6d842406066dac7803d24a597b62da5fae33bcd50c5dae70140c2",
|
||||||
|
"zh:bc4c3b4bfb715beecf5186dfeb91173ef1a9c0b68e8c45cbeee180195bbfa37f",
|
||||||
|
"zh:c741a3fe7d085593a160e79596bd237afc9503c836abcc95fd627554cdf16ec0",
|
||||||
|
"zh:f6763e96485e1ea5b67a33bbd04042e412508b2b06946acf957fb68a314d893e",
|
||||||
|
"zh:fc7144577ea7d6e05c276b54a9f8f8609be7b4d0a128aa45f233a4b0e5cbf090",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
provider "registry.terraform.io/hashicorp/cloudinit" {
|
||||||
|
version = "2.2.0"
|
||||||
|
constraints = ">= 2.0.0, ~> 2.2.0"
|
||||||
|
hashes = [
|
||||||
|
"h1:tQLNREqesrdCQ/bIJnl0+yUK+XfdWzAG0wo4lp10LvM=",
|
||||||
|
"zh:76825122171f9ea2287fd27e23e80a7eb482f6491a4f41a096d77b666896ee96",
|
||||||
|
"zh:795a36dee548e30ca9c9d474af9ad6d29290e0a9816154ad38d55381cd0ab12d",
|
||||||
|
"zh:9200f02cb917fb99e44b40a68936fd60d338e4d30a718b7e2e48024a795a61b9",
|
||||||
|
"zh:a33cf255dc670c20678063aa84218e2c1b7a67d557f480d8ec0f68bc428ed472",
|
||||||
|
"zh:ba3c1b2cd0879286c1f531862c027ec04783ece81de67c9a3b97076f1ce7f58f",
|
||||||
|
"zh:bd575456394428a1a02191d2e46af0c00e41fd4f28cfe117d57b6aeb5154a0fb",
|
||||||
|
"zh:c68dd1db83d8437c36c92dc3fc11d71ced9def3483dd28c45f8640cfcd59de9a",
|
||||||
|
"zh:cbfe34a90852ed03cc074601527bb580a648127255c08589bc3ef4bf4f2e7e0c",
|
||||||
|
"zh:d6ffd7398c6d1f359b96f5b757e77b99b339fbb91df1b96ac974fe71bc87695c",
|
||||||
|
"zh:d9c15285f847d7a52df59e044184fb3ba1b7679fd0386291ed183782683d9517",
|
||||||
|
"zh:f7dd02f6d36844da23c9a27bb084503812c29c1aec4aba97237fec16860fdc8c",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
provider "registry.terraform.io/hashicorp/kubernetes" {
|
||||||
|
version = "2.17.0"
|
||||||
|
constraints = ">= 2.10.0"
|
||||||
|
hashes = [
|
||||||
|
"h1:I1L2R+OPgGSh+P6uBSycvvoyRIey/FqMwSvlJ9ccw0o=",
|
||||||
|
"zh:1cbafea8c404195d8ad2490d75dbeebef131563d3e38dec87231ceb3923a3012",
|
||||||
|
"zh:26d9584423ee77e607999b082de7d9dc3e937934aa83341e0832e7253caf4f51",
|
||||||
|
"zh:333527fc15fb43bbf1898a2f058598c596468a01d88c415627bb617878dc4d4d",
|
||||||
|
"zh:391b8c80e3115af485977d6e949d7260b7fc0b641089b884256bfd36a7077db2",
|
||||||
|
"zh:4d18ba55247486181759d60195777945bcd68e17ccd980820ca18e8a8b94aeb5",
|
||||||
|
"zh:607ae94d85d1c1ed3845bd71095daadea4b2468e16f57fa05c98eab0de6b14ae",
|
||||||
|
"zh:95c6cf22f8ef14e7a4f85e33cff5d6f11056c7880041b71d425d1b5ebbe246e7",
|
||||||
|
"zh:b077edcedb46a313b461ac1e49317872063b3871f2acbe1a50498612cefff387",
|
||||||
|
"zh:c6a7891683e44148b0c928fd4748b7abac727266ab551d679015f5fe8b72d1e6",
|
||||||
|
"zh:e5cebfdf873770c37a4304362003d3fea8d6c2fd819663ad121bc65bb81e4738",
|
||||||
|
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
|
||||||
|
"zh:feb19269e7c0de473ad412b37818b48da0cc91e5c93dd4c77a72676ca97a16b1",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
provider "registry.terraform.io/hashicorp/random" {
|
||||||
|
version = "3.4.3"
|
||||||
|
constraints = "~> 3.4.3"
|
||||||
|
hashes = [
|
||||||
|
"h1:xZGZf18JjMS06pFa4NErzANI98qi59SEcBsOcS2P2yQ=",
|
||||||
|
"zh:41c53ba47085d8261590990f8633c8906696fa0a3c4b384ff6a7ecbf84339752",
|
||||||
|
"zh:59d98081c4475f2ad77d881c4412c5129c56214892f490adf11c7e7a5a47de9b",
|
||||||
|
"zh:686ad1ee40b812b9e016317e7f34c0d63ef837e084dea4a1f578f64a6314ad53",
|
||||||
|
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
|
||||||
|
"zh:84103eae7251384c0d995f5a257c72b0096605048f757b749b7b62107a5dccb3",
|
||||||
|
"zh:8ee974b110adb78c7cd18aae82b2729e5124d8f115d484215fd5199451053de5",
|
||||||
|
"zh:9dd4561e3c847e45de603f17fa0c01ae14cae8c4b7b4e6423c9ef3904b308dda",
|
||||||
|
"zh:bb07bb3c2c0296beba0beec629ebc6474c70732387477a65966483b5efabdbc6",
|
||||||
|
"zh:e891339e96c9e5a888727b45b2e1bb3fcbdfe0fd7c5b4396e4695459b38c8cb1",
|
||||||
|
"zh:ea4739860c24dfeaac6c100b2a2e357106a89d18751f7693f3c31ecf6a996f8d",
|
||||||
|
"zh:f0c76ac303fd0ab59146c39bc121c5d7d86f878e9a69294e29444d4c653786f8",
|
||||||
|
"zh:f143a9a5af42b38fed328a161279906759ff39ac428ebcfe55606e05e1518b93",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
provider "registry.terraform.io/hashicorp/tls" {
|
||||||
|
version = "4.0.4"
|
||||||
|
constraints = ">= 3.0.0, ~> 4.0.4"
|
||||||
|
hashes = [
|
||||||
|
"h1:pe9vq86dZZKCm+8k1RhzARwENslF3SXb9ErHbQfgjXU=",
|
||||||
|
"zh:23671ed83e1fcf79745534841e10291bbf34046b27d6e68a5d0aab77206f4a55",
|
||||||
|
"zh:45292421211ffd9e8e3eb3655677700e3c5047f71d8f7650d2ce30242335f848",
|
||||||
|
"zh:59fedb519f4433c0fdb1d58b27c210b27415fddd0cd73c5312530b4309c088be",
|
||||||
|
"zh:5a8eec2409a9ff7cd0758a9d818c74bcba92a240e6c5e54b99df68fff312bbd5",
|
||||||
|
"zh:5e6a4b39f3171f53292ab88058a59e64825f2b842760a4869e64dc1dc093d1fe",
|
||||||
|
"zh:810547d0bf9311d21c81cc306126d3547e7bd3f194fc295836acf164b9f8424e",
|
||||||
|
"zh:824a5f3617624243bed0259d7dd37d76017097dc3193dac669be342b90b2ab48",
|
||||||
|
"zh:9361ccc7048be5dcbc2fafe2d8216939765b3160bd52734f7a9fd917a39ecbd8",
|
||||||
|
"zh:aa02ea625aaf672e649296bce7580f62d724268189fe9ad7c1b36bb0fa12fa60",
|
||||||
|
"zh:c71b4cd40d6ec7815dfeefd57d88bc592c0c42f5e5858dcc88245d371b4b8b1e",
|
||||||
|
"zh:dabcd52f36b43d250a3d71ad7abfa07b5622c69068d989e60b79b2bb4f220316",
|
||||||
|
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,81 @@
|
||||||
|
|
||||||
|
# Context
|
||||||
|
|
||||||
|
Terraform templates to quickly create an EKS cluster with a managed node group. This is not a reference setup! It's a vanilla setup to be used when attempting to replicate issues and/or to test new features.
|
||||||
|
|
||||||
|
⚠️ Do not use this setup in production.
|
||||||
|
|
||||||
|
## Pre-requisites
|
||||||
|
|
||||||
|
- Terraform v1.3+ installed locally.
|
||||||
|
- an AWS account
|
||||||
|
- the AWS CLI v2.7.0/v1.24.0 or newer, installed and configured
|
||||||
|
- AWS IAM Authenticator
|
||||||
|
- kubectl v1.24.0 or newer
|
||||||
|
|
||||||
|
<details>
|
||||||
|
<summary>Download & Authenticate</summary>
|
||||||
|
|
||||||
|
```bash
|
||||||
|
brew install awscli aws-iam-authenticator terraform
|
||||||
|
```
|
||||||
|
|
||||||
|
Configure & authenticate AWS CLI. This will vary based on your AWS account and IAM setup
|
||||||
|
|
||||||
|
</details>
|
||||||
|
|
||||||
|
## Setup
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Export AWS region & profile env variables
|
||||||
|
export AWS_REGION="eu-west-2" # Replace with your region
|
||||||
|
export AWS_PROFILE="actions-compute" # Replace with your profile
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# You're free to use terraform cloud but you need to update main.tf first
|
||||||
|
terraform init
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Run terraform plan
|
||||||
|
terraform plan
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Verify the plan output from the previous step
|
||||||
|
# Run terraform apply
|
||||||
|
terraform apply
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Retrieve access credentials for the cluster and configure kubectl
|
||||||
|
aws eks --region "${AWS_REGION}" update-kubeconfig \
|
||||||
|
--name "$(terraform output -raw cluster_name)" \
|
||||||
|
--profile "${AWS_PROFILE}"
|
||||||
|
|
||||||
|
# If you get this error: 'NoneType' object is not iterable
|
||||||
|
# Remove the ~/.kube/config file and try again
|
||||||
|
# https://github.com/aws/aws-cli/issues/4843
|
||||||
|
```
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Verify your installation
|
||||||
|
kubectl cluster-info
|
||||||
|
```
|
||||||
|
|
||||||
|
Setup ARC by following [this quick-start guide](https://github.com/actions/actions-runner-controller/tree/master/docs/preview/actions-runner-controller-2).
|
||||||
|
|
||||||
|
### Troubleshooting
|
||||||
|
|
||||||
|
#### dial tcp: lookup api.github.com: i/o timeout
|
||||||
|
|
||||||
|
If you see this error in the controller pod logs:
|
||||||
|
|
||||||
|
```log
|
||||||
|
ERROR AutoscalingRunnerSet Failed to initialize Actions service client for creating a new runner scale set {"autoscalingrunnerset": "arc-runners/arc-runner-set", "error": "failed to get runner registration token: Post \"https://api.github.com/app/installations/33454774/access_tokens\": POST https://api.github.com/app/installations/33454774/access_tokens giving up after 5 attempt(s): Post \"https://api.github.com/app/installations/33454774/access_tokens\": dial tcp: lookup api.github.com: i/o timeout"}
|
||||||
|
```
|
||||||
|
|
||||||
|
This is because the controller pod is not able to resolve the `api.github.com` domain name. This is a good guide for [troubleshooting DNS failures in EKS](https://aws.amazon.com/premiumsupport/knowledge-center/eks-dns-failure/). For a fresh setup this is most likely **a security group configuration problem.**
|
||||||
|
|
||||||
|
The controller could have allocated to a node that cannot reach coredns. You need to allow the DNS (TCP / UDP) traffic to flow between the worker nodes' security groups.
|
||||||
|
|
@ -0,0 +1,82 @@
|
||||||
|
provider "aws" {}
|
||||||
|
|
||||||
|
data "aws_availability_zones" "available" {}
|
||||||
|
|
||||||
|
locals {
|
||||||
|
cluster_name = "arc-e2etests-eks-${random_string.suffix.result}"
|
||||||
|
}
|
||||||
|
|
||||||
|
resource "random_string" "suffix" {
|
||||||
|
length = 8
|
||||||
|
special = false
|
||||||
|
}
|
||||||
|
|
||||||
|
module "vpc" {
|
||||||
|
source = "terraform-aws-modules/vpc/aws"
|
||||||
|
version = "3.19.0"
|
||||||
|
|
||||||
|
name = "arc-e2etests-vpc"
|
||||||
|
|
||||||
|
cidr = "10.0.0.0/16"
|
||||||
|
azs = slice(data.aws_availability_zones.available.names, 0, 3)
|
||||||
|
|
||||||
|
private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
|
||||||
|
public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"]
|
||||||
|
|
||||||
|
enable_nat_gateway = true
|
||||||
|
single_nat_gateway = true
|
||||||
|
enable_dns_hostnames = true
|
||||||
|
|
||||||
|
public_subnet_tags = {
|
||||||
|
"kubernetes.io/cluster/${local.cluster_name}" = "shared"
|
||||||
|
"kubernetes.io/role/elb" = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
private_subnet_tags = {
|
||||||
|
"kubernetes.io/cluster/${local.cluster_name}" = "shared"
|
||||||
|
"kubernetes.io/role/internal-elb" = 1
|
||||||
|
}
|
||||||
|
|
||||||
|
tags = {
|
||||||
|
# Critical: GitHub specific tag
|
||||||
|
"catalog_service" = "actions-runner-controller"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
module "eks" {
|
||||||
|
source = "terraform-aws-modules/eks/aws"
|
||||||
|
version = "19.5.1"
|
||||||
|
|
||||||
|
cluster_name = local.cluster_name
|
||||||
|
cluster_version = "1.24"
|
||||||
|
|
||||||
|
vpc_id = module.vpc.vpc_id
|
||||||
|
subnet_ids = module.vpc.private_subnets
|
||||||
|
cluster_endpoint_public_access = true
|
||||||
|
|
||||||
|
tags = {
|
||||||
|
# Critical: GitHub specific tag
|
||||||
|
# If removed, EC2 instance creation will fail
|
||||||
|
"catalog_service" = "actions-runner-controller"
|
||||||
|
}
|
||||||
|
|
||||||
|
eks_managed_node_group_defaults = {
|
||||||
|
ami_type = "AL2_x86_64"
|
||||||
|
}
|
||||||
|
|
||||||
|
eks_managed_node_groups = {
|
||||||
|
default = {
|
||||||
|
use_custom_launch_template = false
|
||||||
|
}
|
||||||
|
|
||||||
|
primary = {
|
||||||
|
name = "primary-node-group"
|
||||||
|
|
||||||
|
instance_types = ["t3.small"]
|
||||||
|
|
||||||
|
min_size = 1
|
||||||
|
max_size = 3
|
||||||
|
desired_size = 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
output "cluster_endpoint" {
|
||||||
|
description = "Endpoint for EKS control plane"
|
||||||
|
value = module.eks.cluster_endpoint
|
||||||
|
}
|
||||||
|
|
||||||
|
output "cluster_security_group_id" {
|
||||||
|
description = "Security group ids"
|
||||||
|
value = module.eks.cluster_security_group_id
|
||||||
|
}
|
||||||
|
|
||||||
|
output "cluster_name" {
|
||||||
|
description = "Cluster Name"
|
||||||
|
value = module.eks.cluster_name
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,26 @@
|
||||||
|
terraform {
|
||||||
|
|
||||||
|
required_providers {
|
||||||
|
aws = {
|
||||||
|
source = "hashicorp/aws"
|
||||||
|
version = "~> 4.54.0"
|
||||||
|
}
|
||||||
|
|
||||||
|
random = {
|
||||||
|
source = "hashicorp/random"
|
||||||
|
version = "~> 3.4.3"
|
||||||
|
}
|
||||||
|
|
||||||
|
tls = {
|
||||||
|
source = "hashicorp/tls"
|
||||||
|
version = "~> 4.0.4"
|
||||||
|
}
|
||||||
|
|
||||||
|
cloudinit = {
|
||||||
|
source = "hashicorp/cloudinit"
|
||||||
|
version = "~> 2.2.0"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
required_version = "~> 1.3"
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue