From 7d783f3583b0e5429f983b69efa727ad9164cea0 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 8 May 2024 18:46:01 -0400 Subject: [PATCH 01/10] Updating Karpenter module --- modules/karpenter/main.tf | 13 +++++++++++++ modules/karpenter/outputs.tf | 14 ++++++++++++++ modules/karpenter/variables.tf | 22 ++++++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 5d82475b6a..5018ae7c16 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -411,6 +411,19 @@ resource "aws_iam_role_policy_attachment" "controller_additional" { policy_arn = each.value } +# Pod Identity Association + +resource "aws_eks_pod_identity_association" "karpenter" { + # TODO Update this condition in the next breaking change + count = local.create_iam_role && var.enable_pod_identity && var.create_pod_identity_association ? 1 : 0 + + cluster_name = var.cluster_name + namespace = var.namespace + service_account = var.service_account + role_arn = aws_iam_role.controller[0].arn +} + + ################################################################################ # Node Termination Queue ################################################################################ diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf index 164baa142c..0da7b5b658 100644 --- a/modules/karpenter/outputs.tf +++ b/modules/karpenter/outputs.tf @@ -96,3 +96,17 @@ output "instance_profile_unique" { description = "Stable and unique string identifying the IAM instance profile" value = try(aws_iam_instance_profile.this[0].unique_id, null) } + +################################################################################ +# Pod Identity +################################################################################ + +output "namespace" { + description = "Namespace for the Karpenter Pod Identity association" + value = var.namespace +} + +output "service_account" { + description = "Service Account to be associated with Karpenter Pod Identity" + value = var.service_account +} \ No newline at end of file diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index fc79b1a413..ea8ba7872e 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -138,6 +138,28 @@ variable "irsa_assume_role_condition_test" { default = "StringEquals" } +################################################################################ +# Pod Identity +################################################################################ +# TODO: Remove this in the next breaking change +variable "create_pod_identity_association" { + description = "Determines whether to create pod identity association" + type = bool + default = false +} + +variable "namespace" { + description = "Namespace where pod identity will be created" + type = string + default = "kube-system" +} + +variable "service_account" { + description = "Service account where pod identity will be created" + type = string + default = "karpenter" +} + ################################################################################ # Node Termination Queue ################################################################################ From 9b242afae2de2b2c63955a35e477bb5b59386491 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 8 May 2024 18:46:46 -0400 Subject: [PATCH 02/10] Updating Karpenter example --- examples/karpenter/main.tf | 86 ++++++++++++++------------------------ 1 file changed, 32 insertions(+), 54 deletions(-) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 9247d6e1ee..f77ff93338 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -71,53 +71,40 @@ module "eks" { enable_cluster_creator_admin_permissions = true cluster_addons = { - coredns = { - configuration_values = jsonencode({ - computeType = "Fargate" - # Ensure that we fully utilize the minimum amount of resources that are supplied by - # Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html - # Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes - # components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following - # compute configuration that most closely matches the sum of vCPU and memory requests in - # order to ensure pods always have the resources that they need to run. - resources = { - limits = { - cpu = "0.25" - # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - requests = { - cpu = "0.25" - # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - } - }) - } - kube-proxy = {} - vpc-cni = {} + coredns = {} + kube-proxy = {} + vpc-cni = {} + eks-pod-identity-agent = {} } vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets - # Fargate profiles use the cluster primary security group so these are not utilized - create_cluster_security_group = false - create_node_security_group = false - - fargate_profiles = { - karpenter = { - selectors = [ - { namespace = "karpenter" } - ] - } - kube-system = { - selectors = [ - { namespace = "kube-system" } - ] + # eks_managed_node_group_defaults = { + # ami_type = "BOTTLEROCKET_x86_64" + # instance_types = ["t3.large", "t3a.large"] + + # iam_role_attach_cni_policy = true + # } + + eks_managed_node_groups = { + initial = { + instance_types = ["m5.large"] + + min_size = 1 + max_size = 3 + desired_size = 2 + + taints = { + # This Taint aims to keep just EKS Addons and Karpenter running on this MNG + # The pods that do not tolerate this taint should run on nodes created by Karpenter + addons = { + key = "CriticalAddonsOnly" + value = "true" + effect = "NO_SCHEDULE" + }, + } } } @@ -138,9 +125,8 @@ module "karpenter" { cluster_name = module.eks.cluster_name - # EKS Fargate currently does not support Pod Identity - enable_irsa = true - irsa_oidc_provider_arn = module.eks.oidc_provider_arn + enable_pod_identity = true + create_pod_identity_association = true # Used to attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { @@ -162,14 +148,14 @@ module "karpenter_disabled" { ################################################################################ resource "helm_release" "karpenter" { - namespace = "karpenter" + namespace = "kube-system" create_namespace = true name = "karpenter" repository = "oci://public.ecr.aws/karpenter" repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "0.35.1" + version = "0.36.1" wait = false values = [ @@ -178,14 +164,6 @@ resource "helm_release" "karpenter" { clusterName: ${module.eks.cluster_name} clusterEndpoint: ${module.eks.cluster_endpoint} interruptionQueue: ${module.karpenter.queue_name} - serviceAccount: - annotations: - eks.amazonaws.com/role-arn: ${module.karpenter.iam_role_arn} - tolerations: - - key: 'eks.amazonaws.com/compute-type' - operator: Equal - value: fargate - effect: "NoSchedule" EOT ] } From b7cfc49fbdd0b9cbf048bbf6b8ddd1dfe65486e7 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 8 May 2024 18:50:16 -0400 Subject: [PATCH 03/10] Running pre-commit on `modules/karpenter` --- modules/karpenter/README.md | 6 ++++++ modules/karpenter/outputs.tf | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index ae39e6ccb3..a8df252865 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -104,6 +104,7 @@ No modules. | [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | | [aws_eks_access_entry.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | +| [aws_eks_pod_identity_association.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_pod_identity_association) | resource | | [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_policy.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | @@ -135,6 +136,7 @@ No modules. | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created | `bool` | `true` | no | | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `false` | no | | [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_pod\_identity\_association](#input\_create\_pod\_identity\_association) | Determines whether to create pod identity association | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support for IAM role for service accounts | `bool` | `false` | no | | [enable\_pod\_identity](#input\_enable\_pod\_identity) | Determines whether to enable support for EKS pod identity | `bool` | `true` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | @@ -153,6 +155,7 @@ No modules. | [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | | [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` |
[
"karpenter:karpenter"
]
| no | | [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | +| [namespace](#input\_namespace) | Namespace where pod identity will be created | `string` | `"kube-system"` | no | | [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [node\_iam\_role\_attach\_cni\_policy](#input\_node\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | @@ -168,6 +171,7 @@ No modules. | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | | [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | | [rule\_name\_prefix](#input\_rule\_name\_prefix) | Prefix used for all event bridge rules | `string` | `"Karpenter"` | no | +| [service\_account](#input\_service\_account) | Service account where pod identity will be created | `string` | `"karpenter"` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | ## Outputs @@ -182,6 +186,7 @@ No modules. | [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID | | [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile | | [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [namespace](#output\_namespace) | Namespace for the Karpenter Pod Identity association | | [node\_access\_entry\_arn](#output\_node\_access\_entry\_arn) | Amazon Resource Name (ARN) of the node Access Entry | | [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | | [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | The name of the node IAM role | @@ -189,4 +194,5 @@ No modules. | [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue | | [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue | | [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | +| [service\_account](#output\_service\_account) | Service Account to be associated with Karpenter Pod Identity | diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf index 0da7b5b658..839668e7e6 100644 --- a/modules/karpenter/outputs.tf +++ b/modules/karpenter/outputs.tf @@ -109,4 +109,4 @@ output "namespace" { output "service_account" { description = "Service Account to be associated with Karpenter Pod Identity" value = var.service_account -} \ No newline at end of file +} From 2a16bc6f14f04ec48cd34f34433658d90b76ce58 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 8 May 2024 19:09:33 -0400 Subject: [PATCH 04/10] Updating example/karpenter/README` --- examples/karpenter/README.md | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 9ccd42c945..69bc52e832 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -22,10 +22,40 @@ aws eks --region eu-west-1 update-kubeconfig --name ex-karpenter kubectl scale deployment inflate --replicas 5 # You can watch Karpenter's controller logs with -kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller +kubectl logs -f -n kube-system -l app.kubernetes.io/name=karpenter -c controller ``` -You should see a new node named `karpenter.sh/provisioner-name/default` eventually come up in the console; this was provisioned by Karpenter in response to the scaled deployment above. +Validate if the Amazon EKS Addons Pods are running in the Managed Node Group and the `inflate` application Pods are running on Karpenter provisioned Nodes. + +```bash +kubectl get nodes -L karpenter.sh/registered +NAME STATUS ROLES AGE VERSION REGISTERED +ip-10-0-16-155.eu-west-1.compute.internal Ready 100s v1.29.3-eks-ae9a62a true +ip-10-0-3-23.eu-west-1.compute.internal Ready 6m1s v1.29.3-eks-ae9a62a +ip-10-0-41-2.eu-west-1.compute.internal Ready 6m3s v1.29.3-eks-ae9a62a + +kubectl get pods -A -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName + +NAME NODE +inflate-75d744d4c6-nqwz8 ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-nrqnn ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-sp4dx ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-xqzd9 ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-xr6p5 ip-10-0-16-155.eu-west-1.compute.internal +aws-node-mnn7r ip-10-0-3-23.eu-west-1.compute.internal +aws-node-rkmvm ip-10-0-16-155.eu-west-1.compute.internal +aws-node-s4slh ip-10-0-41-2.eu-west-1.compute.internal +coredns-68bd859788-7rcfq ip-10-0-3-23.eu-west-1.compute.internal +coredns-68bd859788-l78hw ip-10-0-41-2.eu-west-1.compute.internal +eks-pod-identity-agent-gbx8l ip-10-0-41-2.eu-west-1.compute.internal +eks-pod-identity-agent-s7vt7 ip-10-0-16-155.eu-west-1.compute.internal +eks-pod-identity-agent-xwgqw ip-10-0-3-23.eu-west-1.compute.internal +karpenter-79f59bdfdc-9q5ff ip-10-0-41-2.eu-west-1.compute.internal +karpenter-79f59bdfdc-cxvhr ip-10-0-3-23.eu-west-1.compute.internal +kube-proxy-7crbl ip-10-0-41-2.eu-west-1.compute.internal +kube-proxy-jtzds ip-10-0-16-155.eu-west-1.compute.internal +kube-proxy-sm42c ip-10-0-3-23.eu-west-1.compute.internal +``` ### Tear Down & Clean-Up From d401b31eda8de6fd682079eb3608ad0514dcf886 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 8 May 2024 19:12:27 -0400 Subject: [PATCH 05/10] Updating module/karpenter/README` --- modules/karpenter/README.md | 13 +++++++------ modules/karpenter/main.tf | 1 - 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index a8df252865..4ddd353e8b 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -11,6 +11,7 @@ In the following example, the Karpenter module will create: - A Node IAM role that Karpenter will use to create an Instance Profile for the nodes to receive IAM permissions - An access entry for the Node IAM role to allow nodes to join the cluster - SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. +- A Pod Identity association to grant Karpenter controller access provided by the IAM Role ```hcl module "eks" { @@ -20,19 +21,19 @@ module "eks" { } module "karpenter" { - source = "terraform-aws-modules/eks/aws//modules/karpenter" + source = "../../modules/karpenter" cluster_name = module.eks.cluster_name - # Attach additional IAM policies to the Karpenter node IAM role + enable_pod_identity = true + create_pod_identity_association = true + + # Used to attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } - tags = { - Environment = "dev" - Terraform = "true" - } + tags = local.tags } ``` diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 5018ae7c16..7954593649 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -423,7 +423,6 @@ resource "aws_eks_pod_identity_association" "karpenter" { role_arn = aws_iam_role.controller[0].arn } - ################################################################################ # Node Termination Queue ################################################################################ From 343e9e794cd5c446ac75d9f29dedf82fe78369e3 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 8 May 2024 19:23:38 -0400 Subject: [PATCH 06/10] Removing unecessary comment --- examples/karpenter/main.tf | 7 ------- 1 file changed, 7 deletions(-) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index f77ff93338..f394c98851 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -81,13 +81,6 @@ module "eks" { subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets - # eks_managed_node_group_defaults = { - # ami_type = "BOTTLEROCKET_x86_64" - # instance_types = ["t3.large", "t3a.large"] - - # iam_role_attach_cni_policy = true - # } - eks_managed_node_groups = { initial = { instance_types = ["m5.large"] From 944481ff7c288b1cd440f015545c6eaf6d603728 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 8 May 2024 20:34:12 -0400 Subject: [PATCH 07/10] Code review --- examples/karpenter/main.tf | 1 - modules/karpenter/README.md | 10 +++++----- modules/karpenter/main.tf | 2 ++ modules/karpenter/outputs.tf | 4 ++-- modules/karpenter/variables.tf | 4 ++-- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index f394c98851..d2054b9f43 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -142,7 +142,6 @@ module "karpenter_disabled" { resource "helm_release" "karpenter" { namespace = "kube-system" - create_namespace = true name = "karpenter" repository = "oci://public.ecr.aws/karpenter" repository_username = data.aws_ecrpublic_authorization_token.token.user_name diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 4ddd353e8b..11804a2b3b 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -21,19 +21,19 @@ module "eks" { } module "karpenter" { - source = "../../modules/karpenter" + source = "terraform-aws-modules/eks/aws//modules/karpenter" cluster_name = module.eks.cluster_name - enable_pod_identity = true - create_pod_identity_association = true - # Used to attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } - tags = local.tags + tags = { + Environment = "dev" + Terraform = "true" + } } ``` diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 7954593649..a9d36fa630 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -421,6 +421,8 @@ resource "aws_eks_pod_identity_association" "karpenter" { namespace = var.namespace service_account = var.service_account role_arn = aws_iam_role.controller[0].arn + + tags = var.tags } ################################################################################ diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf index 839668e7e6..a71d47242d 100644 --- a/modules/karpenter/outputs.tf +++ b/modules/karpenter/outputs.tf @@ -102,11 +102,11 @@ output "instance_profile_unique" { ################################################################################ output "namespace" { - description = "Namespace for the Karpenter Pod Identity association" + description = "Namespace associated with the Karpenter Pod Identity" value = var.namespace } output "service_account" { - description = "Service Account to be associated with Karpenter Pod Identity" + description = "Service Account associated with the Karpenter Pod Identity" value = var.service_account } diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index ea8ba7872e..895367f1df 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -149,13 +149,13 @@ variable "create_pod_identity_association" { } variable "namespace" { - description = "Namespace where pod identity will be created" + description = "Namespace to associate with the Karpenter Pod Identity" type = string default = "kube-system" } variable "service_account" { - description = "Service account where pod identity will be created" + description = "Service account to associate with the Karpenter Pod Identity" type = string default = "karpenter" } From c5eaa9e180b0b4ad78d3ae6dd39a41d1c719a679 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 8 May 2024 20:34:53 -0400 Subject: [PATCH 08/10] pre-commit --- modules/karpenter/README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 11804a2b3b..2ce7443b70 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -30,7 +30,7 @@ module "karpenter" { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } - tags = { + tags = { Environment = "dev" Terraform = "true" } @@ -156,7 +156,7 @@ No modules. | [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | | [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` |
[
"karpenter:karpenter"
]
| no | | [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | -| [namespace](#input\_namespace) | Namespace where pod identity will be created | `string` | `"kube-system"` | no | +| [namespace](#input\_namespace) | Namespace to associate with the Karpenter Pod Identity | `string` | `"kube-system"` | no | | [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [node\_iam\_role\_attach\_cni\_policy](#input\_node\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | @@ -172,7 +172,7 @@ No modules. | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | | [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | | [rule\_name\_prefix](#input\_rule\_name\_prefix) | Prefix used for all event bridge rules | `string` | `"Karpenter"` | no | -| [service\_account](#input\_service\_account) | Service account where pod identity will be created | `string` | `"karpenter"` | no | +| [service\_account](#input\_service\_account) | Service account to associate with the Karpenter Pod Identity | `string` | `"karpenter"` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | ## Outputs @@ -187,7 +187,7 @@ No modules. | [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID | | [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile | | [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | -| [namespace](#output\_namespace) | Namespace for the Karpenter Pod Identity association | +| [namespace](#output\_namespace) | Namespace associated with the Karpenter Pod Identity | | [node\_access\_entry\_arn](#output\_node\_access\_entry\_arn) | Amazon Resource Name (ARN) of the node Access Entry | | [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | | [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | The name of the node IAM role | @@ -195,5 +195,5 @@ No modules. | [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue | | [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue | | [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | -| [service\_account](#output\_service\_account) | Service Account to be associated with Karpenter Pod Identity | +| [service\_account](#output\_service\_account) | Service Account associated with the Karpenter Pod Identity | From 81cc75cbd80579beb51f1f99169374d681c14e89 Mon Sep 17 00:00:00 2001 From: Rodrigo Bersa Date: Wed, 8 May 2024 20:41:08 -0400 Subject: [PATCH 09/10] fix example docs --- examples/karpenter/README.md | 3 +-- examples/karpenter/outputs.tf | 9 --------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 69bc52e832..3d9057891a 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -1,6 +1,6 @@ # Karpenter Example -Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is running on EKS Fargate yet Karpenter is providing compute in the form of EC2 instances. +Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is running on Managed Node Groups on top of Amazon EC2 instances. ## Usage @@ -147,7 +147,6 @@ No inputs. | [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | -| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [karpenter\_event\_rules](#output\_karpenter\_event\_rules) | Map of the event rules created and their attributes | | [karpenter\_iam\_role\_arn](#output\_karpenter\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the controller IAM role | | [karpenter\_iam\_role\_name](#output\_karpenter\_iam\_role\_name) | The name of the controller IAM role | diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index de0e2e6a28..21f84f8a6d 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -164,15 +164,6 @@ output "cloudwatch_log_group_arn" { value = module.eks.cloudwatch_log_group_arn } -################################################################################ -# Fargate Profile -################################################################################ - -output "fargate_profiles" { - description = "Map of attribute maps for all EKS Fargate Profiles created" - value = module.eks.fargate_profiles -} - ################################################################################ # EKS Managed Node Group ################################################################################ From c9bc42a097dc551c56231ca5869e0b004d0f239c Mon Sep 17 00:00:00 2001 From: Bryant Biggs Date: Thu, 9 May 2024 07:54:56 -0400 Subject: [PATCH 10/10] chore: Updates --- examples/karpenter/README.md | 10 +++++++++- examples/karpenter/main.tf | 17 ++++++++--------- examples/karpenter/outputs.tf | 9 +++++++++ modules/karpenter/README.md | 4 ++-- modules/karpenter/main.tf | 3 ++- modules/karpenter/variables.tf | 4 ++-- 6 files changed, 32 insertions(+), 15 deletions(-) diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 3d9057891a..def4eb9f21 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -1,6 +1,6 @@ # Karpenter Example -Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is running on Managed Node Groups on top of Amazon EC2 instances. +Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is provisioned on top of an EKS Managed Node Group. ## Usage @@ -29,13 +29,20 @@ Validate if the Amazon EKS Addons Pods are running in the Managed Node Group and ```bash kubectl get nodes -L karpenter.sh/registered +``` + +```text NAME STATUS ROLES AGE VERSION REGISTERED ip-10-0-16-155.eu-west-1.compute.internal Ready 100s v1.29.3-eks-ae9a62a true ip-10-0-3-23.eu-west-1.compute.internal Ready 6m1s v1.29.3-eks-ae9a62a ip-10-0-41-2.eu-west-1.compute.internal Ready 6m3s v1.29.3-eks-ae9a62a +``` +```sh kubectl get pods -A -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName +``` +```text NAME NODE inflate-75d744d4c6-nqwz8 ip-10-0-16-155.eu-west-1.compute.internal inflate-75d744d4c6-nrqnn ip-10-0-16-155.eu-west-1.compute.internal @@ -147,6 +154,7 @@ No inputs. | [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate | | [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created | | [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups | +| [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created | | [karpenter\_event\_rules](#output\_karpenter\_event\_rules) | Map of the event rules created and their attributes | | [karpenter\_iam\_role\_arn](#output\_karpenter\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the controller IAM role | | [karpenter\_iam\_role\_name](#output\_karpenter\_iam\_role\_name) | The name of the controller IAM role | diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index d2054b9f43..800082ef85 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -41,9 +41,8 @@ data "aws_ecrpublic_authorization_token" "token" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.29" - region = "eu-west-1" + name = "ex-${basename(path.cwd)}" + region = "eu-west-1" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -62,19 +61,19 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_public_access = true + cluster_name = local.name + cluster_version = "1.29" # Gives Terraform identity admin access to cluster which will # allow deploying resources (Karpenter) into the cluster enable_cluster_creator_admin_permissions = true + cluster_endpoint_public_access = true cluster_addons = { coredns = {} + eks-pod-identity-agent = {} kube-proxy = {} vpc-cni = {} - eks-pod-identity-agent = {} } vpc_id = module.vpc.vpc_id @@ -82,10 +81,10 @@ module "eks" { control_plane_subnet_ids = module.vpc.intra_subnets eks_managed_node_groups = { - initial = { + karpenter = { instance_types = ["m5.large"] - min_size = 1 + min_size = 2 max_size = 3 desired_size = 2 diff --git a/examples/karpenter/outputs.tf b/examples/karpenter/outputs.tf index 21f84f8a6d..de0e2e6a28 100644 --- a/examples/karpenter/outputs.tf +++ b/examples/karpenter/outputs.tf @@ -164,6 +164,15 @@ output "cloudwatch_log_group_arn" { value = module.eks.cloudwatch_log_group_arn } +################################################################################ +# Fargate Profile +################################################################################ + +output "fargate_profiles" { + description = "Map of attribute maps for all EKS Fargate Profiles created" + value = module.eks.fargate_profiles +} + ################################################################################ # EKS Managed Node Group ################################################################################ diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 2ce7443b70..ec819ed256 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -8,10 +8,10 @@ Configuration in this directory creates the AWS resources required by Karpenter In the following example, the Karpenter module will create: - An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller +- A Pod Identity association to grant Karpenter controller access provided by the IAM Role - A Node IAM role that Karpenter will use to create an Instance Profile for the nodes to receive IAM permissions - An access entry for the Node IAM role to allow nodes to join the cluster - SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. -- A Pod Identity association to grant Karpenter controller access provided by the IAM Role ```hcl module "eks" { @@ -25,7 +25,7 @@ module "karpenter" { cluster_name = module.eks.cluster_name - # Used to attach additional IAM policies to the Karpenter node IAM role + # Attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { AmazonSSMManagedInstanceCore = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" } diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index a9d36fa630..8a3c9c1b71 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -411,10 +411,11 @@ resource "aws_iam_role_policy_attachment" "controller_additional" { policy_arn = each.value } +################################################################################ # Pod Identity Association +################################################################################ resource "aws_eks_pod_identity_association" "karpenter" { - # TODO Update this condition in the next breaking change count = local.create_iam_role && var.enable_pod_identity && var.create_pod_identity_association ? 1 : 0 cluster_name = var.cluster_name diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index 895367f1df..87238c8389 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -139,9 +139,9 @@ variable "irsa_assume_role_condition_test" { } ################################################################################ -# Pod Identity +# Pod Identity Association ################################################################################ -# TODO: Remove this in the next breaking change +# TODO - Change default to `true` at next breaking change variable "create_pod_identity_association" { description = "Determines whether to create pod identity association" type = bool