diff --git a/examples/karpenter/README.md b/examples/karpenter/README.md index 9ccd42c945..def4eb9f21 100644 --- a/examples/karpenter/README.md +++ b/examples/karpenter/README.md @@ -1,6 +1,6 @@ # Karpenter Example -Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is running on EKS Fargate yet Karpenter is providing compute in the form of EC2 instances. +Configuration in this directory creates an AWS EKS cluster with [Karpenter](https://karpenter.sh/) provisioned for managing compute resource scaling. In the example provided, Karpenter is provisioned on top of an EKS Managed Node Group. ## Usage @@ -22,10 +22,47 @@ aws eks --region eu-west-1 update-kubeconfig --name ex-karpenter kubectl scale deployment inflate --replicas 5 # You can watch Karpenter's controller logs with -kubectl logs -f -n karpenter -l app.kubernetes.io/name=karpenter -c controller +kubectl logs -f -n kube-system -l app.kubernetes.io/name=karpenter -c controller ``` -You should see a new node named `karpenter.sh/provisioner-name/default` eventually come up in the console; this was provisioned by Karpenter in response to the scaled deployment above. +Validate if the Amazon EKS Addons Pods are running in the Managed Node Group and the `inflate` application Pods are running on Karpenter provisioned Nodes. + +```bash +kubectl get nodes -L karpenter.sh/registered +``` + +```text +NAME STATUS ROLES AGE VERSION REGISTERED +ip-10-0-16-155.eu-west-1.compute.internal Ready 100s v1.29.3-eks-ae9a62a true +ip-10-0-3-23.eu-west-1.compute.internal Ready 6m1s v1.29.3-eks-ae9a62a +ip-10-0-41-2.eu-west-1.compute.internal Ready 6m3s v1.29.3-eks-ae9a62a +``` + +```sh +kubectl get pods -A -o custom-columns=NAME:.metadata.name,NODE:.spec.nodeName +``` + +```text +NAME NODE +inflate-75d744d4c6-nqwz8 ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-nrqnn ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-sp4dx ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-xqzd9 ip-10-0-16-155.eu-west-1.compute.internal +inflate-75d744d4c6-xr6p5 ip-10-0-16-155.eu-west-1.compute.internal +aws-node-mnn7r ip-10-0-3-23.eu-west-1.compute.internal +aws-node-rkmvm ip-10-0-16-155.eu-west-1.compute.internal +aws-node-s4slh ip-10-0-41-2.eu-west-1.compute.internal +coredns-68bd859788-7rcfq ip-10-0-3-23.eu-west-1.compute.internal +coredns-68bd859788-l78hw ip-10-0-41-2.eu-west-1.compute.internal +eks-pod-identity-agent-gbx8l ip-10-0-41-2.eu-west-1.compute.internal +eks-pod-identity-agent-s7vt7 ip-10-0-16-155.eu-west-1.compute.internal +eks-pod-identity-agent-xwgqw ip-10-0-3-23.eu-west-1.compute.internal +karpenter-79f59bdfdc-9q5ff ip-10-0-41-2.eu-west-1.compute.internal +karpenter-79f59bdfdc-cxvhr ip-10-0-3-23.eu-west-1.compute.internal +kube-proxy-7crbl ip-10-0-41-2.eu-west-1.compute.internal +kube-proxy-jtzds ip-10-0-16-155.eu-west-1.compute.internal +kube-proxy-sm42c ip-10-0-3-23.eu-west-1.compute.internal +``` ### Tear Down & Clean-Up diff --git a/examples/karpenter/main.tf b/examples/karpenter/main.tf index 9247d6e1ee..800082ef85 100644 --- a/examples/karpenter/main.tf +++ b/examples/karpenter/main.tf @@ -41,9 +41,8 @@ data "aws_ecrpublic_authorization_token" "token" { } locals { - name = "ex-${replace(basename(path.cwd), "_", "-")}" - cluster_version = "1.29" - region = "eu-west-1" + name = "ex-${basename(path.cwd)}" + region = "eu-west-1" vpc_cidr = "10.0.0.0/16" azs = slice(data.aws_availability_zones.available.names, 0, 3) @@ -62,62 +61,42 @@ locals { module "eks" { source = "../.." - cluster_name = local.name - cluster_version = local.cluster_version - cluster_endpoint_public_access = true + cluster_name = local.name + cluster_version = "1.29" # Gives Terraform identity admin access to cluster which will # allow deploying resources (Karpenter) into the cluster enable_cluster_creator_admin_permissions = true + cluster_endpoint_public_access = true cluster_addons = { - coredns = { - configuration_values = jsonencode({ - computeType = "Fargate" - # Ensure that we fully utilize the minimum amount of resources that are supplied by - # Fargate https://docs.aws.amazon.com/eks/latest/userguide/fargate-pod-configuration.html - # Fargate adds 256 MB to each pod's memory reservation for the required Kubernetes - # components (kubelet, kube-proxy, and containerd). Fargate rounds up to the following - # compute configuration that most closely matches the sum of vCPU and memory requests in - # order to ensure pods always have the resources that they need to run. - resources = { - limits = { - cpu = "0.25" - # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - requests = { - cpu = "0.25" - # We are targeting the smallest Task size of 512Mb, so we subtract 256Mb from the - # request/limit to ensure we can fit within that task - memory = "256M" - } - } - }) - } - kube-proxy = {} - vpc-cni = {} + coredns = {} + eks-pod-identity-agent = {} + kube-proxy = {} + vpc-cni = {} } vpc_id = module.vpc.vpc_id subnet_ids = module.vpc.private_subnets control_plane_subnet_ids = module.vpc.intra_subnets - # Fargate profiles use the cluster primary security group so these are not utilized - create_cluster_security_group = false - create_node_security_group = false - - fargate_profiles = { + eks_managed_node_groups = { karpenter = { - selectors = [ - { namespace = "karpenter" } - ] - } - kube-system = { - selectors = [ - { namespace = "kube-system" } - ] + instance_types = ["m5.large"] + + min_size = 2 + max_size = 3 + desired_size = 2 + + taints = { + # This Taint aims to keep just EKS Addons and Karpenter running on this MNG + # The pods that do not tolerate this taint should run on nodes created by Karpenter + addons = { + key = "CriticalAddonsOnly" + value = "true" + effect = "NO_SCHEDULE" + }, + } } } @@ -138,9 +117,8 @@ module "karpenter" { cluster_name = module.eks.cluster_name - # EKS Fargate currently does not support Pod Identity - enable_irsa = true - irsa_oidc_provider_arn = module.eks.oidc_provider_arn + enable_pod_identity = true + create_pod_identity_association = true # Used to attach additional IAM policies to the Karpenter node IAM role node_iam_role_additional_policies = { @@ -162,14 +140,13 @@ module "karpenter_disabled" { ################################################################################ resource "helm_release" "karpenter" { - namespace = "karpenter" - create_namespace = true + namespace = "kube-system" name = "karpenter" repository = "oci://public.ecr.aws/karpenter" repository_username = data.aws_ecrpublic_authorization_token.token.user_name repository_password = data.aws_ecrpublic_authorization_token.token.password chart = "karpenter" - version = "0.35.1" + version = "0.36.1" wait = false values = [ @@ -178,14 +155,6 @@ resource "helm_release" "karpenter" { clusterName: ${module.eks.cluster_name} clusterEndpoint: ${module.eks.cluster_endpoint} interruptionQueue: ${module.karpenter.queue_name} - serviceAccount: - annotations: - eks.amazonaws.com/role-arn: ${module.karpenter.iam_role_arn} - tolerations: - - key: 'eks.amazonaws.com/compute-type' - operator: Equal - value: fargate - effect: "NoSchedule" EOT ] } diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index ae39e6ccb3..ec819ed256 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -8,6 +8,7 @@ Configuration in this directory creates the AWS resources required by Karpenter In the following example, the Karpenter module will create: - An IAM role for use with Pod Identity and a scoped IAM policy for the Karpenter controller +- A Pod Identity association to grant Karpenter controller access provided by the IAM Role - A Node IAM role that Karpenter will use to create an Instance Profile for the nodes to receive IAM permissions - An access entry for the Node IAM role to allow nodes to join the cluster - SQS queue and EventBridge event rules for Karpenter to utilize for spot termination handling, capacity re-balancing, etc. @@ -104,6 +105,7 @@ No modules. | [aws_cloudwatch_event_rule.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_rule) | resource | | [aws_cloudwatch_event_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_event_target) | resource | | [aws_eks_access_entry.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_access_entry) | resource | +| [aws_eks_pod_identity_association.karpenter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_pod_identity_association) | resource | | [aws_iam_instance_profile.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | | [aws_iam_policy.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | @@ -135,6 +137,7 @@ No modules. | [create\_iam\_role](#input\_create\_iam\_role) | Determines whether an IAM role is created | `bool` | `true` | no | | [create\_instance\_profile](#input\_create\_instance\_profile) | Whether to create an IAM instance profile | `bool` | `false` | no | | [create\_node\_iam\_role](#input\_create\_node\_iam\_role) | Determines whether an IAM role is created or to use an existing IAM role | `bool` | `true` | no | +| [create\_pod\_identity\_association](#input\_create\_pod\_identity\_association) | Determines whether to create pod identity association | `bool` | `false` | no | | [enable\_irsa](#input\_enable\_irsa) | Determines whether to enable support for IAM role for service accounts | `bool` | `false` | no | | [enable\_pod\_identity](#input\_enable\_pod\_identity) | Determines whether to enable support for EKS pod identity | `bool` | `true` | no | | [enable\_spot\_termination](#input\_enable\_spot\_termination) | Determines whether to enable native spot termination handling | `bool` | `true` | no | @@ -153,6 +156,7 @@ No modules. | [irsa\_assume\_role\_condition\_test](#input\_irsa\_assume\_role\_condition\_test) | Name of the [IAM condition operator](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_condition_operators.html) to evaluate when assuming the role | `string` | `"StringEquals"` | no | | [irsa\_namespace\_service\_accounts](#input\_irsa\_namespace\_service\_accounts) | List of `namespace:serviceaccount`pairs to use in trust policy for IAM role for service accounts | `list(string)` | 
[
  "karpenter:karpenter"
]
| no | | [irsa\_oidc\_provider\_arn](#input\_irsa\_oidc\_provider\_arn) | OIDC provider arn used in trust policy for IAM role for service accounts | `string` | `""` | no | +| [namespace](#input\_namespace) | Namespace to associate with the Karpenter Pod Identity | `string` | `"kube-system"` | no | | [node\_iam\_role\_additional\_policies](#input\_node\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `map(string)` | `{}` | no | | [node\_iam\_role\_arn](#input\_node\_iam\_role\_arn) | Existing IAM role ARN for the IAM instance profile. Required if `create_iam_role` is set to `false` | `string` | `null` | no | | [node\_iam\_role\_attach\_cni\_policy](#input\_node\_iam\_role\_attach\_cni\_policy) | Whether to attach the `AmazonEKS_CNI_Policy`/`AmazonEKS_CNI_IPv6_Policy` IAM policy to the IAM IAM role. WARNING: If set `false` the permissions must be assigned to the `aws-node` DaemonSet pods via another method or nodes will not be able to join the cluster | `bool` | `true` | no | @@ -168,6 +172,7 @@ No modules. | [queue\_managed\_sse\_enabled](#input\_queue\_managed\_sse\_enabled) | Boolean to enable server-side encryption (SSE) of message content with SQS-owned encryption keys | `bool` | `true` | no | | [queue\_name](#input\_queue\_name) | Name of the SQS queue | `string` | `null` | no | | [rule\_name\_prefix](#input\_rule\_name\_prefix) | Prefix used for all event bridge rules | `string` | `"Karpenter"` | no | +| [service\_account](#input\_service\_account) | Service account to associate with the Karpenter Pod Identity | `string` | `"karpenter"` | no | | [tags](#input\_tags) | A map of tags to add to all resources | `map(string)` | `{}` | no | ## Outputs @@ -182,6 +187,7 @@ No modules. | [instance\_profile\_id](#output\_instance\_profile\_id) | Instance profile's ID | | [instance\_profile\_name](#output\_instance\_profile\_name) | Name of the instance profile | | [instance\_profile\_unique](#output\_instance\_profile\_unique) | Stable and unique string identifying the IAM instance profile | +| [namespace](#output\_namespace) | Namespace associated with the Karpenter Pod Identity | | [node\_access\_entry\_arn](#output\_node\_access\_entry\_arn) | Amazon Resource Name (ARN) of the node Access Entry | | [node\_iam\_role\_arn](#output\_node\_iam\_role\_arn) | The Amazon Resource Name (ARN) specifying the node IAM role | | [node\_iam\_role\_name](#output\_node\_iam\_role\_name) | The name of the node IAM role | @@ -189,4 +195,5 @@ No modules. | [queue\_arn](#output\_queue\_arn) | The ARN of the SQS queue | | [queue\_name](#output\_queue\_name) | The name of the created Amazon SQS queue | | [queue\_url](#output\_queue\_url) | The URL for the created Amazon SQS queue | +| [service\_account](#output\_service\_account) | Service Account associated with the Karpenter Pod Identity | diff --git a/modules/karpenter/main.tf b/modules/karpenter/main.tf index 5d82475b6a..8a3c9c1b71 100644 --- a/modules/karpenter/main.tf +++ b/modules/karpenter/main.tf @@ -411,6 +411,21 @@ resource "aws_iam_role_policy_attachment" "controller_additional" { policy_arn = each.value } +################################################################################ +# Pod Identity Association +################################################################################ + +resource "aws_eks_pod_identity_association" "karpenter" { + count = local.create_iam_role && var.enable_pod_identity && var.create_pod_identity_association ? 1 : 0 + + cluster_name = var.cluster_name + namespace = var.namespace + service_account = var.service_account + role_arn = aws_iam_role.controller[0].arn + + tags = var.tags +} + ################################################################################ # Node Termination Queue ################################################################################ diff --git a/modules/karpenter/outputs.tf b/modules/karpenter/outputs.tf index 164baa142c..a71d47242d 100644 --- a/modules/karpenter/outputs.tf +++ b/modules/karpenter/outputs.tf @@ -96,3 +96,17 @@ output "instance_profile_unique" { description = "Stable and unique string identifying the IAM instance profile" value = try(aws_iam_instance_profile.this[0].unique_id, null) } + +################################################################################ +# Pod Identity +################################################################################ + +output "namespace" { + description = "Namespace associated with the Karpenter Pod Identity" + value = var.namespace +} + +output "service_account" { + description = "Service Account associated with the Karpenter Pod Identity" + value = var.service_account +} diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index fc79b1a413..87238c8389 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -138,6 +138,28 @@ variable "irsa_assume_role_condition_test" { default = "StringEquals" } +################################################################################ +# Pod Identity Association +################################################################################ +# TODO - Change default to `true` at next breaking change +variable "create_pod_identity_association" { + description = "Determines whether to create pod identity association" + type = bool + default = false +} + +variable "namespace" { + description = "Namespace to associate with the Karpenter Pod Identity" + type = string + default = "kube-system" +} + +variable "service_account" { + description = "Service account to associate with the Karpenter Pod Identity" + type = string + default = "karpenter" +} + ################################################################################ # Node Termination Queue ################################################################################