Karpenter

aws eks --region ap-northeast-2 update-kubeconfig --name ws-eks-cluster

kubectl create ns karpenter

private_a=$(aws ec2 describe-subnets --filters "Name=tag:Name,Values=ws-private-a" --query "Subnets[].SubnetId[]" --region ap-northeast-2 --output text) private_b=$(aws ec2 describe-subnets --filters "Name=tag:Name,Values=ws-private-b" --query "Subnets[].SubnetId[]" --region ap-northeast-2 --output text) private_subnet_name=("$private_a" "$private_b") for name in "${private_subnet_name[@]}" do aws ec2 create-tags --resources $name --tags Key=karpenter.sh/discovery,Value=ws-eks-cluster done cluster_sg=$(aws ec2 describe-security-groups --filters "Name=tag:aws:eks:cluster-name,Values=ws-eks-cluster" --query "SecurityGroups[].GroupId[]" --region ap-northeast-2 --output text) aws ec2 create-tags --resources $cluster_sg --tags Key=karpenter.sh/discovery,Value=ws-eks-cluster

export KARPENTER_NAMESPACE=karpenter export CLUSTER_NAME=ws-eks-cluster export KARPENTER_VERSION="1.3.3" export TEMPOUT="$(mktemp)" export AWS_PARTITION="aws" export CLUSTER_ENDPOINT=$(aws eks describe-cluster --name ws-eks-cluster --query "cluster.endpoint" --output text) export AWS_DEFAULT_REGION=ap-northeast-2 export OIDC_ENDPOINT="$(aws eks describe-cluster --name "${CLUSTER_NAME}" \ --query "cluster.identity.oidc.issuer" --output text)" export AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) export K8S_VERSION=$(aws eks describe-cluster --name "${CLUSTER_NAME}" --query "cluster.version" --output text) export ALIAS_VERSION="$(aws ssm get-parameter --name "/aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2023/x86_64/standard/recommended/image_id" --query Parameter.Value | xargs aws ec2 describe-images --query 'Images[0].Name' --image-ids | sed -r 's/^.*(v[[:digit:]]+).*$/\1/')" oidc_id=$(aws eks describe-cluster --name $CLUSTER_NAME --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 5)

eksctl utils associate-iam-oidc-provider \ --region ${AWS_DEFAULT_REGION} \ --cluster ${CLUSTER_NAME} \ --approve

curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ --capabilities CAPABILITY_NAMED_IAM \ --parameter-overrides "ClusterName=${CLUSTER_NAME}"

cat << EOF > karpenter-controller-policy.json { "Statement": [ { "Action": [ "ssm:GetParameter", "ec2:DescribeImages", "ec2:RunInstances", "ec2:DescribeSubnets", "ec2:DescribeSecurityGroups", "ec2:DescribeLaunchTemplates", "ec2:DescribeInstances", "ec2:DescribeInstanceTypes", "ec2:DescribeInstanceTypeOfferings", "ec2:DescribeAvailabilityZones", "ec2:DeleteLaunchTemplate", "ec2:CreateTags", "ec2:CreateLaunchTemplate", "ec2:CreateFleet", "ec2:DescribeSpotPriceHistory", "pricing:GetProducts" ], "Effect": "Allow", "Resource": "*", "Sid": "Karpenter" }, { "Action": "ec2:TerminateInstances", "Condition": { "StringLike": { "ec2:ResourceTag/karpenter.sh/nodepool": "*" } }, "Effect": "Allow", "Resource": "*", "Sid": "ConditionalEC2Termination" }, { "Effect": "Allow", "Action": "iam:PassRole", "Resource": "arn:aws:iam::${AWS_ACCOUNT_ID}:role/ws-eks-node-role", "Sid": "PassNodeIAMRole" }, { "Effect": "Allow", "Action": "eks:DescribeCluster", "Resource": "arn:aws:eks:ap-northeast-2:${AWS_ACCOUNT_ID}:cluster/${CLUSTER_NAME}", "Sid": "EKSClusterEndpointLookup" }, { "Sid": "AllowScopedInstanceProfileCreationActions", "Effect": "Allow", "Resource": "*", "Action": [ "iam:CreateInstanceProfile" ], "Condition": { "StringEquals": { "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", "aws:RequestTag/topology.kubernetes.io/region": "ap-northeast-2" }, "StringLike": { "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" } } }, { "Sid": "AllowScopedInstanceProfileTagActions", "Effect": "Allow", "Resource": "*", "Action": [ "iam:TagInstanceProfile" ], "Condition": { "StringEquals": { "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", "aws:ResourceTag/topology.kubernetes.io/region": "ap-northeast-2", "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", "aws:RequestTag/topology.kubernetes.io/region": "ap-northeast-2" }, "StringLike": { "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*", "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" } } }, { "Sid": "AllowScopedInstanceProfileActions", "Effect": "Allow", "Resource": "*", "Action": [ "iam:AddRoleToInstanceProfile", "iam:RemoveRoleFromInstanceProfile", "iam:DeleteInstanceProfile" ], "Condition": { "StringEquals": { "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", "aws:ResourceTag/topology.kubernetes.io/region": "ap-northeast-2" }, "StringLike": { "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*" } } }, { "Sid": "AllowInstanceProfileReadActions", "Effect": "Allow", "Resource": "*", "Action": "iam:GetInstanceProfile" }, { "Sid": "AllowInterruptionQueueActions", "Effect": "Allow", "Resource": "*", "Action": [ "sqs:DeleteMessage", "sqs:GetQueueUrl", "sqs:ReceiveMessage" ] } ], "Version": "2012-10-17" } EOF

aws iam create-policy \ --policy-name "ws-karpenter-controller-policy" \ --policy-document file://karpenter-controller-policy.json

cat << EOF > trust-policy.json { "Version": "2012-10-17", "Statement": [ { "Effect": "Allow", "Principal": { "Federated": "arn:aws:iam::${AWS_ACCOUNT_ID}:oidc-provider/oidc.eks.ap-northeast-2.amazonaws.com/id/${oidc_id}" }, "Action": "sts:AssumeRoleWithWebIdentity", "Condition": { "StringEquals": { "oidc.eks.ap-northeast-2.amazonaws.com/id/${oidc_id}:aud": "sts.amazonaws.com", "oidc.eks.ap-northeast-2.amazonaws.com/id/${oidc_id}:sub": "system:serviceaccount:karpenter:karpenter" } } } ] } EOF

aws iam create-role \ --role-name "ws-karpenter-controller-role" \ --assume-role-policy-document file://trust-policy.json

aws iam attach-role-policy \ --role-name "ws-karpenter-controller-role" \ --policy-arn "arn:aws:iam::${AWS_ACCOUNT_ID}:policy/ws-karpenter-controller-policy"

helm repo add karpenter https://charts.karpenter.sh/ helm repo update helm template karpenter oci://public.ecr.aws/karpenter/karpenter --version "${KARPENTER_VERSION}" --namespace "${KARPENTER_NAMESPACE}" \ --set "settings.clusterName=${CLUSTER_NAME}" \ --set "settings.interruptionQueue=${CLUSTER_NAME}" \ --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/ws-karpenter-controller-role" \ --set controller.resources.requests.cpu=1 \ --set controller.resources.requests.memory=1Gi \ --set controller.resources.limits.cpu=1 \ --set controller.resources.limits.memory=1Gi > karpenter.yaml

# 위 명령어를 수행하면 karpenter.yaml 파일이 실행되는데, 해당 파일을 열어 # 아래 부분에서 values를 Karpenter에 의해 생성되지 않은 워커 노드그룹 이름으로 # 꼭 바꿔주어야 한다. vim karpenter.yaml affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: karpenter.sh/nodepool operator: DoesNotExist - key: eks.amazonaws.com/nodegroup operator: In values: - ${Karpenter에 생성되지 않은 현재 Worker NodeGroup Name ex. ws-addon} - key: topology.kubernetes.io/region operator: In values: - ap-northeast-2 # karpenter만 올리기 위해 toleration섹션도 찾아서 추가 tolerations: - key: CriticalAddonsOnly operator: Exists - key: "node-role.kubernetes.io/ws-addon" operator: "Exists" effect: "NoSchedule"

cat <<EOF | kubectl apply -f - apiVersion: v1 kind: ConfigMap metadata: name: karpenter-global-settings namespace: karpenter data: clusterName: ${CLUSTER_NAME} clusterEndpoint: ${CLUSTER_ENDPOINT} aws.defaultInstanceProfile: "KarpenterInstanceProfile" EOF

kubectl create -f \ "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml" kubectl create -f \ "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml" kubectl create -f \ "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml" kubectl apply -f karpenter.yaml

cat > nodepool.yaml << 'EOF' apiVersion: karpenter.sh/v1 kind: NodePool metadata: name: default namespace: karpenter spec: template: metadata: labels: nodegroup: emr eks.amazonaws.com/compute-type: emr spec: taints: - key: eks.amazonaws.com/compute-type value: emr effect: NoSchedule requirements: - key: kubernetes.io/arch operator: In values: ["amd64"] - key: kubernetes.io/os operator: In values: ["linux"] - key: karpenter.sh/capacity-type operator: In values: ["on-demand"] - key: "node.kubernetes.io/instance-family" operator: In values: ["m5", "m5d", "c5", "c5d", "c4", "r4"] - key: "node.kubernetes.io/instance-cpu" operator: In values: ["4", "8", "16", "32"] - key: "topology.kubernetes.io/zone" operator: In values: ["ap-northeast-2a", "ap-northeast-2b"] nodeClassRef: group: karpenter.k8s.aws kind: EC2NodeClass name: default expireAfter: 720h # 30 * 24h = 720h limits: cpu: 200 memory: 800Gi disruption: consolidationPolicy: WhenEmpty consolidateAfter: 90s --- #ec2nodeclass(Karpenter가 노드 생성시 참고할 EC2 설정) apiVersion: karpenter.k8s.aws/v1 kind: EC2NodeClass metadata: name: default namespace: karpenter spec: role: "ws-eks-node-role" # replace with your cluster name amiSelectorTerms: - alias: "al2023@latest" subnetSelectorTerms: - tags: karpenter.sh/discovery: "ws-eks-cluster" # replace with your cluster name securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "ws-eks-cluster" # replace with your cluster name blockDeviceMappings: - deviceName: /dev/xvda ebs: volumeSize: 100Gi volumeType: gp3 tags: nodegroup: emr eks.amazonaws.com/compute-type: emr EOF

kubectl apply -f nodepool.yaml