Control Plane

# kubeadm으로 HA Control Plane 구성

# kubeadm-config.yaml - HA 클러스터 설정
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
kubernetesVersion: v1.28.0
controlPlaneEndpoint: "k8s-api.company.com:6443"  # Load Balancer VIP
networking:
  podSubnet: "10.244.0.0/16"
  serviceSubnet: "10.96.0.0/12"
apiServer:
  extraArgs:
    audit-log-path: /var/log/kubernetes/audit.log
    audit-log-maxage: "30"
    audit-log-maxbackup: "10"
    audit-log-maxsize: "100"
    enable-admission-plugins: NodeRestriction,PodSecurityPolicy
  certSANs:
  - "k8s-api.company.com"
  - "10.0.0.100"
  - "master-1"
  - "master-2"
  - "master-3"
controllerManager:
  extraArgs:
    bind-address: "0.0.0.0"
    terminated-pod-gc-threshold: "100"
scheduler:
  extraArgs:
    bind-address: "0.0.0.0"
etcd:
  local:
    dataDir: /var/lib/etcd
    extraArgs:
      quota-backend-bytes: "8589934592"  # 8GB
      auto-compaction-retention: "8"
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: "10.0.0.11"
  bindPort: 6443
nodeRegistration:
  criSocket: unix:///var/run/containerd/containerd.sock
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/control-plane

---
# 첫 번째 Control Plane 노드 초기화
sudo kubeadm init --config kubeadm-config.yaml --upload-certs

# 출력된 join 명령어로 다른 Control Plane 노드 추가
# kubeadm join k8s-api.company.com:6443 --token  \
#   --discovery-token-ca-cert-hash sha256: \
#   --control-plane --certificate-key 

# etcd 백업 (정기적으로 실행)
ETCDCTL_API=3 etcdctl snapshot save /backup/etcd-$(date +%Y%m%d).db \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key

# etcd 백업 검증
etcdctl snapshot status /backup/etcd-20240115.db -w table

# Control Plane 트러블슈팅

# 1. API Server 연결 문제 진단
# 인증서 만료 확인
kubeadm certs check-expiration
# CERTIFICATE                EXPIRES                  RESIDUAL TIME
# admin.conf                 Jan 15, 2025 09:00 UTC   364d
# apiserver                  Jan 15, 2025 09:00 UTC   364d
# apiserver-etcd-client      Jan 15, 2025 09:00 UTC   364d

# 인증서 갱신 (만료 전)
kubeadm certs renew all

# API Server 로그 확인
kubectl logs -n kube-system kube-apiserver-master-1 --tail=100

# crictl로 직접 컨테이너 로그 확인 (kubectl 불가 시)
sudo crictl logs $(sudo crictl ps | grep kube-apiserver | awk '{print $1}')

# 2. Scheduler 문제 진단
# Pending Pod가 스케줄되지 않는 이유 확인
kubectl describe pod  | grep -A 10 Events

# 노드 리소스 상태 확인
kubectl top nodes
kubectl describe node  | grep -A 20 "Allocated resources"

# Scheduler 로그에서 스케줄링 실패 원인 찾기
kubectl logs -n kube-system kube-scheduler-master-1 | grep -i "failed"

# 3. Controller Manager 문제 진단
# ReplicaSet이 Pod를 생성하지 못하는 경우
kubectl logs -n kube-system kube-controller-manager-master-1 | grep -i error

# 컨트롤러별 워크큐 상태 (메트릭)
kubectl get --raw /metrics | grep workqueue

# 4. etcd 문제 진단
# etcd 상태 확인
kubectl exec -it etcd-master-1 -n kube-system -- etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  endpoint status -w table

# etcd 디스크 사용량 확인
kubectl exec -it etcd-master-1 -n kube-system -- etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  endpoint status -w json | jq '.[] | {endpoint: .Endpoint, dbSize: .Status.dbSize}'

# etcd 조각 모음 (디스크 공간 확보)
kubectl exec -it etcd-master-1 -n kube-system -- etcdctl \
  --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  defrag

# 5. HA 클러스터 리더 확인
# Scheduler 리더
kubectl get lease -n kube-system kube-scheduler -o jsonpath='{.spec.holderIdentity}'

# Controller Manager 리더
kubectl get lease -n kube-system kube-controller-manager -o jsonpath='{.spec.holderIdentity}'