📚 BusinessExplain 🏠 Home 📊 Analytics
← Back to Skills

Kubernetes Mastery

⚙️ DevOps & Infrastructure 24 min read v1.0.0
📋 Contents

Kubernetes Mastery

Architecture

Control Plane Components

| Component | Purpose |
|-----------|---------|
| kube-apiserver | REST API entry point; validates and stores data in etcd |
| etcd | Distributed key-value store; single source of truth for cluster state |
| kube-scheduler | Assigns pods to nodes based on resources, affinity, taints |
| kube-controller-manager | Runs controllers (Deployment, ReplicaSet, Node, Job, etc.) |
| cloud-controller-manager | Integrates with cloud provider APIs (LB, volumes, routes) |

Worker Node Components

| Component | Purpose |
|-----------|---------|
| kubelet | Agent ensuring containers run in pods per PodSpec |
| kube-proxy | Maintains network rules (iptables/IPVS) for Service routing |
| Container Runtime | containerd, CRI-O, or Docker (via cri-dockerd) |

etcd Best Practices

  • Run odd number of members (3 or 5) for quorum
  • Use dedicated SSD-backed storage
  • Enable automatic compaction: --auto-compaction-mode=periodic --auto-compaction-retention=5m
  • Backup regularly: etcdctl snapshot save /backup/etcd-snapshot.db
  • Restore: etcdctl snapshot restore /backup/etcd-snapshot.db --data-dir /var/lib/etcd-restore
    • # Check etcd health
ETCDCTL_API=3 etcdctl --endpoints=https://127.0.0.1:2379 \
  --cacert=/etc/kubernetes/pki/etcd/ca.crt \
  --cert=/etc/kubernetes/pki/etcd/server.crt \
  --key=/etc/kubernetes/pki/etcd/server.key \
  endpoint health

    Core Objects

    Pods

    apiVersion: v1
kind: Pod
metadata:
  name: web-app
  labels:
    app: web
    version: v1
spec:
  serviceAccountName: web-sa
  securityContext:
    runAsNonRoot: true
    runAsUser: 1000
    fsGroup: 2000
    seccompProfile:
      type: RuntimeDefault
  containers:
  - name: app
    image: myregistry.io/web-app:1.2.3
    ports:
    - containerPort: 8080
      protocol: TCP
    env:
    - name: LOG_LEVEL
      value: "info"
    - name: DB_PASSWORD
      valueFrom:
        secretKeyRef:
          name: db-creds
          key: password
    resources:
      requests:
        cpu: "100m"
        memory: "128Mi"
      limits:
        cpu: "500m"
        memory: "512Mi"
    livenessProbe:
      httpGet:
        path: /healthz
        port: 8080
      initialDelaySeconds: 15
      periodSeconds: 20
      failureThreshold: 3
    readinessProbe:
      httpGet:
        path: /ready
        port: 8080
      initialDelaySeconds: 5
      periodSeconds: 10
    startupProbe:
      httpGet:
        path: /healthz
        port: 8080
      failureThreshold: 30
      periodSeconds: 10
    volumeMounts:
    - name: config
      mountPath: /etc/config
      readOnly: true
  volumes:
  - name: config
    configMap:
      name: app-config
  topologySpreadConstraints:
  - maxSkew: 1
    topologyKey: topology.kubernetes.io/zone
    whenUnsatisfiable: DoNotSchedule
    labelSelector:
      matchLabels:
        app: web

    Deployments

    apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-deployment
  labels:
    app: web
spec:
  replicas: 3
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: web
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  template:
    metadata:
      labels:
        app: web
    spec:
      containers:
      - name: web
        image: myregistry.io/web-app:1.2.3
        ports:
        - containerPort: 8080
        resources:
          requests:
            cpu: "100m"
            memory: "128Mi"
          limits:
            cpu: "500m"
            memory: "512Mi"

    # Rollout commands
kubectl rollout status deployment/web-deployment
kubectl rollout history deployment/web-deployment
kubectl rollout undo deployment/web-deployment
kubectl rollout undo deployment/web-deployment --to-revision=2
kubectl rollout pause deployment/web-deployment
kubectl rollout resume deployment/web-deployment
kubectl scale deployment/web-deployment --replicas=5

    Services

    # ClusterIP (default - internal only)
apiVersion: v1
kind: Service
metadata:
  name: web-svc
spec:
  type: ClusterIP
  selector:
    app: web
  ports:
  - port: 80
    targetPort: 8080

    

    NodePort (external via node IP)
    
apiVersion: v1
kind: Service
metadata:
  name: web-nodeport
spec:
  type: NodePort
  selector:
    app: web
  ports:
  - port: 80
    targetPort: 8080
    nodePort: 30080

    

    LoadBalancer (cloud provider LB)
    
apiVersion: v1
kind: Service
metadata:
  name: web-lb
spec:
  type: LoadBalancer
  selector:
    app: web
  ports:
  - port: 443
    targetPort: 8080

    

    Headless (for StatefulSets)
    
apiVersion: v1
kind: Service
metadata:
  name: db-headless
spec:
  clusterIP: None
  selector:
    app: db
  ports:
  - port: 5432

    ConfigMaps

    apiVersion: v1
kind: ConfigMap
metadata:
  name: app-config
data:
  # Literal values
  LOG_LEVEL: "info"
  APP_ENV: "production"
  # File-like key
  application.yml: |
    server:
      port: 8080
    logging:
      level: info

    # Create from files/literals
kubectl create configmap app-config --from-literal=LOG_LEVEL=info
kubectl create configmap app-config --from-file=config.yml=path/to/config.yml
kubectl create configmap app-config --from-env-file=.env

    Secrets

    apiVersion: v1
kind: Secret
metadata:
  name: db-creds
type: Opaque
data:
  username: YWRtaW4=           # base64 encoded
  password: c3VwZXJzZWNyZXQ=
stringData:                     # plain text (auto-encoded)
  host: "db.default.svc.cluster.local"

    # Create secrets
kubectl create secret generic db-creds \
  --from-literal=username=admin \
  --from-literal=password=supersecret

    kubectl create secret tls tls-cert \
      --cert=path/to/tls.crt --key=path/to/tls.key
    

    Decrypt a secret
    
kubectl get secret db-creds -o jsonpath='{.data.password}' | base64 -d

    External secret stores (recommended for production)
    

    Use: External Secrets Operator, Sealed Secrets, Vault Sidecar/Agent

    Ingress

    apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: web-ingress
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt-prod
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    nginx.ingress.kubernetes.io/rate-limit: "100"
spec:
  ingressClassName: nginx
  tls:
  - hosts:
    - example.com
    secretName: tls-cert
  rules:
  - host: example.com
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: web-svc
            port:
              number: 80
  - host: api.example.com
    http:
      paths:
      - path: /v1
        pathType: Prefix
        backend:
          service:
            name: api-svc
            port:
              number: 80

    PersistentVolumes & PersistentVolumeClaims

    # StorageClass (dynamic provisioning)
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: fast-ssd
provisioner: pd.csi.storage.gke.io
parameters:
  type: pd-ssd
  replication-type: regional-pd
reclaimPolicy: Retain
volumeBindingMode: WaitForFirstConsumer

    

    PVC
    
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: data-pvc
spec:
  accessModes: [ReadWriteOnce]
  storageClassName: fast-ssd
  resources:
    requests:
      storage: 10Gi

    

    Pod using PVC
    
apiVersion: v1
kind: Pod
metadata:
  name: db-pod
spec:
  volumes:
  - name: data
    persistentVolumeClaim:
      claimName: data-pvc
  containers:
  - name: db
    image: postgres:15
    volumeMounts:
    - name: data
      mountPath: /var/lib/postgresql/data

    Jobs & CronJobs

    apiVersion: batch/v1
kind: Job
metadata:
  name: db-migrate
spec:
  backoffLimit: 3
  activeDeadlineSeconds: 300
  ttlSecondsAfterFinished: 86400
  template:
    spec:
      restartPolicy: Never
      containers:
      - name: migrate
        image: myregistry.io/app:latest
        command: ["./migrate.sh"]

    
apiVersion: batch/v1
kind: CronJob
metadata:
  name: nightly-backup
spec:
  schedule: "0 2   *"
  timeZone: "America/New_York"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 3
  failedJobsHistoryLimit: 1
  startingDeadlineSeconds: 200
  jobTemplate:
    spec:
      backoffLimit: 2
      template:
        spec:
          restartPolicy: OnFailure
          containers:
          - name: backup
            image: myregistry.io/backup:latest
            command: ["/scripts/backup.sh"]

    Namespace Management

    # Create and manage namespaces
kubectl create namespace production
kubectl get namespaces
kubectl describe namespace production

    Set default namespace for context
    
kubectl config set-context --current --namespace=production

    Delete namespace (and all resources within)
    
kubectl delete namespace staging

    apiVersion: v1
kind: Namespace
metadata:
  name: production
  labels:
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/audit: restricted
    pod-security.kubernetes.io/warn: restricted

    Labels & Selectors

    # Label resources
kubectl label pod web-app env=production tier=frontend
kubectl label pod web-app env=staging --overwrite
kubectl label pod web-app tier-

    Select by labels
    
kubectl get pods -l env=production
kubectl get pods -l 'env in (production, staging)'
kubectl get pods -l 'env=production,tier=frontend'
kubectl get pods -l 'env!=development'

    Label-based operations
    
kubectl delete pods -l app=legacy
kubectl cordon nodes -l disktype=hdd

    Resource Quotas & Limits

    apiVersion: v1
kind: ResourceQuota
metadata:
  name: production-quota
  namespace: production
spec:
  hard:
    requests.cpu: "100"
    requests.memory: "200Gi"
    limits.cpu: "200"
    limits.memory: "400Gi"
    pods: "100"
    services: "20"
    persistentvolumeclaims: "50"
    configmaps: "50"
    secrets: "100"

    
apiVersion: v1
kind: LimitRange
metadata:
  name: default-limits
  namespace: production
spec:
  limits:
  - type: Container
    default:
      cpu: "500m"
      memory: "256Mi"
    defaultRequest:
      cpu: "100m"
      memory: "128Mi"
    max:
      cpu: "4"
      memory: "8Gi"
    min:
      cpu: "50m"
      memory: "64Mi"
    maxLimitRequestRatio:
      cpu: "5"
      memory: "3"

    Autoscaling

    Horizontal Pod Autoscaler (HPA)

    apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: web-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: web-deployment
  minReplicas: 3
  maxReplicas: 20
  metrics:
  - type: Resource
    resource:
      name: cpu
      target:
        type: Utilization
        averageUtilization: 70
  - type: Resource
    resource:
      name: memory
      target:
        type: Utilization
        averageUtilization: 80
  - type: Pods
    pods:
      metric:
        name: http_requests_per_second
      target:
        type: AverageValue
        averageValue: "1k"
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60
      policies:
      - type: Percent
        value: 100
        periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
      - type: Percent
        value: 10
        periodSeconds: 60

    Vertical Pod Autoscaler (VPA)

    apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: web-vpa
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: web-deployment
  updatePolicy:
    updateMode: Auto  # Off, Initial, Recreate, Auto
  resourcePolicy:
    containerPolicies:
    - containerName: web
      minAllowed:
        cpu: "50m"
        memory: "64Mi"
      maxAllowed:
        cpu: "2"
        memory: "4Gi"
      controlledResources: ["cpu", "memory"]

    # Install VPA (requires metrics-server)
kubectl apply -f https://github.com/kubernetes/autoscaler/releases/latest/download/vertical-pod-autoscaler.yaml

    Check VPA recommendations
    
kubectl get vpa web-vpa -o yaml | grep -A 10 "recommendation"

    Network Policies

    # Default deny all ingress and egress
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: default-deny-all
  namespace: production
spec:
  podSelector: {}
  policyTypes:
  - Ingress
  - Egress

    

    Allow specific ingress
    
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-web-ingress
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: web
  policyTypes:
  - Ingress
  ingress:
  - from:
    - namespaceSelector:
        matchLabels:
          env: production
    - podSelector:
        matchLabels:
          role: ingress-controller
    ports:
    - protocol: TCP
      port: 8080

    

    Allow egress to DNS and specific services
    
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: allow-web-egress
  namespace: production
spec:
  podSelector:
    matchLabels:
      app: web
  policyTypes:
  - Egress
  egress:
  - to: []
    ports:
    - protocol: UDP
      port: 53    # DNS
  - to:
    - podSelector:
        matchLabels:
          app: db
    ports:
    - protocol: TCP
      port: 5432

    # CNI plugins that support NetworkPolicy: Calico, Cilium, Weave, Antrea

    Flannel does NOT support NetworkPolicy

    RBAC & ServiceAccounts

    apiVersion: v1
kind: ServiceAccount
metadata:
  name: web-sa
  namespace: production
automountServiceAccountToken: false  # Disable auto-mount if not needed

    
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: pod-reader
  namespace: production
rules:

  • apiGroups: [""]
    • 
  resources: ["pods", "pods/log"]
  verbs: ["get", "list", "watch"]

  • apiGroups: [""]
    • 
  resources: ["configmaps"]
  verbs: ["get"]
  resourceNames: ["app-config"]

    
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: web-pod-reader
  namespace: production
subjects:

  • kind: ServiceAccount
    • 
  name: web-sa
  namespace: production
roleRef:
  kind: Role
  name: pod-reader
  apiGroup: rbac.authorization.k8s.io

    

    ClusterRole for read-only across namespaces
    
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: namespace-reader
rules:

  • apiGroups: [""]
    • 
  resources: ["namespaces", "pods"]
  verbs: ["get", "list", "watch"]

    # RBAC debugging
kubectl auth can-i get pods --as=system:serviceaccount:production:web-sa
kubectl auth can-i list secrets --namespace=production --as=system:serviceaccount:production:web-sa
kubectl get rolebindings -o wide
kubectl get clusterrolebindings -o wide
kubectl describe clusterrole admin

    Pod Security Standards

    # Enforce at namespace level (Kubernetes 1.23+)
apiVersion: v1
kind: Namespace
metadata:
  name: production
  labels:
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/audit: baseline
    pod-security.kubernetes.io/warn: restricted

    

    Restricted Pod Security Context (most secure)
    
apiVersion: v1
kind: Pod
metadata:
  name: secure-pod
spec:
  securityContext:
    runAsNonRoot: true
    runAsUser: 1000
    fsGroup: 2000
    seccompProfile:
      type: RuntimeDefault
  containers:
  - name: app
    image: app:latest
    securityContext:
      allowPrivilegeEscalation: false
      readOnlyRootFilesystem: true
      capabilities:
        drop:
        - ALL
    volumeMounts:
    - name: tmp
      mountPath: /tmp
  volumes:
  - name: tmp
    emptyDir: {}

    | Profile | Key Restrictions |
    |---------|-----------------|
    | Privileged | No restrictions (system pods only) |
    | Baseline | No hostPID, hostNetwork, hostIPC; no privileged containers; no hostPath; no adding capabilities |
    | Restricted | Baseline + runAsNonRoot, drop ALL capabilities, no privilege escalation, seccomp=RuntimeDefault, readOnlyRootFilesystem |

    Helm Charts

    Chart Structure

    my-chart/
├── Chart.yaml          # Chart metadata
├── values.yaml         # Default configuration
├── values.schema.json  # JSON schema for values validation
├── charts/             # Chart dependencies
├── templates/          # Kubernetes manifests
│   ├── _helpers.tpl    # Named templates (partials)
│   ├── deployment.yaml
│   ├── service.yaml
│   ├── ingress.yaml
│   ├── configmap.yaml
│   ├── hpa.yaml
│   ├── serviceaccount.yaml
│   └── NOTES.txt       # Post-install instructions
├── templates/tests/    # Chart tests
│   └── test-connection.yaml
└── .helmignore

    Chart.yaml

    apiVersion: v2
name: web-app
description: A production web application
type: application
version: 0.1.0          # Chart version (SemVer 2)
appVersion: "1.2.3"     # Application version
dependencies:

  • name: postgresql
    • 
  version: "12.x.x"
  repository: https://charts.bitnami.com/bitnami
  condition: postgresql.enabled

  • name: redis
    • 
  version: "17.x.x"
  repository: https://charts.bitnami.com/bitnami
  condition: redis.enabled

    values.yaml

    replicaCount: 3

    image:
      repository: myregistry.io/web-app
      pullPolicy: IfNotPresent
      tag: ""
    

    imagePullSecrets: []
    nameOverride: ""
    fullnameOverride: ""
    

    serviceAccount:
      create: true
      annotations: {}
      name: ""
    

    podAnnotations: {}
    podLabels:
      app: web
    

    podSecurityContext:
      runAsNonRoot: true
      runAsUser: 1000
      fsGroup: 2000
    

    securityContext:
      allowPrivilegeEscalation: false
      readOnlyRootFilesystem: true
      capabilities:
        drop:
        - ALL
    

    service:
      type: ClusterIP
      port: 80
      targetPort: 8080
    

    ingress:
      enabled: true
      className: nginx
      annotations:
        cert-manager.io/cluster-issuer: letsencrypt-prod
      hosts:
      - host: example.com
        paths:
        - path: /
          pathType: Prefix
      tls:
      - secretName: web-tls
        hosts:
        - example.com
    

    resources:
      requests:
        cpu: 100m
        memory: 128Mi
      limits:
        cpu: 500m
        memory: 512Mi
    

    autoscaling:
      enabled: true
      minReplicas: 3
      maxReplicas: 20
      targetCPUUtilizationPercentage: 70
      targetMemoryUtilizationPercentage: 80
    

    nodeSelector: {}
    tolerations: []
    affinity: {}
    

    postgresql:
      enabled: true
      auth:
        database: appdb
    

    redis:
      enabled: true
      architecture: standalone

    Template with Helpers

    # templates/_helpers.tpl
{{- define "web-app.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}

    {{- define "web-app.labels" -}}
    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
    {{ include "web-app.selectorLabels" . }}
    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
    app.kubernetes.io/managed-by: {{ .Release.Service }}
    {{- end }}
    

    {{- define "web-app.selectorLabels" -}}
    app.kubernetes.io/name: {{ include "web-app.fullname" . }}
    {{- end }}

    # templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: {{ include "web-app.fullname" . }}
  labels:
    {{- include "web-app.labels" . | nindent 4 }}
spec:
  replicas: {{ .Values.replicaCount }}
  selector:
    matchLabels:
      {{- include "web-app.selectorLabels" . | nindent 6 }}
  template:
    metadata:
      labels:
        {{- include "web-app.selectorLabels" . | nindent 8 }}
        {{- with .Values.podLabels }}
        {{- toYaml . | nindent 8 }}
        {{- end }}
    spec:
      serviceAccountName: {{ include "web-app.fullname" . }}
      securityContext:
        {{- toYaml .Values.podSecurityContext | nindent 8 }}
      containers:
      - name: {{ .Chart.Name }}
        image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
        imagePullPolicy: {{ .Values.image.pullPolicy }}
        ports:
        - containerPort: {{ .Values.service.targetPort }}
        env:
        - name: DB_HOST
          value: "{{ include "web-app.fullname" . }}-postgresql"
        {{- with .Values.extraEnv }}
        {{- toYaml . | nindent 8 }}
        {{- end }}
        resources:
          {{- toYaml .Values.resources | nindent 10 }}
        {{- with .Values.securityContext }}
        securityContext:
          {{- toYaml . | nindent 10 }}
        {{- end }}
      {{- with .Values.nodeSelector }}
      nodeSelector:
        {{- toYaml . | nindent 8 }}
      {{- end }}
      {{- with .Values.affinity }}
      affinity:
        {{- toYaml . | nindent 8 }}
      {{- end }}
      {{- with .Values.tolerations }}
      tolerations:
        {{- toYaml . | nindent 8 }}
      {{- end }}

    Helm Hooks

    # templates/pre-install-hook.yaml
apiVersion: batch/v1
kind: Job
metadata:
  name: {{ include "web-app.fullname" . }}-pre-install
  annotations:
    "helm.sh/hook": pre-install
    "helm.sh/hook-weight": "-5"
    "helm.sh/hook-delete-policy": hook-succeeded
spec:
  template:
    spec:
      restartPolicy: Never
      containers:
      - name: setup
        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
        command: ["/scripts/pre-install.sh"]

    | Hook | Timing |
    |------|--------|
    | pre-install | Before resources are created |
    | post-install | After resources are created |
    | pre-delete | Before deletion |
    | post-delete | After deletion |
    | pre-upgrade | Before upgrade |
    | post-upgrade | After upgrade |
    | pre-rollback | Before rollback |
    | post-rollback | After rollback |
    | test | On helm test |

    Helm Commands

    # Repository management
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo update
helm search repo bitnami/nginx

    Install/Upgrade
    
helm install my-release ./my-chart -n production -f values.yaml
helm upgrade my-release ./my-chart -n production -f values.yaml
helm upgrade --install my-release ./my-chart -n production -f values.yaml  # idempotent

    Rollback
    
helm rollback my-release 1 -n production

    Debug
    
helm template ./my-chart -f values.yaml > rendered.yaml
helm lint ./my-chart
helm diff upgrade my-release ./my-chart -n production  # requires helm-diff plugin

    List & History
    
helm list -n production
helm history my-release -n production

    Uninstall
    
helm uninstall my-release -n production

    Plugin management
    
helm plugin install https://github.com/databus23/helm-diff
helm plugin install https://github.com/helm/helm-secrets

    Kustomize Overlays

    Directory Structure

    base/
├── kustomization.yaml
├── deployment.yaml
├── service.yaml
└── configmap.yaml
overlays/
├── development/
│   ├── kustomization.yaml
│   └── patch-replicas.yaml
├── staging/
│   ├── kustomization.yaml
│   └── patch-resources.yaml
└── production/
    ├── kustomization.yaml
    ├── patch-replicas.yaml
    └── patch-resources.yaml

    Base kustomization.yaml

    apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:

  • deployment.yaml
    • 

  • service.yaml
    • 

  • configmap.yaml
    • 
namespace: my-app
commonLabels:
  app.kubernetes.io/part-of: my-app

    Production Overlay

    apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:

  • ../../base
    • 
patches:

  • path: patch-replicas.yaml
    • 
  target:
    kind: Deployment
    name: web-deployment

  • path: patch-resources.yaml
    • 
images:

  • name: myregistry.io/web-app
    • 
  newTag: v1.2.3
replicas:

  • name: web-deployment
    • 
  count: 5
configMapGenerator:

  • name: app-config
    • 
  behavior: merge
  literals:
  - LOG_LEVEL=warn
  - APP_ENV=production
secretGenerator:

  • name: db-creds
    • 
  literals:
  - password=prod-secret

    Strategic Merge Patch

    # patch-replicas.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-deployment
spec:
  replicas: 5
  template:
    spec:
      resources:
        requests:
          cpu: "200m"
          memory: "256Mi"
        limits:
          cpu: "1"
          memory: "1Gi"

    JSON 6902 Patch

    # patch-toleration.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
patches:

  • patch: |-
    • 
    - op: add
      path: /spec/template/spec/tolerations
      value:
      - key: "dedicated"
        operator: "Equal"
        value: "production"
        effect: "NoSchedule"
  target:
    kind: Deployment
    name: web-deployment

    # Kustomize commands
kubectl kustomize overlays/production/
kubectl apply -k overlays/production/
kubectl diff -k overlays/production/
kubectl delete -k overlays/production/

    Debugging

    Logs

    # Pod logs
kubectl logs web-app-7d9f8b6c4d-xyz12
kubectl logs web-app-7d9f8b6c4d-xyz12 -c sidecar      # specific container
kubectl logs web-app-7d9f8b6c4d-xyz12 --previous         # crashed container
kubectl logs web-app-7d9f8b6c4d-xyz12 -f                 # follow/stream
kubectl logs web-app-7d9f8b6c4d-xyz12 --since=1h         # last hour
kubectl logs web-app-7d9f8b6c4d-xyz12 --tail=100         # last 100 lines

    Deployment logs (all pods)
    
kubectl logs deployment/web-deployment --all-containers=true
kubectl logs -l app=web --all-containers=true --prefix

    Exec & Port-Forward

    # Execute into container
kubectl exec -it web-app-7d9f8b6c4d-xyz12 -- /bin/bash
kubectl exec -it web-app-7d9f8b6c4d-xyz12 -c sidecar -- /bin/sh

    Port forwarding
    
kubectl port-forward pod/web-app-7d9f8b6c4d-xyz12 8080:8080
kubectl port-forward svc/web-svc 8080:80
kubectl port-forward deploy/web-deployment 8080:8080

    Ephemeral debug container (Kubernetes 1.25+)
    
kubectl debug web-app-7d9f8b6c4d-xyz12 -it --image=busybox
kubectl debug web-app-7d9f8b6c4d-xyz12 -it --copy-pod --container=debug --image=nicolaka/netshoot

    Describe & Events

    # Detailed resource info
kubectl describe pod web-app-7d9f8b6c4d-xyz12
kubectl describe node worker-node-1
kubectl describe ingress web-ingress

    Events (cluster-level debugging)
    
kubectl get events --sort-by='.lastTimestamp' -n production
kubectl get events --field-selector type=Warning -n production
kubectl explain pod.spec.containers.livenessProbe  # API field docs

    Common Failure Scenarios

    | Symptom | Debug Command | Common Cause |
    |---------|---------------|--------------|
    | ImagePullBackOff | kubectl describe pod | Wrong image name/tag, missing pull secret |
    | CrashLoopBackOff | kubectl logs --previous | App crash, missing config/secrets |
    | Pending | kubectl describe pod | Insufficient resources, PVC unbound, taints |
    | OOMKilled | kubectl describe pod | Memory limit too low |
    | CreateContainerConfigError | kubectl describe pod | Missing ConfigMap/Secret key |
    | NetworkPolicy blocks | kubectl get networkpolicy | Overly restrictive policy |
    | Evicted | kubectl get events | Node pressure (disk, memory) |

    Monitoring

    Prometheus Operator (kube-prometheus-stack)

    # Install via Helm
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo update
helm install monitoring prometheus-community/kube-prometheus-stack \
  --namespace monitoring --create-namespace \
  --set grafana.adminPassword=admin \
  --set prometheus.prometheusSpec.retention=30d \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.storageClassName=fast-ssd \
  --set prometheus.prometheusSpec.storageSpec.volumeClaimTemplate.spec.resources.requests.storage=50Gi

    ServiceMonitor (Custom Metrics)

    apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  name: web-app
  labels:
    release: monitoring
spec:
  selector:
    matchLabels:
      app: web
  endpoints:
  - port: http
    path: /metrics
    interval: 15s
    scrapeTimeout: 10s

    PrometheusRule (Alerting)

    apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: web-app-alerts
spec:
  groups:
  - name: web-app
    rules:
    - alert: HighErrorRate
      expr: |
        sum(rate(http_requests_total{job="web-app", status=~"5.."}[5m]))
        /
        sum(rate(http_requests_total{job="web-app"}[5m]))
        > 0.05
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: "High error rate on {{ $labels.job }}"
        description: "Error rate is {{ $value | humanizePercentage }}"
    - alert: PodNotReady
      expr: |
        kube_pod_status_phase{phase!="Running"} > 0
      for: 10m
      labels:
        severity: warning

    Grafana Dashboard

    # Access Grafana
kubectl port-forward svc/monitoring-grafana 3000:80 -n monitoring

    Import dashboards: Grafana.com IDs
    

    1860  - Node Exporter Full
    

    15760 - Kubernetes API Server
    

    15758 - Kubernetes Controller Manager
    

    15757 - Kubernetes Scheduler
    

    15756 - Kubernetes Proxy
    

    15755 - Kubernetes Kubelet

    Backup & Restore (Velero)

    # Install Velero
velero install \
  --provider aws \
  --bucket k8s-backups \
  --backup-location-config region=us-east-1 \
  --snapshot-location-config region=us-east-1 \
  --secret-file ./credentials-velero

    Backup operations
    
velero backup create daily-backup --schedule="0 2   *"
velero backup create pre-upgrade-backup --include-namespaces production
velero backup create full-cluster-backup
velero backup get
velero backup describe daily-backup --details

    Schedule backups
    
velero schedule create daily-backup --schedule="0 2   *" --include-namespaces production
velero schedule create hourly-critical --schedule="0    " --include-namespaces production --ttl=72h

    Restore operations
    
velero restore create --from-backup daily-backup
velero restore create --from-backup daily-backup --include-namespaces production
velero restore get
velero restore describe restore-name

    Migrate namespace between clusters
    

    Source cluster:
    
velero backup create migration-backup --include-namespaces production

    Destination cluster:
    
velero restore create --from-backup migration-backup

    Multi-Cluster Management

    Kubefed (Kubernetes Federation)

    # Install kubefed
helm repo add kubefed-charts https://raw.githubusercontent.com/kubernetes-sigs/kubefed/master/charts
helm install kubefed kubefed-charts/kubefed -n kube-federation-system --create-namespace

    Join clusters
    
kubefedctl join cluster-west --cluster-context cluster-west --host-cluster-context cluster-primary
kubefedctl join cluster-east --cluster-context cluster-east --host-cluster-context cluster-primary

    Verify
    
kubectl get kubefedclusters -n kube-federation-system

    Karmada

    # Install Karmada
helm install karmada karmada/karmada --namespace karmada-system --create-namespace

    Join member clusters
    
kubectl karmada join member-cluster-1 --kubeconfig=/path/to/karmada-apiserver.kubeconfig
kubectl karmada join member-cluster-2 --kubeconfig=/path/to/karmada-apiserver.kubeconfig

    Propagate Deployment
    
apiVersion: work.karmada.io/v1alpha2
kind: Deployment
metadata:
  name: web-deployment
  namespace: production
spec:
  replicas: 9
  template:
    spec:
      containers:
      - name: web
        image: myregistry.io/web-app:1.2.3

    
apiVersion: policy.karmada.io/v1alpha1
kind: PropagationPolicy
metadata:
  name: web-propagation
spec:
  resourceSelectors:
  - apiVersion: apps/v1
    kind: Deployment
    name: web-deployment
  placement:
    clusterAffinity:
      clusterNames:
      - member-cluster-1
      - member-cluster-2
    replicaScheduling:
      replicaDivisionPreference: Weighted
      replicaSchedulingType: Divided
      weightPreference:
        staticWeightList:
        - clusterName: member-cluster-1
          weight: 2
        - clusterName: member-cluster-2
          weight: 1

    GitOps with ArgoCD

    Install ArgoCD

    kubectl create namespace argocd
kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml

    Access UI
    
kubectl port-forward svc/argocd-server -n argocd 8080:443

    Login
    
argocd admin initial-password -n argocd
argocd login localhost:8080

    Application Manifest

    apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
  name: web-app
  namespace: argocd
  labels:
    team: platform
spec:
  project: default
  source:
    repoURL: https://github.com/org/kubernetes-manifests.git
    targetRevision: main
    path: overlays/production
    kustomize:
      namePrefix: prod-
  destination:
    server: https://kubernetes.default.svc
    namespace: production
  syncPolicy:
    automated:
      prune: true
      selfHeal: true
      allowEmpty: false
    syncOptions:
    - CreateNamespace=true
    - ServerSideApply=true
    - PrunePropagationPolicy=foreground
    retry:
      limit: 3
      backoff:
        duration: 5s
        factor: 2
        maxDuration: 3m

    ApplicationSet (Multi-Cluster/Multi-Env)

    apiVersion: argoproj.io/v1alpha1
kind: ApplicationSet
metadata:
  name: web-app
spec:
  generators:
  - matrix:
      generators:
      - git:
          repoURL: https://github.com/org/kubernetes-manifests.git
          files:
          - path: "clusters/*/config.json"
      - clusters:
          selector:
            matchLabels:
              env: production
  template:
    metadata:
      name: 'web-app-{{name}}'
    spec:
      project: default
      source:
        repoURL: https://github.com/org/kubernetes-manifests.git
        targetRevision: main
        path: 'overlays/{{environment}}'
      destination:
        server: '{{server}}'
        namespace: production
      syncPolicy:
        automated:
          prune: true
          selfHeal: true

    ArgoCD CLI Commands

    # Application management
argocd app create web-app \
  --repo https://github.com/org/kubernetes-manifests.git \
  --path overlays/production \
  --dest-namespace production \
  --dest-server https://kubernetes.default.svc

    argocd app sync web-app
    argocd app diff web-app
    argocd app history web-app
    argocd app rollback web-app 
    argocd app delete web-app
    

    Refresh & Watch
    
argocd app get web-app --refresh
argocd app watch web-app

    Multiple clusters
    
argocd cluster add production-cluster
argocd cluster list

    Flowcharts & Decision Diagrams

    Pod Lifecycle Decision Tree

    Pod Created → Scheduled? ──No──→ Pending (check node resources, taints, selectors)
                │
               Yes
                ↓
      Container Creating? ──Fail──→ CrashLoopBackOff (check: image pull, configmap, secret mounts)
                │
               Yes
                ↓
          Running → Readiness Probe Pass? ──No──→ Not in Service endpoints (traffic excluded)
                │                              │
               Yes                            Yes (after retry)
                ↓                              ↓
          Serving Traffic ←───────────────── In Service endpoints
                │
           Liveness Probe Fail? ──Yes──→ Pod Restart (restartCount++)
                │
               No
                ↓
          Stable Operation

    Deployment Strategy Decision Flowchart

    What deployment strategy? ────────────────────────────────┐
  │                                                          │
  ├─ Zero-downtime required? ──No──→ Recreate (delete all, then create)
  │                                    (simple, has downtime)
  ├─ Yes → Can you run 2x resources? ──No──→ Rolling Update (default, maxSurge=1)
  │                                            (gradual, resource-efficient)
  ├─ Yes → Need instant rollback? ──Yes──→ Blue-Green (2 full envs)
  │                                       (zero-downtime, instant switch)
  └─ Yes → Need progressive validation? ──Yes──→ Canary (gradual traffic shift)
                                                  (observational, rollback-safe)

    Debugging Decision Tree

    Pod not working? ─┬─ Pending → kubectl describe pod → check events
                   ├─ CrashLoopBackOff → kubectl logs --previous → check exit code
                   ├─ ImagePullBackOff → check image name, secrets, registry access
                   ├─ Not in Service → check readiness probe, labels match selector
                   ├─ OOMKilled → increase resources or fix memory leak
                   └─ Error → kubectl logs + kubectl events → trace stack

    Testing Strategy for Kubernetes

    Pre-Deployment Testing

  • YAML Validation: kubectl apply --dry-run=client -f manifest.yaml
  • Policy Validation: kubectl apply --dry-run=server -f manifest.yaml (checks admission controllers)
  • Helm Testing: helm test (runs Helm test pods)
  • Kustomize Build Verification: kustomize build overlay/ | kubeval -
  • Schema Validation: kubeconform -schema-location 'https://kubernetesjsonschema.dev' manifest.yaml

    • Cluster-Level Testing

    # Test Pod — verifies core services
apiVersion: v1
kind: Pod
metadata:
  name: cluster-health-check
  annotations:
    test: "true"
spec:
  containers:
  - name: dns-test
    image: busybox:1.36
    command: ['sh', '-c', 'nslookup kubernetes.default && echo DNS_OK']
  - name: network-test
    image: busybox:1.36
    command: ['sh', '-c', 'wget -qO- http://kubernetes.default.svc:443/healthz && echo NET_OK']
  restartPolicy: Never

    Integration Testing with k6

    // k6-load-test.js — smoke test for K8s service
import http from 'k6/http';
import { check, sleep } from 'k6';

    export let options = {
      stages: [
        { duration: '30s', target: 20 },   // ramp up
        { duration: '1m',  target: 20 },   // sustain
        { duration: '10s', target: 0 },     // ramp down
      ],
      thresholds: {
        http_req_duration: ['p(95)<500'],  // 95% under 500ms
        http_req_failed: ['rate<0.01'],    // <1% failure
      },
    };
    

    export default function () {
      let res = http.get('http://my-service.default.svc.cluster.local/api/health');
      check(res, { 'status 200': (r) => r.status === 200 });
      sleep(1);
    }

    Chaos Testing with Chaos Mesh

    # Install chaos-mesh
helm repo add chaos-mesh https://charts.chaos-mesh.org
helm install chaos-mesh chaos-mesh/chaos-mesh --namespace chaos-testing --create-namespace

    Pod kill chaos experiment
    
kubectl apply -f - <

    Performance Optimization for Kubernetes

  • Resource Right-Sizing: Use VPA recommendations before setting limits
    •    # Check VPA recommendations
   kubectl get vpa my-app-vpa -o jsonpath='{.status.recommendation.containerRecommendations}'
  • Node Autoscaling: Karpenter for faster node provisioning (vs Cluster Autoscaler)
    •    # Karpenter — 30s node launch vs 5-10min with Cluster Autoscaler
   helm install karpenter oci://public.ecr.aws/karpenter/karpenter --namespace karpenter --create-namespace
  • HPA Tuning: Use custom metrics (not just CPU) for better scaling decisions
  • etcd Performance: etcdctl check perf --load=1000 — ensure <10ms KV write latency
  • Network Performance: Cilium eBPF replaces kube-proxy, reducing latency 30-50%
  • Image Pull Optimization: Use imagePullPolicy: IfNotPresent + registry mirror + kubelet --serialized-image-pull-alpha

    • Best Practices

    Pod Design

  • Always set resource requests and limits — prevents resource starvation and noisy neighbors
  • Set liveness, readiness, and startup probes — use startup probes for slow-starting apps
  • Use PodDisruptionBudget — ensure minimum availability during disruptions
    • apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: web-pdb
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: web
  • Use topologySpreadConstraints — distribute replicas across zones/nodes
  • Pin image tags — never use :latest in production; use specific SHA or semantic version
  • Use imagePullPolicy: IfNotPresent with explicit tags

    • Security

  • Run as non-root — set runAsNonRoot: true, runAsUser: 1000+
  • Drop all capabilities — capabilities: { drop: [ALL] }
  • Read-only root filesystem — readOnlyRootFilesystem: true
  • Disallow privilege escalation — allowPrivilegeEscalation: false
  • Use network policies — default deny all, then allow specific traffic
  • Encrypt Secrets at rest — enable --encryption-provider-config on apiserver
  • Use external secret management — Vault, AWS Secrets Manager via External Secrets Operator
  • Rotate ServiceAccount tokens — Bound ServiceAccount Token Volume (Kubernetes 1.22+)
  • Enable audit logging — --audit-log-path on apiserver
  • Scan images — Trivy, Grype in CI pipeline; OPA/Gatekeeper or Kyverno for admission control

    • Reliability

  • Use anti-affinity — spread pods across nodes/zones
    • affinity:
  podAntiAffinity:
    preferredDuringSchedulingIgnoredDuringExecution:
    - weight: 100
      podAffinityTerm:
        labelSelector:
          matchExpressions:
          - key: app
            operator: In
            values: ["web"]
        topologyKey: topology.kubernetes.io/zone
  • Set revisionHistoryLimit — control rollback depth (default 10)
  • Use maxSurge: 1, maxUnavailable: 0 — zero-downtime rolling updates
  • Configure terminationGracePeriodSeconds appropriately
  • Use preStop hooks for graceful shutdown of connections
  • Test DR regularly — restore from Velero backups quarterly

    • Cost Optimization

  • Set appropriate resource requests — over-provisioning wastes money
  • Use VPA recommender mode to find optimal requests
  • Use spot/preemptible instances with nodeAffinity and proper PDBs
  • Enable cluster autoscaler — scale nodes up/down with demand
  • Use topologySpreadConstraints with maxSkew: 1 for even distribution
  • Right-size persistent volumes — monitor actual usage

    • Common Pitfalls

    | Pitfall | Impact | Solution |
    |---------|--------|----------|
    | Missing resource limits | OOMKills, node instability | Always set requests and limits |
    | Using :latest tag | Unpredictable deployments, rollbacks fail | Pin to specific version or SHA digest |
    | Default LoadBalancer type | Exposes every service publicly | Use ClusterIP + Ingress |
    | No NetworkPolicy | All pods can talk to all pods | Default deny, then allow explicitly |
    | Hardcoded ConfigMap refs | Inflexible across environments | Use Helm values / Kustomize overlays |
    | Missing health probes | Dead pods receive traffic | Set liveness + readiness probes |
    | imagePullPolicy: Always | Slower starts, registry dependency | Use IfNotPresent with pinned tags |
    | Large container images | Slow scheduling, wasted bandwidth | Use distroless or alpine base images |
    | No PDB | Unexpected downtime during node drains | Define PDB with minAvailable or maxUnavailable |
    | Stale kubeconfig | Authentication failures | Rotate certs, use OIDC-based auth |
    | Secret data in etcd | Plaintext secrets at risk | Enable encryption at rest, use external secret stores |
    | Single replica Deployments | No fault tolerance | Minimum 2-3 replicas with anti-affinity |

    Essential CLI Commands Reference

    Cluster Management

    kubectl cluster-info
kubectl get nodes -o wide
kubectl top nodes
kubectl describe node 
kubectl cordon           # Mark unschedulable
kubectl uncordon         # Mark schedulable
kubectl drain  --ignore-daemonsets --delete-emptydir-data
kubectl taint nodes  key=value:NoSchedule
kubectl taint nodes  key=value:NoSchedule-

    Resource Operations

    kubectl get all -n production
kubectl get pods -o wide
kubectl get pods -o jsonpath='{.items[*].metadata.name}'
kubectl get pods -o custom-columns='NAME:.metadata.name,STATUS:.status.phase,NODE:.spec.nodeName'
kubectl get events --sort-by='.lastTimestamp'
kubectl api-resources
kubectl explain pod.spec.containers
kubectl diff -f manifest.yaml

    Context & Namespace

    kubectl config get-contexts
kubectl config use-context production
kubectl config set-context --current --namespace=production
kubectl config current-context

    Dry Run & Validation

    kubectl apply -f manifest.yaml --dry-run=client    # Client-side validation
kubectl apply -f manifest.yaml --dry-run=server    # Server-side validation
kubectl apply -f manifest.yaml --server-side       # Server-side apply (avoid conflicts)

    Debugging Workflow

    # 1. Check pod status
kubectl get pods -n production
kubectl describe pod  -n production

    2. Check logs
    
kubectl logs  -n production
kubectl logs  --previous -n production

    3. Exec into container
    
kubectl exec -it  -n production -- /bin/sh

    4. Check events
    
kubectl get events -n production --sort-by='.lastTimestamp'

    5. Check node resources
    
kubectl top nodes
kubectl describe node 

    6. Check networking
    
kubectl run tmp --image=nicolaka/netshoot --rm -it -- bash

    Inside pod:
    

    nslookup ..svc.cluster.local
    

    curl http://:/healthz
    💬 Ask about Kubernetes Mastery