Scaling & Autoscaling

# hpa.yaml
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: api-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: api-server
  minReplicas: 3
  maxReplicas: 50
  metrics:
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70
    - type: Resource
      resource:
        name: memory
        target:
          type: Utilization
          averageUtilization: 80
  behavior:
    scaleUp:
      stabilizationWindowSeconds: 30
      policies:
        - type: Percent
          value: 100
          periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300
      policies:
        - type: Percent
          value: 10
          periodSeconds: 60

# keda-scaledobject.yaml
apiVersion: keda.sh/v1alpha1
kind: ScaledObject
metadata:
  name: order-processor
  namespace: production
spec:
  scaleTargetRef:
    name: order-processor
  pollingInterval: 15
  cooldownPeriod: 120
  minReplicaCount: 1
  maxReplicaCount: 100
  triggers:
    - type: azure-servicebus
      metadata:
        queueName: orders
        messageCount: "5"
        connectionFromEnv: SERVICEBUS_CONNECTION

# Enable autoscaler on the default node pool
az aks nodepool update \
  --resource-group myResourceGroup \
  --cluster-name myAKSCluster \
  --name nodepool1 \
  --enable-cluster-autoscaler \
  --min-count 2 \
  --max-count 10

# Verify autoscaler status
az aks show -g myResourceGroup -n myAKSCluster \
  --query "agentPoolProfiles[0].{name:name, minCount:minCount, maxCount:maxCount, enableAutoScaling:enableAutoScaling}" \
  -o table

# Name       MinCount    MaxCount    EnableAutoScaling
# ---------  ----------  ----------  -------------------
# nodepool1  2           10          True

# Tune cluster-wide autoscaler settings
az aks update \
  --resource-group myResourceGroup \
  --name myAKSCluster \
  --cluster-autoscaler-profile \
    scan-interval=20s \
    scale-down-delay-after-add=5m \
    scale-down-unneeded-time=5m \
    scale-down-utilization-threshold=0.6 \
    expander=priority

# Verify the profile
az aks show -g myResourceGroup -n myAKSCluster \
  --query "autoScalerProfile" -o json

# Deploy a sample app with resource requests
kubectl create deployment php-apache \
  --image=registry.k8s.io/hpa-example \
  --requests='cpu=200m'
kubectl expose deployment php-apache --port=80

# Create an HPA targeting 50% CPU
kubectl autoscale deployment php-apache \
  --cpu-percent=50 \
  --min=1 \
  --max=20

# Check HPA status
kubectl get hpa php-apache
# NAME         REFERENCE               TARGETS   MINPODS   MAXPODS   REPLICAS   AGE
# php-apache   Deployment/php-apache   0%/50%    1         20        1          30s

# Generate load from a separate terminal
kubectl run load-gen --image=busybox:1.36 --restart=Never -- \
  /bin/sh -c "while true; do wget -q -O- http://php-apache; done"

# Watch scaling in real-time (in your main terminal)
kubectl get hpa php-apache -w

# NAME         REFERENCE               TARGETS    MINPODS   MAXPODS   REPLICAS
# php-apache   Deployment/php-apache   248%/50%   1         20        1
# php-apache   Deployment/php-apache   248%/50%   1         20        5
# php-apache   Deployment/php-apache   76%/50%    1         20        8
# php-apache   Deployment/php-apache   49%/50%    1         20        8

# Stop the load generator
kubectl delete pod load-gen

# After ~5 minutes, HPA scales back down
kubectl get hpa php-apache
# TARGETS   REPLICAS
# 0%/50%    1

# Enable KEDA on your AKS cluster
az aks update \
  --resource-group myResourceGroup \
  --name myAKSCluster \
  --enable-keda

# Verify KEDA operator is running
kubectl get pods -n kube-system -l app=keda-operator
# NAME                             READY   STATUS    RESTARTS
# keda-operator-7c8d6b5f4d-x9k2j  1/1     Running   0

# Check KEDA CRDs are installed
kubectl get crd | grep keda
# scaledobjects.keda.sh
# scaledjobs.keda.sh
# triggerauthentications.keda.sh

# pdb.yaml
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: api-pdb
  namespace: production
spec:
  minAvailable: 2
  selector:
    matchLabels:
      app: api-server

kubectl apply -f pdb.yaml

kubectl get pdb -n production
# NAME      MIN AVAILABLE   MAX UNAVAILABLE   ALLOWED DISRUPTIONS   AGE
# api-pdb   2               N/A               1                     5s

# Step 1: Confirm pods are Pending due to insufficient resources
kubectl get pods --field-selector=status.phase=Pending
kubectl describe pod <pending-pod>
# Look for: "0/5 nodes are available: 5 Insufficient cpu"

# Step 2: Verify autoscaler is enabled and not at max
az aks nodepool show \
  -g myResourceGroup --cluster-name myAKSCluster -n nodepool1 \
  --query "{autoScaling:enableAutoScaling, min:minCount, max:maxCount, current:count}"
# If current == max, increase --max-count

# Step 3: Check Azure VM quota in the region
az vm list-usage --location eastus -o table | Select-String "Standard DS"
# If "CurrentValue" equals "Limit" — request a quota increase

# Step 4: Check for cluster-autoscaler events
kubectl get events --field-selector reason=ScaleUp -A --sort-by='.lastTimestamp'
kubectl get events --field-selector reason=NotTriggerScaleUp -A

# Step 5: Check autoscaler status configmap
kubectl get configmap cluster-autoscaler-status -n kube-system -o yaml
# Look for "ScaleUp" status, "backoff" entries, or "noScaleUp" reasons

# Step 6: If using Spot VMs, check if capacity is available
# Switch expander to allow fallback to regular pools
az aks update -g myResourceGroup -n myAKSCluster \
  --cluster-autoscaler-profile expander=priority

# Step 1: Check if metrics-server is running
kubectl get deployment metrics-server -n kube-system
kubectl top nodes    # Should show CPU/memory for each node
kubectl top pods     # Should show CPU/memory for each pod

# Step 2: If metrics-server is not running or unhealthy
kubectl get pods -n kube-system -l k8s-app=metrics-server
kubectl logs -n kube-system -l k8s-app=metrics-server --tail=50

# Step 3: Verify the deployment has resource requests set
kubectl get deployment api-server -o jsonpath='{.spec.template.spec.containers[0].resources}'
# If empty — HPA cannot calculate %. Add resources.requests:
kubectl set resources deployment api-server --requests=cpu=200m,memory=256Mi

# Step 4: Check HPA events for specific errors
kubectl describe hpa api-hpa
# Look for: "failed to get cpu utilization" or "missing request for cpu"

# Step 5: If using custom metrics, verify the adapter
kubectl get apiservice v1beta1.custom.metrics.k8s.io -o yaml
# If status shows "False" — the custom metrics adapter is broken

# Step 6: After fixing, watch HPA recover
kubectl get hpa api-hpa -w

# Step 1: Check current autoscaler profile
az aks show -g myResourceGroup -n myAKSCluster \
  --query "autoScalerProfile" -o table

# Step 2: Increase scale-down cooldowns
az aks update -g myResourceGroup -n myAKSCluster \
  --cluster-autoscaler-profile \
    scale-down-delay-after-add=15m \
    scale-down-unneeded-time=15m \
    scale-down-utilization-threshold=0.7

# Step 3: Ensure PodDisruptionBudgets are set for critical services
kubectl get pdb -A
# If missing, create PDBs with minAvailable for each stateful/critical service

# Step 4: Annotate pods that should NOT be evicted
kubectl annotate pod important-pod-xyz \
  "cluster-autoscaler.kubernetes.io/safe-to-evict"="false"

# Step 5: Use over-provisioning — deploy low-priority "pause" pods
cat <<EOF | kubectl apply -f -
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
  name: overprovisioning
value: -1
globalDefault: false
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: overprovisioning
spec:
  replicas: 2
  selector:
    matchLabels:
      run: overprovisioning
  template:
    metadata:
      labels:
        run: overprovisioning
    spec:
      priorityClassName: overprovisioning
      containers:
        - name: pause
          image: registry.k8s.io/pause:3.9
          resources:
            requests:
              cpu: "1"
              memory: 2Gi
EOF
# These pods hold capacity. When real pods need space, pause pods are evicted instantly
# — no waiting for new nodes to boot.