Node Pools

# List all node pools in your cluster
az aks nodepool list --resource-group rg-dev --cluster-name dev-cluster -o table

# Example output:
# Name        OsType    VmSize           Count  Mode    OrchestratorVersion
# ----------  --------  ---------------  -----  ------  -------------------
# agentpool   Linux     Standard_D2s_v5  2      System  1.29.2

# Add a user pool for application workloads
az aks nodepool add \
  --resource-group rg-dev \
  --cluster-name dev-cluster \
  --name apppool \
  --mode User \
  --node-count 3 \
  --node-vm-size Standard_D4s_v5 \
  --max-pods 110 \
  --zones 1 2 3 \
  --labels workload=app environment=dev \
  --enable-cluster-autoscaler \
  --min-count 2 \
  --max-count 8 \
  --os-sku AzureLinux

# Verify the pool was added
az aks nodepool list -g rg-dev --cluster-name dev-cluster -o table
kubectl get nodes -l agentpool=apppool

# Add a spot pool for batch/CI workloads (up to 90% cheaper)
az aks nodepool add \
  --resource-group rg-dev \
  --cluster-name dev-cluster \
  --name spotpool \
  --mode User \
  --priority Spot \
  --eviction-policy Delete \
  --spot-max-price -1 \
  --node-count 2 \
  --node-vm-size Standard_D4s_v5 \
  --max-pods 110 \
  --labels workload=batch priority=spot \
  --node-taints "kubernetes.azure.com/scalesetpriority=spot:NoSchedule" \
  --enable-cluster-autoscaler \
  --min-count 0 \
  --max-count 10

# AKS automatically adds the spot taint, but adding it explicitly ensures clarity
# To schedule pods on spot nodes, add this toleration to your pod spec:
# tolerations:
# - key: "kubernetes.azure.com/scalesetpriority"
#   operator: "Equal"
#   value: "spot"
#   effect: "NoSchedule"

# Add a GPU pool for ML workloads (scale to zero when not training)
az aks nodepool add \
  --resource-group rg-dev \
  --cluster-name dev-cluster \
  --name gpupool \
  --mode User \
  --node-count 0 \
  --node-vm-size Standard_NC6s_v3 \
  --node-taints "sku=gpu:NoSchedule" \
  --labels workload=ml accelerator=nvidia \
  --enable-cluster-autoscaler \
  --min-count 0 \
  --max-count 3

# When a pod with matching toleration + GPU resource request appears,
# the autoscaler spins up a GPU node. When job finishes, scales back to 0.

# Manual scale — set exact node count
az aks nodepool scale \
  --resource-group rg-dev \
  --cluster-name dev-cluster \
  --name apppool \
  --node-count 5

# Update autoscaler limits
az aks nodepool update \
  --resource-group rg-dev \
  --cluster-name dev-cluster \
  --name apppool \
  --update-cluster-autoscaler \
  --min-count 3 \
  --max-count 15

# Disable autoscaler (switch to manual)
az aks nodepool update \
  --resource-group rg-dev \
  --cluster-name dev-cluster \
  --name apppool \
  --disable-cluster-autoscaler

# List nodes with their pool, VM size, and zone
kubectl get nodes -o custom-columns=\
NAME:.metadata.name,\
POOL:.metadata.labels.agentpool,\
VM:.metadata.labels.node\\.kubernetes\\.io/instance-type,\
ZONE:.metadata.labels.topology\\.kubernetes\\.io/zone,\
STATUS:.status.conditions[-1].type

# Example output:
# NAME                               POOL       VM                ZONE       STATUS
# aks-agentpool-12345-vmss000000     agentpool  Standard_D2s_v5   eastus-1   Ready
# aks-agentpool-12345-vmss000001     agentpool  Standard_D2s_v5   eastus-2   Ready
# aks-apppool-67890-vmss000000       apppool    Standard_D4s_v5   eastus-1   Ready
# aks-apppool-67890-vmss000001       apppool    Standard_D4s_v5   eastus-2   Ready
# aks-apppool-67890-vmss000002       apppool    Standard_D4s_v5   eastus-3   Ready

# Check taints on a node
kubectl describe node aks-agentpool-12345-vmss000000 | grep -A 3 "Taints:"
# Taints: CriticalAddonsOnly=true:NoSchedule

# List all labels on a specific node pool's nodes
kubectl get nodes -l agentpool=spotpool --show-labels

# deploy-to-apppool.yaml — target the app pool using nodeSelector
apiVersion: apps/v1
kind: Deployment
metadata:
  name: web-api
spec:
  replicas: 3
  selector:
    matchLabels:
      app: web-api
  template:
    metadata:
      labels:
        app: web-api
    spec:
      nodeSelector:
        agentpool: apppool          # targets the apppool node pool
      containers:
      - name: web-api
        image: myacr.azurecr.io/web-api:v1.2
        resources:
          requests:
            cpu: 500m
            memory: 512Mi
          limits:
            cpu: "1"
            memory: 1Gi

# batch-job-on-spot.yaml — schedule job on spot nodes with toleration
apiVersion: batch/v1
kind: Job
metadata:
  name: data-processing
spec:
  template:
    spec:
      nodeSelector:
        agentpool: spotpool           # target spot pool
      tolerations:
      - key: "kubernetes.azure.com/scalesetpriority"
        operator: "Equal"
        value: "spot"
        effect: "NoSchedule"
      containers:
      - name: processor
        image: myacr.azurecr.io/data-processor:v2.0
        resources:
          requests:
            cpu: "2"
            memory: 4Gi
      restartPolicy: OnFailure

# Upgrade a specific node pool to a new K8s version
az aks nodepool upgrade \
  --resource-group rg-dev \
  --cluster-name dev-cluster \
  --name apppool \
  --kubernetes-version 1.30.0

# Upgrade node image only (no K8s version change — just OS patches)
az aks nodepool upgrade \
  --resource-group rg-dev \
  --cluster-name dev-cluster \
  --name apppool \
  --node-image-only

# Check current node image version
az aks nodepool show -g rg-dev --cluster-name dev-cluster -n apppool \
  --query nodeImageVersion -o tsv

# Step 1: Check the pod events
kubectl describe pod <pod-name> | grep -A 10 "Events:"
# Look for: "0/5 nodes are available: 3 node(s) had untolerated taint..."

# Step 2: Check what nodeSelector/tolerations the pod requires
kubectl get pod <pod-name> -o jsonpath='{.spec.nodeSelector}'
kubectl get pod <pod-name> -o jsonpath='{.spec.tolerations}'

# Step 3: Check what taints exist on nodes
kubectl get nodes -o custom-columns=NAME:.metadata.name,TAINTS:.spec.taints

# Example output:
# NAME                            TAINTS
# aks-agentpool-...-vmss000000    [map[effect:NoSchedule key:CriticalAddonsOnly value:true]]
# aks-apppool-...-vmss000000      <none>

# Step 4: Common causes:
# - Pod has nodeSelector for a pool that scaled to zero → wait for autoscaler
# - Pod needs GPU but targets wrong pool → fix nodeSelector
# - All user pools are tainted and pod has no toleration → add toleration
# - Pod requests more CPU/memory than any node can provide → use larger VM size

# Step 5: If autoscaler should scale up but doesn't, check its status
kubectl -n kube-system logs -l app=cluster-autoscaler --tail=50 | grep -i "scale"

# Step 1: Confirm eviction happened
kubectl get events --sort-by='.lastTimestamp' | grep -i "evict\|preempt\|spot"

# Step 2: Check which nodes were affected
kubectl get nodes -l kubernetes.azure.com/scalesetpriority=spot -o wide
# Evicted nodes will be gone; replacements may already be provisioning

# Step 3: Check pod status — pods should reschedule on surviving nodes
kubectl get pods -o wide | grep -v Running

# Step 4: Data loss root cause — spot pods used emptyDir or local volumes
# Fix: Use PersistentVolumeClaims with Azure Disks or Azure Files
# These survive node evictions because the data is on Azure storage, not the VM

# Step 5: Add a PodDisruptionBudget to ensure minimum availability
# apiVersion: policy/v1
# kind: PodDisruptionBudget
# metadata:
#   name: processor-pdb
# spec:
#   minAvailable: 1
#   selector:
#     matchLabels:
#       app: data-processor

# Step 6: Ensure your workload handles SIGTERM gracefully
# Spot evictions send SIGTERM → 30 second grace period → SIGKILL
# Your app should checkpoint or save state within those 30 seconds

# Step 1: Check your current quota usage
az vm list-usage --location eastus -o table | grep -i "total\|standard D\|standard NC"

# Example output:
# CurrentValue  Limit    Name
# 12            20       Total Regional vCPUs
# 8             20       Standard DSv5 Family vCPUs
# 0             0        Standard NCSv3 Family vCPUs  ← GPU quota is 0!

# Step 2: Request a quota increase
# Azure Portal → Subscriptions → Usage + Quotas → Request Increase
# Or via CLI:
az quota create --resource-name "StandardNCSv3Family" \
  --scope "/subscriptions/{sub-id}/providers/Microsoft.Compute/locations/eastus" \
  --limit-object value=12

# Step 3: For GPU VMs, quota increases may take 1-2 business days
# Workaround: try a different region with available capacity

# Step 4: Verify quota was increased before retrying
az vm list-usage --location eastus -o table | grep "NC"

# Step 1: Check if the system pool has the CriticalAddonsOnly taint
kubectl describe node aks-agentpool-12345-vmss000000 | grep -A 3 "Taints:"
# If "Taints: <none>" — the system pool isn't tainted

# Step 2: The default node pool created with az aks create is mode=System
# but doesn't have the taint unless you have a separate user pool
# When only one pool exists, all pods schedule there (including yours)

# Step 3: To enforce separation, make sure you have both pools:
az aks nodepool list -g rg-dev --cluster-name dev-cluster -o table
# If only one pool, add a user pool (see Hands-on section above)

# Step 4: Add the taint to the system pool manually if needed
az aks nodepool update \
  --resource-group rg-dev \
  --cluster-name dev-cluster \
  --name agentpool \
  --node-taints "CriticalAddonsOnly=true:NoSchedule"

# Step 5: Verify pods migrate to the user pool
kubectl get pods -o wide
# Pods without the toleration will be evicted from system nodes
# and rescheduled on user pool nodes