Monitoring & Logging

# Enable Container Insights on an existing cluster
az aks enable-addons -a monitoring -g myRG -n myAKS

# Or specify a particular Log Analytics workspace
WORKSPACE_ID=$(az monitor log-analytics workspace show -g myRG -n myWorkspace --query id -o tsv)
az aks enable-addons -a monitoring -g myRG -n myAKS --workspace-resource-id "$WORKSPACE_ID"

# Enable during cluster creation
az aks create -g myRG -n myAKS \
  --enable-addons monitoring \
  --workspace-resource-id "$WORKSPACE_ID" \
  --node-count 3 --generate-ssh-keys

// Find all OOMKilled containers in the last 24 hours
KubeEvents
| where TimeGenerated > ago(24h)
| where Reason == "OOMKilling"
| project TimeGenerated, Namespace, Name, Message
| order by TimeGenerated desc

// Average CPU usage per node over the last 1 hour
InsightsMetrics
| where TimeGenerated > ago(1h)
| where Name == "cpuUsageNanoCores"
| summarize AvgCPU = avg(Val) by Computer, bin(TimeGenerated, 5m)
| render timechart

// Top 10 pods by memory consumption
InsightsMetrics
| where TimeGenerated > ago(30m)
| where Name == "memoryWorkingSetBytes"
| extend PodName = tostring(parse_json(Tags).pod)
| summarize AvgMemMB = avg(Val / 1024 / 1024) by PodName
| top 10 by AvgMemMB desc

// Container restart count by namespace
KubePodInventory
| where TimeGenerated > ago(24h)
| summarize Restarts = max(ContainerRestartCount) by Namespace, Name
| where Restarts > 0
| order by Restarts desc

# Create an Azure Monitor workspace (Prometheus-compatible)
az monitor account create -g myRG -n myPrometheusWorkspace -l eastus

# Link it to AKS
PROM_ID=$(az monitor account show -g myRG -n myPrometheusWorkspace --query id -o tsv)
az aks update -g myRG -n myAKS \
  --enable-azure-monitor-metrics \
  --azure-monitor-workspace-resource-id "$PROM_ID"

# Create a Grafana instance and link to Prometheus
az grafana create -g myRG -n myGrafana -l eastus
az grafana update -g myRG -n myGrafana \
  --azure-monitor-workspace-integrations "[{azureMonitorWorkspaceResourceId:$PROM_ID}]"

# Pod annotations for Prometheus scraping
metadata:
  annotations:
    prometheus.io/scrape: "true"
    prometheus.io/port: "8080"
    prometheus.io/path: "/metrics"

# Create an action group (email + Teams webhook)
az monitor action-group create \
  -g myRG -n aks-ops-team \
  --short-name AKSOps \
  --email-receiver name=OnCall email=oncall@company.com \
  --webhook-receiver name=Teams uri="https://company.webhook.office.com/webhookb2/..."

# Create a metric alert: node CPU > 80% for 5 minutes
AKS_RESOURCE_ID=$(az aks show -g myRG -n myAKS --query id -o tsv)
az monitor metrics alert create \
  -g myRG -n "aks-node-cpu-high" \
  --scopes "$AKS_RESOURCE_ID" \
  --condition "avg node_cpu_usage_percentage > 80" \
  --window-size 5m \
  --evaluation-frequency 1m \
  --action aks-ops-team \
  --severity 2 \
  --description "AKS node CPU exceeds 80% for 5 minutes"

# Create a log alert: OOMKilled events
az monitor scheduled-query create \
  -g myRG -n "aks-oomkilled-alert" \
  --scopes "$WORKSPACE_ID" \
  --condition "count 'KubeEvents | where Reason == \"OOMKilling\"' > 0" \
  --condition-query "KubeEvents | where TimeGenerated > ago(10m) | where Reason == 'OOMKilling'" \
  --window-size 10 \
  --evaluation-frequency 5 \
  --action-groups aks-ops-team \
  --severity 1

# Enable all control plane log categories
az monitor diagnostic-settings create \
  --name aks-diagnostics \
  --resource "$AKS_RESOURCE_ID" \
  --workspace "$WORKSPACE_ID" \
  --logs '[
    {"category":"kube-apiserver","enabled":true},
    {"category":"kube-controller-manager","enabled":true},
    {"category":"kube-scheduler","enabled":true},
    {"category":"kube-audit","enabled":true},
    {"category":"kube-audit-admin","enabled":true},
    {"category":"cluster-autoscaler","enabled":true},
    {"category":"guard","enabled":true}
  ]'

# Auto-instrument a Java app on AKS using Application Insights agent
apiVersion: apps/v1
kind: Deployment
metadata:
  name: api-service
spec:
  template:
    spec:
      containers:
      - name: api
        image: myacr.azurecr.io/api:v2
        env:
        - name: APPLICATIONINSIGHTS_CONNECTION_STRING
          valueFrom:
            secretKeyRef:
              name: appinsights-secret
              key: connectionString
        - name: JAVA_TOOL_OPTIONS
          value: "-javaagent:/opt/applicationinsights-agent.jar"
        volumeMounts:
        - name: ai-agent
          mountPath: /opt/applicationinsights-agent.jar
          subPath: applicationinsights-agent.jar
      initContainers:
      - name: ai-jar
        image: mcr.microsoft.com/applicationinsights/agent:3.4.0
        command: ['cp', '/agent/applicationinsights-agent.jar', '/opt/']
        volumeMounts:
        - name: ai-agent
          mountPath: /opt
      volumes:
      - name: ai-agent
        emptyDir: {}

// Find pods requesting far more CPU than they use (right-sizing candidates)
InsightsMetrics
| where TimeGenerated > ago(7d)
| where Name == "cpuUsageNanoCores"
| extend PodName = tostring(parse_json(Tags).pod)
| extend Namespace = tostring(parse_json(Tags).namespace)
| summarize P95CPU_mCores = percentile(Val / 1000000, 95) by PodName, Namespace
| join kind=inner (
    KubePodInventory
    | where TimeGenerated > ago(1h)
    | extend CpuRequest_mCores = toint(parse_json(ContainerResourceRequestCPU))
    | project PodName = Name, CpuRequest_mCores
    | distinct PodName, CpuRequest_mCores
) on PodName
| extend WasteRatio = round((CpuRequest_mCores - P95CPU_mCores) / CpuRequest_mCores * 100, 1)
| where WasteRatio > 50
| order by WasteRatio desc

# Real-time resource usage per node
kubectl top nodes

# Real-time CPU/memory per pod in a namespace
kubectl top pods -n production --sort-by=memory

# Check recent events for a specific pod
kubectl describe pod myapp -n production | tail -30

# Stream live logs from a container
kubectl logs -f deployment/api-service -n production -c api

# View events cluster-wide, sorted by time
kubectl get events --sort-by='.lastTimestamp' -A | tail -20

# 1. Create a Log Analytics workspace (if you don't have one)
az monitor log-analytics workspace create -g myRG -n aks-logs -l eastus
WORKSPACE_ID=$(az monitor log-analytics workspace show -g myRG -n aks-logs --query id -o tsv)

# 2. Enable Container Insights
az aks enable-addons -a monitoring -g myRG -n myAKS --workspace-resource-id "$WORKSPACE_ID"

# 3. Verify OMS agent pods are running
kubectl get daemonset ama-logs -n kube-system

# 4. Generate some data — deploy a memory-hungry pod that will get OOMKilled
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: oom-test
spec:
  containers:
  - name: stress
    image: polinux/stress
    resources:
      limits:
        memory: "50Mi"
    command: ["stress"]
    args: ["--vm", "1", "--vm-bytes", "100M", "--vm-hang", "1"]
EOF

# 5. Wait ~2 minutes, then check events
kubectl get events --field-selector involvedObject.name=oom-test

# 6. Go to Azure Portal → Log Analytics → Logs, run this KQL:
#    KubeEvents | where Reason == "OOMKilling" | project TimeGenerated, Name, Message

# 1. Create an action group
az monitor action-group create \
  -g myRG -n "aks-alerts" \
  --short-name AKSAlert \
  --email-receiver name=Admin email=admin@company.com

# 2. Create a metric alert for node CPU > 80%
AKS_ID=$(az aks show -g myRG -n myAKS --query id -o tsv)
az monitor metrics alert create \
  -g myRG -n "node-cpu-high" \
  --scopes "$AKS_ID" \
  --condition "avg node_cpu_usage_percentage > 80" \
  --window-size 5m \
  --evaluation-frequency 1m \
  --action aks-alerts \
  --severity 2

# 3. Generate CPU load to test the alert
cat <<EOF | kubectl apply -f -
apiVersion: v1
kind: Pod
metadata:
  name: cpu-stress
spec:
  containers:
  - name: stress
    image: polinux/stress
    resources:
      requests:
        cpu: "1"
      limits:
        cpu: "2"
    command: ["stress"]
    args: ["--cpu", "4", "--timeout", "600"]
EOF

# 4. Watch node CPU rise
kubectl top nodes
# Wait 5+ minutes for alert to fire, check Azure Portal → Monitor → Alerts

# View node-level resource usage
kubectl top nodes

# View pod-level resource usage sorted by CPU
kubectl top pods -A --sort-by=cpu | head -20

# Describe a specific pod to see events, conditions, and resource usage
kubectl describe pod cpu-stress

# View all warning events in the cluster
kubectl get events -A --field-selector type=Warning --sort-by='.lastTimestamp'

# Compare kubectl data with Azure Monitor
# The "kubectl top" shows real-time, Azure Monitor shows historical trends
# Use kubectl for immediate debugging, Azure Monitor for pattern analysis

# Step 1: Is the monitoring add-on enabled?
az aks show -g myRG -n myAKS --query "addonProfiles.omsagent.enabled"
# If null or false → az aks enable-addons -a monitoring ...

# Step 2: Check OMS agent (ama-logs) DaemonSet status
kubectl get daemonset ama-logs -n kube-system
# Desired=3, Current=3, Ready=3 → agents are running
# Ready=0 → agents are crashing

# Step 3: Check agent pod logs for errors
kubectl logs daemonset/ama-logs -n kube-system --tail=50
# Look for "Unauthorized", "workspace not found", or connectivity errors

# Step 4: Verify the Log Analytics workspace exists and is linked
az aks show -g myRG -n myAKS --query "addonProfiles.omsagent.config.logAnalyticsWorkspaceResourceID"
az monitor log-analytics workspace show --ids "<WORKSPACE_ID>" --query "provisioningState"

# Step 5: Check network connectivity (private cluster scenario)
# OMS agent needs outbound connectivity to:
#   *.ods.opinsights.azure.com
#   *.oms.opinsights.azure.com
#   *.monitoring.azure.com
# If using private link, ensure the Private Link Scope includes these endpoints

# Step 6: Data ingestion delay — wait 10-15 minutes after enabling
# Run a simple KQL to check: Heartbeat | take 5

# Fix: Re-enable the add-on if agent pods aren't created, fix network
# rules if agents can't reach Azure, verify workspace hasn't been deleted.

# Step 1: Check if the alert rule exists and is enabled
az monitor metrics alert list -g myRG -o table
# Verify: state = "Enabled", not "Disabled"

# Step 2: Check alert scope — is it targeting the right AKS resource?
az monitor metrics alert show -g myRG -n "node-cpu-high" \
  --query "{scopes:scopes, condition:criteria.allOf[0]}"

# Step 3: Verify the metric name is correct
# Common mistake: using "cpuUsagePercentage" instead of "node_cpu_usage_percentage"
az monitor metrics list --resource "$AKS_ID" --metric-namespace "Insights.Container/nodes"

# Step 4: Check the action group is configured correctly
az monitor action-group show -g myRG -n aks-alerts \
  --query "{email:emailReceivers, webhook:webhookReceivers}"

# Step 5: Check action group test — send a test notification
az monitor action-group test-notifications create \
  -g myRG --action-group-name aks-alerts \
  --alert-type "budget" \
  --email-receiver name=Admin email=admin@company.com

# Step 6: Check Azure Monitor alert history
# Azure Portal → Monitor → Alerts → check "Alert history"
# If alert fired but email not received → check spam folder, action group config

# Fix: Correct the metric name, enable the rule, verify action group
# receivers, check email/webhook endpoints are valid and reachable.

# Step 1: Check if diagnostic settings exist for the AKS resource
az monitor diagnostic-settings list --resource "$AKS_RESOURCE_ID" -o table
# If empty → no diagnostic settings configured

# Step 2: Create diagnostic settings (see section 7 above)
az monitor diagnostic-settings create \
  --name aks-diagnostics \
  --resource "$AKS_RESOURCE_ID" \
  --workspace "$WORKSPACE_ID" \
  --logs '[{"category":"kube-apiserver","enabled":true},{"category":"kube-audit-admin","enabled":true},{"category":"cluster-autoscaler","enabled":true}]'

# Step 3: Wait 5-10 minutes for logs to start flowing

# Step 4: Verify data is arriving
# KQL: AzureDiagnostics | where Category == "kube-apiserver" | take 5

# Step 5: If diagnostic settings exist but no data appears:
# - Check the workspace is correct and active
# - Verify the categories are set to enabled: true
# - Check for Azure Policy restricting diagnostic settings

# Common misconception: Container Insights (monitoring add-on) does NOT
# collect control plane logs. You must configure diagnostic settings separately.

# Step 6: Query audit logs to find who deleted the namespace
# KQL:
# AzureDiagnostics
# | where Category == "kube-audit-admin"
# | where log_s contains "namespaces" and log_s contains "delete"
# | project TimeGenerated, log_s
# | order by TimeGenerated desc