Debugging Scenarios

# 1. Get the pod name and current status
kubectl get pods -n <namespace>
# NAME          READY   STATUS             RESTARTS   AGE
# myapp-abc12   0/1     CrashLoopBackOff   5          3m

# 2. Check logs (current crash)
kubectl logs myapp-abc12 -n <namespace>

# 3. Check previous crash logs (if container already restarted)
kubectl logs myapp-abc12 -n <namespace> --previous

# 4. Check events and exit code
kubectl describe pod myapp-abc12 -n <namespace>
# Look for:
#   Last State: Terminated
#   Exit Code: 1 (app error), 137 (OOMKilled/SIGKILL), 139 (segfault)

# 5. If logs are empty, check the command
kubectl get pod myapp-abc12 -o jsonpath='{.spec.containers[0].command}' -n <namespace>

# 6. Debug interactively — override entrypoint
kubectl run debug-pod --image=<same-image> -it --rm -- /bin/sh
# Then manually run the app command to see the error

# 1. Describe pod → read Events section
kubectl describe pod <pod-name> -n <namespace>
# Events:
#   Failed to pull image "myrepo/myimage:v9.99": ... not found

# 2. Verify the image exists (locally)
docker pull myrepo/myimage:v9.99
# If this fails, the image doesn't exist or you need auth

# 3. For private registries — create/check imagePullSecrets
kubectl get secrets -n <namespace> | grep docker
kubectl get pod <pod-name> -o jsonpath='{.spec.imagePullSecrets}' -n <namespace>

# 4. Create image pull secret if missing
kubectl create secret docker-registry regcred \
  --docker-server=<registry-url> \
  --docker-username=<user> \
  --docker-password=<pass> \
  -n <namespace>

# 5. Add to deployment spec:
# spec.template.spec.imagePullSecrets:
#   - name: regcred

# 1. Describe the pending pod
kubectl describe pod <pod-name> -n <namespace>
# Events will tell you WHY scheduling failed

# 2. Check node resources
kubectl describe nodes | grep -A5 "Allocated resources"
# See how much capacity is left on each node

# 3. Check node taints
kubectl describe nodes | grep -A3 Taints
# If nodes have taints, pods need matching tolerations

# 4. Check node labels (for nodeSelector)
kubectl get nodes --show-labels

# 5. Check PVC status (if using persistent volumes)
kubectl get pvc -n <namespace>
# STATUS should be "Bound", not "Pending"

# 6. Check ResourceQuota
kubectl get resourcequota -n <namespace>
kubectl describe resourcequota -n <namespace>

# Fixes:
# - Reduce resource requests
# - Add tolerations for node taints
# - Fix nodeSelector labels
# - Provision PersistentVolume
# - Increase ResourceQuota or free up resources

# 1. Verify pods are actually Running and Ready
kubectl get pods -n <namespace> -l <your-selector>
# All should be 1/1 Running

# 2. Check Service exists and has endpoints
kubectl get svc <service-name> -n <namespace>
kubectl get endpoints <service-name> -n <namespace>
# If ENDPOINTS is <none>, the selector doesn't match any pods

# 3. Compare labels
kubectl get pods -n <namespace> --show-labels
kubectl get svc <service-name> -n <namespace> -o jsonpath='{.spec.selector}'
# Must match exactly

# 4. Check the target port
kubectl get svc <service-name> -n <namespace> -o yaml
# spec.ports[].targetPort must match the container's actual port

# 5. Test connectivity from within the cluster
kubectl run test-curl --image=curlimages/curl -it --rm -- \
  curl -s http://<service-name>.<namespace>.svc.cluster.local:<port>

# 6. Test direct pod connectivity
kubectl get endpoints <service-name> -n <namespace>
kubectl run test-curl --image=curlimages/curl -it --rm -- \
  curl -s http://<pod-ip>:<container-port>
# If direct works but service doesn't, it's a service/selector issue

# 1. Verify ingress resource
kubectl get ingress -n <namespace>
kubectl describe ingress <name> -n <namespace>
# Check: host, path, backend service name, port

# 2. Check ingress controller is running
kubectl get pods -n ingress-nginx
# All should be Running

# 3. Check ingress controller logs
kubectl logs -n ingress-nginx -l app.kubernetes.io/name=ingress-nginx

# 4. Verify backend service has endpoints
kubectl get endpoints <backend-svc> -n <namespace>

# 5. For TLS issues
kubectl get secret <tls-secret> -n <namespace>
# Verify the secret exists and has tls.crt + tls.key

# 1. Check node status
kubectl get nodes
# STATUS: NotReady

# 2. Describe the node
kubectl describe node <node-name>
# Look at Conditions section:
#   MemoryPressure, DiskPressure, PIDPressure, Ready

# 3. Check kubelet on the node (SSH required)
systemctl status kubelet
journalctl -u kubelet -f --no-pager | tail -50

# 4. Common causes:
# - kubelet stopped → systemctl restart kubelet
# - Certificate expired → kubeadm certs renew all
# - Disk full → free disk space
# - Network issue → check CNI plugin pods: kubectl get pods -n kube-system

# === POD DEBUGGING ===
kubectl get pods -A                              # All pods, all namespaces
kubectl get pods -o wide                         # Show node & IP
kubectl describe pod <name>                      # Full details + events
kubectl logs <pod>                               # Current container logs
kubectl logs <pod> --previous                    # Previous crash logs
kubectl logs <pod> -c <container>                # Specific container (multi-container)
kubectl logs <pod> -f                            # Stream live logs
kubectl exec -it <pod> -- /bin/sh                # Shell into container
kubectl cp <pod>:/path/file ./local-file         # Copy file from pod

# === SERVICE / NETWORK ===
kubectl get svc,endpoints -n <ns>                # Service + endpoints together
kubectl port-forward svc/<name> 8080:80          # Quick local test

# === CLUSTER / NODES ===
kubectl get nodes -o wide                        # Node status + versions
kubectl top nodes                                # Resource usage (needs metrics-server)
kubectl top pods -n <ns>                         # Pod resource usage
kubectl get events -n <ns> --sort-by=.lastTimestamp  # Recent events

# === ADVANCED ===
kubectl get pods --field-selector=status.phase=Failed  # Filter by status
kubectl get pods -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.phase}{"\n"}{end}'
kubectl api-resources                            # All available resource types
kubectl explain pod.spec.containers              # Built-in docs