Debugging Scenarios

Error: INSTALLATION FAILED: template: mychart/templates/deployment.yaml:25:20:
executing "mychart/templates/deployment.yaml" at <.Values.image.tag>:
nil pointer evaluating interface {}.tag

# Step 1: Reproduce locally
helm template test . 2>&1

# Step 2: Render with debug for more context
helm template test . --debug 2>&1

# Step 3: Render specific template
helm template test . -s templates/deployment.yaml 2>&1

# Step 4: Check values being passed
helm template test . --set image.tag=v1 2>&1
# If this works → the value is missing, not a syntax issue

# Step 1: Check what values Helm actually used
helm get values myapp -n production
helm get values myapp -n production --all   # Includes defaults

# Step 2: Check rendered manifest
helm get manifest myapp -n production | grep replicas

# Step 3: Compare with expected
helm template test . --set replicaCount=3 | grep replicas

# Step 4: Check value merge order
# CLI --set > -f values-prod.yaml > -f values.yaml > chart defaults

Error: UPGRADE FAILED: timed out waiting for the condition

# Step 1: Check pod status
kubectl get pods -n production
kubectl describe pod <failing-pod> -n production

# Step 2: Check events (recent issues)
kubectl get events -n production --sort-by='.lastTimestamp' | tail -20

# Step 3: Check container logs
kubectl logs <pod-name> -n production
kubectl logs <pod-name> -n production --previous  # Previous crash

# Step 4: Check release status
helm status myapp -n production
helm history myapp -n production

Error: UPGRADE FAILED: another operation (install/upgrade/rollback) is in progress

# Step 1: Check current status
helm list -n production
helm status myapp -n production

# Step 2: View history
helm history myapp -n production
# Look for STATUS: pending-upgrade or pending-install

# Step 3: Rollback to last good revision
helm rollback myapp <last-deployed-revision> -n production

# Step 4: If rollback fails, manually fix
# List release secrets
kubectl get secrets -n production -l owner=helm,name=myapp

# Delete the stuck pending secret
kubectl delete secret sh.helm.release.v1.myapp.v<N> -n production

# Step 5: Retry
helm upgrade --install myapp . -n production

Error: failed pre-install: job failed: BackoffLimitExceeded

# Step 1: Find the hook Job
kubectl get jobs -n production
kubectl describe job myapp-db-migrate -n production

# Step 2: Check hook pod logs
kubectl logs job/myapp-db-migrate -n production

# Step 3: Fix the hook (usually wrong command, missing env var, DB unreachable)
# Edit the template, then:

# Step 4: Clean up failed hook
kubectl delete job myapp-db-migrate -n production

# Step 5: Retry
helm upgrade --install myapp . -n production --atomic

Error: rendered manifests contain a resource that already exists.
Unable to continue with install: existing resource conflict: ... ServiceAccount

# Step 1: Find who owns the resource
kubectl get serviceaccount <name> -n production -o yaml | grep -A 3 "annotations"

# Step 2a: If owned by another Helm release → use a unique name
# Check _helpers.tpl fullname template

# Step 2b: If created manually → adopt it into Helm
kubectl annotate serviceaccount <name> -n production \
  meta.helm.sh/release-name=myapp \
  meta.helm.sh/release-namespace=production
kubectl label serviceaccount <name> -n production \
  app.kubernetes.io/managed-by=Helm

# Step 3: Retry
helm upgrade --install myapp . -n production

# Install helm-diff plugin
helm plugin install https://github.com/databus23/helm-diff

# See what will change
helm diff upgrade myapp . -n production -f values-prod.yaml

# Compare revisions
helm diff revision myapp 5 6 -n production

# Common causes of unexpected diff:
# 1. --reuse-values merging old/new values unexpectedly
# 2. Chart upgrade changed default values
# 3. Random generator (password) regenerating each time
#    Fix: Use lookup function or external secret management

# Step 1: Verify Service and Endpoints
kubectl get svc -n production
kubectl get endpoints myapp -n production
# If endpoints list is EMPTY → labels don't match

# Step 2: Compare Service selector with Pod labels
kubectl get svc myapp -n production -o jsonpath='{.spec.selector}'
kubectl get pods -n production --show-labels
# Ensure selectors match pod labels EXACTLY

# Step 3: Test DNS resolution from inside the cluster
kubectl run dns-test --rm -it --image=busybox -n production -- nslookup myapp
kubectl run dns-test --rm -it --image=busybox -n production -- nslookup myapp.production.svc.cluster.local

# Step 4: Test connectivity to the service
kubectl run curl-test --rm -it --image=curlimages/curl -n production -- \
  curl -v http://myapp:80/health

# Step 5: Check NetworkPolicies blocking traffic
kubectl get networkpolicies -n production
kubectl describe networkpolicy -n production
# If NetworkPolicy exists, verify it allows the necessary ingress/egress

# Step 6: For Ingress issues
kubectl describe ingress myapp -n production
kubectl get events -n production | grep ingress
kubectl logs -n ingress-nginx deployment/ingress-nginx-controller

# Step 1: Check pod status and scheduling
kubectl get pods -n production -o wide
kubectl describe pod <pending-pod> -n production
# Look at Events section for:
#   FailedScheduling: 0/5 nodes are available: insufficient cpu
#   FailedScheduling: 0/5 nodes are available: pod has unbound PersistentVolumeClaims

# Step 2: Check node resources
kubectl top nodes
kubectl describe node <node-name> | grep -A 8 "Allocated resources"

# Step 3: Check resource quotas on the namespace
kubectl describe resourcequota -n production
# If quota is exceeded, reduce resource requests or increase quota

# Step 4: Check PodDisruptionBudget blocking rollout
kubectl get pdb -n production
kubectl describe pdb myapp-pdb -n production
# If minAvailable is too high, new pods can't be scheduled during rolling update

# Step 5: Check Deployment rollout strategy
kubectl get deploy myapp -n production -o jsonpath='{.spec.strategy}'
# With RollingUpdate: maxUnavailable=0 + maxSurge=1 is slowest but safest

# Step 6: Monitor rollout progress
kubectl rollout status deployment/myapp -n production
kubectl get events -n production --sort-by='.lastTimestamp' | tail -20

# 5:01 PM — Alert fires: 503 errors on /api/checkout

# Step 1: Is it Helm or K8s?
helm status payment-api -n prod
# STATUS: deployed — Helm thinks it's fine

# Step 2: Check pods
kubectl get pods -n prod -l app.kubernetes.io/name=payment-api
# 2/3 pods in CrashLoopBackOff, 1 still running (old revision)

# Step 3: Why are they crashing?
kubectl logs payment-api-7f8b9-xk2mn -n prod
# Error: STRIPE_API_KEY environment variable not set

# Step 4: What changed?
helm diff revision payment-api 11 12 -n prod
# Reveals: env block restructured, STRIPE_API_KEY moved under a new key

# Step 5: Check values that were applied
helm get values payment-api -n prod
# Missing: payment.stripe.apiKeySecret

# Step 6: Immediate fix — rollback
helm rollback payment-api 11 -n prod
kubectl get pods -n prod -l app.kubernetes.io/name=payment-api
# All 3 pods Running, 503s stop

# Step 7: Monday — Fix properly
# Update values-prod.yaml with the new key structure
# Test with: helm template test . -f values-prod.yaml | grep STRIPE
# Deploy to staging first, then prod with --atomic