Troubleshooting

# Step 1: Is OneAgent process running?
sudo systemctl status oneagent
# OR for containerised:
ps aux | grep oneagent

# Step 2: Check OneAgent logs for connection errors
sudo tail -100 /var/log/dynatrace/oneagent/oneagent*.log | grep -i "error\|warning\|connect"

# Step 3: Test connectivity to Dynatrace endpoint
curl -v https://your-env.live.dynatrace.com/communication \
  -H "Authorization: AgentTechnologyType=OneAgent"
# Expected: HTTP 200 — any other code = network/firewall issue

# Step 4: Verify agent configuration
sudo cat /var/lib/dynatrace/oneagent/agent/config/ruxitagentproc.conf | grep server

# Step 5: Run OneAgent self-diagnostics
sudo /opt/dynatrace/oneagent/agent/lib64/oneagentwatchdog --check

# Step 6: Restart agent if configuration was fixed
sudo systemctl restart oneagent

# Step 1: Check DynaKube status
kubectl get dynakube -n dynatrace
kubectl describe dynakube dynakube -n dynatrace
# Look for: Status.Conditions — any "False" conditions indicate problems

# Step 2: Check Dynatrace Operator is running
kubectl get pods -n dynatrace
# Expected pods: dynakube-oneagent-*, dynakube-activegate-*, dynatrace-operator-*

# Step 3: Check Operator logs for injection errors
kubectl logs -n dynatrace deployment/dynatrace-operator --tail=100 | grep -i error

# Step 4: Verify namespace is configured for injection
kubectl get namespace your-namespace --show-labels
# Should include: dynatrace.com/inject=true OR rely on DynaKube namespaceSelector

# Step 5: Check webhook configuration
kubectl get mutatingwebhookconfiguration | grep dynatrace
kubectl describe mutatingwebhookconfiguration dynatrace-webhook

# Step 6: Check pod annotations for injection
kubectl get pod your-pod-xxxx -o yaml | grep -A5 annotations
# If disabled: remove annotation: instrumentation.dynatrace.com/inject=false

# Step 7: Force re-injection by restarting the pod
kubectl rollout restart deployment/your-service

# AWS Integration — validate via Dynatrace API
curl -s "https://your-env.live.dynatrace.com/api/v1/aws/iamExternalId" \
  -H "Authorization: Api-Token YOUR_TOKEN"

# Check AWS integration status
curl -s "https://your-env.live.dynatrace.com/api/v1/aws" \
  -H "Authorization: Api-Token YOUR_TOKEN" \
  | jq '.[] | {id: .id, status: .status, label: .label}'

# If status = ERROR, check:
# 1. IAM role ARN is correct
# 2. IAM role has the required policies:
#    - CloudWatch:GetMetricStatistics
#    - EC2:DescribeInstances
#    - tag:GetResources

# ActiveGate must be running for cloud integrations
kubectl get pods -n dynatrace | grep activegate
# OR on-prem:
sudo systemctl status dynatracegateway

# Azure — check service principal permissions
# The Dynatrace Azure app registration needs:
# - Monitoring Reader role on subscription
# - Reader role on subscription

# Force metric refresh via API
curl -s -X POST \
  "https://your-env.live.dynatrace.com/api/v1/aws/YOUR_CONFIG_ID/refresh" \
  -H "Authorization: Api-Token YOUR_TOKEN"

# Test if API token is valid
curl -s "https://your-env.live.dynatrace.com/api/v2/metrics?pageSize=1" \
  -H "Authorization: Api-Token YOUR_TOKEN" -o /dev/null -w "%{http_code}"
# 200 = valid, 401 = invalid/expired token, 403 = insufficient scope

# List required scopes for common operations:
# Metrics API v2:           metrics.read
# Problems API v2:          problems.read
# Entities API v2:          entities.read
# Events ingest:            events.ingest
# Settings write (Monaco):  settings.write
# PaaS token (OneAgent):    InstallerDownload

# Create new token via UI: Settings → Access Tokens → Generate new token
# OR via API (requires existing token with tokens.write scope):
curl -s -X POST \
  "https://your-env.live.dynatrace.com/api/v2/apiTokens" \
  -H "Authorization: Api-Token YOUR_ADMIN_TOKEN" \
  -H "Content-Type: application/json" \
  -d '{"name":"ci-pipeline","scopes":["metrics.read","problems.read","events.ingest"]}'