Linux

# Find failed messages in last hour, grouped by error code
tail -f /var/log/messaging/app.log | grep 'FAILED' | \
  awk -F'|' '{print $3, $5, $7}' | \
  sort | uniq -c | sort -rn | head -20

# Extract unique IPs causing errors in time window
sed -n '/2026-06-23 14:/,/2026-06-23 15:/p' /var/log/messaging/app.log | \
  grep 'TIMEOUT' | \
  awk '{print $4}' | sort -u

# Monitor process resource usage
ps aux --sort=-%mem | head -10
watch -n 2 'netstat -an | grep ESTABLISHED | wc -l'

#!/bin/bash
# healthcheck.sh — monitor messaging service and auto-restart

SERVICE="messaging-api"
LOG="/var/log/healthcheck.log"
ENDPOINT="http://localhost:8080/health"
MAX_RESTARTS=3
RESTART_COUNT=0

log() {
  echo "$(date '+%Y-%m-%d %H:%M:%S') $1" >> "$LOG"
}

check_health() {
  HTTP_CODE=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 "$ENDPOINT")
  [ "$HTTP_CODE" = "200" ]
}

while true; do
  if ! check_health; then
    log "ALERT: $SERVICE health check failed (HTTP $HTTP_CODE)"
    if [ $RESTART_COUNT -lt $MAX_RESTARTS ]; then
      systemctl restart "$SERVICE"
      RESTART_COUNT=$((RESTART_COUNT + 1))
      log "Restarted $SERVICE (attempt $RESTART_COUNT/$MAX_RESTARTS)"
      sleep 30
    else
      log "CRITICAL: Max restarts reached. Manual intervention required."
      # Send alert — integrate with PagerDuty/Slack webhook
      curl -s -X POST "$SLACK_WEBHOOK" -d '{"text":"CRITICAL: messaging-api down"}'
      exit 1
    fi
  else
    RESTART_COUNT=0
  fi
  sleep 60
done

## CPU Issues

```bash
top -o %CPU
strace -p <pid> -c          # syscall summary
perf top -p <pid>           # hotspot analysis
```

## Memory Issues

```bash
free -h
cat /proc/<pid>/status | grep -i vm
slabtop -s c                 # kernel slab usage
```

## Disk & I/O

```bash
df -h
iostat -x 1 5
lsof +D /var/log             # what's writing to logs
```

## Network

```bash
ss -tlnp                     # listening ports
netstat -an | grep :8080
tcpdump -i eth0 port 8080 -c 20
dig +trace example.com       # DNS resolution chain
```

## Log Locations (RHEL/Ubuntu)

| Log | Path |
|-----|------|
| System | `/var/log/messages` or `journalctl` |
| Auth | `/var/log/secure` or `/var/log/auth.log` |
| App | `/var/log/<app>/` |
| Kernel | `dmesg` or `/var/log/kern.log` |