Alerting & Incident Response: From Alert to Resolution

# docker-compose-alertmanager.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - ./alerts.yml:/etc/prometheus/alerts.yml
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--alertmanager.url=http://alertmanager:9093'

  alertmanager:
    image: prom/alertmanager:latest
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
    command:
      - '--config.file=/etc/alertmanager/alertmanager.yml'
      - '--web.external-url=http://localhost:9093'

# prometheus.yml addition
rule_files:
  - "alerts.yml"

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

# alerts.yml - Complete alerting rules
groups:
  - name: infrastructure
    interval: 30s
    rules:
      - alert: HighCPUUsage
        expr: 100 - (avg(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 10m
        labels:
          severity: warning
          team: infrastructure
        annotations:
          summary: "High CPU usage on {{ $labels.instance }}"
          description: "CPU usage is {{ $value }}% for more than 10 minutes"
          runbook: "https://runbooks.example.com/high-cpu"

      - alert: CriticalCPUUsage
        expr: 100 - (avg(rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 95
        for: 2m
        labels:
          severity: critical
          team: infrastructure
        annotations:
          summary: "Critical CPU usage on {{ $labels.instance }}"
          description: "CPU usage is {{ $value }}%"

      - alert: HighMemoryUsage
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 90
        for: 5m
        labels:
          severity: warning
          team: infrastructure
        annotations:
          summary: "High memory usage on {{ $labels.instance }}"
          description: "Memory usage is {{ $value }}%"

      - alert: DiskSpaceLow
        expr: (node_filesystem_free_bytes / node_filesystem_size_bytes) * 100 < 10
        for: 5m
        labels:
          severity: critical
          team: infrastructure
        annotations:
          summary: "Low disk space on {{ $labels.instance }} ({{ $labels.mountpoint }})"
          description: "Only {{ $value }}% free"

  - name: application
    interval: 30s
    rules:
      - alert: HighErrorRate
        expr: sum(rate(http_requests_total{status=~"5.."}[5m])) / sum(rate(http_requests_total[5m])) > 0.05
        for: 5m
        labels:
          severity: critical
          team: backend
        annotations:
          summary: "High error rate detected"
          description: "Error rate is {{ $value | humanizePercentage }} for last 5 minutes"

      - alert: SlowResponses
        expr: histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le)) > 2
        for: 10m
        labels:
          severity: warning
          team: backend
        annotations:
          summary: "Slow API responses"
          description: "95th percentile latency is {{ $value }}s"

      - alert: ServiceDown
        expr: up{job="myapp"} == 0
        for: 1m
        labels:
          severity: critical
          team: backend
        annotations:
          summary: "Service {{ $labels.job }} is down"
          description: "{{ $labels.instance }} has been down for 1 minute"

      - alert: NoMetrics
        expr: absent(up{job="myapp"} == 1)
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "No metrics from {{ $labels.job }}"
          description: "No metrics received for 5 minutes"

  - name: business
    interval: 1m
    rules:
      - alert: LowOrderVolume
        expr: rate(orders_created_total[1h]) < 10
        for: 30m
        labels:
          severity: warning
          team: product
        annotations:
          summary: "Low order volume detected"
          description: "Only {{ $value }} orders per hour"

      - alert: HighPaymentFailure
        expr: sum(rate(payment_failures_total[5m])) / sum(rate(payment_attempts_total[5m])) > 0.1
        for: 5m
        labels:
          severity: critical
          team: payments
        annotations:
          summary: "High payment failure rate"
          description: "Payment failure rate is {{ $value | humanizePercentage }}"

# alertmanager.yml
route:
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 12h
  receiver: 'default'
  routes:
    # Critical alerts go to PagerDuty immediately
    - match:
        severity: critical
      receiver: pagerduty
      group_wait: 0s
      continue: true
    
    # Infrastructure warnings go to Slack
    - match:
        severity: warning
        team: infrastructure
      receiver: slack-infra
      group_interval: 5m
    
    # Backend warnings go to dedicated Slack channel
    - match:
        severity: warning
        team: backend
      receiver: slack-backend
    
    # Business alerts during business hours only
    - match:
        severity: warning
        team: product
      receiver: slack-business
      mute_time_intervals:
        - offhours

# Time intervals for muting
mute_time_intervals:
  - name: offhours
    time_intervals:
      - times:
        - start_time: '18:00'
          end_time: '09:00'
        weekdays: ['monday', 'tuesday', 'wednesday', 'thursday', 'friday']
      - times:
        - start_time: '00:00'
          end_time: '23:59'
        weekdays: ['saturday', 'sunday']

# Inhibition rules (suppress less severe alerts when critical exists)
inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'instance']

# Receivers
receivers:
  # Default receiver (if no match)
  - name: 'default'
    email_configs:
      - to: 'ops@example.com'
        from: 'alertmanager@example.com'
        smarthost: 'smtp.example.com:587'
        auth_username: 'alertmanager'
        auth_password: 'password'
        require_tls: true

  # PagerDuty for critical alerts
  - name: 'pagerduty'
    pagerduty_configs:
      - service_key: 'YOUR_PD_SERVICE_KEY'
        severity: 'critical'
        description: '{{ .CommonAnnotations.summary }}'
        details:
          alertname: '{{ .CommonLabels.alertname }}'
          instance: '{{ .CommonLabels.instance }}'
          description: '{{ .CommonAnnotations.description }}'
        images:
          - src: 'https://example.com/grafana/dashboard.png'
            alt: 'Grafana Dashboard'

  # Slack receivers
  - name: 'slack-infra'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/XXX/YYY/ZZZ'
        channel: '#infrastructure-alerts'
        title: '{{ .CommonAnnotations.summary }}'
        text: |
          *Severity:* {{ .CommonLabels.severity }}
          *Description:* {{ .CommonAnnotations.description }}
          *Instance:* {{ .CommonLabels.instance }}
          *Time:* {{ .StartsAt }}
          *Runbook:* {{ .CommonAnnotations.runbook }}
        color: 'warning'
        fields:
          - title: 'Value'
            value: '{{ .CommonLabels.value }}'
            short: true

  - name: 'slack-backend'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/XXX/YYY/ZZZ'
        channel: '#backend-alerts'
        title: 'Backend Alert: {{ .CommonAnnotations.summary }}'
        text: '{{ .CommonAnnotations.description }}'

  - name: 'slack-business'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/XXX/YYY/ZZZ'
        channel: '#business-metrics'
        title: 'Business Alert'
        text: '{{ .CommonAnnotations.description }}'

  # Opsgenie
  - name: 'opsgenie'
    opsgenie_configs:
      - api_key: 'YOUR_OPSGENIE_API_KEY'
        message: '{{ .CommonAnnotations.summary }}'
        description: '{{ .CommonAnnotations.description }}'
        priority: 'P1'
        tags: ['infrastructure', '{{ .CommonLabels.severity }}']
        source: 'Prometheus'

  # Webhook (custom integration)
  - name: 'webhook'
    webhook_configs:
      - url: 'http://webhook-service:8080/alerts'
        send_resolved: true

  # Microsoft Teams
  - name: 'teams'
    webhook_configs:
      - url: 'https://outlook.office.com/webhook/XXX/YYY/ZZZ'
        send_resolved: true

# Create silence via Alertmanager UI
# Alertmanager UI: http://localhost:9093

# Create silence via API
curl -X POST http://localhost:9093/api/v2/silences \
  -H 'Content-Type: application/json' \
  -d '{
    "matchers": [
      {
        "name": "alertname",
        "value": "HighCPUUsage",
        "isRegex": false
      },
      {
        "name": "instance",
        "value": "web-01",
        "isRegex": false
      }
    ],
    "startsAt": "2025-02-25T10:00:00Z",
    "endsAt": "2025-02-25T12:00:00Z",
    "createdBy": "admin",
    "comment": "Scheduled maintenance"
  }'

# List silences
curl http://localhost:9093/api/v2/silences

# Delete silence
curl -X DELETE http://localhost:9093/api/v2/silence/{silence_id}

# opsgenie-schedule.yml (example schedule)
apiVersion: opsgenie.com/v1alpha1
kind: Schedule
metadata:
  name: primary-oncall
spec:
  description: Primary on-call rotation
  timezone: America/New_York
  enabled: true
  ownerTeam: platform
  rotations:
    - name: week1
      startDate: '2025-02-25T09:00:00Z'
      participants:
        - username: alice@example.com
          type: user
      length: 1
      type: weekly
    - name: week2
      startDate: '2025-03-04T09:00:00Z'
      participants:
        - username: bob@example.com
          type: user
      length: 1
      type: weekly

# Escalation policy
apiVersion: opsgenie.com/v1alpha1
kind: Escalation
metadata:
  name: critical-escalation
spec:
  rules:
    - condition: if not acknowledged within 5 minutes
      notify:
        - type: schedule
          name: primary-oncall
    - condition: if not acknowledged within 15 minutes
      notify:
        - type: user
          name: engineering-manager@example.com
    - condition: if not acknowledged within 30 minutes
      notify:
        - type: user
          name: vp-engineering@example.com

# Runbook example (markdown)
# runbooks/high-cpu.md
---
title: High CPU Usage Runbook
severity: warning
tags: [cpu, performance, scaling]
---

# High CPU Usage - Response Runbook

## Symptoms
- CPU usage > 80% for 10+ minutes
- Slow response times
- Potential timeout errors

## Immediate Actions

1. **Check which process is causing high CPU**
   ```bash
   top -c
   # or
   htop
   ```

2. **Check if this is expected behavior**
   - Deployments happening?
   - Traffic spikes?
   - Batch jobs running?

3. **If unexpected, kill the problematic process**
   ```bash
   kill -9 PID
   ```

4. **Scale up if needed**
   ```bash
   kubectl scale deployment myapp --replicas=5
   # or
   docker-compose up --scale app=5
   ```

5. **Restart the service**
   ```bash
   systemctl restart myapp
   # or
   kubectl rollout restart deployment/myapp
   ```

## Escalation
- If not resolved in 15 minutes: Page on-call backup
- If not resolved in 30 minutes: Page engineering manager

## Post-Incident
- Document root cause
- Update runbook if steps were missing
- Create follow-up ticket for permanent fix

## Dashboard
- [Grafana Dashboard](https://grafana.example.com/d/cpu-dashboard)
- [Logs Search](https://kibana.example.com/app/discover)

## Related Alerts
- SlowResponses
- ServiceDown
- HighMemoryUsage

# Post-Mortem: [Incident Title]

**Date:** YYYY-MM-DD
**Duration:** HH:MM - HH:MM (Total: X hours)
**Severity:** [SEV1/SEV2/SEV3]
**Impact:** [Describe user/customer impact]
**Team:** [Team name]

## Timeline
| Time | Event |
|------|-------|
| 14:00 | Initial alert triggered |
| 14:02 | On-call engineer acknowledged |
| 14:10 | Root cause identified |
| 14:30 | Mitigation applied |
| 14:45 | Service recovered |
| 15:00 | Post-mortem initiated |

## Root Cause
[Detailed explanation of what caused the incident]

## Impact Assessment
- **User Impact:** [Number of users affected, duration]
- **Data Loss:** [Any data loss?]
- **Financial Impact:** [Estimated cost if applicable]

## Detection
- **How was it detected?** [Monitoring alert / Customer report / Manual]
- **Why wasn't it detected earlier?** [Missing monitoring / Threshold too high]

## Response
- **What went well?** [List successes]
- **What went poorly?** [List failures]
- **Where did we get stuck?** [Bottlenecks]

## Action Items
| Action | Owner | Due Date | Priority |
|--------|-------|----------|----------|
| Add CPU throttling detection | @engineer | YYYY-MM-DD | High |
| Update runbook with new steps | @engineer | YYYY-MM-DD | Medium |
| Implement auto-scaling | @engineer | YYYY-MM-DD | High |
| Add dashboard panel for error rate | @engineer | YYYY-MM-DD | Low |

## Lessons Learned
1. [Lesson 1]
2. [Lesson 2]
3. [Lesson 3]

## Follow-up
- [ ] Action items completed
- [ ] Runbook updated
- [ ] Monitoring improved
- [ ] Team notified