# ChatApp Alerting Rules
# Prometheus alertmanager configuration
# Each alert includes a runbook_url for operational guidance.

groups:
  - name: chatapp-critical
    interval: 30s
    rules:
      # API down
      - alert: APIDown
        expr: up{job="chatapp-api"} == 0
        for: 1m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "API is down"
          description: "API has been down for more than 1 minute on {{ $labels.instance }}"
          runbook_url: "https://wiki.chatapp.com/runbooks/api-down"

      # Realtime down
      - alert: RealtimeDown
        expr: up{job="chatapp-realtime"} == 0
        for: 1m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "Realtime (WebSocket) is down"
          description: "Realtime service has been down for more than 1 minute"
          runbook_url: "https://wiki.chatapp.com/runbooks/realtime-down"

      # High error rate (5xx)
      - alert: HighErrorRate
        expr: rate(http_requests_total{job="chatapp-api",status=~"5.."}[5m]) / rate(http_requests_total{job="chatapp-api"}[5m]) > 0.05
        for: 5m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "Error rate above 5%"
          description: "API 5xx error rate is {{ $value | humanizePercentage }} on {{ $labels.instance }}"
          runbook_url: "https://wiki.chatapp.com/runbooks/high-error-rate"

      # Database connections exhausted
      - alert: DBConnectionsExhausted
        expr: pg_stat_database_numbackends / pg_settings_max_connections > 0.8
        for: 5m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "Database connections above 80%"
          description: "PostgreSQL connection pool at {{ $value | humanizePercentage }} of max"
          runbook_url: "https://wiki.chatapp.com/runbooks/db-connections-exhausted"

      # High response time (P99)
      - alert: HighResponseTime
        expr: histogram_quantile(0.99, rate(http_request_duration_seconds_bucket{job="chatapp-api"}[5m])) > 2
        for: 10m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "P99 response time above 2 seconds"
          description: "P99 latency is {{ $value }}s on {{ $labels.instance }}"
          runbook_url: "https://wiki.chatapp.com/runbooks/high-response-time"

      # Disk space low
      - alert: DiskSpaceLow
        expr: 100 - (node_filesystem_avail_bytes / node_filesystem_size_bytes * 100) > 85
        for: 10m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Disk usage above 85%"
          description: "Disk {{ $labels.mountpoint }} on {{ $labels.instance }} is {{ $value }}% full"
          runbook_url: "https://wiki.chatapp.com/runbooks/disk-space-low"

      # Job queue backed up
      - alert: JobQueueBackedUp
        expr: bullmq_jobs_waiting > 1000
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Job queue has more than 1000 waiting jobs"
          description: "Queue {{ $labels.queue }} has {{ $value }} waiting jobs"
          runbook_url: "https://wiki.chatapp.com/runbooks/job-queue-backed-up"

      # Memory usage high
      - alert: HighMemoryUsage
        expr: container_memory_working_set_bytes / container_spec_memory_limit_bytes > 0.85
        for: 10m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Memory usage above 85%"
          description: "Container {{ $labels.name }} memory at {{ $value | humanizePercentage }}"
          runbook_url: "https://wiki.chatapp.com/runbooks/high-memory-usage"

      # Redis memory high
      - alert: RedisMemoryHigh
        expr: redis_memory_used_bytes / redis_memory_max_bytes > 0.9
        for: 5m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Redis memory usage above 90%"
          description: "Redis is at {{ $value | humanizePercentage }} of max memory"
          runbook_url: "https://wiki.chatapp.com/runbooks/redis-memory-high"

      # Prometheus target missing
      - alert: PrometheusTargetMissing
        expr: up == 0
        for: 2m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Prometheus target missing: {{ $labels.job }}"
          description: "Target {{ $labels.instance }} (job: {{ $labels.job }}) has been down for more than 2 minutes"
          runbook_url: "https://wiki.chatapp.com/runbooks/prometheus-target-missing"

      # Node exporter down
      - alert: NodeExporterDown
        expr: up{job="node"} == 0
        for: 5m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "Node exporter is down"
          description: "Node exporter on {{ $labels.instance }} has been down for more than 5 minutes"
          runbook_url: "https://wiki.chatapp.com/runbooks/node-exporter-down"