┌─────────────────┐     ┌────────────────┐     ┌─────────────────┐
│   Hystersis API  │────▶│   Prometheus    │────▶│    Grafana      │
│   /metrics       │     │   (Metrics)     │     │   (Dashboards)  │
└─────────────────┘     └───────┬────────┘     └─────────────────┘
                                │
                        ┌───────▼────────┐
                        │  Alertmanager  │
                        │  (Alerts)      │
                        └───────┬────────┘
                                │
                    ┌───────────┼───────────┐
                    ▼           ▼           ▼
               ┌──────┐  ┌──────┐  ┌──────────┐
               │Slack │  │Email │  │PagerDuty │
               └──────┘  └──────┘  └──────────┘

# monitoring/docker-compose.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:v2.48.0
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
      - ./prometheus/alerts:/etc/prometheus/alerts
      - prometheus-data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.retention.time=30d'
      - '--storage.tsdb.retention.size=10GB'

  grafana:
    image: grafana/grafana:10.2.0
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
      - GF_USERS_ALLOW_SIGN_UP=false
    volumes:
      - grafana-data:/var/lib/grafana
      - ./grafana/dashboards:/etc/grafana/provisioning/dashboards
      - ./grafana/datasources:/etc/grafana/provisioning/datasources

  alertmanager:
    image: prom/alertmanager:v0.26.0
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager/alertmanager.yml:/etc/alertmanager/alertmanager.yml

volumes:
  prometheus-data:
  grafana-data:

# monitoring/prometheus/prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

rule_files:
  - /etc/prometheus/alerts/*.yml

alerting:
  alertmanagers:
    - static_configs:
        - targets: ['alertmanager:9093']

scrape_configs:
  - job_name: 'hystersis-api'
    metrics_path: '/metrics'
    static_configs:
      - targets: ['api:8080']
        labels:
          environment: 'production'

  - job_name: 'neo4j'
    static_configs:
      - targets: ['neo4j:2000']

  - job_name: 'redis'
    static_configs:
      - targets: ['redis:6379']

  - job_name: 'qdrant'
    static_configs:
      - targets: ['qdrant:6333']

# monitoring/prometheus/alerts/hystersis.yml
groups:
  - name: hystersis-api
    rules:
      - alert: APIHighErrorRate
        expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High error rate on API"
          description: "Error rate is {{ $value }} requests/sec"

      - alert: APIHighLatency
        expr: histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High API latency"
          description: "P95 latency is {{ $value }}s"

      - alert: CompressionAccuracyDrop
        expr: hystersis_compression_accuracy_retention < 0.95
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Compression accuracy dropped"
          description: "Accuracy is {{ $value }}"

      - alert: MemoryStoreHighUsage
        expr: container_memory_usage_bytes{container="api"} / container_spec_memory_limit_bytes{container="api"} > 0.85
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High memory usage"
          description: "Memory usage is {{ $value | humanizePercentage }}"

# monitoring/alertmanager/alertmanager.yml
route:
  group_by: ['alertname', 'severity']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h
  receiver: 'slack-default'
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty'
    - match:
        severity: warning
      receiver: 'slack-default'

receivers:
  - name: 'slack-default'
    slack_configs:
      - api_url: 'https://hooks.slack.com/services/YOUR/WEBHOOK/URL'
        channel: '#hystersis-alerts'
        title: '{{ .GroupLabels.alertname }}'
        text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'

  - name: 'pagerduty'
    pagerduty_configs:
      - service_key: 'YOUR_PAGERDUTY_KEY'
        severity: '{{ .GroupLabels.severity }}'

# monitoring/grafana/datasources/prometheus.yml
apiVersion: 1
datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true

# monitoring/grafana/dashboards/dashboards.yml
apiVersion: 1
providers:
  - name: 'Hystersis'
    orgId: 1
    folder: 'Hystersis'
    type: file
    disableDeletion: false
    editable: true
    options:
      path: /etc/grafana/provisioning/dashboards
      foldersFromFilesStructure: true

# Start monitoring stack
cd monitoring
docker-compose up -d

# Access Grafana
open http://localhost:3000
# Default: admin/admin

# Check Prometheus targets
open http://localhost:9090/targets

# Verify API metrics
curl http://localhost:8080/metrics | grep hystersis