# docker-compose.monitoring.yml
version: '3.8'
services:
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus:/etc/prometheus
      - prom_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.retention.time=30d'

  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    pid: host
    restart: unless-stopped
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
    depends_on:
      - prometheus

  loki:
    image: grafana/loki:latest
    ports:
      - "3100:3100"
    volumes:
      - ./loki:/etc/loki

  promtail:
    image: grafana/promtail:latest
    volumes:
      - /var/log:/var/log:ro
      - ./promtail-config.yml:/etc/promtail/config.yml

volumes:
  prom_data:
  grafana_data:

# ai_anomaly_detector.py - Simplified dynamic baseline example
import numpy as np
from sklearn.ensemble import IsolationForest

class AIVPSMonitor:
    def __init__(self, window_hours=168):  # 7-day window
        self.window_hours = window_hours
        self.model = IsolationForest(
            contamination=0.05,
            random_state=42
        )
        self.baseline = None

    def fit_baseline(self, metrics_history):
        """Train anomaly detection model with historical data"""
        self.model.fit(metrics_history)
        self.baseline = {
            'cpu_mean': np.mean(metrics_history[:, 0]),
            'cpu_std': np.std(metrics_history[:, 0]),
            'mem_mean': np.mean(metrics_history[:, 1]),
            'mem_std': np.std(metrics_history[:, 1]),
        }

    def detect_anomaly(self, current_metrics):
        """Detect if current metrics are anomalous"""
        score = self.model.score_samples([current_metrics])[0]
        is_anomaly = score < -0.5

        # Also check deviation magnitude
        cpu_deviation = abs(current_metrics[0] - self.baseline['cpu_mean']) / max(self.baseline['cpu_std'], 1)
        mem_deviation = abs(current_metrics[1] - self.baseline['mem_mean']) / max(self.baseline['mem_std'], 1)

        return {
            'is_anomaly': is_anomaly or cpu_deviation > 3 or mem_deviation > 3,
            'anomaly_score': float(score),
            'details': {
                'cpu_deviation_sigma': float(cpu_deviation),
                'mem_deviation_sigma': float(mem_deviation),
            }
        }

# alertmanager.yml
route:
  receiver: 'ai-agent'
  group_by: ['alertname', 'instance']
  routes:
    - match:
        severity: 'critical'
      receiver: 'ai-agent-immediate'
      continue: true
    - match:
        severity: 'warning'
      receiver: 'ai-agent-batch'
      group_wait: 5m

receivers:
  - name: 'ai-agent-immediate'
    webhook_configs:
      - url: 'http://ai-agent:8080/api/v1/alert/immediate'
        send_resolved: true

  - name: 'ai-agent-batch'
    webhook_configs:
      - url: 'http://ai-agent:8080/api/v1/alert/batch'
        send_resolved: true

# root_cause_collector.py
class RootCauseCollector:
    def __init__(self, prometheus_url, loki_url):
        self.prom = PrometheusClient(prometheus_url)
        self.loki = LokiClient(loki_url)

    def gather_context(self, alert, instance):
        """Collect all context needed for root cause analysis"""
        context = {
            'alert_info': alert,
            'timeline': self._get_metric_timeline(instance, alert),
            'recent_changes': self._get_recent_config_changes(instance),
            'logs': self._query_logs(instance, alert),
            'dependencies': self._check_service_dependencies(instance),
            'similar_incidents': self._find_similar_historical_incidents(alert),
        }
        return context

    def _get_metric_timeline(self, instance, alert):
        """Get metric timeline for the past 2 hours"""
        metrics = {
            'cpu': f'avg_over_time(node_cpu_seconds_total{{mode!="idle"}}[5m])',
            'memory': f'avg_over_time(node_memory_MemAvailable_bytes[5m])',
            'disk_io': f'rate(node_disk_io_time_seconds_total[5m])',
            'network': f'rate(node_network_receive_bytes_total[5m])',
        }
        return {k: self.prom.query(v, instance) for k, v in metrics.items()}

    def _query_logs(self, instance, alert, hours=2):
        """Query relevant logs"""
        query = f'{{instance="{instance}"}} |= "error" or |= "warn" or |= "panic"'
        return self.loki.query(query, start=f'-{hours}h')

    def _get_recent_config_changes(self, instance):
        """Check recent configuration changes"""
        return {
            'systemd_changes': self._check_systemd_changes(instance),
            'cron_changes': self._check_cron_changes(instance),
            'deploy_history': self._check_deploy_history(instance),
        }

# llm_root_cause_analyzer.py
import json
from litellm import completion

def analyze_root_cause(context, model="gpt-4o"):
    """Use LLM to analyze root cause"""

    system_prompt = """You are an AI ops expert for SelfVPS. Your task is to analyze
the root cause of server faults and provide specific remediation suggestions.
Please respond in the following format:

1. **Fault Summary**: One-sentence description of what happened
2. **Root Cause Analysis**: Detailed explanation of why this occurred
3. **Confidence**: High/Medium/Low
4. **Remediation Steps**: Specific, executable fix commands
5. **Prevention**: How to avoid this issue recurring

Only output facts and analysis, do not fabricate data."""

    user_prompt = f"""## Alert Information
{json.dumps(context['alert_info'], indent=2)}

## Instance: {context['timeline'].get('instance')}

## Metric Timeline
{json.dumps(context['timeline'], indent=2, default=str)[:3000]}

## Related Logs (Past 2 Hours)
{json.dumps(context['logs'], indent=2)[:2000]}

## Recent Config Changes
{json.dumps(context['recent_changes'], indent=2)[:1000]}

## Historical Similar Incidents
{json.dumps(context['similar_incidents'], indent=2)[:1000]}

Please analyze the root cause and provide remediation advice."""

    response = completion(
        model=model,
        messages=[
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt},
        ],
        temperature=0.3,
        max_tokens=2000,
    )

    return response.choices[0].message.content

# playbooks/memory-leak.yaml
name: "Automatic Memory Leak Remediation"
trigger:
  condition: "memory_usage > 85% AND memory_trend = increasing AND duration > 30m"
  severity: "warning"

steps:
  - name: "Collect Diagnostic Info"
    action: "diagnose"
    params:
      tools: ["top", "free -m", "journalctl --since '30 minutes ago'"]

  - name: "Clear System Cache"
    action: "shell"
    params:
      command: "sync && echo 3 > /proc/sys/vm/drop_caches"
      timeout: 30

  - name: "Restart Affected Service"
    action: "systemctl"
    params:
      service: "{{ target_service }}"
      operation: "restart"

  - name: "Verify Fix Effect"
    action: "verify"
    params:
      metric: "node_memory_MemAvailable_bytes"
      expected_improvement: "10%"
      check_interval: "1m"
      check_duration: "5m"

  - name: "Escalate Alert"
    action: "escalate"
    if_condition: "memory_usage still > 80% after 5m"
    to: "oncall-team"

# safety_guardrails.py
class SelfHealingGuardrails:
    """Safety guardrails for self-healing system"""

    def __init__(self):
        self.blocklist_actions = {
            'rm -rf /',           # Destructive command
            'dd if=/dev/zero',     # Overwrite disk
            'iptables -F',         # Flush firewall rules
            'systemctl stop docker', # Stop container runtime
            'userdel -r root',     # Delete root user
        }
        self.max_concurrent_fixes = 2  # Max 2 concurrent fixes
        self.cooldown_period = 300  # 5-minute cooldown per instance
        self.rollback_enabled = True

    def validate_action(self, action):
        """Validate if remediation action is safe"""
        cmd = action.get('command', '')
        for blocked in self.blocklist_actions:
            if blocked in cmd:
                raise SecurityViolation(f"Blocked dangerous command: {cmd}")

        # Check idempotency of write operations
        if action.get('type') == 'write':
            if not action.get('idempotent', False):
                raise SecurityViolation("Non-idempotent write requires approval")

    def check_cooldown(self, instance):
        """Check if instance is in cooldown period"""
        last_fix = self._get_last_fix_time(instance)
        if last_fix and (time.time() - last_fix) < self.cooldown_period:
            return False  # Still in cooldown
        return True

    def require_approval(self, action):
        """High-risk actions require human approval"""
        risk_score = self._calculate_risk(action)
        if risk_score > 0.7:
            return True  # Requires approval
        return False

# verification_engine.py
class VerificationEngine:
    """Post-fix effect verification"""

    def verify_fix(self, instance, action, expected_outcome):
        """Verify if fix was successful"""
        checks = [
            self._check_metric_recovery(instance, expected_outcome),
            self._check_service_health(instance),
            self._check_log_errors_stopped(instance),
            self._check_user_impact(instance),
        ]

        results = {check['name']: check['passed'] for check in checks}
        all_passed = all(r['passed'] for r in checks)

        if all_passed:
            self._record_success(instance, action)
            return {'status': 'success', 'details': results}
        else:
            if self.guardrails.rollback_enabled:
                self._execute_rollback(instance, action)
            return {'status': 'failed', 'details': results, 'rollback_triggered': True}

# reinforcement_optimizer.py
class SelfHealingOptimizer:
    """Self-healing strategy optimization based on historical data"""

    def optimize_playbook(self, incident_history):
        """Analyze historical events to optimize remediation strategies"""
        # Find most effective remediation actions
        success_rates = {}
        for action_type in incident_history:
            total = len(incident_history[action_type])
            successes = sum(1 for i in incident_history[action_type] if i['success'])
            success_rates[action_type] = successes / total if total > 0 else 0

        # Update Playbook priority order
        optimized_order = sorted(success_rates.items(), key=lambda x: x[1], reverse=True)
        return optimized_order

    def update_baseline(self, new_data):
        """Dynamically update anomaly detection baseline"""
        # Merge new normal-pattern data
        # Retrain Isolation Forest model
        pass

# docker-compose.selfheal.yml
version: '3.8'
services:
  # Data Collection
  prometheus:
    image: prom/prometheus:latest
    ports: ["9090:9090"]
    volumes:
      - ./prometheus:/etc/prometheus
      - prom_data:/prometheus

  grafana:
    image: grafana/grafana:latest
    ports: ["3000:3000"]
    volumes:
      - grafana_data:/var/lib/grafana

  # AI Decision Layer
  ai-agent:
    build: ./ai-agent
    ports: ["8080:8080"]
    environment:
      - OPENAI_API_KEY=***
      - PROMETHEUS_URL=http://prometheus:9090
      - GUARDRAIL_MODE=strict  # strict | relaxed | disabled

  # Execution Layer
  fix-runner:
    build: ./fix-runner
    privileged: true  # Needs root privileges for remediation
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ./playbooks:/etc/playbooks