# alert/normalized_alert.yaml
alert:
  id: "alert-20260810-001"
  source: "prometheus"
  severity: "warning"  # info | warning | critical | emergency
  timestamp: "2026-08-10T14:32:00+08:00"
  labels:
    instance: "vps-web-01"
    service: "nginx"
    team: "infra"
  annotations:
    summary: "CPU usage exceeded 90% for 5 minutes"
    description: "High CPU load detected on vps-web-01, likely caused by traffic spike"
    runbook_url: "/runbooks/high-cpu.md"
  value: 94.2
  threshold: 90

# tickets/ai_aggregator.py
from litellm import completion

def should_merge_alerts(existing_ticket: dict, new_alert: dict) -> bool:
    """Use LLM to determine if a new alert correlates with an existing ticket"""
    
    prompt = f"""
    You are an operations expert. Determine whether the following two alerts 
    are likely caused by the same root cause:

    Existing Ticket:
    - Alert: {existing_ticket['alert']['summary']}
    - Server: {existing_ticket['alert']['labels']['instance']}
    - Status: {existing_ticket['status']}

    New Alert:
    - Alert: {new_alert['annotations']['summary']}
    - Server: {new_alert['labels']['instance']}
    - Time Delta: {calculate_time_delta(existing_ticket, new_alert)}

    Return true if they are likely the same root cause, false otherwise.
    Return only true or false.
    """
    
    response = completion(
        model="qwen2.5:7b",
        messages=[{"role": "user", "content": prompt}],
        temperature=0.1
    )
    
    return "true" in response.choices[0].message.content.lower()

# diagnosis/context_collector.py
import subprocess
import requests
from datetime import datetime, timedelta

class ContextCollector:
    """Collect all contextual data related to a failure"""
    
    def collect(self, ticket: dict) -> dict:
        instance = ticket['alert']['labels']['instance']
        
        return {
            # System resource status
            "system_metrics": self._get_metrics(instance),
            
            # Error logs from the last 1 hour
            "recent_errors": self._get_error_logs(instance, hours=1),
            
            # Related service status
            "service_status": self._get_service_status(instance),
            
            # Recent change events
            "recent_changes": self._get_recent_changes(instance),
            
            # Network connectivity
            "network_status": self._get_network_status(instance),
            
            # Historical similar cases
            "similar_cases": self._search_similar_cases(ticket)
        }
    
    def _get_metrics(self, instance: str) -> dict:
        """Get key metrics from Prometheus"""
        resp = requests.get(
            f"http://prometheus:9090/api/v1/query",
            params={
                "query": f'up{{instance="{instance}"}}',
                "time": datetime.now().isoformat()
            }
        )
        return resp.json()
    
    def _get_error_logs(self, instance: str, hours: int = 1) -> list:
        """Get error logs from Loki"""
        start = (datetime.now() - timedelta(hours=hours)).isoformat()
        resp = requests.get(
            f"http://loki:3100/loki/api/v1/query",
            params={
                "query": f'{{instance="{instance}"}} |= "error"',
                "start": start,
                "limit": 100
            }
        )
        return resp.json().get('data', {}).get('result', [])
    
    def _get_recent_changes(self, instance: str) -> list:
        """Get recent system changes"""
        changes = []
        
        # Docker container changes
        result = subprocess.run(
            ['docker', 'events', '--since', f'{hours}h', '--filter', f'label=instance={instance}'],
            capture_output=True, text=True
        )
        changes.append({"source": "docker", "events": result.stdout.split('\n')})
        
        # Package changes
        result = subprocess.run(
            ['grep', '-E', '(install|remove|upgrade|purge)', '/var/log/dpkg.log'],
            capture_output=True, text=True
        )
        changes.append({"source": "apt", "events": result.stdout.split('\n')})
        
        return changes

# diagnosis/root_cause_analyzer.py
from litellm import completion

class RootCauseAnalyzer:
    """Analyze fault root causes using LLM"""
    
    SYSTEM_PROMPT = """
    You are an experienced SRE engineer skilled in fault diagnosis.
    Your task is to analyze the root cause of VPS failures based on 
    the provided contextual information.

    Analysis framework:
    1. First identify the most direct symptoms
    2. Then trace possible causes leading to the symptoms
    3. Finally determine the most likely root cause through contextual evidence
    4. Provide confidence level and remediation recommendations

    Output in JSON format with these fields:
    - root_cause: Root cause description
    - confidence: Confidence level (0-1)
    - evidence: Supporting evidence list
    - alternative_causes: Other possible causes and why they were ruled out
    - recommended_actions: List of recommended remediation actions
    """
    
    def analyze(self, context: dict, ticket: dict) -> dict:
        user_prompt = f"""
        ## Incident Ticket
        Alert: {ticket['alert']['annotations']['summary']}
        Server: {ticket['alert']['labels']['instance']}
        Severity: {ticket['alert']['severity']}
        
        ## Collected Context
        {self._format_context(context)}
        """
        
        response = completion(
            model="qwen2.5:7b",
            messages=[
                {"role": "system", "content": self.SYSTEM_PROMPT},
                {"role": "user", "content": user_prompt}
            ],
            temperature=0.1,
            response_format={"type": "json_object"}
        )
        
        import json
        return json.loads(response.choices[0].message.content)
    
    def _format_context(self, context: dict) -> str:
        """Format context into readable text"""
        lines = []
        
        lines.append("### System Metrics")
        for metric, value in context.get('system_metrics', {}).items():
            lines.append(f"- {metric}: {value}")
        
        lines.append("\n### Error Logs (Last 1 Hour)")
        for log in context.get('recent_errors', [])[:10]:
            lines.append(f"- {log}")
        
        lines.append("\n### Service Status")
        for service, status in context.get('service_status', {}).items():
            lines.append(f"- {service}: {status}")
        
        lines.append("\n### Recent Changes")
        for change in context.get('recent_changes', []):
            lines.append(f"- [{change['source']}] {change['description']}")
        
        lines.append("\n### Historical Similar Cases")
        for case in context.get('similar_cases', [])[:3]:
            lines.append(f"- {case['title']}: {case['solution']}")
        
        return '\n'.join(lines)

# knowledge/incident_patterns.yaml
patterns:
  - id: "disk-full-logs"
    keywords: ["disk full", "no space left", "log rotation"]
    indicators:
      - node_filesystem_avail_bytes < 1GB
      - rate(node_filesystem_write_bytes_total[1h]) > 5MB/s
    common_causes:
      - name: "Log rotation not configured"
        probability: 0.7
        solution: "Configure logrotate, clean old logs"
      - name: "Application log explosion"
        probability: 0.2
        solution: "Check application logging config, limit log level"
      - name: "Temporary file accumulation"
        probability: 0.1
        solution: "Clean /tmp and cache directories"
    similar_incidents: ["inc-20260715", "inc-20260620"]
    
  - id: "memory-leak"
    keywords: ["OOM", "out of memory", "killed process"]
    indicators:
      - node_memory_MemAvailable_bytes < 500MB
      - rate(process_resident_memory_bytes[1h]) > 0
    common_causes:
      - name: "Java application memory leak"
        probability: 0.5
        solution: "Limit JVM heap size, check for memory leaks"
      - name: "Go application goroutine leak"
        probability: 0.3
        solution: "Analyze goroutine count, check connection pools"
      - name: "Insufficient system memory"
        probability: 0.2
        solution: "Add memory or optimize other processes"
    similar_incidents: ["inc-20260801", "inc-20260710"]

# remediation/executor.py
import subprocess
import logging
from dataclasses import dataclass
from enum import Enum

class RiskLevel(Enum):
    LOW = "low"
    MEDIUM = "medium"
    HIGH = "high"
    CRITICAL = "critical"

@dataclass
class RemediationAction:
    action_id: str
    description: str
    risk_level: RiskLevel
    command: str
    rollback_command: str
    timeout: int = 300

class RemediationExecutor:
    """Safely execute remediation operations"""
    
    def __init__(self, knowledge_base: dict, approval_required: bool = False):
        self.kb = knowledge_base
        self.approval_required = approval_required
        self.logger = logging.getLogger(__name__)
    
    def execute(self, action: RemediationAction, ticket: dict) -> dict:
        """Execute a remediation action"""
        
        # High-risk operations require human approval
        if action.risk_level in [RiskLevel.HIGH, RiskLevel.CRITICAL]:
            if self.approval_required and not self._get_approval(action, ticket):
                return {"status": "pending_approval", "action": action.action_id}
        
        # Pre-execution safety check
        pre_check = self._pre_check(action)
        if not pre_check['passed']:
            return {"status": "pre_check_failed", "reason": pre_check['reason']}
        
        # Log execution start
        self.logger.info(f"Executing {action.action_id}: {action.description}")
        
        try:
            # Execute remediation command
            result = subprocess.run(
                action.command,
                shell=True,
                capture_output=True,
                text=True,
                timeout=action.timeout
            )
            
            if result.returncode == 0:
                # Post-execution verification
                post_check = self._post_check(action, ticket)
                if post_check['passed']:
                    return {
                        "status": "success",
                        "action": action.action_id,
                        "output": result.stdout,
                        "verification": post_check
                    }
                else:
                    # Verification failed, execute rollback
                    self._rollback(action)
                    return {
                        "status": "verified_failed_rolled_back",
                        "action": action.action_id,
                        "rollback_output": result.stdout
                    }
            else:
                return {
                    "status": "execution_failed",
                    "action": action.action_id,
                    "error": result.stderr
                }
                
        except subprocess.TimeoutExpired:
            self._rollback(action)
            return {
                "status": "timeout_rolled_back",
                "action": action.action_id
            }
    
    def _pre_check(self, action: RemediationAction) -> dict:
        """Pre-execution safety check"""
        checks = []
        
        # Disk space check
        result = subprocess.run(
            ['df', '-h', '/'], capture_output=True, text=True
        )
        usage = result.stdout.split('\n')[1].split()[4].replace('%', '')
        if int(usage) > 95:
            return {"passed": False, "reason": f"Disk usage too high: {usage}%"}
        
        return {"passed": True, "checks": checks}
    
    def _post_check(self, action: RemediationAction, ticket: dict) -> dict:
        """Post-execution verification"""
        return {"passed": True, "details": "Service resumed normal operation"}
    
    def _rollback(self, action: RemediationAction):
        """Execute rollback"""
        if action.rollback_command:
            subprocess.run(action.rollback_command, shell=True)
        self.logger.warning(f"Rollback executed for {action.action_id}")
    
    def _get_approval(self, action: RemediationAction, ticket: dict) -> bool:
        """Get human approval (simplified — integrate Slack/DingTalk in production)"""
        return True

# remediation/playbooks/
playbooks:
  disk-full-cleanup:
    name: "Disk Space Cleanup"
    risk: "low"
    triggers:
      - "node_filesystem_avail_bytes < 1GB"
    actions:
      - description: "Clean systemd journals"
        command: "journalctl --vacuum-time=3d"
        rollback: "none"
      - description: "Clean apt cache"
        command: "apt-get clean && apt-get autoclean"
        rollback: "none"
      - description: "Clean old kernels"
        command: "apt-get autoremove --purge"
        rollback: "none"
      - description: "Clean log files"
        command: "find /var/log -name '*.gz' -delete && find /var/log -name '*.old' -delete"
        rollback: "none"
    verification:
      - "node_filesystem_avail_bytes > 2GB"
    
  service-restart:
    name: "Service Restart"
    risk: "low"
    triggers:
      - "service_state != running"
    actions:
      - description: "Restart specified service"
        command: "systemctl restart {{service_name}}"
        rollback: "systemctl start {{service_name}}"
    verification:
      - "systemctl is-active {{service_name}} == active"
    
  memory-pressure-relief:
    name: "Memory Pressure Relief"
    risk: "medium"
    triggers:
      - "node_memory_MemAvailable_bytes < 500MB"
    actions:
      - description: "Drop page cache"
        command: "echo 3 > /proc/sys/vm/drop_caches"
        rollback: "none"
      - description: "Restart memory-leaking service"
        command: "systemctl restart {{problem_service}}"
        rollback: "systemctl start {{problem_service}}"
    verification:
      - "node_memory_MemAvailable_bytes > 1GB"

# reports/incident_report.py
from datetime import datetime

class IncidentReportGenerator:
    """Generate incident reports"""
    
    def generate(self, ticket: dict, diagnosis: dict, remediation: dict) -> str:
        """Generate a complete incident report"""
        
        report = f"""
# Incident Report

## Basic Information
- **Incident ID**: {ticket['id']}
- **Occurrence Time**: {ticket['alert']['timestamp']}
- **Duration**: {self._calc_duration(ticket)}
- **Impact Scope**: {ticket['alert']['labels']['instance']}
- **Severity**: {ticket['alert']['severity']}

## Symptom
{ticket['alert']['annotations']['summary']}

## Root Cause Analysis
**Root Cause**: {diagnosis['root_cause']}
**Confidence**: {diagnosis['confidence']:.0%}
**Supporting Evidence**:
{self._format_evidence(diagnosis['evidence'])}

## Resolution Process
1. **Alert Received**: {ticket['alert']['timestamp']}
2. **AI Diagnosis Completed**: {datetime.now().isoformat()}
3. **Remediation Executed**: {remediation.get('action', 'N/A')}
4. **Remediation Result**: {remediation.get('status', 'N/A')}

## Remediation Actions
{self._format_remediation(remediation)}

## Lessons Learned
- **Problem Category**: {diagnosis.get('category', 'unknown')}
- **Auto-Remediable**: {"Yes" if remediation.get('status') == 'success' else "No"}
- **Improvement Suggestions**: {self._generate_suggestions(ticket, diagnosis)}

## Appendix
### Original Alert
{ticket['alert']['annotations']['description']}

### Related Logs
{self._format_logs(ticket.get('logs', []))}
"""
        return report

# metrics/mttr_tracking.yaml
metrics:
  mttr_by_severity:
    critical:
      target: 15m
      current_avg: 12m
      trend: "down"  # Improving
    warning:
      target: 30m
      current_avg: 22m
      trend: "stable"
    info:
      target: 60m
      current_avg: 45m
      trend: "down"
  
  auto_remediation_rate: 0.73  # 73% of incidents auto-remediated
  escalation_rate: 0.15        # 15% require human intervention
  false_positive_rate: 0.08    # 8% false positive rate
  
  top_resolution_patterns:
    - pattern: "disk-full-cleanup"
      count: 45
      avg_time: "3m"
    - pattern: "service-restart"
      count: 32
      avg_time: "1m"
    - pattern: "memory-pressure-relief"
      count: 18
      avg_time: "5m"

# docker-compose.yml
version: "3.8"

services:
  ticket-service:
    build: ./ticket-service
    ports:
      - "8080:8080"
    environment:
      - DB_URL=postgres://user:pass@db:5432/tickets
      - LLM_ENDPOINT=http://llm-server:8000/v1
      - PROMETHEUS_URL=http://prometheus:9090
      - LOKI_URL=http://loki:3100
    depends_on:
      - db
      - llm-server
  
  llm-server:
    image: ghcr.io/someone/qwen2.5-7b-instruct:latest
    ports:
      - "8000:8000"
    volumes:
      - ./models:/models
    environment:
      - MODEL_PATH=/models/qwen2.5-7b-instruct
  
  db:
    image: postgres:16
    environment:
      - POSTGRES_USER=user
      - POSTGRES_PASSWORD=pass
      - POSTGRES_DB=tickets
    volumes:
      - pgdata:/var/lib/postgresql/data
  
  alertmanager:
    image: prom/alertmanager:latest
    volumes:
      - ./alertmanager:/config
    command: --config.file=/config/alertmanager.yml

volumes:
  pgdata:

# alertmanager.yml
route:
  receiver: 'ticket-webhook'
  group_by: ['alertname', 'instance']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 4h

receivers:
  - name: 'ticket-webhook'
    webhook_configs:
      - url: 'http://ticket-service:8080/api/v1/alerts'
        send_resolved: true

  - name: 'dingtalk-critical'
    dingtalk_configs:
      - webhook: 'https://oapi.dingtalk.com/robot/send?token=xxx'
        message: '{{ template "dingtalk.message" . }}'

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['instance', 'alertname']