# prometheus/alerts/vps-alerts.yml
groups:
  - name: vps-critical
    rules:
      - alert: HighCPUUsage
        expr: cpu_usage_percent > 90
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "VPS {{ $labels.instance }} CPU 使用率超过 90%"
          description: "当前值: {{ $value }}%, 持续 5 分钟"

      - alert: DiskSpaceLow
        expr: disk_free_percent < 10
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "VPS {{ $labels.instance }} 磁盘空间不足"

      - alert: MemoryLeakDetected
        expr: increase(memory_usage_bytes[1h]) > 1073741824
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "VPS {{ $labels.instance }} 内存增长异常，疑似泄漏"

# loki/alerts/log-alerts.yml
alerts:
  - name: ErrorRateSpike
    query: sum(rate({app="web"} |= "error" [5m])) by (instance) > 10
    for: 5m
    severity: critical

  - name: AuthFailureBurst
    query: sum(rate({app="ssh"} |=~ "Failed password" [5m])) > 5
    for: 2m
    severity: warning

// traces/alerts.go
func CheckTraceAnomaly(span *model.Span) bool {
    duration := span.Duration.Milliseconds()
    if duration > 5000 { // 5秒阈值
        return true
    }
    if span.Status.Code != model SpanStatus_OK {
        return true
    }
    return false
}

# incident_context_collector.py
class IncidentContextCollector:
    def collect(self, alert: Alert) -> Context:
        return Context(
            # 时间窗口
            time_range=alert.timestamp - timedelta(hours=1),
            
            # 指标数据
            metrics=self.fetch_metrics(alert.instance, alert.metric),
            
            # 日志快照
            logs=self.fetch_logs(alert.instance, alert.log_pattern),
            
            # 变更记录
            changes=self.fetch_changes(alert.instance, hours=24),
            
            # 拓扑关系
            topology=self.get_service_topology(alert.service),
            
            # 历史故障
            similar_incidents=self.search_similar_incidents(alert)
        )

# root_cause_analyzer.py
from llama_index import PromptTemplate
from openai import OpenAI

class RootCauseAnalyzer:
    def __init__(self):
        self.client = OpenAI(base_url="http://localhost:11434/v1")
        self.rag = RAGRetriever()
    
    ANALYSIS_PROMPT = """
    你是一个经验丰富的 SRE 工程师。请分析以下 VPS 故障信息，找出根因。

    ## 告警信息
    {alert}

    ## 指标数据
    {metrics}

    ## 日志快照
    {logs}

    ## 近期变更
    {changes}

    ## 历史相似故障
    {similar_incidents}

    请按照以下格式输出：
    1. 根因判断（高/中/低置信度）
    2. 根因解释
    3. 推荐修复方案
    4. 风险评估
    """

    def analyze(self, context: Context) -> AnalysisResult:
        # RAG 检索相似故障案例
        similar_cases = self.rag.retrieve(context, top_k=3)
        
        # 构建 LLM 提示
        prompt = self.ANALYSIS_PROMPT.format(
            alert=context.alert,
            metrics=context.metrics,
            logs=context.logs,
            changes=context.changes,
            similar_incidents=similar_cases
        )
        
        # LLM 分析
        response = self.client.chat.completions.create(
            model="llama3",
            messages=[{"role": "user", "content": prompt}],
            temperature=0.1
        )
        
        return self.parse_analysis(response.choices[0].message.content)

# knowledge_base.py
from qdrant_client import QdrantClient
from llama_index import Document, VectorStoreIndex

class IncidentKnowledgeBase:
    def __init__(self):
        self.client = QdrantClient("localhost", port=6333)
        self.index = VectorStoreIndex.from_documents(self.load_documents())
    
    def load_documents(self):
        """加载历史故障案例"""
        documents = []
        for incident in self.fetch_historical_incidents():
            doc = Document(
                text=f"告警: {incident.alert}\n根因: {incident.root_cause}\n修复: {incident.remediation}",
                metadata={"incident_id": incident.id, "category": incident.category}
            )
            documents.append(doc)
        return documents
    
    def retrieve(self, query: str, top_k: int = 3) -> List[str]:
        """检索相似故障案例"""
        retriever = self.index.as_retriever(similarity_top_k=top_k)
        return [doc.text for doc in retriever.retrieve(query)]

# remediation_strategies.yml
strategies:
  - id: high_cpu
    name: "CPU 过高响应"
    conditions:
      - metric: cpu_usage_percent
        operator: ">"
        value: 90
        duration: 5m
    actions:
      - type: restart_service
        target: "{{ service_name }}"
        dry_run: false
      - type: scale_up
        target: "vps"
        params:
          cpu: "+2 cores"
    rollback:
      - type: restart_service
        target: "{{ service_name }}"

  - id: disk_full
    name: "磁盘空间清理"
    conditions:
      - metric: disk_free_percent
        operator: "<"
        value: 10
    actions:
      - type: cleanup_logs
        target: "/var/log"
        params:
          max_age: "7d"
      - type: rotate_logs
        target: "app"
    rollback:
      - type: restore_logs
        target: "backup"

  - id: memory_leak
    name: "内存泄漏响应"
    conditions:
      - metric: memory_growth_rate
        operator: ">"
        value: "1GB/h"
    actions:
      - type: restart_service
        target: "{{ leaking_service }}"
        params:
          graceful: true
      - type: notify
        target: "ops-team"
        channel: "slack"
        message: "检测到内存泄漏，已重启服务"

# decision_engine.py
class DecisionEngine:
    def __init__(self):
        self.strategies = self.load_strategies()
        self.llm = OpenAI(base_url="http://localhost:11434/v1")
    
    DECISION_PROMPT = """
    你是一个运维决策专家。根据以下故障上下文和可用策略，选择最优修复方案。

    ## 故障上下文
    {context}

    ## 可用策略
    {strategies}

    ## 约束条件
    - 服务可用性要求：99.9%
    - 最大停机时间：5 分钟
    - 禁止操作：生产数据库直接修改

    请输出：
    1. 选择的策略 ID
    2. 选择理由
    3. 预期效果
    4. 风险等级
    """

    def decide(self, context: Context, available_strategies: List[str]) -> Decision:
        prompt = self.DECISION_PROMPT.format(
            context=context,
            strategies="\n".join(available_strategies)
        )
        
        response = self.llm.chat.completions.create(
            model="llama3",
            messages=[{"role": "user", "content": prompt}],
            temperature=0.1
        )
        
        return self.parse_decision(response.choices[0].message.content)

# remediation_executor.py
class RemediationExecutor:
    def __init__(self):
        self.ansible = AnsibleExecutor()
        self.ssh = SSHClient()
        self.webhook = WebhookClient()
    
    def execute(self, decision: Decision) -> ExecutionResult:
        # 1. 预检查
        if not self.pre_check(decision):
            return ExecutionResult(status="blocked", reason="预检查失败")
        
        # 2. 创建快照（回滚点）
        snapshot_id = self.create_snapshot(decision.target)
        
        try:
            # 3. 执行修复
            result = self.run_action(decision.action)
            
            # 4. 验证效果
            if self.validate_fix(decision, result):
                return ExecutionResult(status="success", snapshot_id=snapshot_id)
            else:
                raise FixValidationFailed("修复后指标未改善")
                
        except Exception as e:
            # 5. 自动回滚
            self.rollback(snapshot_id)
            return ExecutionResult(status="rolled_back", error=str(e))
    
    def pre_check(self, decision: Decision) -> bool:
        """执行前安全检查"""
        checks = [
            self.check_service_status(decision.target),
            self.check_dependency_health(decision.target),
            self.check_time_window(decision)
        ]
        return all(checks)

# safety_constraints.yml
constraints:
  # 时间窗口
  maintenance_window:
    allowed: "02:00-06:00"
    blocked: "08:00-22:00"
  
  # 操作白名单
  allowed_actions:
    - restart_service
    - rotate_logs
    - scale_up
    - clear_cache
  
  # 禁止操作
  forbidden_actions:
    - delete_database
    - modify_schema
    - change_network_config
  
  # 权限分级
  permission_levels:
    critical:
      - ops-lead
      - oncall-engineer
    warning:
      - all-engineers
    info:
      - read-only

# rollback_handler.py
class RollbackHandler:
    def rollback(self, snapshot_id: str, reason: str = "") -> bool:
        """执行回滚到快照状态"""
        # 1. 停止当前修复
        self.stop_action()
        
        # 2. 恢复快照
        result = self.restore_snapshot(snapshot_id)
        
        # 3. 验证回滚
        if self.verify_rollback(snapshot_id):
            # 4. 通知团队
            self.notify_team(
                type="rollback",
                snapshot_id=snapshot_id,
                reason=reason
            )
            return True
        
        return False

# validation_engine.py
class ValidationEngine:
    def validate(self, incident: Incident, result: ExecutionResult) -> bool:
        # 1. 指标恢复检查
        metrics_ok = self.check_metrics_restored(incident, result)
        
        # 2. 服务健康检查
        health_ok = self.check_service_health(incident.target)
        
        # 3. 用户感知检查
        user_ok = self.check_user_impact(incident)
        
        # 4. 长期稳定性检查（24小时）
        stability_ok = self.check_long_term_stability(incident)
        
        return all([metrics_ok, health_ok, user_ok, stability_ok])
    
    def check_metrics_restored(self, incident: Incident, result: ExecutionResult) -> bool:
        """检查关键指标是否恢复到正常范围"""
        for metric in incident.related_metrics:
            current = self.get_current_value(metric)
            if current > metric.threshold:
                return False
        return True

# learning_loop.py
class LearningLoop:
    def learn(self, incident: Incident, result: ExecutionResult):
        """从故障响应中学习，优化策略"""
        
        # 1. 记录完整事件
        self.record_incident(incident, result)
        
        # 2. 评估策略有效性
        effectiveness = self.assess_strategy_effectiveness(incident, result)
        
        # 3. 更新策略库
        if effectiveness > 0.8:
            self.boost_strategy(incident.strategy_id)
        elif effectiveness < 0.3:
            self.downgrade_strategy(incident.strategy_id)
        
        # 4. 发现新模式
        new_patterns = self.detect_new_patterns(incident)
        if new_patterns:
            self.update_knowledge_base(new_patterns)
        
        # 5. 生成复盘报告
        self.generate_postmortem(incident, result)

# main.tf
resource "null_resource" "incident_response_system" {
    provisioner "local-exec" {
        command = <<-EOT
            docker-compose up -d
            # 部署可观测性栈
            kubectl apply -f k8s/prometheus.yml
            kubectl apply -f k8s/loki.yml
            kubectl apply -f k8s/jaeger.yml
            # 部署 AI 引擎
            kubectl apply -f k8s/llm-agent.yml
        EOT
    }
}

# docker-compose.yml
version: '3.8'
services:
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus_data:/prometheus
  
  loki:
    image: grafana/loki:latest
    ports:
      - "3100:3100"
  
  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
  
  qdrant:
    image: qdrant/qdrant:latest
    ports:
      - "6333:6333"
  
  ollama:
    image: ollama/ollama:latest
    ports:
      - "11434:11434"
    volumes:
      - ollama_data:/root/.ollama

volumes:
  prometheus_data:
  ollama_data:

# alertmanager.yml
receivers:
  - name: 'slack-notifications'
    slack_configs:
      - channel: '#vps-incidents'
        send_resolved: true
        title: '{{ .CommonAnnotations.summary }}'
        text: '{{ .CommonAnnotations.description }}'

route:
  receiver: 'slack-notifications'
  group_by: ['alertname', 'instance']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h