# prometheus/alerts/vps-alerts.yml
groups:
  - name: vps-critical
    rules:
      - alert: HighCPUUsage
        expr: cpu_usage_percent > 90
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "VPS {{ $labels.instance }} CPU usage exceeds 90%"
          description: "Current value: {{ $value }}%, sustained for 5 minutes"

      - alert: DiskSpaceLow
        expr: disk_free_percent < 10
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "VPS {{ $labels.instance }} disk space running low"

      - alert: MemoryLeakDetected
        expr: increase(memory_usage_bytes[1h]) > 1073741824
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "VPS {{ $labels.instance }} abnormal memory growth, possible leak"

# loki/alerts/log-alerts.yml
alerts:
  - name: ErrorRateSpike
    query: sum(rate({app="web"} |= "error" [5m])) by (instance) > 10
    for: 5m
    severity: critical

  - name: AuthFailureBurst
    query: sum(rate({app="ssh"} |=~ "Failed password" [5m])) > 5
    for: 2m
    severity: warning

// traces/alerts.go
func CheckTraceAnomaly(span *model.Span) bool {
    duration := span.Duration.Milliseconds()
    if duration > 5000 { // 5 second threshold
        return true
    }
    if span.Status.Code != model.SpanStatus_OK {
        return true
    }
    return false
}

# incident_context_collector.py
class IncidentContextCollector:
    def collect(self, alert: Alert) -> Context:
        return Context(
            # Time window
            time_range=alert.timestamp - timedelta(hours=1),
            
            # Metric data
            metrics=self.fetch_metrics(alert.instance, alert.metric),
            
            # Log snapshot
            logs=self.fetch_logs(alert.instance, alert.log_pattern),
            
            # Change records
            changes=self.fetch_changes(alert.instance, hours=24),
            
            # Topology relationships
            topology=self.get_service_topology(alert.service),
            
            # Historical incidents
            similar_incidents=self.search_similar_incidents(alert)
        )

# root_cause_analyzer.py
from llama_index import PromptTemplate
from openai import OpenAI

class RootCauseAnalyzer:
    def __init__(self):
        self.client = OpenAI(base_url="http://localhost:11434/v1")
        self.rag = RAGRetriever()
    
    ANALYSIS_PROMPT = """
    You are an experienced SRE engineer. Analyze the following VPS incident information and identify the root cause.

    ## Alert Information
    {alert}

    ## Metric Data
    {metrics}

    ## Log Snapshot
    {logs}

    ## Recent Changes
    {changes}

    ## Historical Similar Incidents
    {similar_incidents}

    Please output in the following format:
    1. Root cause judgment (high/medium/low confidence)
    2. Root cause explanation
    3. Recommended remediation plan
    4. Risk assessment
    """

    def analyze(self, context: Context) -> AnalysisResult:
        # RAG retrieve similar incident cases
        similar_cases = self.rag.retrieve(context, top_k=3)
        
        # Build LLM prompt
        prompt = self.ANALYSIS_PROMPT.format(
            alert=context.alert,
            metrics=context.metrics,
            logs=context.logs,
            changes=context.changes,
            similar_incidents=similar_cases
        )
        
        # LLM analysis
        response = self.client.chat.completions.create(
            model="llama3",
            messages=[{"role": "user", "content": prompt}],
            temperature=0.1
        )
        
        return self.parse_analysis(response.choices[0].message.content)

# knowledge_base.py
from qdrant_client import QdrantClient
from llama_index import Document, VectorStoreIndex

class IncidentKnowledgeBase:
    def __init__(self):
        self.client = QdrantClient("localhost", port=6333)
        self.index = VectorStoreIndex.from_documents(self.load_documents())
    
    def load_documents(self):
        """Load historical incident cases"""
        documents = []
        for incident in self.fetch_historical_incidents():
            doc = Document(
                text=f"Alert: {incident.alert}\nRoot Cause: {incident.root_cause}\nRemediation: {incident.remediation}",
                metadata={"incident_id": incident.id, "category": incident.category}
            )
            documents.append(doc)
        return documents
    
    def retrieve(self, query: str, top_k: int = 3) -> List[str]:
        """Retrieve similar incident cases"""
        retriever = self.index.as_retriever(similarity_top_k=top_k)
        return [doc.text for doc in retriever.retrieve(query)]

# remediation_strategies.yml
strategies:
  - id: high_cpu
    name: "High CPU Response"
    conditions:
      - metric: cpu_usage_percent
        operator: ">"
        value: 90
        duration: 5m
    actions:
      - type: restart_service
        target: "{{ service_name }}"
        dry_run: false
      - type: scale_up
        target: "vps"
        params:
          cpu: "+2 cores"
    rollback:
      - type: restart_service
        target: "{{ service_name }}"

  - id: disk_full
    name: "Disk Space Cleanup"
    conditions:
      - metric: disk_free_percent
        operator: "<"
        value: 10
    actions:
      - type: cleanup_logs
        target: "/var/log"
        params:
          max_age: "7d"
      - type: rotate_logs
        target: "app"
    rollback:
      - type: restore_logs
        target: "backup"

  - id: memory_leak
    name: "Memory Leak Response"
    conditions:
      - metric: memory_growth_rate
        operator: ">"
        value: "1GB/h"
    actions:
      - type: restart_service
        target: "{{ leaking_service }}"
        params:
          graceful: true
      - type: notify
        target: "ops-team"
        channel: "slack"
        message: "Memory leak detected, service restarted"

# decision_engine.py
class DecisionEngine:
    def __init__(self):
        self.strategies = self.load_strategies()
        self.llm = OpenAI(base_url="http://localhost:11434/v1")
    
    DECISION_PROMPT = """
    You are an operations decision expert. Based on the following incident context and available strategies, select the optimal remediation plan.

    ## Incident Context
    {context}

    ## Available Strategies
    {strategies}

    ## Constraints
    - Service availability requirement: 99.9%
    - Maximum downtime: 5 minutes
    - Forbidden operations: Direct modification of production database

    Please output:
    1. Selected strategy ID
    2. Selection rationale
    3. Expected effect
    4. Risk level
    """

    def decide(self, context: Context, available_strategies: List[str]) -> Decision:
        prompt = self.DECISION_PROMPT.format(
            context=context,
            strategies="\n".join(available_strategies)
        )
        
        response = self.llm.chat.completions.create(
            model="llama3",
            messages=[{"role": "user", "content": prompt}],
            temperature=0.1
        )
        
        return self.parse_decision(response.choices[0].message.content)

# remediation_executor.py
class RemediationExecutor:
    def __init__(self):
        self.ansible = AnsibleExecutor()
        self.ssh = SSHClient()
        self.webhook = WebhookClient()
    
    def execute(self, decision: Decision) -> ExecutionResult:
        # 1. Pre-check
        if not self.pre_check(decision):
            return ExecutionResult(status="blocked", reason="Pre-check failed")
        
        # 2. Create snapshot (rollback point)
        snapshot_id = self.create_snapshot(decision.target)
        
        try:
            # 3. Execute fix
            result = self.run_action(decision.action)
            
            # 4. Validate effect
            if self.validate_fix(decision, result):
                return ExecutionResult(status="success", snapshot_id=snapshot_id)
            else:
                raise FixValidationFailed("Metrics did not improve after fix")
                
        except Exception as e:
            # 5. Automatic rollback
            self.rollback(snapshot_id)
            return ExecutionResult(status="rolled_back", error=str(e))
    
    def pre_check(self, decision: Decision) -> bool:
        """Execute safety check"""
        checks = [
            self.check_service_status(decision.target),
            self.check_dependency_health(decision.target),
            self.check_time_window(decision)
        ]
        return all(checks)

# safety_constraints.yml
constraints:
  # Time window
  maintenance_window:
    allowed: "02:00-06:00"
    blocked: "08:00-22:00"
  
  # Operation whitelist
  allowed_actions:
    - restart_service
    - rotate_logs
    - scale_up
    - clear_cache
  
  # Forbidden operations
  forbidden_actions:
    - delete_database
    - modify_schema
    - change_network_config
  
  # Permission levels
  permission_levels:
    critical:
      - ops-lead
      - oncall-engineer
    warning:
      - all-engineers
    info:
      - read-only

# rollback_handler.py
class RollbackHandler:
    def rollback(self, snapshot_id: str, reason: str = "") -> bool:
        """Execute rollback to snapshot state"""
        # 1. Stop current fix
        self.stop_action()
        
        # 2. Restore snapshot
        result = self.restore_snapshot(snapshot_id)
        
        # 3. Verify rollback
        if self.verify_rollback(snapshot_id):
            # 4. Notify team
            self.notify_team(
                type="rollback",
                snapshot_id=snapshot_id,
                reason=reason
            )
            return True
        
        return False

# validation_engine.py
class ValidationEngine:
    def validate(self, incident: Incident, result: ExecutionResult) -> bool:
        # 1. Metric recovery check
        metrics_ok = self.check_metrics_restored(incident, result)
        
        # 2. Service health check
        health_ok = self.check_service_health(incident.target)
        
        # 3. User perception check
        user_ok = self.check_user_impact(incident)
        
        # 4. Long-term stability check (24 hours)
        stability_ok = self.check_long_term_stability(incident)
        
        return all([metrics_ok, health_ok, user_ok, stability_ok])
    
    def check_metrics_restored(self, incident: Incident, result: ExecutionResult) -> bool:
        """Check if key metrics have returned to normal range"""
        for metric in incident.related_metrics:
            current = self.get_current_value(metric)
            if current > metric.threshold:
                return False
        return True

# learning_loop.py
class LearningLoop:
    def learn(self, incident: Incident, result: ExecutionResult):
        """Learn from incident response, optimize strategies"""
        
        # 1. Record complete event
        self.record_incident(incident, result)
        
        # 2. Evaluate strategy effectiveness
        effectiveness = self.assess_strategy_effectiveness(incident, result)
        
        # 3. Update strategy library
        if effectiveness > 0.8:
            self.boost_strategy(incident.strategy_id)
        elif effectiveness < 0.3:
            self.downgrade_strategy(incident.strategy_id)
        
        # 4. Detect new patterns
        new_patterns = self.detect_new_patterns(incident)
        if new_patterns:
            self.update_knowledge_base(new_patterns)
        
        # 5. Generate postmortem report
        self.generate_postmortem(incident, result)

# main.tf
resource "null_resource" "incident_response_system" {
    provisioner "local-exec" {
        command = <<-EOT
            docker-compose up -d
            # Deploy observability stack
            kubectl apply -f k8s/prometheus.yml
            kubectl apply -f k8s/loki.yml
            kubectl apply -f k8s/jaeger.yml
            # Deploy AI engine
            kubectl apply -f k8s/llm-agent.yml
        EOT
    }
}

# docker-compose.yml
version: '3.8'
services:
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus_data:/prometheus
  
  loki:
    image: grafana/loki:latest
    ports:
      - "3100:3100"
  
  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD}
  
  qdrant:
    image: qdrant/qdrant:latest
    ports:
      - "6333:6333"
  
  ollama:
    image: ollama/ollama:latest
    ports:
      - "11434:11434"
    volumes:
      - ollama_data:/root/.ollama

volumes:
  prometheus_data:
  ollama_data:

# alertmanager.yml
receivers:
  - name: 'slack-notifications'
    slack_configs:
      - channel: '#vps-incidents'
        send_resolved: true
        title: '{{ .CommonAnnotations.summary }}'
        text: '{{ .CommonAnnotations.description }}'

route:
  receiver: 'slack-notifications'
  group_by: ['alertname', 'instance']
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h