# Route all logs to a unified location
sudo rsyslog -n /etc/rsyslog.conf

# Or use Docker's JSON logging driver
docker service create \
  --log-driver json-file \
  --log-opt max-size=10m \
  --log-opt max-file=3 \
  myapp:latest

import json
from datetime import datetime, timedelta
from collections import defaultdict

class LogPreprocessor:
    """Log preprocessing: deduplication, classification, time-window aggregation"""
    
    def __init__(self):
        self.severity_map = {
            'emerg': 0, 'alert': 1, 'crit': 2, 'err': 3,
            'warning': 4, 'notice': 5, 'info': 6, 'debug': 7
        }
    
    def normalize(self, log_line):
        """Normalize log lines, extracting key information"""
        patterns = [
            r'^(\w+\s+\d+\s+[\d:]+)\s+(\S+)\s+(\S+?):\s+(.*)',  # syslog
            r'(\d{4}-\d{2}-\d{T}\d{2}:\d{2}:\d{2}[.\d+Z-]+)\s+(.*?)\s+(\w+)\s+(.*)',  # ISO
        ]
        
        for pattern in patterns:
            import re
            match = re.search(pattern, log_line)
            if match:
                groups = match.groups()
                return {
                    'timestamp': groups[0],
                    'host': groups[1] if len(groups) > 1 else 'unknown',
                    'service': groups[2] if len(groups) > 2 else 'system',
                    'message': groups[-1],
                    'raw': log_line
                }
        return {'message': log_line, 'raw': log_line}
    
    def deduplicate(self, logs):
        """Deduplicate based on message content"""
        seen = {}
        result = []
        for log in logs:
            msg = log.get('message', '')[:200]
            if msg not in seen:
                seen[msg] = 0
            seen[msg] += 1
            if seen[msg] == 1:
                result.append({**log, 'count': 1})
            else:
                result[-1]['count'] = seen[msg]
        return result
    
    def aggregate_by_window(self, logs, window_minutes=5):
        """Aggregate logs by time window"""
        windows = defaultdict(list)
        for log in logs:
            ts = log.get('timestamp', '')
            try:
                dt = datetime.fromisoformat(ts.replace('Z', '+00:00'))
                window_key = dt.replace(
                    minute=(dt.minute // window_minutes) * window_minutes,
                    second=0, microsecond=0
                )
                windows[str(window_key)].append(log)
            except (ValueError, TypeError):
                windows['unknown'].append(log)
        return dict(windows)

SYSTEM_PROMPT = """You are an experienced Linux system operations engineer and security analyst.
Your task is to analyze provided system logs for potential issues and output in the following format:

1. [Summary] One-sentence overview of the core issue found
2. [Severity] P0(Critical) / P1(High) / P2(Medium) / P3(Low)
3. [Impact Scope] Affected services/users/functions
4. [Root Cause Analysis] Detailed explanation of possible causes
5. [Evidence Chain] Key log entries supporting the judgment
6. [Fix Recommendations] Specific, actionable remediation steps
7. [Prevention] How to avoid similar issues in the future

Note: Analyze only based on provided logs. Do not speculate about information not present.
If no obvious anomalies are detected, explicitly state "No known issue patterns detected.""""

USER_PROMPT_TEMPLATE = """Below are summarized logs collected from the VPS during {time_range}:

Server Information:
- OS: {os_info}
- Kernel: {kernel_version}
- Uptime: {uptime}
- Load Average: {load_average}

--- Log Content ---
{log_content}

--- Known Alerts ---
{alerts}

Please begin analysis."""

import openai

class LogAnalyzer:
    """AI-powered intelligent log analyzer"""
    
    def __init__(self, api_key, model="gpt-4o"):
        self.client = openai.OpenAI(api_key=api_key)
        self.model = model
    
    def analyze_logs(self, server_info, log_entries, alerts=None):
        """Analyze logs and return a structured diagnostic report"""
        
        log_text = self._format_logs(log_entries)
        
        user_prompt = USER_PROMPT_TEMPLATE.format(
            time_range=server_info.get('time_range', 'last 30 minutes'),
            os_info=server_info.get('os', 'Linux'),
            kernel_version=server_info.get('kernel', 'unknown'),
            uptime=server_info.get('uptime', 'unknown'),
            load_average=server_info.get('load_average', 'unknown'),
            log_content=log_text,
            alerts=alerts or "No known alerts"
        )
        
        response = self.client.chat.completions.create(
            model=self.model,
            messages=[
                {"role": "system", "content": SYSTEM_PROMPT},
                {"role": "user", "content": user_prompt}
            ],
            temperature=0.3,  # Lower temperature for consistent results
            max_tokens=2000
        )
        
        return self._parse_analysis(response.choices[0].message.content)
    
    def _format_logs(self, logs):
        """Format log list into readable text"""
        formatted = []
        for log in logs[:200]:  # Limit count to stay within token limits
            severity = log.get('severity', 'INFO').upper()
            timestamp = log.get('timestamp', '?')
            service = log.get('service', 'system')
            message = log.get('message', '')[:300]
            count = log.get('count', 1)
            
            entry = f"[{timestamp}] [{severity}] {service}: {message}"
            if count > 1:
                entry += f" (repeated {count} times)"
            formatted.append(entry)
        
        return "\n".join(formatted)
    
    def _parse_analysis(self, analysis_text):
        """Parse LLM analysis into structured data"""
        result = {
            'summary': '',
            'severity': 'P3',
            'impact': '',
            'root_cause': '',
            'evidence': [],
            'fix_steps': [],
            'prevention': ''
        }
        
        current_section = None
        section_map = {
            'Summary': 'summary',
            'Severity': 'severity',
            'Impact': 'impact',
            'Root Cause': 'root_cause',
            'Evidence': 'evidence',
            'Fix': 'fix_steps',
            'Prevention': 'prevention'
        }
        
        for line in analysis_text.split('\n'):
            line = line.strip()
            for en_title, key in section_map.items():
                if en_title in line:
                    current_section = key
                    result[key] = line.split(en_title)[-1].strip().lstrip(':- ')
                    break
        
        return result

def deep_dive_analysis(analyzer, server_id, date_range, keywords=None):
    """
    Deep analysis of logs within a specified time range
    
    Args:
        analyzer: LogAnalyzer instance
        server_id: Server identifier
        date_range: Time range, e.g., ('2026-07-20', '2026-07-22')
        keywords: Keywords of interest
    """
    # 1. Fetch historical logs
    logs = fetch_history_logs(server_id, date_range, keywords)
    
    # 2. Group by error type
    error_groups = group_by_error_type(logs)
    
    # 3. Construct context-rich analysis request
    context = {
        'server_id': server_id,
        'time_range': f"{date_range[0]} to {date_range[1]}",
        'total_events': len(logs),
        'error_count': sum(1 for l in logs if l.get('severity') in ['err', 'crit', 'alert', 'emerg']),
        'top_errors': error_groups[:10],
        'recent_deployments': get_recent_deployments(server_id, date_range),
        'config_changes': get_config_changes(server_id, date_range)
    }
    
    # 4. Send to LLM for deep analysis
    report = analyzer.analyze_logs(context, logs)
    
    return report

class AlertManager:
    """Intelligent alerting and remediation based on AI analysis"""
    
    def __init__(self):
        self.notification_channels = {
            'slack': self._send_to_slack,
            'email': self._send_email,
            'webhook': self._send_webhook
        }
    
    def process_analysis(self, analysis, server_info):
        """Process AI analysis results, decide on alerting and remediation"""
        
        severity = analysis.get('severity', 'P3')
        
        if severity in ['P0', 'P1']:
            self.send_alert(analysis, server_info, priority='critical')
            if severity == 'P0':
                self.try_auto_fix(analysis, server_info)
        
        elif severity == 'P2':
            self.send_alert(analysis, server_info, priority='normal')
        
        self.save_to_knowledge_base(analysis, server_info)
    
    def send_alert(self, analysis, server_info, priority='normal'):
        """Send alert notification"""
        message = f"""🔧 **VPS AI Diagnostic Report**

**Server**: {server_info.get('hostname')} ({server_info.get('ip')})
**Severity**: {analysis.get('severity', 'Unknown')}
**Summary**: {analysis.get('summary', 'None')}

**Root Cause**:
{analysis.get('root_cause', 'N/A')}

**Fix Steps**:
{chr(10).join('- ' + step for step in analysis.get('fix_steps', []))}

---
Auto-generated by selfvps.net AI Log Analysis Engine
"""
        
        for channel_name, channel_fn in self.notification_channels.items():
            channel_fn(message, priority=priority)
    
    def try_auto_fix(self, analysis, server_info):
        """Attempt to execute AI-recommended auto-remediation"""
        fix_steps = analysis.get('fix_steps', [])
        
        for step in fix_steps:
            if self._is_safe_command(step):
                result = self._execute_command(step, server_info)
                print(f"Executing fix: {step} → {result}")
            else:
                print(f"⚠️ Skipping unsafe command: {step}")
    
    def _is_safe_command(self, command_str):
        """Check if a command is safe to execute"""
        dangerous_patterns = ['rm -rf /', 'dd if=', '> /dev/sda', ':(){ :|:& };:']
        return not any(p in command_str for p in dangerous_patterns)

# Run locally with Ollama
ollama pull llama3.2

# Call via API
curl http://localhost:11434/api/generate -d '{
  "model": "llama3.2",
  "prompt": "Analyze the following system logs...",
  "stream": false
}'

class KnowledgeBase:
    """Operations knowledge base — accumulating historical diagnostic experience"""
    
    def store_case(self, log_pattern, diagnosis, fix, effectiveness):
        case = {
            'pattern_hash': hash(log_pattern),
            'diagnosis': diagnosis,
            'fix_applied': fix,
            'effectiveness': effectiveness,
            'timestamp': datetime.now().isoformat(),
            'server_type': self._classify_server()
        }
        self.db.collection('cases').insert(case)
    
    def find_similar(self, new_logs):
        """Find similar historical cases"""
        pattern_hash = hash(new_logs)
        similar = self.db.collection('cases').find({
            'pattern_hash': {'$near': pattern_hash},
            'effectiveness': {'$gt': 0.8}
        }).limit(5)
        return similar

def correlated_analysis(server_logs_dict):
    """Cross-server log correlation analysis"""
    unified_timeline = merge_timelines(server_logs_dict)
    correlations = find_temporal_correlations(unified_timeline)
    
    prompt = f"""
    The following events occurred simultaneously across multiple servers:
    
    {correlations}
    
    Possible correlated scenarios:
    1. Same network change affected all servers
    2. Upstream service failure caused cascading downstream impact
    3. Scheduled tasks competing for resources at the same time
    4. External attack targeting a specific service
    """
    
    return llm_client.generate(prompt)

# Custom Prometheus exporter rule
rules:
  - alert: HighErrorRate
    expr: rate(http_requests_total{status=~"5.."}[5m]) > 0.05
    for: 2m
    annotations:
      summary: "HTTP 5xx error rate exceeds 5%"
      ai_analysis: "Trigger AI log analysis"

def incremental_analyze(current_logs, previous_state):
    """Incremental analysis: only analyze newly added anomalies"""
    new_logs = filter_since(current_logs, previous_state['last_analysis_time'])
    anomaly_windows = detect_anomaly_windows(new_logs)
    
    for window in anomaly_windows:
        report = llm_analyze(window.logs)
        if report['severity'] in ['P0', 'P1']:
            send_alert(report)
    
    previous_state['last_analysis_time'] = datetime.now()
    return previous_state

from celery import Celery

celery_app = Celery('log_analyzer', broker='redis://localhost:6379/0')

@celery_app.task
def async_log_analysis(server_id, log_batch):
    """Asynchronously execute log analysis"""
    analyzer = LogAnalyzer(api_key=os.environ['LLM_API_KEY'])
    result = analyzer.analyze_logs(load_server_info(server_id), log_batch)
    
    save_analysis_result(server_id, result)
    if result['severity'] in ['P0', 'P1']:
        trigger_alert.s(result, server_id).delay()
    
    return result['severity']