top -bn1 | head -20
vmstat 1 5
mpstat -P ALL 1 3
pidstat -u 1 5

# Collection config example
scrape_configs:
  - job_name: 'cpu_deep_monitor'
    scrape_interval: 5s
    metrics_path: '/metrics'
    static_configs:
      - targets: ['node-exporter:9100']
    metric_relabel_configs:
      # Extract deep metrics like CPU frequency, temperature
      - source_labels: [__name__]
        regex: 'cpu_seconds_total|cpufreq_*|thermal_*'
        action: keep

# AI analysis pseudo-code
async def analyze_cpu_bottleneck(metrics: MetricStream) -> TuningRecommendation:
    # 1. Anomaly detection
    anomaly_score = isolation_forest.predict(metrics.cpu_usage)
    
    # 2. Pattern classification
    pattern = classify_cpu_pattern(metrics)  # burst/steady/cyclic
    
    # 3. Process-level localization
    hot_process = await identify_hot_process(metrics.pid_stats)
    
    # 4. Generate tuning recommendations
    recommendations = await llm_generate_recommendation(
        pattern=pattern,
        hot_process=hot_process,
        current_config=get_sysctl_config()
    )
    
    return recommendations

# Memory anomaly detection example
from sklearn.ensemble import IsolationForest

def detect_memory_anomaly(history: pd.DataFrame) -> dict:
    """Detect memory usage anomalies using Isolation Forest"""
    features = history[['mem_used_ratio', 'swap_used_ratio', 
                        'cache_ratio', 'buffer_ratio', 'oom_kill_count']]
    
    model = IsolationForest(contamination=0.05, random_state=42)
    predictions = model.fit_predict(features)
    
    # Identify anomaly points
    anomalies = features[predictions == -1]
    
    return {
        "anomaly_detected": len(anomalies) > 0,
        "current_state": {
            "mem_used": history.iloc[-1]['mem_used_ratio'],
            "swap_pressure": history.iloc[-1]['swap_used_ratio'],
            "cache_efficiency": history.iloc[-1]['cache_ratio']
        },
        "trend": identify_memory_trend(history),
        "recommendation": generate_tuning_advice(anomalies, history)
    }

# Memory leak detection alert rule
groups:
  - name: memory_leak_detection
    rules:
      - alert: MemoryLeakPotential
        expr: |
          rate(process_virtual_memory_bytes[1h]) > 1048576
          and rate(process_resident_memory_bytes[1h]) > 524288
          and rate(process_resident_memory_bytes[1h] offset 6h) < rate(process_resident_memory_bytes[1h])
        for: 30m
        labels:
          severity: warning
        annotations:
          summary: "Potential memory leak detected in {{ $labels.instance }}"
          description: "Process memory growing at {{ $value | humanize }} bytes/sec for 30 minutes"
          ai_action: "Collect dump and analyze leak stack trace; AI will auto-generate analysis report"

# I/O bottleneck comprehensive analysis
async def analyze_io_bottleneck(metrics: MetricStream) -> IoAnalysis:
    # Key metrics
    iops = metrics.disk_io_operations_per_sec
    latency = metrics.disk_read_latency_avg
    throughput = metrics.disk_read_bytes_per_sec
    util = metrics.disk_utilization
    
    # AI analysis: distinguish different types of I/O problems
    if util > 0.9 and latency > 10:
        # Disk saturated—likely random reads
        if iops > 10000 and throughput < 50 * MB:
            return IoAnalysis(
                root_cause="Random I/O intensive, disk IOPS insufficient",
                suggestions=[
                    "Consider migrating to SSD/NVMe storage",
                    "Switch I/O scheduler to `noop` or `mq-deadline`",
                    "Optimize application I/O pattern, increase sequential reads"
                ],
                risk_level="low"
            )
    
    elif latency > 20 and util < 0.5:
        # High latency but low utilization—likely I/O scheduler issue
        return IoAnalysis(
            root_cause="Improper I/O scheduler configuration or I/O fragmentation",
            suggestions=[
                "Check current I/O scheduler: cat /sys/block/sda/queue/scheduler",
                "Try switching to mq-deadline: echo mq-deadline > /sys/block/sda/queue/scheduler",
                "Check filesystem mount options, add noatime"
            ],
            risk_level="medium"
        )
    
    return IoAnalysis(root_cause="I/O normal", suggestions=[], risk_level="none")

# I/O tuning execution config
io_tuning:
  pre_check:
    - command: "cat /sys/block/*/queue/scheduler"
      expected_pattern: ".*"
    - command: "df -h"
      threshold: "usage < 90%"
  
  tuning_actions:
    - name: "Switch I/O scheduler to mq-deadline"
      command: "echo mq-deadline > /sys/block/sda/queue/scheduler"
      validate:
        command: "cat /sys/block/sda/queue/scheduler"
        expected: "mq-deadline"
      rollback:
        command: "echo bfq > /sys/block/sda/queue/scheduler"
    
    - name: "Optimize mount options"
      command: "mount -o remount,noatime,nodiratime /data"
      validate:
        command: "mount | grep '/data'"
        expected_pattern: "noatime"
      rollback:
        command: "mount -o remount,relatime /data"
  
  post_check:
    - command: "iostat -x 1 3"
      metrics: ["await", "svctm", "%util"]
      improvement_threshold: 20%  # At least 20% improvement

# Network tuning AI recommendation engine
def recommend_network_tuning(business_type: str, traffic_profile: dict) -> dict:
    """Recommend network parameters based on business type and traffic profile"""
    
    recommendations = {}
    
    if business_type in ("web_server", "api_gateway"):
        # Web/API: high concurrent connections
        recommendations.update({
            "net.core.somaxconn": {"value": 65535, "reason": "Increase backlog queue"},
            "net.ipv4.tcp_max_syn_backlog": {"value": 65535, "reason": "Enlarge SYN queue"},
            "net.ipv4.tcp_tw_reuse": {"value": 1, "reason": "Enable TIME_WAIT socket reuse"},
            "net.ipv4.ip_local_port_range": {"value": "1024 65535", "reason": "Expand available port range"},
        })
    
    elif business_type in ("database", "redis"):
        # Database: high throughput, low latency
        recommendations.update({
            "net.core.rmem_max": {"value": 16777216, "reason": "Increase receive buffer"},
            "net.core.wmem_max": {"value": 16777216, "reason": "Increase send buffer"},
            "net.ipv4.tcp_rmem": {"value": "4096 87380 16777216", "reason": "Auto-tune receive window"},
            "net.ipv4.tcp_wmem": {"value": "4096 65536 16777216", "reason": "Auto-tune send window"},
            "net.ipv4.tcp_congestion_control": {"value": "bbr", "reason": "Enable BBR congestion control"},
        })
    
    elif business_type in ("cdn", "proxy"):
        # CDN/proxy: high throughput
        recommendations.update({
            "net.ipv4.tcp_window_scaling": {"value": 1, "reason": "Enable TCP window scaling"},
            "net.ipv4.tcp_timestamps": {"value": 1, "reason": "Enable timestamps (PAWS protection)"},
            "net.core.netdev_max_backlog": {"value": 5000, "reason": "Increase NIC receive queue"},
        })
    
    # AI evaluates risk based on current config
    risk_assessment = assess_risk(recommendations, get_current_config())
    
    return {
        "recommendations": recommendations,
        "risk_assessment": risk_assessment,
        "rollback_plan": generate_rollback_plan(recommendations)
    }

# Auto-select optimal TCP congestion control algorithm
def select_tcp_congestion_control() -> str:
    """AI auto-selects optimal TCP congestion control algorithm"""
    
    # Detect network environment
    latency = measure_ping_latency()
    packet_loss = measure_packet_loss()
    bandwidth = measure_bandwidth()
    
    if latency < 10 and packet_loss < 0.01:
        return "cubic"   # Low-latency LAN environment
    elif bandwidth > 1 * Gbps:
        return "bbr"     # High-bandwidth, BBR performs best
    elif packet_loss > 0.05:
        return "cubic"   # High loss environment, Cubic more stable
    else:
        return "bbr"     # Default recommendation: BBR

# Core tuning agent logic
class PerformanceTuningAgent:
    def __init__(self, llm_client, metric_client, config_store):
        self.llm = llm_client
        self.metrics = metric_client
        self.config = config_store
        self.rollback_log = []
    
    async def tune(self, target: str, current_metrics: dict) -> TuningResult:
        """Execute intelligent tuning"""
        
        # Step 1: Establish baseline
        baseline = await self.capture_baseline(target)
        
        # Step 2: AI analysis & plan generation
        recommendation = await self.llm.generate_tuning_plan(
            target=target,
            metrics=current_metrics,
            baseline=baseline,
            history=self.config.get_tuning_history(target)
        )
        
        # Step 3: Risk assessment
        risk = await self.assess_risk(recommendation)
        if risk.severity == "high":
            return TuningResult(status="rejected", reason=risk.reason)
        
        # Step 4: Gray execution
        if risk.severity in ("low", "medium"):
            result = await self.execute_with_rollback(
                recommendation, baseline
            )
        else:
            result = await self.execute_manual_approval(recommendation)
        
        return result
    
    async def execute_with_rollback(self, plan, baseline):
        """Execution with rollback guarantee"""
        rollback_commands = []
        
        try:
            for action in plan.actions:
                # Record rollback command
                rollback_commands.append(action.rollback)
                # Execute tuning
                await self.execute(action.command)
                # Validate
                if not await self.validate(action):
                    raise TuningError(f"Validation failed for {action.name}")
            
            return TuningResult(status="success", improvement=plan.estimated_improvement)
            
        except Exception as e:
            # Auto-rollback
            await self.rollback(rollback_commands)
            return TuningResult(status="rolled_back", error=str(e))

# Tuning effect evaluation rules
evaluation:
  metrics_to_track:
    - name: cpu_usage_avg
      window: ["5m", "15m", "1h"]
    - name: latency_p99
      window: ["5m", "15m", "1h"]
    - name: throughput_rps
      window: ["5m", "15m"]
    - name: error_rate
      window: ["5m"]
  
  success_criteria:
    - metric: latency_p99
      improvement: "> 10%"
    - metric: cpu_usage_avg
      target: "< 70%"
    - metric: error_rate
      target: "< 0.1%"
  
  rollback_trigger:
    - metric: error_rate
      threshold: "> 1%"
      action: "immediate_rollback"
    - metric: cpu_usage_avg
      change: "> +50%"
      action: "immediate_rollback"

# docker-compose.yml
version: '3.8'

services:
  # Observability infrastructure
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus:/etc/prometheus
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--storage.tsdb.retention.time=30d'
  
  victoria-metrics:
    image: victoria/victoria-metrics:latest
    ports:
      - "8428:8428"
    volumes:
      - vmdata:/var/lib/victoria-metrics
  
  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana
  
  # Data collection
  node-exporter:
    image: prom/node-exporter:latest
    container_name: node-exporter
    pid: host
    restart: unless-stopped
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'
  
  # AI analysis engine
  ai-tuning-engine:
    build: ./ai-tuning-engine
    environment:
      - LLM_API_KEY=${LLM_API_KEY}
      - PROMETHEUS_URL=http://prometheus:9090
      - VM_URL=http://victoria-metrics:8428
    volumes:
      - ./tuning-rules:/etc/tuning-rules
      - ./tuning-logs:/var/log/tuning
    depends_on:
      - prometheus
      - victoria-metrics

# ai-tuning-engine/main.py
import asyncio
import json
from datetime import datetime, timedelta
from prometheus_api_client import PrometheusConnect
from llama_index import LLM, QueryEngine
from sklearn.ensemble import IsolationForest
import numpy as np

class VPSPerformanceTuner:
    def __init__(self):
        self.prom = PrometheusConnect(
            url="http://prometheus:9090",
            disable_ssl=True
        )
        self.llm = LLM(model="gpt-4", api_key=os.environ["LLM_API_KEY"])
        self.anomaly_detector = IsolationForest(contamination=0.05)
    
    async def run_tuning_cycle(self):
        """Main tuning loop"""
        while True:
            try:
                # 1. Collect current metrics
                metrics = await self.collect_metrics()
                
                # 2. Anomaly detection
                anomalies = await self.detect_anomalies(metrics)
                
                if anomalies:
                    # 3. AI root cause analysis
                    root_cause = await self.analyze_root_cause(anomalies, metrics)
                    
                    # 4. Generate tuning plan
                    plan = await self.generate_tuning_plan(root_cause, metrics)
                    
                    # 5. Risk assessment & execution
                    await self.execute_with_safety(plan)
                
                await asyncio.sleep(300)  # Every 5 minutes
                
            except Exception as e:
                logger.error(f"Tuning cycle error: {e}")
                await asyncio.sleep(60)
    
    async def collect_metrics(self) -> dict:
        """Collect full-stack performance metrics"""
        return {
            "cpu": {
                "user": self.prom.custom_query(
                    '100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100)'
                )[0].value,
                "iowait": self.prom.custom_query(
                    'avg by(instance) (rate(node_cpu_seconds_total{mode="iowait"}[5m])) * 100'
                )[0].value,
                "context_switches": self.prom.custom_query(
                    'rate(node_context_switches_total[5m])'
                )[0].value,
            },
            "memory": {
                "used_ratio": self.prom.custom_query(
                    '(1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100'
                )[0].value,
                "swap_used": self.prom.custom_query(
                    '(node_memory_SwapTotal_bytes - node_memory_SwapFree_bytes) / node_memory_SwapTotal_bytes * 100'
                )[0].value,
            },
            "io": {
                "read_latency": self.prom.custom_query(
                    'rate(node_disk_read_time_seconds_total[5m]) / rate(node_disk_reads_completed_total[5m])'
                )[0].value,
                "write_latency": self.prom.custom_query(
                    'rate(node_disk_write_time_seconds_total[5m]) / rate(node_disk_writes_completed_total[5m])'
                )[0].value,
                "iops": self.prom.custom_query(
                    'rate(node_disk_reads_completed_total[5m]) + rate(node_disk_writes_completed_total[5m])'
                )[0].value,
            },
            "network": {
                "retransmits": self.prom.custom_query(
                    'rate(node_network_transmit_packets_total[5m])'
                )[0].value,
                "errors": self.prom.custom_query(
                    'rate(node_network_transmit_errs_total[5m])'
                )[0].value,
            }
        }

# Tuning frequency limits
MAX_TUNING_OPERATIONS_PER_HOUR = 3
MAX_TUNING_OPERATIONS_PER_DAY = 10

def should_tune(target: str) -> bool:
    """Check if tuning should proceed"""
    recent_tunings = get_tuning_history(target, hours=1)
    daily_tunings = get_tuning_history(target, hours=24)
    
    if len(recent_tunings) >= MAX_TUNING_OPERATIONS_PER_HOUR:
        logger.warning("Throttling: too many tunings in last hour")
        return False
    if len(daily_tunings) >= MAX_TUNING_OPERATIONS_PER_DAY:
        logger.warning("Throttling: too many tunings today")
        return False
    
    return True