# Collect with Node Exporter + Prometheus
docker run -d \
  --name node-exporter \
  --pid=host \
  --network=host \
  -v "/proc:/host/proc:ro" \
  -v "/sys:/host/sys:ro" \
  -v "/:/rootfs:ro" \
  prom/node-exporter:latest

# docker-compose.yml — App monitoring config
services:
  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    ports:
      - "9090:9090"

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3001:3000"
    volumes:
      - grafana_data:/var/lib/grafana

# prometheus.yml
global:
  scrape_interval: 15s
  evaluation_interval: 30s

scrape_configs:
  - job_name: 'vps-infra'
    static_configs:
      - targets: ['node-exporter:9100']

  - job_name: 'docker-containers'
    docker_sd_configs:
      - host: unix:///var/run/docker.sock
    metrics_path: /metrics
    relabel_configs:
      - source_labels: [__meta_docker_container_name]
        regex: '/(.*)'
        target_label: container

# features.py — Feature engineering
import pandas as pd
import numpy as np
from datetime import timedelta

def extract_features(df):
    """
    Extract key features from time series data
    
    df should contain:
    - timestamp: datetime
    - cpu_usage: CPU percentage
    - memory_usage: Memory percentage
    - network_in: Inbound bandwidth
    - network_out: Outbound bandwidth
    - active_connections: Active connections
    """
    df = df.sort_values('timestamp')
    
    # === Temporal Features ===
    df['hour_of_day'] = df['timestamp'].dt.hour
    df['day_of_week'] = df['timestamp'].dt.dayofweek
    df['is_weekend'] = (df['day_of_week'] >= 5).astype(int)
    df['is_business_hours'] = ((df['hour_of_day'] >= 9) & 
                                  (df['hour_of_day'] <= 18)).astype(int)
    
    # === Lag Features (past N time points) ===
    for lag in [1, 2, 3, 6, 12, 24]:
        df[f'cpu_lag_{lag}h'] = df['cpu_usage'].shift(lag)
        df[f'memory_lag_{lag}h'] = df['memory_usage'].shift(lag)
    
    # === Rolling Statistics ===
    for window in [3, 6, 12, 24]:
        df[f'cpu_roll_mean_{window}h'] = df['cpu_usage'].rolling(window).mean()
        df[f'cpu_roll_std_{window}h'] = df['cpu_usage'].rolling(window).std()
        df[f'memory_roll_mean_{window}h'] = df['memory_usage'].rolling(window).mean()
    
    # === Difference Features (rate of change) ===
    df['cpu_diff'] = df['cpu_usage'].diff()
    df['cpu_diff_pct'] = df['cpu_usage'].pct_change()
    df['network_diff'] = df['network_in'].diff()
    
    # === Cyclical Features ===
    # Sin/cos encoding for hour, preserving periodicity
    df['hour_sin'] = np.sin(2 * np.pi * df['hour_of_day'] / 24)
    df['hour_cos'] = np.cos(2 * np.pi * df['hour_of_day'] / 24)
    df['day_sin'] = np.sin(2 * np.pi * df['day_of_week'] / 7)
    df['day_cos'] = np.cos(2 * np.pi * df['day_of_week'] / 7)
    
    # Drop NaN rows from lagging
    df = df.dropna()
    
    return df

# predictor.py — AI prediction engine
import numpy as np
import pandas as pd
from prophet import Prophet
import tensorflow as tf
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.preprocessing import StandardScaler
from datetime import timedelta

class AIPredictor:
    """
    Hybrid AI Predictor
    
    Combines three models:
    - Prophet: excels at capturing seasonality and trends
    - LSTM: captures non-linear temporal dependencies
    - GBR/RF: handles multi-dimensional features well
    """
    
    def __init__(self, forecast_horizon=24):
        self.forecast_horizon = forecast_horizon  # Predict next 24 hours
        self.prophet_model = None
        self.lstm_model = None
        self.rf_model = None
        self.scaler = StandardScaler()
        
    def train_prophet(self, df):
        """Train Prophet model — time series decomposition"""
        prophet_df = df[['timestamp', 'cpu_usage']].copy()
        prophet_df.columns = ['ds', 'y']
        
        self.prophet_model = Prophet(
            daily_seasonality=True,
            weekly_seasonality=True,
            yearly_seasonality=False,
            changepoint_prior_scale=0.05,
            seasonality_prior_scale=10
        )
        self.prophet_model.fit(prophet_df)
        print("✅ Prophet model trained")
        
    def train_lstm(self, df, sequence_length=48):
        """Train LSTM — deep learning for time series"""
        values = df['cpu_usage'].values.reshape(-1, 1)
        scaled = self.scaler.fit_transform(values)
        
        X, y = [], []
        for i in range(sequence_length, len(scaled)):
            X.append(scaled[i-sequence_length:i])
            y.append(scaled[i])
        
        X, y = np.array(X), np.array(y)
        
        model = tf.keras.Sequential([
            tf.keras.layers.LSTM(64, return_sequences=True, 
                                input_shape=(sequence_length, 1)),
            tf.keras.layers.Dropout(0.2),
            tf.keras.layers.LSTM(32, return_sequences=False),
            tf.keras.layers.Dropout(0.2),
            tf.keras.layers.Dense(16, activation='relu'),
            tf.keras.layers.Dense(1)
        ])
        
        model.compile(optimizer='adam', loss='mse')
        model.fit(X, y, epochs=30, batch_size=32, verbose=0)
        
        self.lstm_model = model
        print("✅ LSTM model trained")
        
    def train_gradient_boosting(self, df):
        """Train Gradient Boosting — feature-based prediction"""
        feature_cols = [col for col in df.columns 
                       if col not in ['timestamp', 'cpu_usage']]
        
        X = df[feature_cols].values
        y = df['cpu_usage'].values
        
        self.rf_model = GradientBoostingRegressor(
            n_estimators=200,
            max_depth=5,
            learning_rate=0.1,
            random_state=42
        )
        self.rf_model.fit(X, y)
        print("✅ Gradient Boosting model trained")
        
    def predict_next_24h(self, last_data_points):
        """
        Fuse predictions from all three models
        
        Returns: (predictions, confidence_intervals)
        """
        # Prophet prediction
        future_df = self.prophet_model.make_future_dataframe(periods=24, freq='h')
        prophet_pred = self.prophet_model.predict(future_df)
        prophet_forecast = prophet_pred['yhat'].tail(24).values
        
        # LSTM prediction (rolling)
        lstm_predictions = []
        current_seq = last_data_points[-48:].values.reshape(-1, 1)
        for _ in range(24):
            current_seq = current_seq.reshape(1, 48, 1)
            pred = self.lstm_model.predict(current_seq, verbose=0)[0][0]
            pred = self.scaler.inverse_transform([[pred]])[0][0]
            lstm_predictions.append(pred)
            current_seq = np.append(current_seq[0][1:], pred)
        
        # Weighted fusion
        weights = {'prophet': 0.4, 'lstm': 0.3, 'gb': 0.3}
        
        # GB prediction (baseline from last feature vector)
        gb_pred = self.rf_model.predict(last_data_points.iloc[[-1]].values)[0]
        gb_forecast = np.full(24, gb_pred)
        
        fused = (weights['prophet'] * prophet_forecast + 
                weights['lstm'] * np.array(lstm_predictions) +
                weights['gb'] * gb_forecast)
        
        # Confidence intervals based on model variance
        std_dev = np.std([prophet_forecast, lstm_predictions, [gb_pred]*24], axis=0)
        upper_bound = fused + 1.96 * std_dev
        lower_bound = fused - 1.96 * std_dev
        
        return fused, lower_bound, upper_bound

# autoscaler.py — Auto-scaling controller
import requests
import json
from datetime import datetime, timedelta

class VPSAutoscaler:
    """
    VPS Auto-scaling Controller
    
    Automatically adjusts resource configuration based on AI predictions.
    Supports multiple cloud provider APIs.
    """
    
    def __init__(self, config):
        self.config = config
        self.cloud_provider = config.get('provider', 'hetzner')
        self.min_vcpus = config.get('min_vcpus', 1)
        self.max_vcpus = config.get('max_vcpus', 8)
        self.min_memory_gb = config.get('min_memory_gb', 1)
        self.max_memory_gb = config.get('max_memory_gb', 16)
        
    def evaluate_scaling_action(self, predictions, current_usage):
        """
        Evaluate whether scaling is needed and by how much.
        
        Args:
            predictions: Next 24 hours predicted values
            current_usage: Current resource utilization
            
        Returns:
            dict with action, target specs, urgency level
        """
        peak_prediction = np.max(predictions[:6])  # Next 6h peak
        avg_prediction = np.mean(predictions[:6])
        
        # 30% buffer for safety margin
        buffer_factor = 1.3
        
        required_cpu = int(np.ceil(avg_prediction * buffer_factor / 10))
        required_memory = int(np.ceil(peak_prediction * buffer_factor / 20))
        
        # Clamp to min/max
        required_cpu = max(self.min_vcpus, min(self.max_vcpus, required_cpu))
        required_memory = max(self.min_memory_gb, 
                             min(self.max_memory_gb, required_memory))
        
        current_cpu = current_usage.get('current_vcpus', 2)
        current_memory = current_usage.get('current_memory_gb', 2)
        
        if required_cpu > current_cpu or required_memory > current_memory:
            action = 'scale_up'
            urgency = 'high' if (required_cpu > current_cpu * 2) else 'medium'
        elif required_cpu < current_cpu * 0.7 and required_memory < current_memory * 0.7:
            action = 'scale_down'
            urgency = 'low'
        else:
            action = 'no_change'
            urgency = 'none'
            
        return {
            'action': action,
            'target_vcpus': required_cpu,
            'target_memory_gb': required_memory,
            'urgency': urgency,
            'reason': f"Predicted peak: CPU {peak_prediction:.1f}%, Memory {np.max(predictions[:6]):.1f}%"
        }
    
    def execute_scaling(self, scaling_decision):
        """Execute the scaling decision"""
        action = scaling_decision['action']
        
        if action == 'no_change':
            print(f"ℹ️ No scaling needed: {scaling_decision['reason']}")
            return {'status': 'skipped'}
        
        print(f"🔄 Executing scaling: {action}")
        print(f"   Target: CPU={scaling_decision['target_vcpus']} cores, "
              f"Memory={scaling_decision['target_memory_gb']}GB")
        print(f"   Reason: {scaling_decision['reason']}")
        
        if action == 'scale_up':
            return self._scale_up(scaling_decision)
        elif action == 'scale_down':
            return self._scale_down(scaling_decision)
    
    def _scale_up(self, decision):
        """Scale up — move to higher-tier instance"""
        if self.cloud_provider == 'hetzner':
            return self._hetzner_upgrade(decision)
        elif self.cloud_provider == 'aws':
            return self._aws_upgrade(decision)
        return {'status': 'unsupported_provider'}
    
    def _hetzner_upgrade(self, decision):
        """Hetzner upgrade strategy"""
        target_plan = self._find_cheapest_plan(
            vcpus=decision['target_vcpus'],
            memory_gb=decision['target_memory_gb']
        )
        
        # Execution steps:
        # 1. Provision new instance
        # 2. Migrate data via rsync
        # 3. Switch DNS
        # 4. Decommission old instance
        print(f"📋 Hetzner upgrade plan: {target_plan}")
        print("   ⚠️ Requires manual confirmation or automated migration script")
        return {'status': 'planned', 'plan': target_plan}
    
    def _find_cheapest_plan(self, vcpus, memory_gb):
        """Find the cheapest matching plan"""
        plans = [
            {'name': 'cx22', 'vcpus': 2, 'memory': 4, 'price_usd': 4.50},
            {'name': 'cx32', 'vcpus': 4, 'memory': 8, 'price_usd': 9.50},
            {'name': 'cx42', 'vcpus': 4, 'memory': 16, 'price_usd': 15.20},
            {'name': 'cx52', 'vcpus': 8, 'memory': 16, 'price_usd': 29.50},
        ]
        for plan in plans:
            if plan['vcpus'] >= vcpus and plan['memory'] >= memory_gb:
                return plan
        return plans[-1]

    def _scale_down(self, decision):
        """Scale down — move to lower-tier instance"""
        print("📋 Downgrade plan generation...")
        return {'status': 'planned'}

#!/bin/bash
# deploy-autoscaler.sh — One-click AI predictive scaling deployment

set -euo pipefail

echo "🤖 Deploying AI predictive scaling system..."

# 1. Create project directory
PROJECT_DIR="/opt/ai-autoscaler"
mkdir -p $PROJECT_DIR/{data,models,logs,scripts}

# 2. Install dependencies
pip3 install prophet tensorflow scikit-learn pandas numpy requests

# 3. Create systemd service
cat > /etc/systemd/system/ai-autoscaler.service << 'EOF'
[Unit]
Description=AI VPS Autoscaler
After=network.target prometheus.service

[Service]
Type=simple
User=root
WorkingDirectory=/opt/ai-autoscaler
ExecStart=/usr/bin/python3 /opt/ai-autoscaler/scripts/predict_and_scale.py
Restart=always
RestartSec=60
StandardOutput=append:/opt/ai-autoscaler/logs/autoscaler.log
StandardError=append:/opt/ai-autoscaler/logs/autoscaler-error.log

[Install]
WantedBy=multi-user.target
EOF

# 4. Create Cron schedule
crontab -l 2>/dev/null | grep -v "ai-autoscaler" | crontab -
(crontab -l 2>/dev/null; echo "*/15 * * * * /opt/ai-autoscaler/scripts/predict_and_scale.py >> /opt/ai-autoscaler/logs/cron.log 2>&1") | crontab -

# 5. Start service
systemctl daemon-reload
systemctl enable ai-autoscaler
systemctl start ai-autoscaler

echo "✅ AI predictive scaling system deployed!"
echo "   Status: systemctl status ai-autoscaler"
echo "   Logs: tail -f /opt/ai-autoscaler/logs/autoscaler.log"

#!/usr/bin/env python3
# predict_and_scale.py — Daily prediction and scaling decisions

import sys
import os
import json
import requests
import numpy as np
import pandas as pd
from datetime import datetime, timedelta
from pathlib import Path

sys.path.insert(0, str(Path(__file__).parent))

from predictor import AIPredictor
from autoscaler import VPSAutoscaler
from features import extract_features

# === Configuration ===
PROMETHEUS_URL = os.getenv('PROMETHEUS_URL', 'http://localhost:9090')
AUTOSCALER_CONFIG = {
    'provider': 'hetzner',
    'min_vcpus': 1,
    'max_vcpus': 4,
    'min_memory_gb': 1,
    'max_memory_gb': 8,
}
TELEGRAM_BOT_TOKEN = os.getenv('TELEGRAM_BOT_TOKEN', '')
TELEGRAM_CHAT_ID = os.getenv('TELEGRAM_CHAT_ID', '')

def fetch_prometheus_data(metric, start_hours=168):
    """Fetch metric data from Prometheus"""
    end = datetime.now()
    start = end - timedelta(hours=start_hours)
    
    query = f'{metric}{{instance=~".*"}}'
    url = f'{PROMETHEUS_URL}/api/v1/query_range'
    params = {
        'query': query,
        'start': int(start.timestamp()),
        'end': int(end.timestamp()),
        'step': '300',  # 5-minute granularity
    }
    
    resp = requests.get(url, params=params, timeout=10)
    resp.raise_for_status()
    data = resp.json()['data']
    
    if not data['result']:
        print("⚠️ No data from Prometheus, using defaults")
        return None
    
    timestamps = data['result'][0]['values']
    values = [float(v[1]) for v in timestamps]
    times = [datetime.fromtimestamp(int(v[0])) for v in timestamps]
    
    df = pd.DataFrame({'timestamp': times, 'cpu_usage': values})
    return df

def get_current_resource_usage():
    """Get current resource utilization from Node Exporter"""
    try:
        resp = requests.get('http://localhost:9100/metrics', timeout=5)
        metrics = {}
        for line in resp.text.split('\n'):
            if line.startswith('#'):
                continue
            parts = line.split()
            if len(parts) >= 2:
                key = parts[0].split('{')[0]
                metrics[key] = float(parts[1])
        
        cpu_idle = metrics.get('node_cpu_seconds_total{mode="idle"}', 0)
        cpu_usage = 100 - cpu_idle
        mem_total = metrics.get('node_memory_MemTotal_bytes', 1)
        mem_available = metrics.get('node_memory_MemAvailable_bytes', mem_total)
        memory_usage = (1 - mem_available / mem_total) * 100
        
        return {
            'current_vcpus': 2,
            'current_memory_gb': 4,
            'cpu_usage_percent': cpu_usage,
            'memory_usage_percent': memory_usage,
        }
    except Exception as e:
        print(f"⚠️ Failed to get current resources: {e}")
        return {'current_vcpus': 2, 'current_memory_gb': 4}

def send_notification(message):
    """Send Telegram notification"""
    if not TELEGRAM_BOT_TOKEN or not TELEGRAM_CHAT_ID:
        return
    
    url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
    requests.post(url, json={
        'chat_id': TELEGRAM_CHAT_ID,
        'text': message,
        'parse_mode': 'HTML'
    }, timeout=10)

def main():
    print(f"\n{'='*60}")
    print(f"🤖 AI Predictive Scaling — {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"{'='*60}\n")
    
    # Step 1: Fetch historical data
    print("📊 Step 1/4: Fetching historical data...")
    history_df = fetch_prometheus_data('node_cpu_seconds_total')
    if history_df is None or len(history_df) < 168:
        print("⚠️ Less than 7 days of history, generating demo data")
        hours = pd.date_range(end=datetime.now(), periods=168, freq='h')
        np.random.seed(42)
        cpu_data = 30 + 20*np.sin(2*np.pi*hours.hour/24) + \
                   15*np.sin(2*np.pi*hours.dayofweek/7) + \
                   np.random.normal(0, 5, 168)
        history_df = pd.DataFrame({
            'timestamp': hours,
            'cpu_usage': np.clip(cpu_data, 5, 95)
        })
    
    print(f"   ✅ Collected {len(history_df)} data points")
    
    # Step 2: Feature engineering
    print("\n🔧 Step 2/4: Feature engineering...")
    features_df = extract_features(history_df)
    print(f"   ✅ Generated {len(features_df.columns)} features")
    
    # Step 3: Train and predict
    print("\n🧠 Step 3/4: AI prediction...")
    predictor = AIPredictor(forecast_horizon=24)
    predictor.train_prophet(features_df)
    predictor.train_gradient_boosting(features_df)
    
    predictions, lower, upper = predictor.predict_next_24h(features_df)
    
    print(f"   ✅ 24-hour forecast:")
    peak_hour_idx = np.argmax(predictions[:6])
    peak_value = predictions[peak_hour_idx]
    print(f"   📈 Next 6h peak: {peak_value:.1f}% (expected at {peak_hour_idx}:00)")
    print(f"   📉 Next 6h average: {np.mean(predictions[:6]):.1f}%")
    
    # Step 4: Evaluate and act
    print("\n⚙️  Step 4/4: Evaluating scaling decision...")
    current_usage = get_current_resource_usage()
    autoscaler = VPSAutoscaler(AUTOSCALER_CONFIG)
    
    decision = autoscaler.evaluate_scaling_action(predictions, current_usage)
    print(f"   📋 Decision: {decision['action']} (urgency: {decision['urgency']})")
    print(f"   💬 Reason: {decision['reason']}")
    
    result = autoscaler.execute_scaling(decision)
    
    # Generate report
    report = {
        'timestamp': datetime.now().isoformat(),
        'prediction': {
            'peak_6h': float(peak_value),
            'avg_6h': float(np.mean(predictions[:6])),
            'forecast': [float(p) for p in predictions],
        },
        'decision': decision,
        'result': result,
    }
    
    # Save report
    log_dir = Path('/opt/ai-autoscaler/logs')
    log_dir.mkdir(parents=True, exist_ok=True)
    with open(log_dir / f"decision-{datetime.now().strftime('%Y%m%d-%H%M%S')}.json", 'w') as f:
        json.dump(report, f, indent=2, default=str)
    
    # Send notification
    notification_msg = format_notification(report)
    send_notification(notification_msg)
    
    print(f"\n{'='*60}")
    print(f"✅ Scaling cycle completed")
    print(f"   Report saved to: {log_dir}")
    print(f"{'='*60}\n")

def format_notification(report):
    """Format notification message"""
    msg = f"<b>🤖 AI Scaling Report</b>\n\n"
    msg += f"⏰ {report['timestamp']}\n"
    msg += f"📈 Predicted peak (6h): {report['prediction']['peak_6h']:.1f}%\n"
    msg += f"📊 Decision: {report['decision']['action']}\n"
    msg += f"💬 {report['decision']['reason']}"
    return msg

if __name__ == '__main__':
    main()

# Cold start protection
if days_of_history < 7:
    buffer_factor = 1.5  # More conservative
    mode = 'monitoring_only'  # Monitor only, no execution
elif days_of_history < 14:
    buffer_factor = 1.4
    mode = 'approved_auto'  # Auto after approval
else:
    buffer_factor = 1.3
    mode = 'fully_auto'  # Fully automatic

# dashboard-config.json — Key panels
panels:
  - title: "CPU Prediction vs Actual"
    type: graph
    queries:
      - prediction: "predictor_cpu_forecast"
      - actual: "node_cpu_usage_actual"
  
  - title: "Scaling Decision History"
    type: table
    columns:
      - Timestamp
      - Decision Type
      - Predicted Peak
      - Execution Result
  
  - title: "Cost Trend"
    type: stat
    metrics:
      - monthly_spend
      - predicted_savings
      - roi_percentage