# promtail.yaml base configuration
scrape_configs:
  - job_name: nginx
    static_configs:
      - targets: ['localhost']
        labels:
          job: nginx-access
          __path__: /var/log/nginx/access.log
  - job_name: app
    static_configs:
      - targets: ['localhost']
        labels:
          job: myapp
          __path__: /var/log/myapp/*.log

# log_classifier.py — AI Log Classifier
from openai import OpenAI

client = OpenAI(base_url="http://localhost:11434/v1", api_key="ollama")

CATEGORY_PROMPT = """
Analyze the following log snippet and determine its category and severity:
- Category: access/error/security/performance/config/debug
- Severity: info/warning/critical
- Needs long-term retention: yes/no

Log content: {log_sample}

Return JSON: {{"category": "...", "severity": "...", "retain": true/false}}
"""

def classify_log(line: str) -> dict:
    response = client.chat.completions.create(
        model="qwen2.5:7b",
        messages=[{"role": "user", "content": CATEGORY_PROMPT.format(log_sample=line[:500])}]
    )
    return json.loads(response.choices[0].message.content)

# retention_predictor.py
import numpy as np
from sklearn.ensemble import GradientBoostingRegressor
import joblib

class RetentionPredictor:
    """Predict optimal log retention period based on historical data"""
    
    def __init__(self):
        self.model = GradientBoostingRegressor(n_estimators=100, max_depth=4)
        self.fitted = False
    
    def extract_features(self, log_meta: dict) -> list:
        """Extract features affecting retention period"""
        return [
            log_meta['daily_volume_mb'],       # Daily log volume
            log_meta['error_rate'],             # Error rate
            log_meta['compliance_required'],   # Compliance requirement
            log_meta['debug_ratio'],           # Debug log ratio
            log_meta['business_type'],         # Business type encoding
            log_meta['search_frequency'],      # Historical query frequency
        ]
    
    def predict_retention_days(self, log_meta: dict) -> int:
        features = self.extract_features(log_meta)
        if self.fitted:
            days = int(self.model.predict([features])[0])
            return max(7, min(2555, days))  # 7 days ~ 7 years
        # Default strategy
        return self._default_strategy(log_meta)
    
    def _default_strategy(self, meta: dict) -> int:
        if meta.get('compliance_required'):
            return 2555  # 7 years
        if meta.get('error_rate', 0) > 0.05:
            return 365   # High error rate → 1 year retention
        if meta['daily_volume_mb'] > 100:
            return 30    # Large logs → fast rotation
        return 90      # Default 90 days

# policy_generator.py
POLICY_PROMPT = """
You are a VPS log management expert. Generate a log retention policy based on:

Log type: {log_type}
Daily size: {daily_size} MB
Current disk usage: {disk_usage}%
Compliance requirement: {compliance}
Historical query frequency (last 30 days): {query_freq} times
Business importance: {business_importance}

Output JSON policy:
{{
  "retention_hot": N,      # Hot tier retention (ms-level retrieval)
  "retention_warm": N,     # Warm tier retention (second-level retrieval)
  "retention_cold": N,     # Cold tier retention (minute-level retrieval)
  "compression": "...",    # Compression algorithm
  "delete_after": N        # Final deletion (days)
}}
"""

def generate_policy(log_info: dict) -> dict:
    response = client.chat.completions.create(
        model="qwen2.5:7b",
        messages=[{"role": "user", "content": POLICY_PROMPT.format(**log_info)}]
    )
    return json.loads(response.choices[0].message.content)

# Loki tiered storage configuration
schema_config:
  configs:
    - from: 2024-01-01
      store: tsdb
      object_store: s3
      schema: v13
      index:
        prefix: index_
        period: 24h

storage_config:
  aws:
    bucketnames: vps-logs-bucket
    region: ap-east-1
    endpoint: s3.ap-east-1.amazonaws.com
  filesystem:
    directory: /var/log/loki/chunks

#!/bin/bash
# smart_tier_migration.sh — AI-driven log tier migration

set -euo pipefail

LOKI_API="http://localhost:3100"
COLD_STORAGE="/mnt/cold-storage/logs"
WARM_STORAGE="/mnt/warm-storage/logs"
HOT_STORAGE="/var/log/loki"

# Get AI-recommended migration policy
MIGRATION_POLICY=$(python3 << 'PYEOF'
import json
from retention_predictor import RetentionPredictor

predictor = RetentionPredictor()
policies = {}

for log_type in ['nginx-access', 'nginx-error', 'app-debug', 'app-error', 'syslog']:
    meta = {
        'daily_volume_mb': 50 if 'access' in log_type else 5,
        'error_rate': 0.02 if 'error' in log_type else 0.001,
        'compliance_required': log_type in ['app-error'],
        'debug_ratio': 0.8 if 'debug' in log_type else 0.1,
        'business_type': 1 if 'app' in log_type else 0,
        'search_frequency': 100 if 'access' in log_type else 5,
    }
    policies[log_type] = {
        'hot_days': 7,
        'warm_days': predictor.predict_retention_days(meta) // 3,
        'cold_days': predictor.predict_retention_days(meta),
    }

print(json.dumps(policies, indent=2))
PYEOF
)

echo "Migration policy: $MIGRATION_POLICY" | jq -c 'to_entries[]' | while read -r entry; do
    log_type=$(echo "$entry" | jq -r '.key')
    warm_days=$(echo "$entry" | jq -r '.value.warm_days')
    
    # Migrate logs exceeding hot tier to warm tier
    find "$HOT_STORAGE/$log_type" -name "*.gz" -mtime +${warm_days} -exec mv {} "$WARM_STORAGE/" \;
    
    # Compress and move logs exceeding warm tier to cold storage
    find "$WARM_STORAGE/$log_type" -name "*.gz" -mtime +$((warm_days * 2)) -exec gzip -c {} >> "$COLD_STORAGE/${log_type}_$(date +%Y%m%d).tar.gz" \;
done

echo "Tier migration completed at $(date)"

# capacity_forecaster.py
from statsmodels.tsa.holtwinters import ExponentialSmoothing
import numpy as np

class DiskForecaster:
    """Holt-Winters based disk capacity forecasting"""
    
    def __init__(self, history_days=60):
        self.history_days = history_days
        self.data = self._load_history()
    
    def _load_history(self):
        np.random.seed(42)
        return np.array([10 + i*0.5 + np.random.normal(0, 1) 
                        for i in range(self.history_days)])
    
    def predict_until_full(self, total_capacity_gb: float) -> int:
        """Predict when disk will be full"""
        model = ExponentialSmoothing(
            self.data, 
            trend='add', 
            seasonal='add',
            seasonal_periods=7
        )
        fit = model.fit()
        forecast = fit.forecast(365)
        capacity = total_capacity_gb * 1024
        
        for day, usage in enumerate(forecast):
            if usage > capacity:
                return day
        return 365
    
    def optimize_cost(self, current_cost_per_gb: float, 
                     cold_cost_per_gb: float) -> dict:
        """Calculate tiered storage cost optimization"""
        projected_growth = np.mean(np.diff(self.data[-7:]))
        cold_capable_days = 90
        hot_data_mb = np.mean(self.data[-7:]) * 7
        
        return {
            'daily_growth_mb': round(float(projected_growth), 2),
            'days_until_full': int(self.predict_until_full(100)),
            'hot_storage_mb': round(float(hot_data_mb), 2),
            'estimated_monthly_savings': round(
                float(cold_capable_days * np.mean(self.data[-7:]) * 
                      (current_cost_per_gb - cold_cost_per_gb) / 1024), 2
            ),
            'recommendation': self._generate_recommendation(projected_growth, cold_capable_days)
        }

# docker-compose.yml — AI Log Lifecycle Management
version: '3.8'

services:
  promtail:
    image: grafana/promtail:latest
    volumes:
      - /var/log:/var/log:ro
      - ./promtail.yaml:/etc/promtail/config.yml:ro

  loki:
    image: grafana/loki:3.2.0
    ports:
      - "3100:3100"
    volumes:
      - lochi-data:/loki
    command: -config.file=/etc/loki/local-config.yaml

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      - GF_INSTALL_PLUGINS=grafana-lokiexplore-app
    volumes:
      - grafana-data:/var/lib/grafana

  log-lifecycle-agent:
    build: ./agent
    volumes:
      - ./agent:/app
      - /var/log:/host-log:ro
      - ./policies:/app/policies
    environment:
      - OLLAMA_HOST=http://ollama:11434
      - LOKI_API=http://loki:3100
    depends_on:
      - loki
      - ollama
    schedule: "0 */6 * * *"

  ollama:
    image: ollama/ollama:latest
    ports:
      - "11434:11434"
    volumes:
      - ollama-data:/root/.ollama
    command: serve

  prometheus:
    image: prom/prometheus:latest
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prom-data:/prometheus
    ports:
      - "9090:9090"

volumes:
  lochi-data:
  grafana-data:
  ollama-data:
  prom-data:

# agent/main.py — Log Lifecycle Management AI Agent
import os
import json
from datetime import datetime
from pathlib import Path
import requests

from log_classifier import classify_log
from retention_predictor import RetentionPredictor
from capacity_forecaster import DiskForecaster
from policy_generator import generate_policy

OLLAMA_HOST = os.getenv("OLLAMA_HOST", "http://localhost:11434")
LOKI_API = os.getenv("LOKI_API", "http://localhost:3100")
POLICY_DIR = Path("/app/policies")
HOST_LOG_DIR = Path("/host-log")

def analyze_and_update_policies():
    """Main analysis loop: collect → classify → generate policy → execute"""
    print(f"[{datetime.now()}] Starting log lifecycle analysis...")
    
    predictor = RetentionPredictor()
    forecaster = DiskForecaster()
    
    # 1. Scan all log directories
    log_types = {}
    for log_dir in HOST_LOG_DIR.iterdir():
        if not log_dir.is_dir():
            continue
        files = list(log_dir.glob("*.log*"))
        if not files:
            continue
        
        total_size = sum(f.stat().st_size for f in files)
        sample_lines = []
        for f in files[:3]:
            with open(f) as fp:
                for i, line in enumerate(fp):
                    if i < 3:
                        sample_lines.append(line.strip())
                    if i >= 10:
                        break
        
        # 2. AI classification
        categories = {}
        for line in sample_lines:
            cat = classify_log(line)
            cat_key = cat.get('category', 'unknown')
            categories[cat_key] = categories.get(cat_key, 0) + 1
        
        log_types[log_dir.name] = {
            'size_mb': round(total_size / 1024 / 1024, 2),
            'file_count': len(files),
            'categories': categories,
        }
    
    # 3. Generate policies for each log type
    policies = {}
    for log_type, meta in log_types.items():
        policy_meta = {
            'daily_volume_mb': meta['size_mb'] / 7,
            'error_rate': meta['categories'].get('error', 0) / max(1, sum(meta['categories'].values())),
            'compliance_required': any(k in log_type for k in ['payment', 'auth', 'audit']),
            'debug_ratio': meta['categories'].get('debug', 0) / max(1, sum(meta['categories'].values())),
            'business_type': 1 if 'app' in log_type else 0,
            'search_frequency': 50 if 'access' in log_type else 5,
        }
        
        llm_policy = generate_policy({
            'log_type': log_type,
            'daily_size': policy_meta['daily_volume_mb'],
            'disk_usage': 45,
            'compliance': 'yes' if policy_meta['compliance_required'] else 'no',
            'query_freq': policy_meta['search_frequency'],
            'business_importance': 'high' if policy_meta['business_type'] else 'medium',
        })
        
        ml_retention = predictor.predict_retention_days(policy_meta)
        
        policies[log_type] = {
            'llm_policy': llm_policy,
            'ml_retention_days': ml_retention,
            'generated_at': datetime.now().isoformat(),
        }
    
    # 4. Save policies
    POLICY_DIR.mkdir(parents=True, exist_ok=True)
    with open(POLICY_DIR / "current_policies.json", "w") as f:
        json.dump(policies, f, indent=2)
    
    # 5. Cost optimization report
    cost_report = forecaster.optimize_cost(0.10, 0.02)
    
    report = {
        'log_policies': policies,
        'cost_optimization': cost_report,
        'generated_at': datetime.now().isoformat(),
    }
    
    with open(POLICY_DIR / "cost_report.json", "w") as f:
        json.dump(report, f, indent=2)
    
    print(f"[{datetime.now()}] Analysis complete. {len(policies)} log types processed.")
    print(f"  Estimated monthly savings: ${cost_report['estimated_monthly_savings']:.2f}")

if __name__ == "__main__":
    analyze_and_update_policies()

# crontab — AI Log Lifecycle Management
# Re-evaluate policies every 6 hours
0 */6 * * * cd /opt/log-lifecycle-agent && python3 main.py >> /var/log/log-lifecycle-agent.log 2>&1

# Execute tier migration daily at 2 AM
0 2 * * * /opt/log-lifecycle-agent/smart_tier_migration.sh

# Generate weekly cost report
0 3 * * 0 /opt/log-lifecycle-agent/generate_weekly_report.sh

# When AI detects anomalous patterns, auto-adjust retention policy
def on_anomaly_detected(log_type: str, anomaly_type: str):
    """Anomaly detection triggers retention policy adjustment"""
    policy_path = POLICY_DIR / "current_policies.json"
    with open(policy_path) as f:
        policies = json.load(f)
    
    # Security-related logs auto-extend retention
    if 'security' in anomaly_type or 'intrusion' in anomaly_type:
        policies[log_type]['ml_retention_days'] = max(
            policies[log_type]['ml_retention_days'], 365
        )
        policies[log_type]['auto_extended'] = True
        policies[log_type]['extend_reason'] = f"Security anomaly: {anomaly_type}"
        
        with open(policy_path, "w") as f:
            json.dump(policies, f, indent=2)
        
        print(f"[ALERT] Extended retention for {log_type} to 365 days")

# Alertmanager rules
groups:
  - name: log_lifecycle
    rules:
      - alert: LogVolumeAnomaly
        expr: rate(log_volume_bytes[1h]) > 3 * avg(rate(log_volume_bytes[24h]))
        for: 10m
        annotations:
          summary: "Abnormal log volume spike — {{ $labels.job }}"
      
      - alert: StorageTierMigrationFailed
        expr: log_tier_migration_errors > 0
        for: 5m
        annotations:
          summary: "Log tier migration failed"