#!/bin/bash
# backup-behavior-analyzer.sh - Collect server behavior data

COLLECT_DIR="/var/lib/backup-analyzer"
mkdir -p "$COLLECT_DIR/daily" "$COLLECT_DIR/hourly"

# Collect write activity for the current hour
HOUR=$(date +%Y%m%d-%H)
echo "$(date +%s)" > "$COLLECT_DIR/hourly/$HOUR.timestamp"

# Count file changes in the last hour (via inotify or diff)
find /etc /var/www /home -mmin -60 -type f 2>/dev/null | wc -l > "$COLLECT_DIR/hourly/$HOUR.changes"

# Estimate disk write volume
iostat -x 1 5 | awk '/^sd/ {print $NF}' | tail -1 > "$COLLECT_DIR/hourly/$HOUR.write_mb"

# Mark business peak hours (heuristic-based)
# Simple heuristic: 9-18 on weekdays = peak
DAY=$(date +%u)
HOUR_INT=$(date +%H)
if [ "$DAY" -ge 1 ] && [ "$DAY" -le 5 ] && [ "$HOUR_INT" -ge 9 ] && [ "$HOUR_INT" -le 18 ]; then
    echo "peak" > "$COLLECT_DIR/hourly/$HOUR.period"
else
    echo "offpeak" > "$COLLECT_DIR/hourly/$HOUR.period"
fi

#!/usr/bin/env python3
"""backup-behavior-ai.py - Generate backup strategy recommendations from historical data"""

import json
import os
import glob
from datetime import datetime, timedelta

class BackupBehaviorAnalyzer:
    def __init__(self, data_dir="/var/lib/backup-analyzer"):
        self.data_dir = data_dir
        self.history = self._load_history()

    def _load_history(self):
        """Load 7-day behavior data"""
        history = []
        for f in sorted(glob.glob(f"{self.data_dir}/hourly/*.changes")):
            day = os.path.basename(f).replace('.changes', '')
            try:
                with open(f) as fh:
                    changes = int(fh.read().strip())
            except:
                changes = 0
            history.append({"day": day, "changes": changes})
        return history

    def analyze_pattern(self):
        """Analyze file change patterns, identify peak/off-peak periods"""
        if len(self.history) < 7:
            return {"status": "insufficient_data", "message": "Need at least 7 days of data"}

        # Calculate average daily changes and standard deviation
        changes = [h["changes"] for h in self.history]
        avg_changes = sum(changes) / len(changes)
        variance = sum((c - avg_changes) ** 2 for c in changes) / len(changes)
        std_dev = variance ** 0.5

        # Identify high-change and low-change days
        high_change_days = sum(1 for c in changes if c > avg_changes + std_dev)
        low_change_days = sum(1 for c in changes if c < avg_changes - std_dev)

        # Smart backup frequency recommendation
        if avg_changes > 50:
            suggested_freq = "every_6h"      # High activity → every 6 hours
        elif avg_changes > 20:
            suggested_freq = "daily"          # Medium → daily
        else:
            suggested_freq = "weekly"         # Low → weekly

        # RPO (Recovery Point Objective) recommendation
        if high_change_days >= 4:
            rpo_hours = 6
        elif high_change_days >= 2:
            rpo_hours = 12
        else:
            rpo_hours = 24

        return {
            "avg_daily_changes": round(avg_changes, 1),
            "std_dev": round(std_dev, 1),
            "high_change_days": high_change_days,
            "low_change_days": low_change_days,
            "suggested_frequency": suggested_freq,
            "recommended_rpo_hours": rpo_hours,
            "confidence": min(1.0, len(self.history) / 30)
        }

    def get_optimal_schedule(self):
        """Generate optimal backup schedule"""
        pattern = self.analyze_pattern()

        schedule = {
            "frequency": pattern.get("suggested_frequency", "daily"),
            "rpo_hours": pattern.get("recommended_rpo_hours", 24),
            "retention": {
                "hourly": 24,      # Keep 24 hourly snapshots
                "daily": 30,       # Keep 30 days daily
                "weekly": 12,      # Keep 12 weeks weekly
                "monthly": 6       # Keep 6 months monthly
            },
            "offpeak_only": True,   # Execute full backups during off-peak
            "ai_confidence": pattern.get("confidence", 0)
        }

        return schedule

# Usage example
analyzer = BackupBehaviorAnalyzer()
schedule = analyzer.get_optimal_schedule()
print(json.dumps(schedule, indent=2, ensure_ascii=False))

#!/bin/bash
# backup-strategy-engine.sh - Execute backup strategy based on AI analysis

STRATEGY_FILE="/etc/backup-analyzer/strategy.json"
LOG_FILE="/var/log/backup-strategy.log"

log() {
    echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >> "$LOG_FILE"
}

# Read AI-generated strategy
if [ ! -f "$STRATEGY_FILE" ]; then
    log "Strategy file not found, using defaults"
    FREQUENCY="daily"
    RPO_HOURS=24
else
    FREQUENCY=$(python3 -c "import json; print(json.load(open('$STRATEGY_FILE'))['frequency'])")
    RPO_HOURS=$(python3 -c "import json; print(json.load(open('$STRATEGY_FILE'))['rpo_hours'])")
fi

log "Current strategy: frequency=$FREQUENCY, rpo=$RPO_HOURS hours"

# Execute backup based on strategy
execute_backup() {
    local backup_type=$1
    local target=$2
    local timestamp=$(date +%Y%m%d_%H%M%S)

    log "Starting ${backup_type} backup: ${target}"

    # Use rsync + hardlinks for incremental backup
    BACKUP_DEST="/backup/vps/${backup_type}/${timestamp}"
    mkdir -p "$BACKUP_DEST"

    case $backup_type in
        "full")
            # Full backup: back up all critical directories
            rsync -av --delete \
                --exclude='proc' --exclude='sys' --exclude='dev' \
                /etc/ "$BACKUP_DEST/etc/"
            rsync -av --delete /var/www/ "$BACKUP_DEST/var-www/"
            rsync -av --delete /home/ "$BACKUP_DEST/home/"
            # Database dumps
            mysqldump --all-databases -u root -p$(cat /etc/mysql/.root_pass) \
                > "$BACKUP_DEST/databases/full.sql" 2>/dev/null
            pg_dumpall -U postgres > "$BACKUP_DEST/databases/postgres.sql" 2>/dev/null
            ;;
        "incremental")
            # Incremental: only backed up changed files
            rsync -av --delete \
                --files-from=<(find /etc /var/www /home -mmin -$((RPO_HOURS * 60)) -type f 2>/dev/null) \
                / "$BACKUP_DEST/files/" 2>/dev/null
            ;;
    esac

    # Generate checksums
    find "$BACKUP_DEST" -type f ! -name "*.sha256" -exec sha256sum {} \; \
        > "$BACKUP_DEST/checksums.sha256" 2>/dev/null

    log "${backup_type} backup completed: $BACKUP_DEST"
    echo "$BACKUP_DEST"
}

# Main scheduling logic
NOW_HOUR=$(date +%H)
CURRENT_DAY=$(date +%u)  # 1=Monday, 7=Sunday

case $FREQUENCY in
    "every_6h")
        if [ $((10#$NOW_HOUR % 6)) -eq 0 ]; then
            execute_backup "incremental" "all"
        fi
        # Full backup daily at 3 AM on Mondays
        if [ $NOW_HOUR -eq 3 ] && [ $CURRENT_DAY -eq 1 ]; then
            execute_backup "full" "all"
        fi
        ;;
    "daily")
        if [ $NOW_HOUR -eq 3 ]; then
            execute_backup "incremental" "all"
        fi
        ;;
    "weekly")
        if [ $NOW_HOUR -eq 3 ] && [ $CURRENT_DAY -eq 1 ]; then
            execute_backup "full" "all"
        fi
        ;;
esac

#!/bin/bash
# backup-verify.sh - Automatically verify backup integrity

BACKUP_ROOT="/backup/vps"
ALERT_CHANNEL="${BACKUP_ALERT_URL:-}"  # webhook URL

verify_backup() {
    local backup_dir=$1
    local status="OK"
    local issues=""

    # Check if backup directory exists
    if [ ! -d "$backup_dir" ]; then
        echo "FAIL: Backup directory missing: $backup_dir"
        return 1
    fi

    # Check checksums
    if [ -f "$backup_dir/checksums.sha256" ]; then
        cd "$backup_dir" || return 1
        if sha256sum -c checksums.sha256 --quiet 2>&1; then
            echo "CHECKSUM: OK"
        else
            status="FAIL"
            issues="$issues\n- Checksum mismatch"
            echo "CHECKSUM: FAIL"
        fi
    else
        status="WARN"
        issues="$issues\n- Missing checksum file"
        echo "CHECKSUM: SKIPPED"
    fi

    # Check critical files
    critical_files=("$backup_dir/etc/passwd" "$backup_dir/etc/shadow")
    for f in "${critical_files[@]}"; do
        if [ -f "$f" ]; then
            echo "CRITICAL FILE: $(basename $f) OK"
        else
            status="WARN"
            issues="$issues\n- Missing critical file: $f"
        fi
    done

    # Check database files are parseable
    if [ -f "$backup_dir/databases/full.sql" ]; then
        sql_size=$(stat -c%s "$backup_dir/databases/full.sql" 2>/dev/null || echo 0)
        if [ "$sql_size" -gt 0 ]; then
            echo "DATABASE: OK ($sql_size bytes)"
        else
            status="FAIL"
            issues="$issues\n- Database backup is empty"
        fi
    fi

    # Check backup size is reasonable
    total_size=$(du -sm "$backup_dir" 2>/dev/null | cut -f1)
    if [ -n "$total_size" ]; then
        if [ "$total_size" -lt 1 ]; then
            status="FAIL"
            issues="$issues\n- Abnormal backup size: ${total_size}MB"
        else
            echo "SIZE: ${total_size}MB"
        fi
    fi

    if [ "$status" = "FAIL" ]; then
        echo "VERIFICATION RESULT: FAILED"
        echo -e "$issues" | while read line; do echo "  ISSUE:$line"; done
        if [ -n "$ALERT_CHANNEL" ]; then
            curl -s -X POST "$ALERT_CHANNEL" \
                -H "Content-Type: application/json" \
                -d "{\"text\":\"🚨 Backup verification failed: $backup_dir$issues\"}" 2>/dev/null
        fi
        return 1
    else
        echo "VERIFICATION RESULT: PASSED"
        return 0
    fi
}

# Scan all backups from the last 7 days
find "$BACKUP_ROOT" -maxdepth 3 -name "checksums.sha256" -printf '%h\n' 2>/dev/null | \
    while read dir; do
        age_days=$(( ($(date +%s) - $(stat -c%Y "$dir/checksums.sha256")) / 86400 ))
        if [ "$age_days" -le 7 ]; then
            echo "=== Verifying: $dir (age: ${age_days} days) ==="
            verify_backup "$dir"
            echo ""
        fi
    done

#!/bin/bash
# backup-restore-drill.sh - Automated restore drill

DRILL_CONTAINER="backup-drill-$(date +%Y%m%d%H%M%S)"
DRILL_DIR="/tmp/backup-drill"
LATEST_BACKUP=$(find /backup/vps/full -maxdepth 1 -type d -newer /tmp/.last_drill 2>/dev/null | sort -r | head -1)

if [ -z "$LATEST_BACKUP" ]; then
    LATEST_BACKUP=$(ls -td /backup/vps/full/*/ 2>/dev/null | head -1)
fi

if [ -z "$LATEST_BACKUP" ]; then
    echo "No full backup available for restore drill"
    exit 1
fi

echo "=== Backup Restore Drill ==="
echo "Using backup: $LATEST_BACKUP"

# Create temporary container for restore test
docker run -d --name "$DRILL_CONTAINER" \
    --privileged \
    -v "$LATEST_BACKUP/etc:/mnt/etc:ro" \
    -v "$LATEST_BACKUP/var-www:/mnt/var-www:ro" \
    -v "$LATEST_BACKUP/databases:/mnt/databases:ro" \
    ubuntu:22.04 sleep 300

sleep 5

# Validate restore inside container
DRILL_RESULTS="$DRILL_DIR/results-$(date +%s).txt"
mkdir -p "$DRILL_DIR"

docker exec "$DRILL_CONTAINER" bash -c '
    echo "=== Restore Drill Report ==="
    echo "Time: '"$(date)"'"
    echo ""

    # Check /etc restore
    echo "--- System Config Restore ---"
    if [ -f /mnt/etc/passwd ]; then
        user_count=$(wc -l < /mnt/etc/passwd)
        echo "✅ passwd file OK ($user_count users)"
    else
        echo "❌ passwd file missing"
    fi

    if [ -f /mnt/etc/hosts ]; then
        echo "✅ hosts file OK"
    else
        echo "❌ hosts file missing"
    fi

    # Check website files
    echo ""
    echo "--- Website Files Restore ---"
    if [ -d /mnt/var-www ]; then
        file_count=$(find /mnt/var-www -type f 2>/dev/null | wc -l)
        total_size=$(du -sh /mnt/var-www 2>/dev/null | cut -f1)
        echo "✅ Website files: $file_count files, $total_size"
    else
        echo "⚠️ Website directory not found"
    fi

    # Check databases
    echo ""
    echo "--- Database Restore ---"
    if [ -f /mnt/databases/full.sql ]; then
        sql_size=$(stat -c%s /mnt/databases/full.sql)
        if [ "$sql_size" -gt 100 ]; then
            echo "✅ Database backup valid ($sql_size bytes)"
        else
            echo "❌ Database backup too small, possibly corrupted"
        fi
    else
        echo "⚠️ No database backup file"
    fi

    echo ""
    echo "=== Drill Complete ==="
' > "$DRILL_RESULTS" 2>&1

# Cleanup container
docker rm -f "$DRILL_CONTAINER" >/dev/null 2>&1

# Output results
cat "$DRILL_RESULTS"

# Save results
cp "$DRILL_RESULTS" /var/log/backup-drills/
touch /tmp/.last_drill

echo ""
echo "Drill report saved to: $DRILL_RESULTS"

#!/bin/bash
# backup-distribute.sh - AI-driven multi-cloud backup distribution

BACKUP_SOURCE="/backup/vps"
RCLONE_REMOTE="remote"  # Pre-configured rclone remote

# AI decision: which backups go to which cloud
# Rule: Latest 3 full backups → all clouds; incremental → at least one cloud

get_latest_backups() {
    find "$BACKUP_SOURCE/full" -maxdepth 1 -type d -mtime -30 | sort -r | head -3
}

distribute_to_cloud() {
    local backup_dir=$1
    local cloud=$2
    local backup_name=$(basename "$backup_dir")

    echo "Distributing $backup_name → $cloud ..."

    # Sync with rclone, with encryption
    rclone sync "$backup_dir" "${RCLONE_REMOTE}:${backup_name}-${cloud}" \
        --progress \
        --transfers=4 \
        --checkers=8 \
        --log-file="/var/log/rclone-${cloud}.log" \
        --log-level=INFO \
        --drive-chunk-size=64M \
        --bwlimit=10M \
        2>&1 | tee -a "/var/log/backup-distribute.log"

    if [ $? -eq 0 ]; then
        echo "✅ $backup_name → $cloud synced successfully"
    else
        echo "❌ $backup_name → $cloud sync failed"
        send_alert "Backup distribution failed: $backup_name → $cloud"
    fi
}

# Get latest 3 full backups
for backup in $(get_latest_backups); do
    distribute_to_cloud "$backup" "backblaze-b2"
    distribute_to_cloud "$backup" "aws-s3"
    distribute_to_cloud "$backup" "local-nas"
done

# Incremental backups go to cheapest storage only
latest_incremental=$(find "$BACKUP_SOURCE/incremental" -maxdepth 1 -type d -mtime -7 | sort -r | head -1)
if [ -n "$latest_incremental" ]; then
    distribute_to_cloud "$latest_incremental" "cheapest-storage"
fi

# ~/.config/rclone/rclone.conf
[backblaze-b2]
type = b2
account = YOUR_B2_ACCOUNT
key = YOUR_B2_KEY
hard_delete = true

[aws-s3]
type = s3
provider = AWS
access_key_id = YOUR_AWS_KEY
secret_access_key = YOUR_AWS_SECRET
region = us-east-1
storage_class = STANDARD

[local-nas]
type = local
server = true
port = 5572

#!/bin/bash
# backup-dashboard.sh - Generate backup status dashboard

echo "╔══════════════════════════════════════════════╗"
echo "║     AI Smart Backup System - Status Dashboard║"
echo "╚══════════════════════════════════════════════╝"
echo ""

# 1. Backup statistics
echo "📊 Backup Overview"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
FULL_COUNT=$(find /backup/vps/full -maxdepth 1 -type d 2>/dev/null | wc -l)
INCR_COUNT=$(find /backup/vps/incremental -maxdepth 1 -type d 2>/dev/null | wc -l)
TOTAL_SIZE=$(du -sh /backup/vps 2>/dev/null | cut -f1)
echo "  Full backups:     $FULL_COUNT"
echo "  Incremental:      $INCR_COUNT"
echo "  Total storage:    $TOTAL_SIZE"

LATEST_FULL=$(ls -td /backup/vps/full/*/ 2>/dev/null | head -1)
if [ -n "$LATEST_FULL" ]; then
    LATEST_TIME=$(stat -c%y "$LATEST_FULL" | cut -d. -f1)
    echo "  Latest full:      $LATEST_TIME"
else
    echo "  Latest full:      None"
fi

LATEST_INCR=$(ls -td /backup/vps/incremental/*/ 2>/dev/null | head -1)
if [ -n "$LATEST_INCR" ]; then
    LATEST_TIME=$(stat -c%y "$LATEST_INCR" | cut -d. -f1)
    echo "  Latest incremental: $LATEST_TIME"
else
    echo "  Latest incremental: None"
fi

echo ""

# 2. Recent verification results
echo "🔍 Verification Status"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
VERIFY_LOG="/var/log/backup-verify.log"
if [ -f "$VERIFY_LOG" ]; then
    LAST_RESULT=$(tail -5 "$VERIFY_LOG" | grep -o "PASSED\|FAILED" | tail -1)
    if [ "$LAST_RESULT" = "PASSED" ]; then
        echo "  Last verification: ✅ Passed"
    elif [ "$LAST_RESULT" = "FAILED" ]; then
        echo "  Last verification: ❌ Failed"
    else
        echo "  Last verification: No recent results"
    fi
else
    echo "  Verification log: Not available"
fi

# 3. Restore drills
echo ""
echo "🔄 Restore Drills"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
DRILL_LOG="/var/log/backup-drills"
if [ -d "$DRILL_LOG" ] && [ "$(ls -A $DRILL_LOG 2>/dev/null)" ]; then
    LAST_DRILL=$(ls -t "$DRILL_LOG" | head -1)
    echo "  Last drill: $LAST_DRILL"
    grep -q "Drill Complete" "$DRILL_LOG/$LAST_DRILL" 2>/dev/null && \
        echo "  Status: ✅ Completed"
else
    echo "  Last drill: Never executed"
fi

# 4. Cloud sync status
echo ""
echo "☁️  Cloud Sync"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
for cloud in backblaze-b2 aws-s3 local-nas; do
    RCLONE_LOG="/var/log/rclone-${cloud}.log"
    if [ -f "$RCLONE_LOG" ]; then
        LAST_LINE=$(tail -1 "$RCLONE_LOG")
        if echo "$LAST_LINE" | grep -qi "ok\|success"; then
            echo "  $cloud: ✅ Sync OK"
        else
            echo "  $cloud: ⚠️ Needs review"
        fi
    else
        echo "  $cloud: ⏸️ Not executed"
    fi
done

# 5. AI strategy recommendations
echo ""
echo "🤖 AI Strategy Recommendations"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
python3 /etc/backup-analyzer/behavior-analyzer.py 2>/dev/null | python3 -c "
import sys, json
try:
    data = json.load(sys.stdin)
    if 'suggested_frequency' in data:
        freq_map = {'every_6h': 'Every 6 hours', 'daily': 'Daily', 'weekly': 'Weekly'}
        print(f\"  Suggested frequency: {freq_map.get(data['suggested_frequency'], data['suggested_frequency'])}\")
        print(f\"  Recommended RPO: {data.get('recommended_rpo_hours', 'N/A')} hours\")
        print(f\"  Data confidence: {int(data.get('confidence', 0) * 100)}%\")
    else:
        print(f\"  {data.get('message', 'Analyzing...')}\")
except:
    print('  Strategy analysis temporarily unavailable')
"

echo ""
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo "Generated: $(date '+%Y-%m-%d %H:%M:%S')"

apt-get update && apt-get install -y \
    rsync rclone jq python3 \
    inotify-tools \
    mysql-client postgresql-client

mkdir -p /backup/vps/{full,incremental}
mkdir -p /etc/backup-analyzer
mkdir -p /var/lib/backup-analyzer/{daily,hourly}
mkdir -p /var/log/backup-drills

chmod +x /usr/local/bin/backup-{behavior,strategy,verify,restore,distribute,dashboard}.sh

crontab -e

# First run: generate initial strategy
python3 /etc/backup-analyzer/backup-behavior-ai.py > /etc/backup-analyzer/strategy.json

# View dashboard
/usr/local/bin/backup-dashboard.sh

#!/usr/bin/env python3
"""llm-backup-decider.py - Use LLM to analyze backup status and generate natural language reports"""

import subprocess
import json
import requests

def get_backup_status():
    """Get current backup status"""
    status = {}
    status["latest_full"] = subprocess.getoutput(
        "ls -td /backup/vps/full/*/ 2>/dev/null | head -1"
    ).strip()
    status["total_size"] = subprocess.getoutput(
        "du -sh /backup/vps 2>/dev/null"
    ).split()[0]
    status["disk_usage"] = subprocess.getoutput(
        "df -h /backup | tail -1"
    ).split()[4]
    return status

def ask_llm_for_advice(status):
    """Ask LLM for backup strategy advice"""
    prompt = f"""You are a senior DevOps engineer. Here is my VPS backup status:

{json.dumps(status, indent=2)}

Please analyze and answer:
1. Is the current backup strategy reasonable?
2. Should backup frequency be adjusted?
3. Is storage space sufficient?
4. Any improvement suggestions?

Respond concisely with bullet points."""

    try:
        response = requests.post(
            "http://localhost:11434/api/generate",
            json={
                "model": "qwen2.5:7b",
                "prompt": prompt,
                "stream": False
            }
        )
        return response.json()["response"]
    except Exception as e:
        return f"LLM request failed: {e}"

# Execute
status = get_backup_status()
advice = ask_llm_for_advice(status)
print(advice)