# Prometheus告警规则
groups:
  - name: system_alerts
    rules:
      # CPU告警
      - alert: HighCPUUsage
        expr: 100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "CPU使用率过高"
          description: "实例 {{ $labels.instance }} CPU使用率超过80%，当前值 {{ $value }}%"

      # 内存告警
      - alert: HighMemoryUsage
        expr: (1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "内存使用率过高"
          description: "实例 {{ $labels.instance }} 内存使用率超过85%，当前值 {{ $value }}%"

      # 磁盘告警
      - alert: HighDiskUsage
        expr: (1 - node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 > 85
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "磁盘使用率过高"
          description: "实例 {{ $labels.instance }} 磁盘使用率超过85%，当前值 {{ $value }}%"

      # 负载告警
      - alert: HighLoadAverage
        expr: node_load1 > count by(instance) (node_cpu_seconds_total{mode="idle"}) * 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "系统负载过高"
          description: "实例 {{ $labels.instance }} 负载超过CPU核心数的2倍"

groups:
  - name: application_alerts
    rules:
      # 响应时间告警
      - alert: HighResponseTime
        expr: histogram_quantile(0.99, rate(http_request_duration_seconds_bucket[5m])) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "响应时间过长"
          description: "P99响应时间超过1秒"

      # 错误率告警
      - alert: HighErrorRate
        expr: rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m]) > 0.05
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "错误率过高"
          description: "5xx错误率超过5%"

# Alertmanager配置
inhibit_rules:
  # 如果有Critical告警，抑制同实例的Warning告警
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['instance']

# 维护期间静默告警
# 通过Alertmanager Web UI设置静默

# 或通过API
curl -X POST http://alertmanager:9093/api/v2/silences -d '{
  "matchers": [{"name": "instance", "value": "server1"}],
  "startsAt": "2026-05-10T22:00:00Z",
  "endsAt": "2026-05-11T06:00:00Z",
  "createdBy": "admin",
  "comment": "计划维护"
}'

# 安装Glances
pip install 'glances[web]'

# 启动Web模式
glances -w --bind 0.0.0.0 -p 61208

# 访问
# http://server_ip:61208

# docker-compose-monitoring.yml
version: '3.8'

services:
  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    volumes:
      - grafana_data:/var/lib/grafana

  alertmanager:
    image: prom/alertmanager:latest
    ports:
      - "9093:9093"
    volumes:
      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml

  node-exporter:
    image: prom/node-exporter:latest
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.sysfs=/host/sys'
      - '--path.rootfs=/rootfs'

volumes:
  prometheus_data:
  grafana_data:

#!/bin/bash
# health_check.sh - 系统健康检查脚本

# 颜色定义
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

echo "=== 系统健康检查 ==="
echo "时间: $(date)"
echo "主机: $(hostname)"
echo ""

# CPU检查
echo "--- CPU ---"
CPU_USAGE=$(top -bn1 | grep "Cpu(s)" | awk '{print $2}')
CPU_INT=${CPU_USAGE%.*}
if [ $CPU_INT -gt 90 ]; then
    echo -e "${RED}CPU使用率: ${CPU_USAGE}% (严重)${NC}"
elif [ $CPU_INT -gt 70 ]; then
    echo -e "${YELLOW}CPU使用率: ${CPU_USAGE}% (警告)${NC}"
else
    echo -e "${GREEN}CPU使用率: ${CPU_USAGE}% (正常)${NC}"
fi

# 内存检查
echo "--- 内存 ---"
MEM_TOTAL=$(free -m | awk 'NR==2{print $2}')
MEM_USED=$(free -m | awk 'NR==2{print $3}')
MEM_PERCENT=$((MEM_USED * 100 / MEM_TOTAL))
if [ $MEM_PERCENT -gt 90 ]; then
    echo -e "${RED}内存使用率: ${MEM_PERCENT}% (严重)${NC}"
elif [ $MEM_PERCENT -gt 70 ]; then
    echo -e "${YELLOW}内存使用率: ${MEM_PERCENT}% (警告)${NC}"
else
    echo -e "${GREEN}内存使用率: ${MEM_PERCENT}% (正常)${NC}"
fi

# 磁盘检查
echo "--- 磁盘 ---"
df -h | awk 'NR>1 {
    gsub(/%/, "", $5)
    if ($5 > 90) printf "\033[0;31m%s: %s%% (严重)\033[0m\n", $6, $5
    else if ($5 > 70) printf "\033[1;33m%s: %s%% (警告)\033[0m\n", $6, $5
    else printf "\033[0;32m%s: %s%% (正常)\033[0m\n", $6, $5
}'

# 负载检查
echo "--- 负载 ---"
LOAD=$(cat /proc/loadavg | awk '{print $1}')
CORES=$(nproc)
LOAD_INT=${LOAD%.*}
if [ $LOAD_INT -gt $((CORES * 2)) ]; then
    echo -e "${RED}负载: ${LOAD} (严重，CPU核心数: ${CORES})${NC}"
elif [ $LOAD_INT -gt $CORES ]; then
    echo -e "${YELLOW}负载: ${LOAD} (警告，CPU核心数: ${CORES})${NC}"
else
    echo -e "${GREEN}负载: ${LOAD} (正常，CPU核心数: ${CORES})${NC}"
fi

# Swap检查
echo "--- Swap ---"
SWAP_TOTAL=$(free -m | awk 'NR==3{print $2}')
SWAP_USED=$(free -m | awk 'NR==3{print $3}')
if [ $SWAP_TOTAL -gt 0 ]; then
    SWAP_PERCENT=$((SWAP_USED * 100 / SWAP_TOTAL))
    if [ $SWAP_PERCENT -gt 50 ]; then
        echo -e "${RED}Swap使用率: ${SWAP_PERCENT}% (严重)${NC}"
    elif [ $SWAP_PERCENT -gt 20 ]; then
        echo -e "${YELLOW}Swap使用率: ${SWAP_PERCENT}% (警告)${NC}"
    else
        echo -e "${GREEN}Swap使用率: ${SWAP_PERCENT}% (正常)${NC}"
    fi
else
    echo -e "${GREEN}Swap: 未启用${NC}"
fi

echo ""
echo "=== 检查完成 ==="

#!/bin/bash
# perf_collect.sh - 性能数据采集脚本

LOG_DIR="/var/log/performance"
mkdir -p $LOG_DIR

INTERVAL=${1:-60}  # 默认60秒
DURATION=${2:-3600}  # 默认1小时

LOG_FILE="$LOG_DIR/perf_$(date +%Y%m%d_%H%M%S).csv"

echo "开始性能数据采集..."
echo "间隔: ${INTERVAL}秒"
echo "持续: ${DURATION}秒"
echo "输出: $LOG_FILE"

# CSV头
echo "timestamp,cpu_user,cpu_system,cpu_idle,mem_used,mem_free,swap_used,disk_read,disk_write,net_rx,net_tx" > $LOG_FILE

END_TIME=$(($(date +%s) + DURATION))

while [ $(date +%s) -lt $END_TIME ]; do
    TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
    
    # CPU
    CPU=$(mpstat 1 1 | tail -1)
    CPU_USER=$(echo $CPU | awk '{print $3}')
    CPU_SYSTEM=$(echo $CPU | awk '{print $5}')
    CPU_IDLE=$(echo $CPU | awk '{print $12}')
    
    # 内存
    MEM=$(free -m | awk 'NR==2{print $3,$4}')
    MEM_USED=$(echo $MEM | awk '{print $1}')
    MEM_FREE=$(echo $MEM | awk '{print $2}')
    
    # Swap
    SWAP_USED=$(free -m | awk 'NR==3{print $3}')
    
    # 磁盘
    DISK=$(iostat -d sda 1 1 | tail -1)
    DISK_READ=$(echo $DISK | awk '{print $3}')
    DISK_WRITE=$(echo $DISK | awk '{print $4}')
    
    # 网络
    NET=$(sar -n DEV 1 1 | grep "eth0" | tail -1)
    NET_RX=$(echo $NET | awk '{print $5}')
    NET_TX=$(echo $NET | awk '{print $6}')
    
    echo "$TIMESTAMP,$CPU_USER,$CPU_SYSTEM,$CPU_IDLE,$MEM_USED,$MEM_FREE,$SWAP_USED,$DISK_READ,$DISK_WRITE,$NET_RX,$NET_TX" >> $LOG_FILE
    
    sleep $INTERVAL
done

echo "采集完成: $LOG_FILE"

#!/bin/bash
# alert_notify.sh - 告警通知脚本

# 配置
WEBHOOK_URL="https://hooks.slack.com/services/xxx/yyy/zzz"
EMAIL="admin@example.com"
ALERT_LOG="/var/log/alerts.log"

# 参数
LEVEL=$1      # critical, warning, info
SUBJECT=$2    # 告警主题
MESSAGE=$3    # 告警内容

# 记录日志
echo "$(date '+%Y-%m-%d %H:%M:%S') [$LEVEL] $SUBJECT: $MESSAGE" >> $ALERT_LOG

# 发送Slack通知
send_slack() {
    local color
    case $LEVEL in
        critical) color="#FF0000" ;;
        warning)  color="#FFA500" ;;
        *)        color="#36A64F" ;;
    esac
    
    curl -X POST -H 'Content-type: application/json' \
        --data "{
            \"attachments\": [{
                \"color\": \"$color\",
                \"title\": \"[$LEVEL] $SUBJECT\",
                \"text\": \"$MESSAGE\",
                \"footer\": \"$(hostname)\",
                \"ts\": $(date +%s)
            }]
        }" \
        $WEBHOOK_URL
}

# 发送邮件通知
send_email() {
    echo "$MESSAGE" | mail -s "[$LEVEL] $SUBJECT" $EMAIL
}

# 根据级别发送通知
case $LEVEL in
    critical)
        send_slack
        send_email
        ;;
    warning)
        send_slack
        ;;
    *)
        send_slack
        ;;
esac

#!/bin/bash
# daily_check.sh - 每日巡检脚本

REPORT_FILE="/var/log/daily_check_$(date +%Y%m%d).txt"

{
    echo "=== 每日巡检报告 ==="
    echo "时间: $(date)"
    echo "主机: $(hostname)"
    echo ""

    echo "1. 系统负载"
    uptime
    echo ""

    echo "2. CPU使用率"
    top -bn1 | head -5
    echo ""

    echo "3. 内存使用"
    free -h
    echo ""

    echo "4. 磁盘使用"
    df -h
    echo ""

    echo "5. 磁盘IO"
    iostat -x 1 1
    echo ""

    echo "6. 网络连接"
    ss -s
    echo ""

    echo "7. 服务状态"
    systemctl list-units --state=failed
    echo ""

    echo "8. 最近错误日志"
    journalctl -p err --since today --no-pager | tail -20
    echo ""

    echo "=== 巡检完成 ==="
} > $REPORT_FILE

echo "巡检报告: $REPORT_FILE"

# CPU问题
top -o %CPU                    # 找CPU大户
pidstat -u 1 5                 # 进程CPU统计
perf top                       # CPU性能分析

# 内存问题
top -o %MEM                    # 找内存大户
pmap -x <PID>                  # 进程内存映射
vmstat 1 5                     # 内存统计

# 磁盘问题
iostat -x 1 5                  # 磁盘IO统计
iotop -o                       # 找IO大户
lsof +D /path                  # 打开的文件

# 网络问题
iftop -i eth0                  # 网络流量
ss -tuln                       # 监听端口
netstat -an | awk '{print $5}' | sort | uniq -c | sort -rn  # 连接统计

# 1. 收集历史数据
# 使用SAR收集30天数据
sar -u -f /var/log/sysstat/sa$(date +%d) > /tmp/cpu_history.txt
sar -r -f /var/log/sysstat/sa$(date +%d) > /tmp/mem_history.txt
sar -d -f /var/log/sysstat/sa$(date +%d) > /tmp/disk_history.txt

# 2. 分析趋势
# 使用Python分析
python3 << EOF
import pandas as pd
import matplotlib.pyplot as plt

# 读取数据
df = pd.read_csv('/tmp/cpu_history.txt', sep='\s+')

# 分析趋势
print(df.describe())

# 绘图
df.plot()
plt.savefig('/tmp/cpu_trend.png')
EOF

# 3. 预测未来需求
# 基于历史增长率预测
# CPU需求 = 当前使用 * (1 + 增长率)^月数

# 1. 备份当前配置
cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak.$(date +%Y%m%d)

# 2. 记录当前状态
systemctl status nginx > /tmp/nginx_before.txt

# 3. 测试新配置
nginx -t

# 4. 准备回滚方案
echo "回滚命令: cp /etc/nginx/nginx.conf.bak.$(date +%Y%m%d) /etc/nginx/nginx.conf && systemctl restart nginx"

# 1. 检查服务状态
systemctl status nginx

# 2. 检查日志
journalctl -u nginx --since "5 minutes ago"

# 3. 测试功能
curl -I http://localhost

# 4. 监控指标
# 观察CPU、内存、响应时间是否有异常

# prometheus.yml
global:
  scrape_interval: 15s

# 命令行参数
--storage.tsdb.retention.time=30d
--storage.tsdb.retention.size=50GB

# /etc/sysstat/sysstat
HISTORY=28      # 保留28天
COMPRESSAFTER=10  # 10天后压缩

#!/bin/bash
# backup_monitoring.sh - 监控数据备份脚本

BACKUP_DIR="/backup/monitoring"
DATE=$(date +%Y%m%d)

mkdir -p $BACKUP_DIR

# 备份Prometheus数据
tar czf $BACKUP_DIR/prometheus_$DATE.tar.gz /var/lib/prometheus/

# 备份Grafana数据
tar czf $BACKUP_DIR/grafana_$DATE.tar.gz /var/lib/grafana/

# 备份SAR数据
tar czf $BACKUP_DIR/sar_$DATE.tar.gz /var/log/sysstat/

# 备份配置文件
tar czf $BACKUP_DIR/config_$DATE.tar.gz /etc/prometheus/ /etc/grafana/

# 清理30天前的备份
find $BACKUP_DIR -name "*.tar.gz" -mtime +30 -delete

echo "备份完成: $BACKUP_DIR"

#!/bin/bash
# cleanup_monitoring.sh - 监控数据清理脚本

# 清理旧的SAR数据
find /var/log/sysstat/ -name "sa*" -mtime +30 -delete
find /var/log/sysstat/ -name "sar*" -mtime +30 -delete

# 清理旧的nmon数据
find /var/log/nmon/ -name "*.nmon" -mtime +30 -delete

# 清理旧的性能日志
find /var/log/performance/ -name "*.csv" -mtime +30 -delete

# 清理旧的监控日志
find /var/log/ -name "monitoring_*.log" -mtime +30 -delete

echo "清理完成"

# 限制Prometheus访问
# 使用反向代理 + 认证
server {
    listen 80;
    server_name prometheus.example.com;

    auth_basic "Prometheus";
    auth_basic_user_file /etc/nginx/.htpasswd;

    location / {
        proxy_pass http://localhost:9090;
    }
}

# 防火墙规则
iptables -A INPUT -p tcp --dport 9090 -s 10.0.0.0/8 -j ACCEPT
iptables -A INPUT -p tcp --dport 9090 -j DROP

# TLS配置
tls_server_config:
  cert_file: /etc/prometheus/tls/cert.pem
  key_file: /etc/prometheus/tls/key.pem

# 不要监控包含密码的环境变量
# 不要监控包含密钥的配置文件
# 不要监控包含个人信息的日志

# 日志脱敏示例
import re

def mask_sensitive(text):
    # 手机号
    text = re.sub(r'1[3-9]\d{9}', '1**********', text)
    # 身份证号
    text = re.sub(r'\d{17}[\dXx]', '*******************', text)
    # 邮箱
    text = re.sub(r'[\w.]+@[\w.]+', '***@***.***', text)
    return text

# 分析告警统计
cat /var/log/alerts.log | awk '{print $3}' | sort | uniq -c | sort -rn

# 分析误报率
# 误报率 = 无需处理的告警 / 总告警数