# prometheus/alert-rules.yaml
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: dqlite-alerts
spec:
  groups:
    - name: dqlite
      rules:
        # 节点存活检查
        - alert: DqliteNodeDown
          expr: up{job="dqlite"} == 0
          for: 1m
          labels:
            severity: critical
          annotations:
            summary: "dqlite 节点 {{ $labels.instance }} 不可用"
            description: "节点已离线超过 1 分钟"

        # Leader 缺失
        - alert: DqliteNoLeader
          expr: dqlite_raft_leader_id == 0
          for: 5s
          labels:
            severity: critical
          annotations:
            summary: "dqlite 集群无 Leader"

        # 写延迟过高
        - alert: DqliteHighWriteLatency
          expr: histogram_quantile(0.99, dqlite_write_latency_seconds_bucket) > 0.01
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "dqlite 写延迟 P99 > 10ms"

        # 磁盘空间
        - alert: DqliteDiskSpaceHigh
          expr: (dqlite_disk_used_bytes / dqlite_disk_total_bytes) > 0.8
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "dqlite 磁盘使用率 > 80%"

        # Raft 日志滞后
        - alert: DqliteRaftLogLag
          expr: dqlite_raft_log_entries - dqlite_raft_commit_index > 1000
          for: 5m
          labels:
            severity: warning
          annotations:
            summary: "Raft 日志滞后 > 1000 条"

#!/bin/bash
# daily-check.sh - dqlite 日常巡检脚本

set -euo pipefail

echo "=== dqlite Daily Check - $(date) ==="

# 1. 检查节点存活
echo -e "\n[1] Node Health:"
for node in "127.0.0.1:9001" "127.0.0.1:9002" "127.0.0.1:9003"; do
    if nc -z -w2 ${node/:/ } 2>/dev/null; then
        echo "  ✓ $node - UP"
    else
        echo "  ✗ $node - DOWN"
    fi
done

# 2. 检查 Leader
echo -e "\n[2] Leader Status:"
# 通过客户端查询集群信息
echo "  (执行集群状态查询...)"

# 3. 检查磁盘使用
echo -e "\n[3] Disk Usage:"
for dir in /var/lib/dqlite/node{1,2,3}; do
    if [ -d "$dir" ]; then
        usage=$(du -sh "$dir" 2>/dev/null | cut -f1)
        echo "  $dir: $usage"
    fi
done

# 4. 检查日志
echo -e "\n[4] Recent Errors:"
journalctl -u dqlite --since "24 hours ago" --priority=err --no-pager | tail -5

# 5. 检查数据库大小
echo -e "\n[5] Database Size:"
find /var/lib/dqlite -name "*.db" -exec ls -lh {} \;

echo -e "\n=== Check Complete ==="

#!/bin/bash
# backup.sh - dqlite 备份脚本

set -euo pipefail

BACKUP_DIR="/backup/dqlite/$(date +%Y%m%d_%H%M%S)"
mkdir -p "$BACKUP_DIR"

echo "Starting dqlite backup to $BACKUP_DIR..."

# 方式 1：使用 dqlite dump 工具
dqlite-dump \
    --address "127.0.0.1:9001" \
    --output "$BACKUP_DIR/dump.sql"

# 方式 2：复制数据文件（需要先暂停写入或使用一致性快照）
# 停止一个 Follower 节点，复制其数据目录
# systemctl stop dqlite-node-3
# cp -r /var/lib/dqlite/node3/* "$BACKUP_DIR/"
# systemctl start dqlite-node-3

# 压缩
gzip "$BACKUP_DIR/dump.sql"

# 上传到远程存储
# aws s3 sync "$BACKUP_DIR" "s3://backups/dqlite/$(basename $BACKUP_DIR)/"

# 清理 30 天前的备份
find /backup/dqlite -maxdepth 1 -mtime +30 -exec rm -rf {} \;

echo "Backup completed: $BACKUP_DIR"

#!/bin/bash
# restore.sh - dqlite 恢复脚本

set -euo pipefail

BACKUP_FILE="$1"
TARGET_ADDRESS="${2:-127.0.0.1:9001}"

if [ ! -f "$BACKUP_FILE" ]; then
    echo "Error: Backup file not found: $BACKUP_FILE"
    exit 1
fi

echo "=== dqlite Restore Procedure ==="
echo "Backup: $BACKUP_FILE"
echo "Target: $TARGET_ADDRESS"
echo ""
echo "WARNING: This will overwrite existing data!"
read -p "Continue? (yes/no): " confirm

if [ "$confirm" != "yes" ]; then
    echo "Aborted."
    exit 0
fi

# 1. 停止所有节点
echo "[1] Stopping all dqlite nodes..."
# systemctl stop dqlite-node-1 dqlite-node-2 dqlite-node-3

# 2. 清空数据目录
echo "[2] Clearing data directories..."
# rm -rf /var/lib/dqlite/node1/* /var/lib/dqlite/node2/* /var/lib/dqlite/node3/*

# 3. 在第一个节点恢复数据
echo "[3] Restoring data..."
# dqlite-restore --backup "$BACKUP_FILE" --data-dir /var/lib/dqlite/node1

# 4. 启动第一个节点
echo "[4] Starting first node..."
# systemctl start dqlite-node-1

# 5. 等待节点就绪
echo "[5] Waiting for node to be ready..."
# sleep 5

# 6. 重新初始化集群
echo "[6] Re-initializing cluster..."
# dqlite-add-node --leader 127.0.0.1:9001 --id 2 --address 127.0.0.1:9002
# dqlite-add-node --leader 127.0.0.1:9001 --id 3 --address 127.0.0.1:9003

# 7. 启动其他节点
echo "[7] Starting remaining nodes..."
# systemctl start dqlite-node-2 dqlite-node-3

echo ""
echo "Restore completed. Please verify data integrity."

#!/bin/bash
# rolling-restart.sh - 滚动重启 dqlite 集群

set -euo pipefail

NODES=("dqlite-node-1" "dqlite-node-2" "dqlite-node-3")

for node in "${NODES[@]}"; do
    echo "=== Restarting $node ==="

    # 1. 确认集群健康（当前节点不是唯一 Leader）
    echo "[1] Checking cluster health..."
    # ... 检查逻辑 ...

    # 2. 如果是 Leader，先转移
    echo "[2] Transferring leadership if needed..."
    # ... 转移逻辑 ...

    # 3. 停止节点
    echo "[3] Stopping $node..."
    systemctl stop "$node"

    # 4. 等待集群稳定
    echo "[4] Waiting for cluster to stabilize..."
    sleep 5

    # 5. 启动节点
    echo "[5] Starting $node..."
    systemctl start "$node"

    # 6. 等待节点加入集群
    echo "[6] Waiting for node to rejoin..."
    sleep 10

    # 7. 验证节点状态
    echo "[7] Verifying node status..."
    # ... 验证逻辑 ...

    echo "=== $node restarted successfully ==="
    echo ""
done

echo "All nodes restarted."