# 完整配置验证
nagios -v /usr/local/nagios/etc/nagios.cfg

# 输出解读：
# Total Warnings: 0    ← 警告数量（建议修复）
# Total Errors:   0    ← 错误数量（必须修复）
# Things look okay - No serious problems were detected during the pre-flight check

# 当配置文件很多时，逐步排除问题

# 方法一：注释部分配置文件
# 在 nagios.cfg 中临时注释掉部分 cfg_file/cfg_dir 行
# 逐步取消注释，定位问题文件

# 方法二：二分法
# 将 conf.d 目录分成两半，分别验证
mv /etc/nagios/conf.d/hosts /tmp/hosts.bak
nagios -v /etc/nagios/nagios.cfg  # 如果通过，问题在 hosts 目录
mv /tmp/hosts.bak /etc/nagios/conf.d/hosts

# 方法三：使用脚本逐文件验证
#!/bin/bash
for f in /etc/nagios/conf.d/**/*.cfg; do
    # 临时引入单个文件
    echo "cfg_file=$f" > /tmp/test.cfg
    cat /etc/nagios/nagios.cfg.base >> /tmp/test.cfg
    result=$(nagios -v /tmp/test.cfg 2>&1 | grep "Errors")
    echo "$f: $result"
done

# 查看 Nagios 内存中的对象缓存
cat /var/log/nagios/objects.cache | head -100

# 搜索特定主机
grep -A 20 "host_name.*web-server-01" /var/log/nagios/objects.cache

# 搜索特定服务
grep -A 20 "service_description.*HTTP" /var/log/nagios/objects.cache

# 检查对象数量
grep "define host {" /var/log/nagios/objects.cache | wc -l
grep "define service {" /var/log/nagios/objects.cache | wc -l
grep "define command {" /var/log/nagios/objects.cache | wc -l

# 查看通知日志
grep "NOTIFICATION:" /var/log/nagios/nagios.log | tail -20

# 查看通知启用/禁用状态
grep "Notifications" /var/log/nagios/nagios.log | tail -10

# 查看特定主机/服务的通知历史
grep "HOST NOTIFICATION" /var/log/nagios/nagios.log | grep "web-server-01" | tail -20
grep "SERVICE NOTIFICATION" /var/log/nagios/nagios.log | grep "HTTP" | tail -20

# 通过 CGI 查看通知历史
# Web 界面 → Reports → Notifications

# 测试邮件通知命令
/usr/bin/printf "%b" "Test notification from Nagios\nTime: $(date)" | \
    /usr/bin/mail -s "Test Nagios Notification" admin@example.com

# 测试通知宏替换
# 通过命令文件强制发送通知
echo "[$(date +%s)] SEND_CUSTOM_HOST_NOTIFICATION;web-server-01;2;admin;Test notification" \
    >> /var/log/nagios/rw/nagios.cmd

# 参数说明：
# SEND_CUSTOM_HOST_NOTIFICATION;主机名;通知类型;作者;备注
# 通知类型: 0=NONE, 1=PROBLEM, 2=ACKNOWLEDGEMENT

# 退出码含义
# 0 = OK
# 1 = WARNING
# 2 = CRITICAL
# 3 = UNKNOWN
# 126 = Permission denied (插件无执行权限)
# 127 = Command not found (插件路径错误)
# 128+N = 被信号 N 终止 (如 130 = SIGINT, 137 = SIGKILL)

# 检查插件权限
ls -la /usr/local/nagios/libexec/check_http
# -rwxr-xr-x 1 nagios nagios ... ← 需要执行权限

# 修复权限
chmod +x /usr/local/nagios/libexec/check_custom.sh
chown nagios:nagios /usr/local/nagios/libexec/check_custom.sh

# 以 nagios 用户身份测试插件
sudo -u nagios /usr/local/nagios/libexec/check_ping -H 192.168.1.1 -w 100,20% -c 500,60%

# 查看退出码
sudo -u nagios /usr/local/nagios/libexec/check_ping -H 192.168.1.1 -w 100,20% -c 500,60%
echo $?  # 应该返回 0, 1, 2, 或 3

# 调试 NRPE 远程插件
/usr/local/nagios/libexec/check_nrpe -H remote-host -c check_disk -a '-w 20% -c 10% -p /'

# 使用 strace 跟踪插件系统调用
strace -f /usr/local/nagios/libexec/check_http -H www.example.com 2>&1 | grep -i error

# Nagios 全局超时配置（nagios.cfg）
service_check_timeout=60
host_check_timeout=30
event_handler_timeout=30
notification_timeout=30

# 插件自身超时
# 检查命令定义中是否有 -t 参数
check_command check_http!-H host -t 30

# 测试插件超时
timeout 10 /usr/local/nagios/libexec/check_http -H slow-server.example.com
echo $?  # 124 = timeout

# NRPE 超时（nrpe.cfg）
command_timeout=60
connection_timeout=300

#!/bin/bash
# 在插件中添加调试输出

DEBUG_FILE="/tmp/check_debug.log"

debug() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') $1" >> $DEBUG_FILE
}

debug "Starting check for $HOST"
debug "Parameters: warn=$WARNING crit=$CRITICAL"

# ... 检查逻辑 ...

debug "Result: value=$VALUE state=$STATE"

# 注意：调试输出只写入文件，不要输出到 stdout
# stdout 只输出状态信息和性能数据

# 查看 Nagios 性能统计
# Web 界面 → Reports → Performance Info

# 关键指标
# - Service Check Execution Time: 服务检查执行时间
# - Service Check Latency: 服务检查延迟
# - Host Check Execution Time: 主机检查执行时间
# - Host Check Latency: 主机检查延迟
# - Active Service Checks: 每分钟主动服务检查数
# - Active Host Checks: 每分钟主动主机检查数

# 命令行查看状态
grep "service_check_execution_time" /var/log/nagios/nagios.log | tail -10
grep "service_check_latency" /var/log/nagios/nagios.log | tail -10

# 1. 调整检查调度
service_inter_check_delay_method=s  # 自动调度
service_interleave_factor=s         # 自动交错

# 2. 限制并行检查数
max_parallel_service_checks=0       # 0=不限制（根据服务器能力）

# 3. 使用被动检查减少主动检查
passive_checks_enabled=1
active_checks_enabled=0  # 对于只能被动检查的服务

# 4. 调整检查间隔
# 关键服务：check_interval=1
# 一般服务：check_interval=5
# 次要服务：check_interval=15

# 5. 使用 check_freshness 替代频繁主动检查
check_freshness=1
freshness_threshold=300

# 6. 禁用不必要的功能
process_performance_data=0  # 如果不需要性能数据
enable_flap_detection=0     # 如果不需要抖动检测

# 7. 优化日志
log_level=LOG_NONE          # 减少日志量
log_external_commands=0     # 不记录外部命令
log_passive_checks=0        # 不记录被动检查

# 使用 Nagios 监控自身
# check_nagios 插件
define command {
    command_name    check_nagios
    command_line    $USER1$/check_nagios -F /var/log/nagios/status.dat -C /usr/local/nagios/bin/nagios
}

define service {
    use                     generic-service
    host_name               localhost
    service_description     Nagios Process
    check_command           check_nagios
}

# 检查 Nagios 日志文件新鲜度
define command {
    command_name    check_nagios_log
    command_line    $USER1$/check_file_age -f /var/log/nagios/nagios.log -w 300 -c 600
}

# 检查命令管道
define service {
    use                     generic-service
    host_name               localhost
    service_description     Nagios Command Pipe
    check_command           check_nagios_pipe
}

define command {
    command_name    check_nagios_pipe
    command_line    /bin/test -p /var/log/nagios/rw/nagios.cmd && echo "OK: Command pipe exists" || echo "CRITICAL: Command pipe missing"
}

# 查看最新的状态变化
grep "SERVICE ALERT" /var/log/nagios/nagios.log | tail -20

# 查看通知
grep "SERVICE NOTIFICATION" /var/log/nagios/nagios.log | tail -20
grep "HOST NOTIFICATION" /var/log/nagios/nagios.log | tail -20

# 查看外部命令
grep "EXTERNAL COMMAND" /var/log/nagios/nagios.log | tail -20

# 查看错误
grep -i "error" /var/log/nagios/nagios.log | tail -20

# 查看警告
grep -i "warning" /var/log/nagios/nagios.log | tail -20

# 统计每小时的告警数量
grep "SERVICE ALERT" /var/log/nagios/nagios.log | awk '{print $1}' | cut -d: -f1,2 | sort | uniq -c

# 统计告警最多的主机
grep "HOST ALERT" /var/log/nagios/nagios.log | awk '{print $6}' | sort | uniq -c | sort -rn | head -10

# 统计告警最多的服务
grep "SERVICE ALERT" /var/log/nagios/nagios.log | awk '{print $6}' | sort | uniq -c | sort -rn | head -10

# 查看特定时间段的日志
awk '/\[2024-01-01 00:00:00\]/,/\[2024-01-02 00:00:00\]/' /var/log/nagios/nagios.log

# 查看特定主机的所有事件
grep "web-server-01" /var/log/nagios/nagios.log | tail -50

# /etc/logrotate.d/nagios
/var/log/nagios/nagios.log /var/log/nagios/archives/*.log {
    daily
    rotate 90
    compress
    delaycompress
    missingok
    notifempty
    create 664 nagios nagios
    sharedscripts
    postrotate
        /bin/kill -HUP $(cat /var/run/nagios.pid 2>/dev/null) 2>/dev/null || true
    endscript
}

# 实时查看日志
tail -f /var/log/nagios/nagios.log

# 实时过滤告警
tail -f /var/log/nagios/nagios.log | grep --line-buffered "ALERT"

# 实时过滤特定主机
tail -f /var/log/nagios/nagios.log | grep --line-buffered "web-server-01"

# 使用 multitail 同时监控多个日志
multitail /var/log/nagios/nagios.log /var/log/httpd/error_log

# 原因：认证配置或文件权限问题

# 检查 htpasswd 文件
ls -la /usr/local/nagios/etc/htpasswd.users
cat /usr/local/nagios/etc/htpasswd.users  # 确认用户名存在

# 检查 Apache 配置
apachectl configtest

# 检查 SELinux
getenforce
ls -Z /usr/local/nagios/share/

# 修复 SELinux
restorecon -Rv /usr/local/nagios/share/
setsebool -P httpd_can_network_connect 1

# 检查 cgi.cfg 权限
grep "authorized_for" /usr/local/nagios/etc/cgi.cfg

# 检查 CGI 执行权限
ls -la /usr/local/nagios/sbin/

# 检查 Apache CGI 配置
# httpd.conf 或 nagios.conf
<Directory "/usr/local/nagios/sbin">
    Options ExecCGI
    AddHandler cgi-script .cgi
</Directory>

# 查看 Apache 错误日志
tail -f /var/log/httpd/error_log

# 检查 Apache 运行状态
systemctl status httpd

# 检查端口监听
ss -tlnp | grep :80
ss -tlnp | grep :443

# 检查防火墙
firewall-cmd --list-all

# 检查 Nagios 配置文件
apachectl configtest

# 测试 CGI 访问
curl -u nagiosadmin:password http://localhost/nagios/cgi-bin/status.cgi

# 症状：Nagios 启动失败或状态异常

# 检查状态文件
ls -la /var/log/nagios/status.dat
cat /var/log/nagios/status.dat | head -20

# 解决方案：删除状态文件重新启动
systemctl stop nagios
rm -f /var/log/nagios/status.dat
rm -f /var/log/nagios/objects.cache
rm -f /var/log/nagios/retention.dat
systemctl start nagios

# 检查命令管道
ls -la /var/log/nagios/rw/nagios.cmd
# 应该显示 prw-rw---- (命名管道)

# 检查权限
# 需要 nagios:nagcmd 所有者和 660 权限

# 修复权限
chown nagios:nagcmd /var/log/nagios/rw/nagios.cmd
chmod 660 /var/log/nagios/rw/nagios.cmd

# 测试写入
echo "[$(date +%s)] TEST_COMMAND" >> /var/log/nagios/rw/nagios.cmd

# Connection refused
# 原因：NRPE 未启动或防火墙阻止

# 检查 NRPE 运行状态
systemctl status nrpe
ss -tlnp | grep 5666

# 检查防火墙
firewall-cmd --list-ports
iptables -L -n | grep 5666

# Connection timeout
# 原因：网络不通或防火墙阻止

# 测试网络连通性
nc -zv remote-host 5666
telnet remote-host 5666

# SSL handshake failed
# 原因：SSL 版本不匹配或证书问题

# 检查 NRPE SSL 配置
grep "ssl_version" /etc/nagios/nrpe.cfg

# 强制指定 SSL 版本
check_nrpe -H host --ssl=TLSv1.2+

# 禁用 SSL（仅调试，不推荐生产使用）
check_nrpe -H host --no-ssl

# Return code 255: 命令不存在
# 检查被监控主机的 nrpe.cfg 中的命令定义
grep "check_disk" /etc/nagios/nrpe.cfg

# Return code 126: 权限不足
# 检查插件权限
ls -la /usr/local/nagios/libexec/check_disk

# Return code 127: 路径错误
# 检查插件路径
which check_disk
ls /usr/local/nagios/libexec/

# 1. Nagios 无法启动
nagios -v /etc/nagios/nagios.cfg  # 检查配置
systemctl status nagios            # 检查状态
journalctl -u nagios -n 50         # 检查系统日志
rm -f /var/log/nagios/status.dat   # 清除状态文件

# 2. 通知风暴
echo "[$(date +%s)] DISABLE_NOTIFICATIONS" >> /var/log/nagios/rw/nagios.cmd
# 排查根因后重新启用
echo "[$(date +%s)] ENABLE_NOTIFICATIONS" >> /var/log/nagios/rw/nagios.cmd

# 3. 磁盘空间不足
du -sh /var/log/nagios/*
find /var/log/nagios/archives/ -name "*.log.gz" -mtime +90 -delete
du -sh /var/log/nagios/var/perfdata/*

# 4. 性能严重下降
# 临时禁用非关键检查
echo "[$(date +%s)] DISABLE_SERVICE_CHECKS;dev-server-01;*" >> /var/log/nagios/rw/nagios.cmd