大海
86 lines
2.6 KiB
Bash
Executable File
86 lines
2.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# watchdog.sh — 远程记忆服务器健康监控
|
|
#
|
|
# 每2小时执行一次（由 cron 调用）
|
|
# 连续 3 次失败 → 告警
|
|
#
|
|
# 用法:
|
|
# bash scripts/watchdog.sh # 单次检查
|
|
# bash scripts/watchdog.sh status # 查看历史
|
|
|
|
set -euo pipefail
|
|
|
|
MEMORY_SERVER="http://111.229.145.18"
|
|
HISTORY_FILE="$HOME/.openclaw/memory/watchdog-history.log"
|
|
ALERT_FILE="$HOME/.openclaw/memory/watchdog-alert.log"
|
|
FAIL_COUNT_FILE="$HOME/.openclaw/memory/watchdog-failures"
|
|
THRESHOLD=3
|
|
|
|
mkdir -p "$(dirname "$HISTORY_FILE")"
|
|
|
|
check() {
|
|
local ts=$(date '+%Y-%m-%d %H:%M:%S')
|
|
local result=""
|
|
local stats=""
|
|
local search_test=""
|
|
|
|
# 1. 健康检查
|
|
stats=$(curl -s --max-time 5 "$MEMORY_SERVER/api/v2/stats" 2>/dev/null) || true
|
|
if [ -z "$stats" ]; then
|
|
result="DOWN (stats unreachable)"
|
|
else
|
|
# 2. 搜索功能检查
|
|
search_test=$(curl -s --max-time 5 -X POST "$MEMORY_SERVER/api/v2/search" \
|
|
-H "Content-Type: application/json" \
|
|
-d '{"query":"健康检查","project":"xiaolong","limit":1}' 2>/dev/null) || true
|
|
if echo "$search_test" | grep -q 'results'; then
|
|
result="OK"
|
|
# 提取记忆数
|
|
local mem_count=$(echo "$stats" | node -e "const d=JSON.parse(require('fs').readFileSync('/dev/stdin','utf8'));console.log((d.projects.find(p=>p.project.includes('xiaolong'))||{}).memories||0)" 2>/dev/null || echo 0)
|
|
result="OK ($mem_count memories)"
|
|
else
|
|
result="DEGRADED (search failed)"
|
|
fi
|
|
fi
|
|
|
|
echo "$ts | $result" >> "$HISTORY_FILE"
|
|
|
|
# 3. 失败计数
|
|
if echo "$result" | grep -qE 'DOWN|DEGRADED'; then
|
|
local failures=0
|
|
[ -f "$FAIL_COUNT_FILE" ] && failures=$(cat "$FAIL_COUNT_FILE")
|
|
failures=$((failures + 1))
|
|
echo "$failures" > "$FAIL_COUNT_FILE"
|
|
|
|
if [ "$failures" -ge "$THRESHOLD" ]; then
|
|
echo "$ts | ALERT: $result (连续失败 $failures 次)" >> "$ALERT_FILE"
|
|
echo "🚨 警告: 远程记忆服务器 $result (连续 $failures 次)"
|
|
# 可以在这里加 webhook 通知
|
|
fi
|
|
echo "⚠️ $result"
|
|
else
|
|
echo "0" > "$FAIL_COUNT_FILE"
|
|
echo "✅ $result"
|
|
fi
|
|
|
|
echo "$ts | watchdog: $result"
|
|
}
|
|
|
|
status() {
|
|
echo "📊 远程服务器监控历史"
|
|
echo ""
|
|
echo "最近 10 次检查:"
|
|
[ -f "$HISTORY_FILE" ] && tail -10 "$HISTORY_FILE" | sed 's/^/ /' || echo " (无记录)"
|
|
echo ""
|
|
echo "失败计数:"
|
|
[ -f "$FAIL_COUNT_FILE" ] && echo " 连续失败: $(cat "$FAIL_COUNT_FILE") 次" || echo " 0 次"
|
|
echo ""
|
|
echo "告警历史:"
|
|
[ -f "$ALERT_FILE" ] && tail -5 "$ALERT_FILE" | sed 's/^/ /' || echo " (无)"
|
|
}
|
|
|
|
case "${1:-check}" in
|
|
check) check ;;
|
|
status) status ;;
|
|
esac
|