mirror of
https://github.com/Ark0N/Codeman.git
synced 2026-10-06 07:29:42 +02:00
fix: rock-solid idle detection in respawn controller
- Add 300-char rolling window to catch working patterns split across PTY chunks - Check completion message BEFORE working patterns (priority fix) - Clear rolling window on completion message (transition point) - Increase working pattern absence threshold from 3s to 8s - Add Session.isWorking safety check before confirming idle - Add 20+ more working patterns (Compiling, Building, Processing, etc.) - Make AI idle checker prompt more conservative (err toward WORKING) - Update documentation Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
arkon
Executable
+124
@@ -0,0 +1,124 @@
|
||||
#!/bin/bash
|
||||
# Respawn Controller Monitor - 12 hour monitoring script
|
||||
# Usage: ./scripts/monitor-respawn.sh <session_id> [duration_hours]
|
||||
|
||||
SESSION_ID="${1:-5ecdf859-3605-42d8-b0a2-f8f0a1b5ff98}"
|
||||
DURATION_HOURS="${2:-12}"
|
||||
ISSUES_FILE="/home/arkon/default/claudeman/respawn-monitor-issues.md"
|
||||
FIXES_FILE="/home/arkon/default/claudeman/respawn-monitor-fixes.md"
|
||||
API_URL="https://localhost:3000"
|
||||
POLL_INTERVAL=60 # seconds
|
||||
|
||||
# Calculate end time
|
||||
END_TIME=$(($(date +%s) + DURATION_HOURS * 3600))
|
||||
|
||||
log_issue() {
|
||||
local severity="$1"
|
||||
local message="$2"
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "" >> "$ISSUES_FILE"
|
||||
echo "### [$severity] $timestamp" >> "$ISSUES_FILE"
|
||||
echo "$message" >> "$ISSUES_FILE"
|
||||
echo "" >> "$ISSUES_FILE"
|
||||
}
|
||||
|
||||
add_fix() {
|
||||
local issue="$1"
|
||||
local fix="$2"
|
||||
local timestamp=$(date '+%Y-%m-%d %H:%M:%S')
|
||||
echo "" >> "$FIXES_FILE"
|
||||
echo "### Issue: $issue" >> "$FIXES_FILE"
|
||||
echo "**Detected:** $timestamp" >> "$FIXES_FILE"
|
||||
echo "" >> "$FIXES_FILE"
|
||||
echo "**Proposed Fix:**" >> "$FIXES_FILE"
|
||||
echo "$fix" >> "$FIXES_FILE"
|
||||
echo "" >> "$FIXES_FILE"
|
||||
echo "---" >> "$FIXES_FILE"
|
||||
}
|
||||
|
||||
# State tracking
|
||||
PREV_STATE=""
|
||||
STATE_STUCK_COUNT=0
|
||||
PREV_CYCLE=0
|
||||
LAST_ACTIVITY=0
|
||||
AI_CHECK_ERRORS=0
|
||||
|
||||
echo "Starting respawn monitoring for session $SESSION_ID"
|
||||
echo "Duration: $DURATION_HOURS hours"
|
||||
echo "Issues logged to: $ISSUES_FILE"
|
||||
echo "Fixes logged to: $FIXES_FILE"
|
||||
|
||||
while [ $(date +%s) -lt $END_TIME ]; do
|
||||
# Get respawn status
|
||||
RESP=$(curl -sk "$API_URL/api/sessions/$SESSION_ID" 2>/dev/null)
|
||||
|
||||
if [ -z "$RESP" ]; then
|
||||
log_issue "ERROR" "API not responding - server may be down"
|
||||
add_fix "API not responding" "Check if claudeman-web service is running:\n\`\`\`bash\nsystemctl --user status claudeman-web\nsystemctl --user restart claudeman-web\n\`\`\`"
|
||||
sleep $POLL_INTERVAL
|
||||
continue
|
||||
fi
|
||||
|
||||
# Parse response
|
||||
STATE=$(echo "$RESP" | jq -r '.respawn.state // "null"')
|
||||
CYCLE=$(echo "$RESP" | jq -r '.respawn.cycleCount // 0')
|
||||
ENABLED=$(echo "$RESP" | jq -r '.respawnEnabled // false')
|
||||
TOKENS=$(echo "$RESP" | jq -r '.respawn.config.lastTokenCount // 0')
|
||||
AI_STATUS=$(echo "$RESP" | jq -r '.respawn.detection.aiCheck.status // "unknown"')
|
||||
AI_ERRORS=$(echo "$RESP" | jq -r '.respawn.detection.aiCheck.consecutiveErrors // 0')
|
||||
TIME_SINCE_ACTIVITY=$(echo "$RESP" | jq -r '.respawn.timeSinceActivity // 0')
|
||||
CONFIDENCE=$(echo "$RESP" | jq -r '.respawn.detection.confidenceLevel // 0')
|
||||
SESSION_STATUS=$(echo "$RESP" | jq -r '.status // "unknown"')
|
||||
|
||||
# Check for issues
|
||||
|
||||
# 1. Session not running
|
||||
if [ "$SESSION_STATUS" = "stopped" ] || [ "$SESSION_STATUS" = "error" ]; then
|
||||
log_issue "CRITICAL" "Session status is '$SESSION_STATUS' - respawn may not function correctly"
|
||||
add_fix "Session stopped/error" "Check session health and restart if needed:\n\`\`\`bash\ncurl -sk $API_URL/api/sessions/$SESSION_ID\n# If needed, create new session or restart\n\`\`\`"
|
||||
fi
|
||||
|
||||
# 2. Respawn disabled unexpectedly
|
||||
if [ "$ENABLED" = "false" ] && [ "$PREV_STATE" != "" ] && [ "$PREV_STATE" != "null" ]; then
|
||||
log_issue "WARNING" "Respawn appears to have been disabled (was in state: $PREV_STATE)"
|
||||
add_fix "Respawn disabled" "Re-enable respawn controller:\n\`\`\`bash\ncurl -sk -X POST $API_URL/api/sessions/$SESSION_ID/respawn/enable\n\`\`\`"
|
||||
fi
|
||||
|
||||
# 3. State stuck for too long (>30 minutes in same state)
|
||||
if [ "$STATE" = "$PREV_STATE" ] && [ "$STATE" != "watching" ]; then
|
||||
STATE_STUCK_COUNT=$((STATE_STUCK_COUNT + 1))
|
||||
if [ $STATE_STUCK_COUNT -ge 30 ]; then # 30 minutes
|
||||
log_issue "WARNING" "State stuck in '$STATE' for $STATE_STUCK_COUNT minutes. Detection: confidence=$CONFIDENCE, timeSinceActivity=${TIME_SINCE_ACTIVITY}ms"
|
||||
add_fix "State stuck in $STATE" "The respawn controller may be stuck. Options:\n1. Check terminal for blocking prompts\n2. Send input to unstick:\n\`\`\`bash\ncurl -sk -X POST $API_URL/api/sessions/$SESSION_ID/input -d '{\"input\": \"\\\\r\"}'\n\`\`\`\n3. Restart respawn:\n\`\`\`bash\ncurl -sk -X POST $API_URL/api/sessions/$SESSION_ID/respawn/stop\ncurl -sk -X POST $API_URL/api/sessions/$SESSION_ID/respawn/start\n\`\`\`"
|
||||
STATE_STUCK_COUNT=0 # Reset to avoid spam
|
||||
fi
|
||||
else
|
||||
STATE_STUCK_COUNT=0
|
||||
fi
|
||||
|
||||
# 4. AI check errors
|
||||
if [ "$AI_ERRORS" -gt 2 ] && [ "$AI_ERRORS" -gt "$AI_CHECK_ERRORS" ]; then
|
||||
log_issue "WARNING" "AI idle check has $AI_ERRORS consecutive errors (status: $AI_STATUS)"
|
||||
add_fix "AI check errors" "AI idle checker is failing. Check:\n1. Claude CLI is working\n2. Sufficient context available\n3. Consider disabling AI check temporarily:\n\`\`\`bash\ncurl -sk -X POST $API_URL/api/sessions/$SESSION_ID/respawn/config -H 'Content-Type: application/json' -d '{\"aiIdleCheckEnabled\": false}'\n\`\`\`"
|
||||
AI_CHECK_ERRORS=$AI_ERRORS
|
||||
fi
|
||||
|
||||
# 5. Cycle completed - log success
|
||||
if [ "$CYCLE" -gt "$PREV_CYCLE" ] && [ "$PREV_CYCLE" -gt 0 ]; then
|
||||
log_issue "INFO" "Respawn cycle $CYCLE completed successfully (tokens: $TOKENS)"
|
||||
fi
|
||||
|
||||
# 6. Very high time since activity (>10 minutes) while not in watching state
|
||||
if [ "$TIME_SINCE_ACTIVITY" -gt 600000 ] && [ "$STATE" != "watching" ]; then
|
||||
log_issue "WARNING" "No activity for ${TIME_SINCE_ACTIVITY}ms while in state '$STATE' - may be stuck"
|
||||
fi
|
||||
|
||||
# Update tracking vars
|
||||
PREV_STATE="$STATE"
|
||||
PREV_CYCLE="$CYCLE"
|
||||
|
||||
sleep $POLL_INTERVAL
|
||||
done
|
||||
|
||||
log_issue "INFO" "Monitoring completed after $DURATION_HOURS hours"
|
||||
echo "Monitoring complete. Check $ISSUES_FILE and $FIXES_FILE"
|
||||
Reference in New Issue
Block a user