diff --git a/.omc/state/hud-state.json b/.omc/state/hud-state.json new file mode 100644 index 00000000..a95d7bbb --- /dev/null +++ b/.omc/state/hud-state.json @@ -0,0 +1,6 @@ +{ + "timestamp": "2026-02-27T09:41:29.668Z", + "backgroundTasks": [], + "sessionStartTimestamp": "2026-02-27T08:56:19.621Z", + "sessionId": "06550954-0d4b-4961-8ccd-af4f724d7078" +} \ No newline at end of file diff --git a/.omc/state/hud-stdin-cache.json b/.omc/state/hud-stdin-cache.json new file mode 100644 index 00000000..33d4e72a --- /dev/null +++ b/.omc/state/hud-stdin-cache.json @@ -0,0 +1 @@ +{"session_id":"06550954-0d4b-4961-8ccd-af4f724d7078","transcript_path":"/home/ubuntu/.claude/projects/-home-ubuntu/06550954-0d4b-4961-8ccd-af4f724d7078.jsonl","cwd":"/home/ubuntu/ccbot-fork","model":{"id":"claude-sonnet-4-6","display_name":"Sonnet 4.6"},"workspace":{"current_dir":"/home/ubuntu/ccbot-fork","project_dir":"/home/ubuntu","added_dirs":[]},"version":"2.1.62","output_style":{"name":"default"},"cost":{"total_cost_usd":7.7672263,"total_duration_ms":1261862,"total_api_duration_ms":560088,"total_lines_added":17,"total_lines_removed":5},"context_window":{"total_input_tokens":166531,"total_output_tokens":25798,"context_window_size":200000,"current_usage":{"input_tokens":160,"output_tokens":722,"cache_creation_input_tokens":0,"cache_read_input_tokens":146816},"used_percentage":73,"remaining_percentage":27},"exceeds_200k_tokens":false} \ No newline at end of file diff --git a/.ultra/queues/.gitkeep b/.ultra/queues/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/.ultra/routines/.gitkeep b/.ultra/routines/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/.ultra/schemas/autoloop.json b/.ultra/schemas/autoloop.json new file mode 100644 index 00000000..26b8ebea --- /dev/null +++ b/.ultra/schemas/autoloop.json @@ -0,0 +1,37 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Autoloop State", + "type": "object", + "required": ["active", "lastHeartbeat", "currentCycle"], + "properties": { + "active": {"type": "boolean"}, + "startedAt": {"type": "string", "format": "date-time"}, + "lastHeartbeat": {"type": "string", "format": "date-time"}, + "currentCycle": {"type": "number"}, + "cycleTime": {"type": "number"}, + "queues": { + "type": "object", + "properties": { + "intake": {"type": "number"}, + "inProgress": {"type": "number"}, + "review": {"type": "number"}, + "completed": {"type": "number"}, + "failed": {"type": "number"} + } + }, + "health": { + "type": "object", + "properties": { + "status": {"type": "string"}, + "diskUsagePercent": {"type": "number"}, + "memoryUsagePercent": {"type": "number"}, + "testsPassing": {"type": "boolean"}, + "gitClean": {"type": "boolean"} + } + }, + "lastActions": { + "type": "array", + "items": {"type": "string"} + } + } +} diff --git a/.ultra/schemas/domain.json b/.ultra/schemas/domain.json new file mode 100644 index 00000000..60a86214 --- /dev/null +++ b/.ultra/schemas/domain.json @@ -0,0 +1,70 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Domain Configuration", + "type": "object", + "required": ["domain", "workspace", "autoloop"], + "properties": { + "domainId": {"type": "string"}, + "domain": { + "type": "object", + "required": ["id", "name", "type"], + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + "type": {"type": "string"}, + "description": {"type": "string"} + } + }, + "workspace": { + "type": "object", + "required": ["language", "framework"], + "properties": { + "language": {"type": "string"}, + "framework": {"type": "string"}, + "packageManager": {"type": "string"}, + "testing": {"type": "string"}, + "versionControl": {"type": "string"} + } + }, + "agents": { + "type": "array", + "items": {"type": "string"} + }, + "routing": { + "type": "object", + "properties": { + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "pattern": {"type": "string"}, + "agent": {"type": "string"} + } + } + } + } + }, + "routines": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "schedule": {"type": "string"}, + "enabled": {"type": "boolean"}, + "command": {"type": "string"} + }, + "required": ["name", "schedule", "enabled"] + } + }, + "autoloop": { + "type": "object", + "properties": { + "cycleTime": {"type": "number"}, + "enabled": {"type": "boolean"}, + "concurrentTasks": {"type": "number"} + } + } + } +} diff --git a/.ultra/schemas/queue.json b/.ultra/schemas/queue.json new file mode 100644 index 00000000..09562b11 --- /dev/null +++ b/.ultra/schemas/queue.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Task Queue", + "type": "object", + "required": ["intake", "in_progress", "review", "completed", "failed"], + "properties": { + "intake": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": {"type": "string"}, + "type": {"type": "string"}, + "title": {"type": "string"}, + "description": {"type": "string"}, + "priority": {"type": "string"}, + "status": {"type": "string"}, + "createdAt": {"type": "string", "format": "date-time"}, + "owner": {"type": "string"}, + "assignedAt": {"type": "string", "format": "date-time"} + } + } + }, + "in_progress": {"type": "array"}, + "review": {"type": "array"}, + "completed": {"type": "array"}, + "failed": {"type": "array"} + } +} diff --git a/.ultra/schemas/routine.json b/.ultra/schemas/routine.json new file mode 100644 index 00000000..9e0d446d --- /dev/null +++ b/.ultra/schemas/routine.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Routine Task", + "type": "object", + "required": ["name", "schedule", "enabled", "command"], + "properties": { + "name": {"type": "string"}, + "schedule": {"type": "string"}, + "enabled": {"type": "boolean"}, + "command": {"type": "string"}, + "timeout": {"type": "number"}, + "retryOnFailure": {"type": "boolean"}, + "maxRetries": {"type": "number"}, + "alertOnFailure": {"type": "boolean"}, + "continueOnFailure": {"type": "boolean"}, + "lastRun": {"type": "string", "format": "date-time"}, + "lastResult": {"type": "string"} + } +} diff --git a/.ultra/state/.gitkeep b/.ultra/state/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md new file mode 100644 index 00000000..053a6e8b --- /dev/null +++ b/TROUBLESHOOTING.md @@ -0,0 +1,87 @@ +# Troubleshooting Guide + +## Session Not Being Tracked + +### Symptom: "No active users for session" in logs + +This means ccbot is tracking the session but can't find users to deliver messages to. Check: + +1. **session_map.json format** - Must be object format, not string: + ```json + { + "ccbot:@8": { + "session_id": "uuid-here", + "cwd": "/home/ubuntu", + "window_name": "ubuntu" + } + } + ``` + + **Wrong format (string value):** + ```json + { + "ccbot:@8": "uuid-here" + } + ``` + +2. **state.json window_states must have cwd populated:** + ```json + { + "window_states": { + "@8": { + "session_id": "uuid-here", + "cwd": "/home/ubuntu", ← MUST be set + "window_name": "ubuntu" + } + } + } + ``` + + The `resolve_session_for_window()` function returns `None` when `cwd` is empty. + +### How to Fix + +1. Find the correct session ID: + ```bash + ls -lt ~/.claude/projects/-home-ubuntu/*.jsonl | head -5 + ``` + +2. Update session_map.json with correct format: + ```bash + nano ~/.ccbot/session_map.json + ``` + +3. Restart ccbot: + ```bash + killall ccbot && ccbot run & + ``` + +## Status Messages "Brewing" / "Forging" + +These are from `status_polling.py` which polls the terminal status line every 1 second. + +To disable: Edit `src/ccbot/bot.py` line 1545-1546 and comment out: +```python +# _status_poll_task = asyncio.create_task(status_poll_loop(application.bot)) +# logger.info("Status polling task started") +logger.info("Status polling DISABLED") +``` + +## Thinking Messages in Telegram + +Internal Claude thinking blocks have `content_type="thinking"`. The fix in `handle_new_message()` skips these: + +```python +if msg.content_type == "thinking": + logger.debug(f"Skipping thinking message for session {msg.session_id}") + return +``` + +## Wrong Session Tracked + +If ccbot is tracking the wrong session: +1. Check which session is active in your tmux window +2. Find the session file with the actual content you're working on +3. Update session_map.json with the correct session_id +4. Clear monitor_state.json: `echo '{"tracked_sessions": {}}' > ~/.ccbot/monitor_state.json` +5. Restart ccbot diff --git a/docs/CCBOT_FAILURES_AND_FIXES.md b/docs/CCBOT_FAILURES_AND_FIXES.md new file mode 100644 index 00000000..3a8ceb96 --- /dev/null +++ b/docs/CCBOT_FAILURES_AND_FIXES.md @@ -0,0 +1,347 @@ +# ccbot Failure Analysis and Fixes + +This document catalogs the reasons ccbot has failed, the fixes applied, and lessons learned for future debugging and maintenance. + +## Table of Contents +- [Rate Limiting Issues](#rate-limiting-issues) +- [Session Tracking Problems](#session-tracking-problems) +- [Token Swapping Procedure](#token-swapping-procedure) +- [Historical Fixes](#historical-fixes) +- [Prevention Strategies](#prevention-strategies) + +--- + +## Rate Limiting Issues + +### Problem: Telegram API Rate Limit Blocking + +**Symptoms:** +``` +telegram.ext.AIORateLimiter - INFO - Rate limit hit. Retrying after 37470.100000 seconds +``` +- Bot appears online but doesn't respond to any messages +- Existing sessions stop receiving updates +- Messages sent to the bot are not delivered +- Rate limit can last 10+ hours + +**Root Cause:** +The Telegram Bot API has rate limits on bot API calls. When a bot sends too many requests too quickly (especially sending a large response in many small chunks), it hits the rate limit and gets temporarily blocked. + +The rate limit is **per bot token** - other bots and Telegram services are unaffected. + +**Detection:** +```bash +# Check logs for rate limit errors +pm2 logs ccbot --lines 100 | grep -i "rate limit" + +# Check if bot is actually running +pm2 status ccbot +ps aux | grep ccbot +``` + +**Fix Applied: Token Swapping** + +When rate limited, swap to a new bot token: + +1. **Create a new bot on Telegram:** + - Message `@BotFather` + - Send `/newbot` + - Follow prompts to name your bot + - Copy the new token + +2. **Update the configuration:** + ```bash + # Update the .env file with new token + nano ~/.ccbot/.env + # Replace TELEGRAM_BOT_TOKEN=... + + # Or use automated edit + sed -i 's/TELEGRAM_BOT_TOKEN=.*/TELEGRAM_BOT_TOKEN=your_new_token/' ~/.ccbot/.env + ``` + +3. **Clear thread bindings** (critical step): + ```bash + # Edit state.json to clear old thread bindings + nano ~/.ccbot/state.json + # Change "thread_bindings" section to: "thread_bindings": {} + ``` + +4. **Restart ccbot:** + ```bash + pm2 restart ccbot + ``` + +5. **Reinitialize in Telegram:** + - Find your new bot on Telegram + - Send `/start` to each topic + - Create new topics/sessions as needed + +**Why Clearing Thread Bindings is Necessary:** + +Thread bindings contain Telegram message thread IDs that are **specific to each bot token**. When you swap tokens: +- Old thread IDs don't exist for the new bot +- Bot tries to send messages to non-existent threads +- Results in "Message thread not found" errors + +By clearing `thread_bindings` in `state.json`, the bot can create fresh bindings for the new token. + +--- + +## Session Tracking Problems + +### Problem: Session Map Race Condition + +**Symptoms:** +- "Session not in session_map" errors when binding new topics +- Window picker fails to bind to existing windows +- Session monitor can't find newly created sessions + +**Root Cause:** +When creating a new Telegram topic and using the window picker to select an existing tmux window, ccbot would bind the thread **before** the SessionStart hook had time to write the session_map.json entry. + +**Fix Applied:** +Added `wait_for_session_map_entry()` call in the window picker callback handler (`bot.py` line ~1141) to ensure the SessionStart hook completes before binding. + +**Testing:** +See `docs/TESTING_FIX.md` for complete testing procedure. + +### Problem: Wrong Session ID Format + +**Symptoms:** +``` +WARNING ccbot.hook: Invalid session_id format: test-123 +``` + +**Root Cause:** +The hook validates session IDs to ensure they're proper UUIDs. Test or malformed session IDs are rejected. + +**Fix:** +Ensure session IDs are proper UUIDs (e.g., `5bdaf9e8-0f61-452b-9894-1f4b611f1c1a`) + +### Problem: Session Not Being Tracked + +**Symptoms:** +- "No active users for session" in logs +- Messages not reaching Telegram + +**Root Causes:** + +1. **Incorrect session_map.json format:** + + **Wrong (string value):** + ```json + { + "ccbot:@8": "uuid-here" + } + ``` + + **Correct (object format):** + ```json + { + "ccbot:@8": { + "session_id": "uuid-here", + "cwd": "/home/ubuntu", + "window_name": "ubuntu" + } + } + ``` + +2. **Missing cwd in state.json:** + ```json + { + "window_states": { + "@8": { + "session_id": "uuid-here", + "cwd": "/home/ubuntu", // MUST be set + "window_name": "ubuntu" + } + } + } + ``` + + The `resolve_session_for_window()` function returns `None` when `cwd` is empty. + +**Fix:** +1. Find the correct session ID: + ```bash + ls -lt ~/.claude/projects/-home-ubuntu/*.jsonl | head -5 + ``` + +2. Update session_map.json with correct format: + ```bash + nano ~/.ccbot/session_map.json + ``` + +3. Restart ccbot: + ```bash + pm2 restart ccbot + ``` + +--- + +## Token Swapping Procedure + +Complete token swap workflow for when rate limiting occurs: + +```bash +#!/bin/bash +# Token swap procedure for ccbot + +# 1. Get new token from @BotFather on Telegram +NEW_TOKEN="your_new_token_here" + +# 2. Backup current state +cp ~/.ccbot/.env ~/.ccbot/.env.backup +cp ~/.ccbot/state.json ~/.ccbot/state.json.backup + +# 3. Update token in .env +sed -i "s/TELEGRAM_BOT_TOKEN=.*/TELEGRAM_BOT_TOKEN=$NEW_TOKEN/" ~/.ccbot/.env + +# 4. Clear thread bindings from state.json +python3 << 'EOF' +import json +with open('/home/ubuntu/.ccbot/state.json', 'r') as f: + state = json.load(f) +state['thread_bindings'] = {} +with open('/home/ubuntu/.ccbot/state.json', 'w') as f: + json.dump(state, f, indent=2) +EOF + +# 5. Restart ccbot +pm2 restart ccbot + +echo "Token swapped. Please send /start to your new bot on Telegram." +``` + +--- + +## Historical Fixes + +### 1. Startup Cleanup +**Issue:** Tracked sessions persisting after being closed +**Fix:** Added cleanup on startup to remove sessions not present in session_map +**Commit:** `3178d75` + +### 2. ANSI Escape Code Stripping +**Issue:** ANSI codes appearing in Telegram messages +**Fix:** Strip ANSI escape codes from parsed message text +**Commit:** `70183a0` + +### 3. Corrupted Byte Offset Recovery +**Issue:** Monitor state corrupted causing read failures +**Fix:** Auto-reset byte offset when file size is smaller than stored offset +**Commit:** `c769cc0` + +### 4. Status Polling Disable Option +**Issue:** "Brewing"/"Forging" status messages flooding chat +**Fix:** Made status polling optional, skip thinking messages +**Commit:** `09b463f` + +### 5. Session Map Cleanup +**Issue:** Stale entries in session_map.json +**Fix:** Clean up stale session_map entries on startup +**Commit:** `3178d75` + +### 6. Window ID Resolution +**Issue:** Using find_window_by_name causing incorrect window matching +**Fix:** Use find_window_by_id instead in usage_command +**Commit:** `2b99b8c` + +--- + +## Prevention Strategies + +### Rate Limiting Prevention + +1. **Batch Messages:** Send fewer, larger messages instead of many small ones +2. **Adjust Rate Limits:** Configure `AIORateLimiter` settings in `bot.py` +3. **Monitor Usage:** Track message frequency to stay under limits +4. **Queue Management:** Ensure message queue doesn't flood + +### Session Tracking Best Practices + +1. **Always use the hook:** Don't manually edit session_map.json unless necessary +2. **Validate format:** Ensure session_map.json uses object format, not strings +3. **Check cwd:** Verify state.json window_states have cwd populated +4. **Clean restart:** Use proper restart procedure (pm2 restart) to ensure clean state + +### Monitoring and Debugging + +**Key log locations:** +```bash +# PM2 logs +~/.pm2/logs/ccbot-out.log +~/.pm2/logs/ccbot-error.log + +# ccbot logs +~/.ccbot/ccbot.log +~/.ccbot/ccbot.out +``` + +**Health check script:** +```bash +#!/bin/bash +echo "=== ccbot Health Check ===" +echo "" + +# Check if running +pm2 status ccbot | grep online && echo "✓ ccbot is running" || echo "✗ ccbot is NOT running" + +# Check for rate limits +echo "" +echo "=== Rate Limit Check ===" +tail -100 ~/.pm2/logs/ccbot-error.log | grep -i "rate limit" && echo "✗ RATE LIMITED" || echo "✓ No rate limit issues" + +# Check for thread errors +echo "" +echo "=== Thread Binding Check ===" +tail -100 ~/.pm2/logs/ccbot-error.log | grep -i "thread not found" && echo "✗ Thread binding errors" || echo "✓ No thread binding errors" + +# Check session map +echo "" +echo "=== Session Map Status ===" +cat ~/.ccbot/session_map.json | python3 -m json.tool > /dev/null 2>&1 && echo "✓ session_map.json valid" || echo "✗ session_map.json INVALID" +``` + +### State File Management + +**Backup before changes:** +```bash +# Create backup +cp ~/.ccbot/state.json ~/.ccbot/state.json.backup.$(date +%Y%m%d_%H%M%S) + +# Restore if needed +cp ~/.ccbot/state.json.backup.YYYYMMDD_HHMMSS ~/.ccbot/state.json +pm2 restart ccbot +``` + +--- + +## Quick Reference: Common Issues + +| Symptom | Likely Cause | Fix | +|---------|--------------|-----| +| Bot not responding | Rate limited | Swap token | +| "Thread not found" | Wrong token | Swap token + clear bindings | +| "Session not in session_map" | Race condition | Wait for hook (fixed in latest) | +| No messages in Telegram | No active users | Check session_map format | +| Wrong session tracked | Bad session_id | Update session_map.json | +| "Session not ready" | Hook not fired | Check hook installation | + +--- + +## Contributing + +When adding new fixes: +1. Document the issue with clear symptoms +2. Identify the root cause +3. Document the fix with code references +4. Add testing procedures +5. Update this document + +## Related Documentation + +- `TROUBLESHOOTING.md` - General troubleshooting guide +- `docs/TESTING_FIX.md` - Testing procedures for specific fixes +- `.claude/rules/architecture.md` - System architecture +- `.claude/rules/topic-architecture.md` - Topic mapping details diff --git a/docs/TESTING_FIX.md b/docs/TESTING_FIX.md new file mode 100644 index 00000000..57411593 --- /dev/null +++ b/docs/TESTING_FIX.md @@ -0,0 +1,82 @@ +# Testing the Session Map Race Condition Fix + +## Problem +When creating a new Telegram topic and using the window picker to select an existing tmux window, ccbot would bind the thread before the SessionStart hook had time to write the session_map.json entry. This caused "Session not in session_map" errors. + +## Fix +Added `wait_for_session_map_entry()` call in the window picker callback handler (bot.py line ~1141) to ensure the SessionStart hook completes before binding. + +## Testing Procedure + +### 1. Install from Fork +```bash +# Uninstall existing ccbot +pipx uninstall ccbot + +# Install from fork with editable mode (for testing) +cd ~/ccbot-fork +pipx install . +``` + +### 2. Restart ccbot +```bash +cd ~/ccbot-fork +./scripts/restart.sh +``` + +### 3. Test Window Picker Flow + +#### Scenario A: Select existing window (NEW TOPIC) +1. Create a NEW topic in Telegram (not previously bound) +2. Send any message to the new topic +3. ccbot should respond with window picker UI showing available windows +4. Select a window (e.g., "ubuntu") +5. **Expected behavior**: + - No error about "Session not in session_map" + - Topic binds successfully to the window + - Topic renames to match window name + - Pending message (if any) is forwarded to the window + +#### Scenario B: Auto-create new session (no existing windows) +1. Create a NEW topic in Telegram +2. Kill all existing tmux windows (except ccbot's own): `tmux kill-window -t @8` +3. Send any message to the new topic +4. ccbot should offer directory browser for creating new window +5. **Expected behavior**: Works as before (this flow wasn't broken) + +### 4. Verify session_map.json +```bash +cat ~/.ccbot/session_map.json +``` + +Should contain entries for all bound windows, including newly bound ones. + +### 5. Check logs for any warnings +```bash +journalctl -u ccbot -n 50 --no-pager +``` + +Should NOT see: +- "Session map entry not found for window" +- "Session not ready" errors (unless genuine timeout) + +## Success Criteria +✅ Window picker successfully binds to existing windows +✅ No "Session not in session_map" errors +✅ Topic renames to match window name +✅ Pending messages are forwarded correctly +✅ session_map.json contains the new binding + +## Rollback +If issues occur: +```bash +cd ~/ccbot-fork +git revert HEAD +./scripts/restart.sh +``` + +Or uninstall from fork and reinstall from PyPI: +```bash +pipx uninstall ccbot +pipx install ccbot +``` diff --git a/scripts/health-check.sh b/scripts/health-check.sh new file mode 100755 index 00000000..d94d0560 --- /dev/null +++ b/scripts/health-check.sh @@ -0,0 +1,162 @@ +#!/bin/bash +# ccbot Health Check Script +# Quick diagnostic tool for common ccbot issues + +# Don't exit on errors - we want to see all checks +set +e + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "=== ccbot Health Check ===" +echo "" + +# Check for multiple ccbot instances (IMPORTANT - causes conflicts) +echo -n "► Checking for multiple ccbot instances... " +# Only count actual Python ccbot processes, not shell/ssh/ccbot-fork processes +CCBOT_PROCS=$(ps aux | grep -E 'python.*ccbot|ccbot.*\.venv/bin/ccbot' | grep -v 'health-check\|grep' | wc -l) +CCBOT_PIDS=$(ps aux | grep -E 'python.*ccbot|ccbot.*\.venv/bin/ccbot' | grep -v 'health-check\|grep' | awk '{print $2}') +if [ "$CCBOT_PROCS" -eq 0 ]; then + echo -e "${RED}✗ No ccbot instances found${NC}" + echo " Fix: Start ccbot in tmux: tmux send-keys -t ccbot:__main__ 'ccbot run' Enter" +elif [ "$CCBOT_PROCS" -gt 1 ]; then + echo -e "${RED}✗ $CCBOT_PROCS ccbot instances running (CONFLICT!)${NC}" + echo " ${YELLOW}This causes 'terminated by other getUpdates request' errors${NC}" + echo " Fix: Kill all but one instance:" + echo "$CCBOT_PIDS" | while read pid; do echo " kill $pid"; done +else + echo -e "${GREEN}✓ Exactly 1 ccbot instance running${NC}" + CCBOT_PID=$(echo "$CCBOT_PIDS" | head -1) + echo " PID: $CCBOT_PID" +fi + +# Check if running in tmux +echo "" +echo -n "► Checking if ccbot is running in tmux... " +if tmux has-session -t ccbot 2>/dev/null; then + if tmux capture-pane -t ccbot:__main__ -p 2>/dev/null | grep -q "ccbot\."; then + echo -e "${GREEN}✓ ccbot running in tmux ccbot:__main__${NC}" + else + echo -e "${YELLOW}⚠ tmux session exists but ccbot may not be running in __main__${NC}" + fi +else + echo -e "${YELLOW}⚠ No tmux session 'ccbot' found${NC}" + echo " (ccbot may be running via PM2 or directly)" +fi + +# Check for rate limits (check both PM2 logs and ccbot logs) +echo "" +echo -n "► Checking for rate limit issues... " +if tail -100 ~/.pm2/logs/ccbot-error.log 2>/dev/null | grep -qi "rate limit\|retry after"; then + echo -e "${RED}✗ RATE LIMITED DETECTED (PM2 logs)${NC}" + echo " Action needed: Swap bot token (see docs/CCBOT_FAILURES_AND_FIXES.md)" + tail -5 ~/.pm2/logs/ccbot-error.log 2>/dev/null | grep -i "rate limit\|retry after" | head -1 | sed 's/^/ /' +elif tail -100 ~/.ccbot/ccbot.log 2>/dev/null | grep -qi "rate limit\|retry after"; then + echo -e "${RED}✗ RATE LIMITED DETECTED (ccbot logs)${NC}" + echo " Action needed: Swap bot token (see docs/CCBOT_FAILURES_AND_FIXES.md)" + tail -5 ~/.ccbot/ccbot.log 2>/dev/null | grep -i "rate limit\|retry after" | head -1 | sed 's/^/ /' +else + echo -e "${GREEN}✓ No rate limit issues${NC}" +fi + +# Check for recent Telegram bot conflicts (last 5 minutes) +echo "" +echo -n "► Checking for recent Telegram bot conflicts... " +if find ~/.ccbot/ccbot.log -mmin -5 2>/dev/null | grep -q . && tail -100 ~/.ccbot/ccbot.log 2>/dev/null | grep -qi "Conflict.*terminated by other getUpdates"; then + echo -e "${RED}✗ Recent bot conflict detected${NC}" + echo " ${YELLOW}This means multiple bot instances are polling Telegram${NC}" + echo " Fix: Ensure only one ccbot instance is running" +else + echo -e "${GREEN}✓ No recent bot conflicts${NC}" +fi + +# Check for thread errors +echo "" +echo -n "► Checking for thread binding errors... " +if tail -100 ~/.pm2/logs/ccbot-error.log 2>/dev/null | grep -q "thread not found"; then + THREAD_ERRORS=$(tail -100 ~/.pm2/logs/ccbot-error.log 2>/dev/null | grep -c "thread not found" || true) + echo -e "${YELLOW}⚠ $THREAD_ERRORS thread binding errors in PM2 logs${NC}" + echo " This usually means you need to swap bot token or clear thread bindings" +elif tail -100 ~/.ccbot/ccbot.out 2>/dev/null | grep -q "thread not found\|Message thread not found"; then + echo -e "${YELLOW}⚠ Thread binding errors in ccbot logs${NC}" + echo " This usually means you need to swap bot token or clear thread bindings" +else + echo -e "${GREEN}✓ No thread binding errors${NC}" +fi + +# Check session map +echo "" +echo -n "► Checking session_map.json format... " +if cat ~/.ccbot/session_map.json 2>/dev/null | python3 -m json.tool > /dev/null 2>&1; then + echo -e "${GREEN}✓ session_map.json is valid JSON${NC}" + # Check for object format vs string format + if grep -qE '"[^"]+":\s*"[a-f0-9-]{36}"' ~/.ccbot/session_map.json 2>/dev/null; then + echo -e " ${YELLOW}⚠ Warning: session_map may use string format (should be object format)${NC}" + fi +else + echo -e "${RED}✗ session_map.json is INVALID${NC}" + echo " Fix: Check format (see TROUBLESHOOTING.md)" +fi + +# Check state.json +echo "" +echo -n "► Checking state.json format... " +if cat ~/.ccbot/state.json 2>/dev/null | python3 -m json.tool > /dev/null 2>&1; then + echo -e "${GREEN}✓ state.json is valid JSON${NC}" + + # Check if cwd is populated + if grep -q '"cwd":\s*""' ~/.ccbot/state.json 2>/dev/null; then + echo -e " ${YELLOW}⚠ Warning: Some window_states have empty cwd${NC}" + fi + + # Check thread_bindings for stale IDs + THREAD_COUNT=$(python3 -c "import json; s=json.load(open('/home/ubuntu/.ccbot/state.json')); print(sum(len(v) for v in s.get('thread_bindings', {}).values()))" 2>/dev/null || echo "0") + echo " Thread bindings: $THREAD_COUNT" +else + echo -e "${RED}✗ state.json is INVALID${NC}" +fi + +# Check session files +echo "" +echo -n "► Checking for active session files... " +SESSION_COUNT=$(ls -1 ~/.claude/projects/*/*.jsonl 2>/dev/null | wc -l) +if [ "$SESSION_COUNT" -gt 0 ]; then + echo -e "${GREEN}✓ Found $SESSION_COUNT session files${NC}" + echo " Latest sessions:" + ls -lt ~/.claude/projects/*/*.jsonl 2>/dev/null | head -3 | awk '{print " " $NF " (" $6 " " $7 " " $8 ")"}' +else + echo -e "${YELLOW}⚠ No session files found${NC}" +fi + +# Check tmux session +echo "" +echo -n "► Checking tmux session 'ccbot'... " +if tmux has-session -t ccbot 2>/dev/null; then + echo -e "${GREEN}✓ tmux session exists${NC}" + WINDOW_COUNT=$(tmux list-windows -t ccbot 2>/dev/null | wc -l) + echo " Windows: $WINDOW_COUNT" +else + echo -e "${RED}✗ tmux session 'ccbot' not found${NC}" + echo " Fix: tmux new-session -d -s ccbot" +fi + +# Summary +echo "" +echo "=== Summary ===" +if [ "$CCBOT_PROCS" -eq 0 ]; then + echo -e "${RED}Action required: Start ccbot${NC}" + echo " Run: tmux send-keys -t ccbot:__main__ 'ccbot run' Enter" +elif [ "$CCBOT_PROCS" -gt 1 ]; then + echo -e "${RED}Action required: Kill duplicate ccbot instances${NC}" + echo " Multiple instances cause Telegram API conflicts" +elif tail -100 ~/.ccbot/ccbot.log 2>/dev/null | grep -qi "rate limit\|retry after"; then + echo -e "${RED}Action required: Swap bot token${NC}" + echo " See docs/CCBOT_FAILURES_AND_FIXES.md for token swap procedure" +else + echo -e "${GREEN}All checks passed! ccbot appears healthy.${NC}" +fi + +echo "" +echo "For detailed troubleshooting, see: docs/CCBOT_FAILURES_AND_FIXES.md" diff --git a/src/ccbot/bot.py b/src/ccbot/bot.py index 1b218934..1c8aa973 100644 --- a/src/ccbot/bot.py +++ b/src/ccbot/bot.py @@ -1138,6 +1138,22 @@ async def callback_handler(update: Update, context: ContextTypes.DEFAULT_TYPE) - await query.answer(f"Window '{display}' no longer exists", show_alert=True) return + # Wait for SessionStart hook to write session_map entry + # This ensures the session is fully initialized before binding + # (fixes race condition where bind_thread executes before hook completes) + if not await session_manager.wait_for_session_map_entry(selected_wid): + logger.warning( + "Session map entry not found for window %s (user=%d, thread=%d)", + selected_wid, + user.id, + _get_thread_id(update), + ) + await query.answer( + "Session not ready. Please try selecting the window again.", + show_alert=True, + ) + return + thread_id = _get_thread_id(update) if thread_id is None: await query.answer("Not in a topic", show_alert=True) @@ -1410,10 +1426,15 @@ async def handle_new_message(msg: NewMessage, bot: Bot) -> None: Messages are queued per-user to ensure status messages always appear last. Routes via thread_bindings to deliver to the correct topic. """ + # Skip internal thinking messages - don't send to Telegram + if msg.content_type == "thinking": + logger.debug(f"Skipping thinking message for session {msg.session_id}") + return + status = "complete" if msg.is_complete else "streaming" logger.info( f"handle_new_message [{status}]: session={msg.session_id}, " - f"text_len={len(msg.text)}" + f"content_type={msg.content_type}, text_len={len(msg.text)}" ) # Find users whose thread-bound window matches this session @@ -1536,9 +1557,11 @@ async def message_callback(msg: NewMessage) -> None: session_monitor = monitor logger.info("Session monitor started") - # Start status polling task + # Status polling disabled - status messages ("Brewed", "Forging") are annoying + # To re-enable, uncomment the next two lines: _status_poll_task = asyncio.create_task(status_poll_loop(application.bot)) logger.info("Status polling task started") + # logger.info("Status polling DISABLED") async def post_shutdown(application: Application) -> None: diff --git a/src/ccbot/terminal_parser.py b/src/ccbot/terminal_parser.py index 288f8635..1ecf64c2 100644 --- a/src/ccbot/terminal_parser.py +++ b/src/ccbot/terminal_parser.py @@ -2,8 +2,8 @@ Parses captured tmux pane content to detect: - Interactive UIs (AskUserQuestion, ExitPlanMode, Permission Prompt, - RestoreCheckpoint) via regex-based UIPattern matching with top/bottom - delimiters. + RestoreCheckpoint, TrustFolder) via regex-based UIPattern matching with + top/bottom delimiters. - Status line (spinner characters + working text) by scanning from bottom up. All Claude Code text patterns live here. To support a new UI type or @@ -97,6 +97,18 @@ class UIPattern: re.compile(r"^\s*Type to filter"), ), ), + UIPattern( + name="TrustFolder", + top=( + re.compile(r"^\s*Accessing workspace:"), + re.compile(r"^\s*Quick safety check:"), + ), + bottom=( + re.compile(r"^\s*Enter to confirm\s*·\s*Esc to cancel"), + re.compile(r"^\s*Enter to confirm"), + ), + min_gap=3, + ), ] diff --git a/src/ccbot/tmux_manager.py b/src/ccbot/tmux_manager.py index eae2702a..84b5b7a0 100644 --- a/src/ccbot/tmux_manager.py +++ b/src/ccbot/tmux_manager.py @@ -67,15 +67,40 @@ def get_or_create_session(self) -> libtmux.Session: if session: return session - # Create new session with main window named specifically - session = self.server.new_session( - session_name=self.session_name, - start_directory=str(Path.home()), - ) - # Rename the default window to the main window name - if session.windows: - session.windows[0].rename_window(config.tmux_main_window_name) - return session + # Try using libtmux first + try: + session = self.server.new_session( + session_name=self.session_name, + start_directory=str(Path.home()), + ) + # If successful and has windows, rename the first window + if session and hasattr(session, 'windows') and session.windows: + session.windows[0].rename_window(config.tmux_main_window_name) + return session + except libtmux.exc.TmuxSessionExists: + # Session already exists, just get it + logger.info(f"Session {self.session_name} already exists, retrieving it") + import time + for _ in range(5): # Retry a few times + session = self.get_session() + if session: + return session + time.sleep(0.2) + raise Exception(f"Session exists but could not retrieve it") + except Exception as e: + # Other libtmux failures, try subprocess + logger.warning(f"libtmux failed: {e}, using subprocess") + import subprocess + subprocess.run(['tmux', 'new-session', '-d', '-s', self.session_name]) + # Get the newly created session + self._server = None + for _ in range(5): + session = self.get_session() + if session: + return session + import time + time.sleep(0.2) + raise Exception(f"Failed to retrieve session after subprocess creation") async def list_windows(self) -> list[TmuxWindow]: """List all windows in the session with their working directories. diff --git a/src/ccbot/tmux_manager.py.bak b/src/ccbot/tmux_manager.py.bak new file mode 100644 index 00000000..eae2702a --- /dev/null +++ b/src/ccbot/tmux_manager.py.bak @@ -0,0 +1,404 @@ +"""Tmux session/window management via libtmux. + +Wraps libtmux to provide async-friendly operations on a single tmux session: + - list_windows / find_window_by_name: discover Claude Code windows. + - capture_pane: read terminal content (plain or with ANSI colors). + - send_keys: forward user input or control keys to a window. + - create_window / kill_window: lifecycle management. + +All blocking libtmux calls are wrapped in asyncio.to_thread(). + +Key class: TmuxManager (singleton instantiated as `tmux_manager`). +""" + +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass +from pathlib import Path + +import libtmux + +from .config import config + +logger = logging.getLogger(__name__) + + +@dataclass +class TmuxWindow: + """Information about a tmux window.""" + + window_id: str + window_name: str + cwd: str # Current working directory + pane_current_command: str = "" # Process running in active pane + + +class TmuxManager: + """Manages tmux windows for Claude Code sessions.""" + + def __init__(self, session_name: str | None = None): + """Initialize tmux manager. + + Args: + session_name: Name of the tmux session to use (default from config) + """ + self.session_name = session_name or config.tmux_session_name + self._server: libtmux.Server | None = None + + @property + def server(self) -> libtmux.Server: + """Get or create tmux server connection.""" + if self._server is None: + self._server = libtmux.Server() + return self._server + + def get_session(self) -> libtmux.Session | None: + """Get the tmux session if it exists.""" + try: + return self.server.sessions.get(session_name=self.session_name) + except Exception: + return None + + def get_or_create_session(self) -> libtmux.Session: + """Get existing session or create a new one.""" + session = self.get_session() + if session: + return session + + # Create new session with main window named specifically + session = self.server.new_session( + session_name=self.session_name, + start_directory=str(Path.home()), + ) + # Rename the default window to the main window name + if session.windows: + session.windows[0].rename_window(config.tmux_main_window_name) + return session + + async def list_windows(self) -> list[TmuxWindow]: + """List all windows in the session with their working directories. + + Returns: + List of TmuxWindow with window info and cwd + """ + + def _sync_list_windows() -> list[TmuxWindow]: + windows = [] + session = self.get_session() + + if not session: + return windows + + for window in session.windows: + name = window.window_name or "" + # Skip the main window (placeholder window) + if name == config.tmux_main_window_name: + continue + + try: + # Get the active pane's current path and command + pane = window.active_pane + if pane: + cwd = pane.pane_current_path or "" + pane_cmd = pane.pane_current_command or "" + else: + cwd = "" + pane_cmd = "" + + windows.append( + TmuxWindow( + window_id=window.window_id or "", + window_name=name, + cwd=cwd, + pane_current_command=pane_cmd, + ) + ) + except Exception as e: + logger.debug(f"Error getting window info: {e}") + + return windows + + return await asyncio.to_thread(_sync_list_windows) + + async def find_window_by_name(self, window_name: str) -> TmuxWindow | None: + """Find a window by its name. + + Args: + window_name: The window name to match + + Returns: + TmuxWindow if found, None otherwise + """ + windows = await self.list_windows() + for window in windows: + if window.window_name == window_name: + return window + logger.debug("Window not found by name: %s", window_name) + return None + + async def find_window_by_id(self, window_id: str) -> TmuxWindow | None: + """Find a window by its tmux window ID (e.g. '@0', '@12'). + + Args: + window_id: The tmux window ID to match + + Returns: + TmuxWindow if found, None otherwise + """ + windows = await self.list_windows() + for window in windows: + if window.window_id == window_id: + return window + logger.debug("Window not found by id: %s", window_id) + return None + + async def capture_pane(self, window_id: str, with_ansi: bool = False) -> str | None: + """Capture the visible text content of a window's active pane. + + Args: + window_id: The window ID to capture + with_ansi: If True, capture with ANSI color codes + + Returns: + The captured text, or None on failure. + """ + if with_ansi: + # Use async subprocess to call tmux capture-pane -e for ANSI colors + try: + proc = await asyncio.create_subprocess_exec( + "tmux", + "capture-pane", + "-e", + "-p", + "-t", + window_id, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + if proc.returncode == 0: + return stdout.decode("utf-8") + logger.error( + f"Failed to capture pane {window_id}: {stderr.decode('utf-8')}" + ) + return None + except Exception as e: + logger.error(f"Unexpected error capturing pane {window_id}: {e}") + return None + + # Original implementation for plain text - wrap in thread + def _sync_capture() -> str | None: + session = self.get_session() + if not session: + return None + try: + window = session.windows.get(window_id=window_id) + if not window: + return None + pane = window.active_pane + if not pane: + return None + lines = pane.capture_pane() + return "\n".join(lines) if isinstance(lines, list) else str(lines) + except Exception as e: + logger.error(f"Failed to capture pane {window_id}: {e}") + return None + + return await asyncio.to_thread(_sync_capture) + + async def send_keys( + self, window_id: str, text: str, enter: bool = True, literal: bool = True + ) -> bool: + """Send keys to a specific window. + + Args: + window_id: The window ID to send to + text: Text to send + enter: Whether to press enter after the text + literal: If True, send text literally. If False, interpret special keys + like "Up", "Down", "Left", "Right", "Escape", "Enter". + + Returns: + True if successful, False otherwise + """ + if literal and enter: + # Split into text + delay + Enter via libtmux. + # Claude Code's TUI sometimes interprets a rapid-fire Enter + # (arriving in the same input batch as the text) as a newline + # rather than submit. A 500ms gap lets the TUI process the + # text before receiving Enter. + def _send_literal(chars: str) -> bool: + session = self.get_session() + if not session: + logger.error("No tmux session found") + return False + try: + window = session.windows.get(window_id=window_id) + if not window: + logger.error(f"Window {window_id} not found") + return False + pane = window.active_pane + if not pane: + logger.error(f"No active pane in window {window_id}") + return False + pane.send_keys(chars, enter=False, literal=True) + return True + except Exception as e: + logger.error(f"Failed to send keys to window {window_id}: {e}") + return False + + def _send_enter() -> bool: + session = self.get_session() + if not session: + return False + try: + window = session.windows.get(window_id=window_id) + if not window: + return False + pane = window.active_pane + if not pane: + return False + pane.send_keys("", enter=True, literal=False) + return True + except Exception as e: + logger.error(f"Failed to send Enter to window {window_id}: {e}") + return False + + # Claude Code's ! command mode: send "!" first so the TUI + # switches to bash mode, wait 1s, then send the rest. + if text.startswith("!"): + if not await asyncio.to_thread(_send_literal, "!"): + return False + rest = text[1:] + if rest: + await asyncio.sleep(1.0) + if not await asyncio.to_thread(_send_literal, rest): + return False + else: + if not await asyncio.to_thread(_send_literal, text): + return False + await asyncio.sleep(0.5) + return await asyncio.to_thread(_send_enter) + + # Other cases: special keys (literal=False) or no-enter + def _sync_send_keys() -> bool: + session = self.get_session() + if not session: + logger.error("No tmux session found") + return False + + try: + window = session.windows.get(window_id=window_id) + if not window: + logger.error(f"Window {window_id} not found") + return False + + pane = window.active_pane + if not pane: + logger.error(f"No active pane in window {window_id}") + return False + + pane.send_keys(text, enter=enter, literal=literal) + return True + + except Exception as e: + logger.error(f"Failed to send keys to window {window_id}: {e}") + return False + + return await asyncio.to_thread(_sync_send_keys) + + async def kill_window(self, window_id: str) -> bool: + """Kill a tmux window by its ID.""" + + def _sync_kill() -> bool: + session = self.get_session() + if not session: + return False + try: + window = session.windows.get(window_id=window_id) + if not window: + return False + window.kill() + logger.info("Killed window %s", window_id) + return True + except Exception as e: + logger.error(f"Failed to kill window {window_id}: {e}") + return False + + return await asyncio.to_thread(_sync_kill) + + async def create_window( + self, + work_dir: str, + window_name: str | None = None, + start_claude: bool = True, + ) -> tuple[bool, str, str, str]: + """Create a new tmux window and optionally start Claude Code. + + Args: + work_dir: Working directory for the new window + window_name: Optional window name (defaults to directory name) + start_claude: Whether to start claude command + + Returns: + Tuple of (success, message, window_name, window_id) + """ + # Validate directory first + path = Path(work_dir).expanduser().resolve() + if not path.exists(): + return False, f"Directory does not exist: {work_dir}", "", "" + if not path.is_dir(): + return False, f"Not a directory: {work_dir}", "", "" + + # Create window name, adding suffix if name already exists + final_window_name = window_name if window_name else path.name + + # Check for existing window name + base_name = final_window_name + counter = 2 + while await self.find_window_by_name(final_window_name): + final_window_name = f"{base_name}-{counter}" + counter += 1 + + # Create window in thread + def _create_and_start() -> tuple[bool, str, str, str]: + session = self.get_or_create_session() + try: + # Create new window + window = session.new_window( + window_name=final_window_name, + start_directory=str(path), + ) + + wid = window.window_id or "" + + # Start Claude Code if requested + if start_claude: + pane = window.active_pane + if pane: + pane.send_keys(config.claude_command, enter=True) + + logger.info( + "Created window '%s' (id=%s) at %s", + final_window_name, + wid, + path, + ) + return ( + True, + f"Created window '{final_window_name}' at {path}", + final_window_name, + wid, + ) + + except Exception as e: + logger.error(f"Failed to create window: {e}") + return False, f"Failed to create window: {e}", "", "" + + return await asyncio.to_thread(_create_and_start) + + +# Global instance with default session name +tmux_manager = TmuxManager()