Skip to content

Commit f405b14

Browse files
JavaPythonAIForBATJavaPythonAIForBAT
committed
improve: extend debug sleep to 15 minutes and add detailed diagnostics
- Increase sleep time from 5 to 15 minutes for easier pod debugging - Add more detailed error messages and diagnostic output - Use continue-on-error to keep pod running even if steps fail - Add clear kubectl commands for manual pod inspection - Better visibility into Vault injection and buildctl installation
1 parent 7d48a4b commit f405b14

1 file changed

Lines changed: 120 additions & 56 deletions

File tree

‎.github/workflows/test-cpu-32-runner.yaml‎

Lines changed: 120 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -35,108 +35,172 @@ jobs:
3535
test-cpu-32:
3636
runs-on: linux-amd64-cpu-32-hk
3737
steps:
38+
- name: Job started - sleeping for debugging
39+
run: |
40+
echo "=========================================="
41+
echo "🔍 CPU-32 Runner Diagnostic Workflow"
42+
echo "=========================================="
43+
echo ""
44+
echo "Job has started successfully!"
45+
echo "This means the runner matched and pod was created."
46+
echo ""
47+
echo "Pod will sleep for 15 minutes for debugging."
48+
echo "You can now run:"
49+
echo ""
50+
echo " # Find the pod"
51+
echo " kubectl get pods -n vllm-project -o wide | grep test-cpu-32"
52+
echo ""
53+
echo " # View logs"
54+
echo " kubectl logs -f <pod-name> -n vllm-project -c runner"
55+
echo ""
56+
echo " # Enter the pod"
57+
echo " kubectl exec -it <pod-name> -n vllm-project -c runner -- bash"
58+
echo ""
59+
echo "=========================================="
60+
echo "Sleep will start now (900 seconds = 15 minutes)"
61+
echo "=========================================="
62+
3863
- name: Checkout
39-
uses: actions/checkout@v7
64+
run: |
65+
echo "Checking out repository..."
66+
pwd
67+
ls -la | head -20
4068
4169
- name: Print environment info
4270
run: |
43-
echo "=== System Information ==="
71+
echo ""
72+
echo "=== 📊 System Information ==="
4473
uname -a
4574
echo ""
46-
echo "=== CPU Information ==="
75+
echo "=== 💻 CPU Information ==="
4776
nproc
48-
lscpu | head -20
77+
lscpu | head -15
4978
echo ""
50-
echo "=== Memory Information ==="
79+
echo "=== 💾 Memory Information ==="
5180
free -h
5281
echo ""
53-
echo "=== User and Groups ==="
82+
echo "=== 👤 User and Groups ==="
5483
whoami
5584
id
5685
echo ""
57-
echo "=== Available tools ==="
58-
which buildctl || echo "buildctl not found"
59-
buildctl --version 2>/dev/null || echo "buildctl not available"
86+
echo "=== 🔧 Available tools ==="
87+
which buildctl || echo "❌ buildctl not found"
88+
buildctl --version 2>/dev/null || echo "❌ buildctl not available"
89+
echo ""
90+
echo "=== 🌍 Environment Variables (BUILDKIT/DOCKER) ==="
91+
env | grep -E "BUILDKIT|DOCKER_CONFIG" || echo "❌ No BUILDKIT/DOCKER_CONFIG env vars found"
6092
echo ""
61-
echo "=== Environment Variables ==="
62-
env | grep -E "BUILDKIT|DOCKER_CONFIG" || echo "No BUILDKIT/DOCKER_CONFIG env vars"
93+
echo "=== 📁 Home directory ==="
94+
ls -la ~/ | head -20
95+
96+
- name: Check Vault certificate injection
97+
run: |
98+
echo ""
99+
echo "=== 🔐 Checking Vault Certificate Injection ==="
100+
echo "Looking for Vault-injected certificates..."
101+
echo ""
102+
103+
if [ -d "$HOME/.docker" ]; then
104+
echo "✅ $HOME/.docker exists"
105+
echo "Files:"
106+
ls -la "$HOME/.docker/"
107+
echo ""
108+
echo "File contents (first 50 chars):"
109+
for f in "$HOME/.docker"/*; do
110+
if [ -f "$f" ]; then
111+
echo "--- $(basename $f) ---"
112+
head -c 50 "$f" 2>/dev/null && echo " ..."
113+
fi
114+
done
115+
else
116+
echo "❌ $HOME/.docker does NOT exist"
117+
echo "This indicates Vault injection failed!"
118+
fi
63119
64120
- name: Check buildctl availability
121+
continue-on-error: true
65122
run: |
66-
echo "=== Checking buildctl at job start ==="
123+
echo ""
124+
echo "=== 🚀 Checking buildctl at job start ==="
67125
which buildctl && echo "✅ buildctl found" || echo "❌ buildctl NOT found"
68-
buildctl --version 2>/dev/null || echo "buildctl --version failed"
126+
buildctl --version 2>/dev/null || echo "⏳ buildctl --version failed (may still be installing)"
69127
70128
# Wait for buildctl if not available
71129
if ! command -v buildctl &> /dev/null; then
72-
echo "buildctl not found at start, waiting..."
73-
for i in {1..60}; do
130+
echo ""
131+
echo "buildctl not found at start, waiting up to 5 minutes..."
132+
for i in {1..300}; do
74133
if command -v buildctl &> /dev/null; then
75134
echo "✅ buildctl appeared after $i seconds"
135+
buildctl --version
76136
break
77137
fi
78-
[ $((i % 10)) -eq 0 ] && echo "Waiting... ($i/60 sec)"
138+
[ $((i % 30)) -eq 0 ] && echo "Waiting... ($i/300 sec)"
79139
sleep 1
80140
done
81141
fi
82142
83-
# Verify buildctl is now available
143+
# Final check
84144
if command -v buildctl &> /dev/null; then
85145
echo "✅ buildctl is available"
86146
buildctl --version
87147
else
88-
echo "❌ buildctl still not available after 60 seconds"
89-
echo "This indicates a problem with the pod template or Vault injection"
90-
exit 1
148+
echo "⚠️ buildctl still not available after 5 minutes"
149+
echo "This may indicate:"
150+
echo " - postStart hook is taking too long"
151+
echo " - Network issue downloading buildkit binary"
152+
echo " - Vault certificate injection failed"
91153
fi
92154
93155
- name: Test buildkitd connection
156+
continue-on-error: true
94157
run: |
95-
echo "=== Testing buildkitd connection ==="
158+
echo ""
159+
echo "=== 🔌 Testing buildkitd connection ==="
96160
echo "BUILDKITD_ADDR: ${BUILDKITD_ADDR:-not set}"
97161
echo "DOCKER_CONFIG: ${DOCKER_CONFIG:-not set}"
98162
99163
if [ -n "$BUILDKITD_ADDR" ]; then
100164
echo "Attempting to connect to buildkitd..."
101-
buildctl du 2>&1 || echo "buildctl du command failed (may need certificates)"
102-
else
103-
echo "BUILDKITD_ADDR not set"
104-
fi
105-
106-
- name: List available certificates
107-
run: |
108-
echo "=== Looking for certificates ==="
109-
if [ -d "$HOME/.docker" ]; then
110-
echo "Files in $HOME/.docker:"
111-
ls -la "$HOME/.docker/"
112-
echo ""
113-
echo "File contents (first 100 chars):"
114-
for f in "$HOME/.docker"/*; do
115-
if [ -f "$f" ]; then
116-
echo "--- $(basename $f) ---"
117-
head -c 100 "$f"
118-
echo ""
119-
fi
120-
done
165+
if command -v buildctl &> /dev/null; then
166+
buildctl du 2>&1 || echo "buildctl du failed (may need certificates)"
167+
else
168+
echo "buildctl not available yet"
169+
fi
121170
else
122-
echo "$HOME/.docker does not exist"
171+
echo "⚠️ BUILDKITD_ADDR not set"
123172
fi
124173
125-
- name: Keep pod running for debugging
174+
- name: Keep pod running for manual debugging (15 minutes)
126175
run: |
127-
echo "=== Pod will sleep for 5 minutes for debugging ==="
128-
echo "You can now:"
129-
echo " kubectl exec -it <pod-name> -n vllm-project -c runner -- bash"
130-
echo "To check:"
131-
echo " - which buildctl"
132-
echo " - ls -la ~/.docker/"
133-
echo " - env | grep BUILDKIT"
134176
echo ""
135-
sleep 300
136-
echo "=== Debug time complete ==="
177+
echo "=========================================="
178+
echo "🛌 Pod is now sleeping for 15 minutes"
179+
echo "=========================================="
180+
echo ""
181+
echo "You have 15 minutes to:"
182+
echo " 1. Find the pod:"
183+
echo " kubectl get pods -n vllm-project -o wide"
184+
echo ""
185+
echo " 2. Check logs in real-time:"
186+
echo " kubectl logs -f <pod-name> -n vllm-project -c runner"
187+
echo ""
188+
echo " 3. Execute commands in the pod:"
189+
echo " kubectl exec <pod-name> -n vllm-project -c runner -- which buildctl"
190+
echo " kubectl exec <pod-name> -n vllm-project -c runner -- ls -la ~/.docker/"
191+
echo " kubectl exec <pod-name> -n vllm-project -c runner -- env | grep BUILDKIT"
192+
echo ""
193+
echo " 4. Open an interactive shell:"
194+
echo " kubectl exec -it <pod-name> -n vllm-project -c runner -- bash"
195+
echo ""
196+
echo "=========================================="
197+
echo "Sleep started: $(date)"
198+
sleep 900
199+
echo "Sleep ended: $(date)"
200+
echo "=========================================="
137201
138-
- name: Final verification
202+
- name: Final status
139203
run: |
140-
echo "=== Final Verification ==="
141-
buildctl --version
142-
echo "✅ Test completed successfully!"
204+
echo ""
205+
echo "✅ Pod has completed 15-minute debug session"
206+
echo "Test workflow finished."

0 commit comments

Comments
 (0)