Skip to content

Commit 568d1a6

Browse files
jinzhongjiajinzhongjia
committed
fix(ci): detect test failures by output content, not exit code
nvim returns non-zero exit codes when error messages are printed, even for expected errors in error handling tests. Changed test runner to check for actual '[31mFail.*||' patterns in output instead.
1 parent 4470cd4 commit 568d1a6

1 file changed

Lines changed: 125 additions & 124 deletions

File tree

‎run_tests.sh‎

Lines changed: 125 additions & 124 deletions
Original file line numberDiff line numberDiff line change
@@ -15,61 +15,65 @@ FILTER=""
1515
TEST_TYPE="all"
1616

1717
print_usage() {
18-
echo "Usage: $0 [OPTIONS]"
19-
echo "Options:"
20-
echo " -f, --filter PATTERN Filter tests by pattern (matches test descriptions)"
21-
echo " -t, --type TYPE Test type: all, minimal, unit, replay, or specific file path"
22-
echo " -h, --help Show this help message"
23-
echo ""
24-
echo "Examples:"
25-
echo " $0 # Run all tests"
26-
echo " $0 -f \"Timer\" # Run tests matching 'Timer'"
27-
echo " $0 -t unit # Run only unit tests"
28-
echo " $0 -t replay # Run only replay tests"
29-
echo " $0 -t tests/unit/timer_spec.lua # Run specific test file"
30-
echo " $0 -f \"creates a new timer\" -t unit # Filter unit tests"
18+
echo "Usage: $0 [OPTIONS]"
19+
echo "Options:"
20+
echo " -f, --filter PATTERN Filter tests by pattern (matches test descriptions)"
21+
echo " -t, --type TYPE Test type: all, minimal, unit, replay, or specific file path"
22+
echo " -h, --help Show this help message"
23+
echo ""
24+
echo "Examples:"
25+
echo " $0 # Run all tests"
26+
echo " $0 -f \"Timer\" # Run tests matching 'Timer'"
27+
echo " $0 -t unit # Run only unit tests"
28+
echo " $0 -t replay # Run only replay tests"
29+
echo " $0 -t tests/unit/timer_spec.lua # Run specific test file"
30+
echo " $0 -f \"creates a new timer\" -t unit # Filter unit tests"
3131
}
3232

3333
while [[ $# -gt 0 ]]; do
34-
case $1 in
35-
-f | --filter)
36-
FILTER="$2"
37-
shift 2
38-
;;
39-
-t | --type)
40-
TEST_TYPE="$2"
41-
shift 2
42-
;;
43-
-h | --help)
44-
print_usage
45-
exit 0
46-
;;
47-
*)
48-
echo "Unknown option: $1"
49-
print_usage
50-
exit 1
51-
;;
52-
esac
34+
case $1 in
35+
-f | --filter)
36+
FILTER="$2"
37+
shift 2
38+
;;
39+
-t | --type)
40+
TEST_TYPE="$2"
41+
shift 2
42+
;;
43+
-h | --help)
44+
print_usage
45+
exit 0
46+
;;
47+
*)
48+
echo "Unknown option: $1"
49+
print_usage
50+
exit 1
51+
;;
52+
esac
5353
done
5454

5555
# Clean test output by removing Errors lines
5656
clean_output() {
57-
echo "$1" | grep -v "\[31mErrors : "
57+
echo "$1" | grep -v "\[31mErrors : "
5858
}
5959

6060
# Build filter option for plenary
6161
FILTER_OPTION=""
6262
if [ -n "$FILTER" ]; then
63-
FILTER_OPTION=", filter = '$FILTER'"
63+
FILTER_OPTION=", filter = '$FILTER'"
6464
fi
6565

6666
if [ -n "$FILTER" ]; then
67-
echo -e "${YELLOW}Running tests for opencode.nvim (filter: '$FILTER')${NC}"
67+
echo -e "${YELLOW}Running tests for opencode.nvim (filter: '$FILTER')${NC}"
6868
else
69-
echo -e "${YELLOW}Running tests for opencode.nvim${NC}"
69+
echo -e "${YELLOW}Running tests for opencode.nvim${NC}"
7070
fi
7171
echo "------------------------------------------------"
7272

73+
output_contains_failed_tests() {
74+
echo "$1" | grep -q "\[31mFail.*||"
75+
}
76+
7377
# Run tests based on type
7478
minimal_status=0
7579
unit_status=0
@@ -79,69 +83,66 @@ unit_output=""
7983
replay_output=""
8084

8185
if [ "$TEST_TYPE" = "all" ] || [ "$TEST_TYPE" = "minimal" ]; then
82-
# Run minimal tests
83-
minimal_output=$(nvim --headless -u tests/minimal/init.lua -c "lua require('plenary.test_harness').test_directory('./tests/minimal', {minimal_init = './tests/minimal/init.lua', sequential = true$FILTER_OPTION})" 2>&1)
84-
minimal_status=$?
85-
clean_output "$minimal_output"
86-
87-
if [ $minimal_status -eq 0 ]; then
88-
echo -e "${GREEN}✓ Minimal tests passed${NC}"
89-
else
90-
echo -e "${RED}✗ Minimal tests failed${NC}"
91-
fi
92-
echo "------------------------------------------------"
86+
# Run minimal tests
87+
minimal_output=$(nvim --headless -u tests/minimal/init.lua -c "lua require('plenary.test_harness').test_directory('./tests/minimal', {minimal_init = './tests/minimal/init.lua', sequential = true$FILTER_OPTION})" 2>&1)
88+
clean_output "$minimal_output"
89+
90+
if output_contains_failed_tests "$minimal_output"; then
91+
minimal_status=1
92+
echo -e "${RED}✗ Minimal tests failed${NC}"
93+
else
94+
echo -e "${GREEN}✓ Minimal tests passed${NC}"
95+
fi
96+
echo "------------------------------------------------"
9397
fi
9498

9599
if [ "$TEST_TYPE" = "all" ] || [ "$TEST_TYPE" = "unit" ]; then
96-
# Run unit tests
97-
unit_output=$(nvim --headless -u tests/minimal/init.lua -c "lua require('plenary.test_harness').test_directory('./tests/unit', {minimal_init = './tests/minimal/init.lua'$FILTER_OPTION})" 2>&1)
98-
unit_status=$?
99-
clean_output "$unit_output"
100-
101-
if [ $unit_status -eq 0 ]; then
102-
echo -e "${GREEN}✓ Unit tests passed${NC}"
103-
else
104-
echo -e "${RED}✗ Unit tests failed${NC}"
105-
fi
106-
echo "------------------------------------------------"
100+
# Run unit tests
101+
unit_output=$(nvim --headless -u tests/minimal/init.lua -c "lua require('plenary.test_harness').test_directory('./tests/unit', {minimal_init = './tests/minimal/init.lua'$FILTER_OPTION})" 2>&1)
102+
clean_output "$unit_output"
103+
104+
if output_contains_failed_tests "$unit_output"; then
105+
unit_status=1
106+
echo -e "${RED}✗ Unit tests failed${NC}"
107+
else
108+
echo -e "${GREEN}✓ Unit tests passed${NC}"
109+
fi
110+
echo "------------------------------------------------"
107111
fi
108112

109113
if [ "$TEST_TYPE" = "all" ] || [ "$TEST_TYPE" = "replay" ]; then
110-
# Run replay tests
111-
replay_output=$(nvim --headless -u tests/minimal/init.lua -c "lua require('plenary.test_harness').test_directory('./tests/replay', {minimal_init = './tests/minimal/init.lua'$FILTER_OPTION})" 2>&1)
112-
replay_status=$?
113-
clean_output "$replay_output"
114-
115-
if [ $replay_status -eq 0 ]; then
116-
echo -e "${GREEN}✓ Replay tests passed${NC}"
117-
else
118-
echo -e "${RED}✗ Replay tests failed${NC}"
119-
fi
120-
echo "------------------------------------------------"
114+
# Run replay tests
115+
replay_output=$(nvim --headless -u tests/minimal/init.lua -c "lua require('plenary.test_harness').test_directory('./tests/replay', {minimal_init = './tests/minimal/init.lua'$FILTER_OPTION})" 2>&1)
116+
clean_output "$replay_output"
117+
118+
if output_contains_failed_tests "$replay_output"; then
119+
replay_status=1
120+
echo -e "${RED}✗ Replay tests failed${NC}"
121+
else
122+
echo -e "${GREEN}✓ Replay tests passed${NC}"
123+
fi
124+
echo "------------------------------------------------"
121125
fi
122126

123127
# Handle specific test file
124128
if [ "$TEST_TYPE" != "all" ] && [ "$TEST_TYPE" != "minimal" ] && [ "$TEST_TYPE" != "unit" ] && [ "$TEST_TYPE" != "replay" ]; then
125-
# Assume it's a specific test file path
126-
if [ -f "$TEST_TYPE" ]; then
127-
specific_output=$(nvim --headless -u tests/minimal/init.lua -c "lua require('plenary.test_harness').test_directory('./$TEST_TYPE', {minimal_init = './tests/minimal/init.lua'$FILTER_OPTION})" 2>&1)
128-
specific_status=$?
129-
clean_output "$specific_output"
130-
131-
if [ $specific_status -eq 0 ]; then
132-
echo -e "${GREEN}✓ Specific test passed${NC}"
133-
else
134-
echo -e "${RED}✗ Specific test failed${NC}"
135-
fi
136-
echo "------------------------------------------------"
137-
138-
# Use specific test output for failure analysis
139-
unit_output="$specific_output"
140-
unit_status=$specific_status
141-
else
142-
echo -e "${RED}Error: Test file '$TEST_TYPE' not found${NC}"
143-
exit 1
144-
fi
129+
if [ -f "$TEST_TYPE" ]; then
130+
specific_output=$(nvim --headless -u tests/minimal/init.lua -c "lua require('plenary.test_harness').test_directory('./$TEST_TYPE', {minimal_init = './tests/minimal/init.lua'$FILTER_OPTION})" 2>&1)
131+
clean_output "$specific_output"
132+
133+
if output_contains_failed_tests "$specific_output"; then
134+
unit_status=1
135+
echo -e "${RED}✗ Specific test failed${NC}"
136+
else
137+
echo -e "${GREEN}✓ Specific test passed${NC}"
138+
fi
139+
echo "------------------------------------------------"
140+
141+
unit_output="$specific_output"
142+
else
143+
echo -e "${RED}Error: Test file '$TEST_TYPE' not found${NC}"
144+
exit 1
145+
fi
145146
fi
146147

147148
# Check for any failures
@@ -150,40 +151,40 @@ $unit_output
150151
$replay_output"
151152

152153
if [ $minimal_status -ne 0 ] || [ $unit_status -ne 0 ] || [ $replay_status -ne 0 ] || echo "$all_output" | grep -q "\[31mFail.*||"; then
153-
echo -e "\n${RED}======== TEST FAILURES SUMMARY ========${NC}"
154-
155-
# Extract and format failures
156-
failures_file=$(mktemp)
157-
echo "$all_output" | grep -B 0 -A 6 "\[31mFail.*||" >"$failures_file"
158-
failure_count=$(grep -c "\[31mFail.*||" "$failures_file")
159-
160-
echo -e "${RED}Found $failure_count failing test(s):${NC}\n"
161-
162-
# Process the output line by line
163-
test_name=""
164-
while IFS= read -r line; do
165-
# Remove ANSI color codes
166-
clean_line=$(echo "$line" | sed -E 's/\x1B\[[0-9;]*[mK]//g')
167-
168-
if [[ "$clean_line" == *"Fail"*"||"* ]]; then
169-
# Extract test name
170-
test_name=$(echo "$clean_line" | sed -E 's/.*Fail.*\|\|\s*(.*)/\1/')
171-
echo -e "${RED}FAILED TEST:${NC} $test_name"
172-
elif [[ "$clean_line" == *"/Users/"*".lua:"*": "* ]]; then
173-
# This is an error message with file:line
174-
echo -e " ${RED}ERROR:${NC} $clean_line"
175-
elif [[ "$clean_line" == *"stack traceback"* ]]; then
176-
# Stack trace header
177-
echo -e " ${YELLOW}TRACE:${NC} $clean_line"
178-
elif [[ "$clean_line" == *"in function"* ]]; then
179-
# Stack trace details
180-
echo -e " $clean_line"
181-
fi
182-
done <"$failures_file"
183-
184-
rm -f "$failures_file"
185-
exit 1
154+
echo -e "\n${RED}======== TEST FAILURES SUMMARY ========${NC}"
155+
156+
# Extract and format failures
157+
failures_file=$(mktemp)
158+
echo "$all_output" | grep -B 0 -A 6 "\[31mFail.*||" >"$failures_file"
159+
failure_count=$(grep -c "\[31mFail.*||" "$failures_file")
160+
161+
echo -e "${RED}Found $failure_count failing test(s):${NC}\n"
162+
163+
# Process the output line by line
164+
test_name=""
165+
while IFS= read -r line; do
166+
# Remove ANSI color codes
167+
clean_line=$(echo "$line" | sed -E 's/\x1B\[[0-9;]*[mK]//g')
168+
169+
if [[ "$clean_line" == *"Fail"*"||"* ]]; then
170+
# Extract test name
171+
test_name=$(echo "$clean_line" | sed -E 's/.*Fail.*\|\|\s*(.*)/\1/')
172+
echo -e "${RED}FAILED TEST:${NC} $test_name"
173+
elif [[ "$clean_line" == *"/Users/"*".lua:"*": "* ]]; then
174+
# This is an error message with file:line
175+
echo -e " ${RED}ERROR:${NC} $clean_line"
176+
elif [[ "$clean_line" == *"stack traceback"* ]]; then
177+
# Stack trace header
178+
echo -e " ${YELLOW}TRACE:${NC} $clean_line"
179+
elif [[ "$clean_line" == *"in function"* ]]; then
180+
# Stack trace details
181+
echo -e " $clean_line"
182+
fi
183+
done <"$failures_file"
184+
185+
rm -f "$failures_file"
186+
exit 1
186187
else
187-
echo -e "${GREEN}All tests passed successfully!${NC}"
188-
exit 0
188+
echo -e "${GREEN}All tests passed successfully!${NC}"
189+
exit 0
189190
fi

0 commit comments

Comments
 (0)