diff --git a/.github/workflows/GnuComment.yml b/.github/workflows/GnuComment.yml new file mode 100644 index 0000000..9000625 --- /dev/null +++ b/.github/workflows/GnuComment.yml @@ -0,0 +1,83 @@ +name: GnuComment + +# Post the GNU grep testsuite comparison (produced by the GnuTests workflow) +# as a comment on the pull request. + +on: + workflow_run: + workflows: ["GnuTests"] + types: + - completed + +permissions: {} +jobs: + post-comment: + permissions: + actions: read # to list workflow run artifacts + pull-requests: write # to comment on the pr + + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' + steps: + - name: 'Download artifact' + uses: actions/github-script@v7 + with: + script: | + // List all artifacts from GnuTests + var artifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: ${{ github.event.workflow_run.id }}, + }); + + // Download the "comment" artifact, which contains a PR number (NR) and result.txt + var matchArtifact = artifacts.data.artifacts.filter((artifact) => { + return artifact.name == "comment" + })[0]; + + if (!matchArtifact) { + console.log('No comment artifact found'); + return; + } + + var download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + var fs = require('fs'); + fs.writeFileSync('${{ github.workspace }}/comment.zip', Buffer.from(download.data)); + - run: unzip comment.zip || echo "Failed to unzip comment artifact" + + - name: 'Comment on PR' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + var fs = require('fs'); + + // Check if files exist + if (!fs.existsSync('./NR')) { + console.log('No NR file found, skipping comment'); + return; + } + if (!fs.existsSync('./result.txt')) { + console.log('No result.txt file found, skipping comment'); + return; + } + + var issue_number = Number(fs.readFileSync('./NR')); + var content = fs.readFileSync('./result.txt'); + + if (content.toString().trim().length > 7) { // 7 because we have backquote + \n + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issue_number, + body: 'GNU grep testsuite comparison:\n```\n' + content + '```' + }); + } else { + console.log('Comment content too short, skipping'); + } diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml new file mode 100644 index 0000000..5f5913e --- /dev/null +++ b/.github/workflows/GnuTests.yml @@ -0,0 +1,180 @@ +name: GnuTests + +# Run the upstream GNU grep testsuite against the Rust grep implementation to +# track and guard byte-for-byte compatibility. See util/run-gnu-testsuite.sh. + +on: + pull_request: + push: + branches: + - '*' + +# End the current execution if there is a new changeset in the PR. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + TEST_FULL_SUMMARY_FILE: 'grep-gnu-full-result.json' + +jobs: + native: + name: Run GNU grep testsuite + runs-on: ubuntu-24.04 + steps: + - name: Checkout code (grep) + uses: actions/checkout@v4 + with: + path: 'grep' + persist-credentials: false + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: "./grep -> target" + + - name: Fetch GNU grep testsuite + shell: bash + run: | + ## Download and extract the upstream GNU grep release tarball + mkdir -p gnu.grep + cd gnu.grep + bash ../grep/util/fetch-gnu.sh + + - name: Build Rust grep binary + shell: bash + run: | + cd 'grep' + cargo build --release + + - name: Run GNU grep testsuite + shell: bash + run: | + cd 'grep' + export GNU_GREP_DIR="../gnu.grep" + ./util/run-gnu-testsuite.sh --json-output "${{ env.TEST_FULL_SUMMARY_FILE }}" || true + + - name: Upload full json results + uses: actions/upload-artifact@v4 + with: + name: grep-gnu-full-result + path: grep/${{ env.TEST_FULL_SUMMARY_FILE }} + if-no-files-found: warn + + aggregate: + needs: [native] + permissions: + actions: read + contents: read + pull-requests: read + name: Aggregate GNU test results + runs-on: ubuntu-24.04 + steps: + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + TEST_SUMMARY_FILE='grep-gnu-result.json' + outputs TEST_SUMMARY_FILE + + - name: Checkout code (grep) + uses: actions/checkout@v4 + with: + path: 'grep' + persist-credentials: false + + - name: Retrieve reference artifacts + uses: dawidd6/action-download-artifact@v6 + continue-on-error: true + with: + workflow: GnuTests.yml + branch: "${{ env.DEFAULT_BRANCH }}" + workflow_conclusion: completed + path: "reference" + if_no_artifact_found: warn + + - name: Download full json results + uses: actions/download-artifact@v4 + with: + name: grep-gnu-full-result + path: results + + - name: Extract/summarize testing info + id: summary + shell: bash + run: | + ## Extract/summarize testing info + outputs() { step_id="${{ github.action }}"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo "${var}=${!var}" >> $GITHUB_OUTPUT; done; } + + RESULT_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}" + if [[ ! -f "$RESULT_FILE" ]]; then + echo "::error ::Result file $RESULT_FILE not found" + find results -type f || true + exit 1 + fi + + TOTAL=$(jq -r '.summary.total // 0' "$RESULT_FILE") + PASS=$(jq -r '.summary.passed // 0' "$RESULT_FILE") + FAIL=$(jq -r '.summary.failed // 0' "$RESULT_FILE") + SKIP=$(jq -r '.summary.skipped // 0' "$RESULT_FILE") + + output="GNU grep tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / SKIP: $SKIP" + echo "${output}" + if [[ "$FAIL" -gt 0 ]]; then + echo "::warning ::${output}" + fi + + outputs TOTAL PASS FAIL SKIP + + - name: Compare test failures VS reference + shell: bash + run: | + ## Compare current results against the reference summary from the default branch + REF_SUMMARY_FILE='reference/grep-gnu-full-result/${{ env.TEST_FULL_SUMMARY_FILE }}' + CURRENT_SUMMARY_FILE="results/${{ env.TEST_FULL_SUMMARY_FILE }}" + IGNORE_INTERMITTENT="grep/.github/workflows/ignore-intermittent.txt" + + # Set up comment directory for the GnuComment workflow. + COMMENT_DIR="reference/comment" + mkdir -p ${COMMENT_DIR} + echo ${{ github.event.number }} > ${COMMENT_DIR}/NR + COMMENT_LOG="${COMMENT_DIR}/result.txt" + : > "${COMMENT_LOG}" + + COMPARISON_RESULT=0 + if test -f "${REF_SUMMARY_FILE}"; then + python3 grep/util/compare_test_results.py \ + --ignore-file "${IGNORE_INTERMITTENT}" \ + --output "${COMMENT_LOG}" \ + "${CURRENT_SUMMARY_FILE}" "${REF_SUMMARY_FILE}" || COMPARISON_RESULT=$? + else + echo "::warning ::Skipping test comparison; no prior reference summary at '${REF_SUMMARY_FILE}'." + fi + + if [ ${COMPARISON_RESULT} -eq 1 ]; then + echo "::error ::Found new non-intermittent test failures" + UPLOAD_EXIT=1 + else + echo "::notice ::No new test failures detected" + UPLOAD_EXIT=0 + fi + echo "UPLOAD_EXIT=${UPLOAD_EXIT}" >> $GITHUB_ENV + + - name: Upload comparison log (for GnuComment workflow) + if: success() || failure() + uses: actions/upload-artifact@v4 + with: + name: comment + path: reference/comment/ + + - name: Report test results + if: success() || failure() + shell: bash + run: | + echo "::notice ::GNU grep testsuite: TOTAL ${{ steps.summary.outputs.TOTAL }} / PASS ${{ steps.summary.outputs.PASS }} / FAIL ${{ steps.summary.outputs.FAIL }} / SKIP ${{ steps.summary.outputs.SKIP }}" + # Fail the job if the comparison found new non-intermittent regressions. + exit "${UPLOAD_EXIT:-0}" diff --git a/.github/workflows/ignore-intermittent.txt b/.github/workflows/ignore-intermittent.txt new file mode 100644 index 0000000..7889f3d --- /dev/null +++ b/.github/workflows/ignore-intermittent.txt @@ -0,0 +1,7 @@ +# List of intermittent test names to ignore in result comparisons +# Format: one test name per line, lines starting with # are comments +# +# Add test names that are known to be flaky or environment-dependent +# Example: +# basic_substitution +# line_address_test diff --git a/util/compare_test_results.py b/util/compare_test_results.py new file mode 100755 index 0000000..7ceb3ff --- /dev/null +++ b/util/compare_test_results.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +""" +Compare the current GNU test results to the last results gathered from the main branch to +highlight if a PR is making the results better/worse. +Don't exit with error code if all failing tests are in the ignore-intermittent.txt list. +""" + +import json +import sys +import argparse +from pathlib import Path + + +def load_ignore_list(ignore_file): + """Load list of intermittent test names to ignore from file.""" + ignore_set = set() + if ignore_file and Path(ignore_file).exists(): + with open(ignore_file, "r") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + ignore_set.add(line) + return ignore_set + + +def extract_test_results(json_data): + """Extract test results from JSON data.""" + if not json_data or "summary" not in json_data: + return {"total": 0, "passed": 0, "failed": 0, "skipped": 0}, [] + + summary = json_data["summary"] + tests = json_data.get("tests", []) + + # Extract failed test names + failed_tests = [] + for test in tests: + if test.get("status") == "FAIL": + failed_tests.append(test.get("name", "unknown")) + + return summary, failed_tests + + +def compare_results(current_file, reference_file, ignore_file=None, output_file=None): + """Compare current results with reference results.""" + # Load ignore list + ignore_set = load_ignore_list(ignore_file) + + # Load JSON files + try: + with open(current_file, "r") as f: + current_data = json.load(f) + current_summary, current_failed = extract_test_results(current_data) + except Exception as e: + print(f"Error loading current results: {e}") + return 1 + + try: + with open(reference_file, "r") as f: + reference_data = json.load(f) + reference_summary, reference_failed = extract_test_results(reference_data) + except Exception as e: + print(f"Error loading reference results: {e}") + return 1 + + # Calculate differences + pass_diff = int(current_summary.get("passed", 0)) - int( + reference_summary.get("passed", 0) + ) + fail_diff = int(current_summary.get("failed", 0)) - int( + reference_summary.get("failed", 0) + ) + total_diff = int(current_summary.get("total", 0)) - int( + reference_summary.get("total", 0) + ) + + # Find new failures and improvements + current_failed_set = set(current_failed) + reference_failed_set = set(reference_failed) + + new_failures = current_failed_set - reference_failed_set + improvements = reference_failed_set - current_failed_set + + # Filter out intermittent failures + non_intermittent_new_failures = new_failures - ignore_set + + # Check if results are identical (no changes) + no_changes = ( + pass_diff == 0 + and fail_diff == 0 + and total_diff == 0 + and not new_failures + and not improvements + ) + + # If no changes, write empty output to prevent comment posting + if no_changes: + with open(output_file, "w") as f: + f.write("") + return 0 + + # Prepare output message + output_lines = [] + + # Show current vs reference numbers for debugging + output_lines.append("Test results comparison:") + output_lines.append( + f" Current: TOTAL: {current_summary.get('total', 0)} / PASSED: {current_summary.get('passed', 0)} / FAILED: {current_summary.get('failed', 0)} / SKIPPED: {current_summary.get('skipped', 0)}" + ) + output_lines.append( + f" Reference: TOTAL: {reference_summary.get('total', 0)} / PASSED: {reference_summary.get('passed', 0)} / FAILED: {reference_summary.get('failed', 0)} / SKIPPED: {reference_summary.get('skipped', 0)}" + ) + output_lines.append("") + + # Summary of changes + if pass_diff != 0 or fail_diff != 0 or total_diff != 0: + output_lines.append("Changes from main branch:") + output_lines.append(f" TOTAL: {total_diff:+d}") + output_lines.append(f" PASSED: {pass_diff:+d}") + output_lines.append(f" FAILED: {fail_diff:+d}") + output_lines.append("") + + # New failures + if new_failures: + output_lines.append(f"New test failures ({len(new_failures)}):") + for test in sorted(new_failures): + if test in ignore_set: + output_lines.append(f" - {test} (intermittent)") + else: + output_lines.append(f" - {test}") + output_lines.append("") + + # Improvements + if improvements: + output_lines.append(f"Test improvements ({len(improvements)}):") + for test in sorted(improvements): + output_lines.append(f" + {test}") + output_lines.append("") + + # Write output + output_text = "\n".join(output_lines) + if output_file: + with open(output_file, "w") as f: + f.write(output_text) + else: + print(output_text) + + # Return appropriate exit code + if non_intermittent_new_failures: + print( + f"ERROR: Found {len(non_intermittent_new_failures)} new non-intermittent test failures" + ) + return 1 + + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Compare GNU test results") + parser.add_argument("current", help="Current test results JSON file") + parser.add_argument("reference", help="Reference test results JSON file") + parser.add_argument( + "--ignore-file", help="File containing intermittent test names to ignore" + ) + parser.add_argument("--output", help="Output file for comparison results") + + args = parser.parse_args() + + return compare_results(args.current, args.reference, args.ignore_file, args.output) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/util/fetch-gnu.sh b/util/fetch-gnu.sh new file mode 100755 index 0000000..f1d92c6 --- /dev/null +++ b/util/fetch-gnu.sh @@ -0,0 +1,17 @@ +#!/bin/bash -e +# This file is part of the uutils grep package. +# +# For the full copyright and license information, please view the LICENSE +# file that was distributed with this source code. +# +# Download and extract the upstream GNU grep release tarball into the current +# directory. Run it from an (empty) directory that will hold the GNU grep tree, +# e.g.: +# +# mkdir -p ../gnu.grep && (cd ../gnu.grep && bash ../grep/util/fetch-gnu.sh) +# +# The extracted tree ships a ready-to-use gnulib test framework under tests/ +# (init.sh + init.cfg + the extensionless test scripts), which +# util/run-gnu-testsuite.sh drives against the Rust grep binary. +ver="3.12" +curl -L "https://ftp.gnu.org/gnu/grep/grep-${ver}.tar.xz" | tar --strip-components=1 -xJf - diff --git a/util/run-gnu-testsuite.sh b/util/run-gnu-testsuite.sh new file mode 100755 index 0000000..0158d0b --- /dev/null +++ b/util/run-gnu-testsuite.sh @@ -0,0 +1,359 @@ +#!/bin/bash +# This file is part of the uutils grep package. +# +# For the full copyright and license information, please view the LICENSE +# file that was distributed with this source code. +# +# Run the upstream GNU grep testsuite against the Rust grep implementation. +# +# Unlike GNU coreutils, we do *not* build GNU grep here. Instead we reuse the +# gnulib test framework (tests/init.sh + tests/init.cfg) shipped in the GNU grep +# release tarball and inject our Rust `grep` binary via PATH, replicating the +# environment that tests/Makefile.am's TESTS_ENVIRONMENT would normally set up. +# Each test is classified by its gnulib exit code: 0 = PASS, 77 = SKIP, anything +# else = FAIL (timeouts and framework failures count as FAIL). +# +# Get the GNU grep sources with: +# mkdir -p ../gnu.grep && (cd ../gnu.grep && bash ../grep/util/fetch-gnu.sh) +# +# Usage: ./util/run-gnu-testsuite.sh [options] +# +# Options: +# -h, --help Show this help message +# -v, --verbose Show diagnostics for failing/skipped tests +# -q, --quiet Only print failures and the final summary +# --json-output FILE Write results to FILE as JSON +# +# Environment variables: +# GNU_GREP_DIR Path to the extracted GNU grep source tree +# (default: ../gnu.grep) +# RUN_EXPENSIVE_TESTS Set to "yes" to run expensive tests (default: no) +# PER_TEST_TIMEOUT Per-test timeout in seconds (default: 30) + +# Don't exit on failure since test failures are expected. +set -o pipefail + +# Configuration +RUST_GREP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GNU_GREP_DIR="${GNU_GREP_DIR:-${RUST_GREP_DIR}/../gnu.grep}" +GNU_TESTS_DIR="" +VERBOSE=false +QUIET=false +JSON_OUTPUT_FILE="" +PER_TEST_TIMEOUT="${PER_TEST_TIMEOUT:-30}" +DETAILED_RESULTS=() + +# Statistics +TOTAL_TESTS=0 +PASSED_TESTS=0 +FAILED_TESTS=0 +SKIPPED_TESTS=0 + +usage() { + echo "Usage: $0 [options]" + echo + echo "Options:" + echo " -h, --help Show this help message" + echo " -v, --verbose Show diagnostics for failing/skipped tests" + echo " -q, --quiet Only print failures and the final summary" + echo " --json-output FILE Write results to FILE as JSON" + echo + echo "Environment variables:" + echo " GNU_GREP_DIR Path to the extracted GNU grep source tree" + echo " (default: ../gnu.grep)" + echo " RUN_EXPENSIVE_TESTS Set to 'yes' to run expensive tests" + echo " PER_TEST_TIMEOUT Per-test timeout in seconds (default: 30)" + echo + echo "Setup:" + echo " mkdir -p ../gnu.grep && (cd ../gnu.grep && bash ../grep/util/fetch-gnu.sh)" +} + +log_info() { [[ "$QUIET" != "true" ]] && echo "[INFO] $1"; return 0; } +log_success() { [[ "$QUIET" != "true" ]] && echo "[PASS] $1"; return 0; } +log_skip() { [[ "$QUIET" != "true" ]] && echo "[SKIP] $1"; return 0; } +log_warning() { echo "[WARN] $1"; } +log_error() { echo "[FAIL] $1"; } + +# Generate JSON output (schema shared with ../sed so compare_test_results.py works). +generate_json_output() { + cd "$RUST_GREP_DIR" || return + + local timestamp + timestamp=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + local rust_version + rust_version=$(cargo metadata --no-deps --format-version 1 2>/dev/null | jq -r '.packages[0].version // "unknown"') + + local tests_json="[]" + if [[ ${#DETAILED_RESULTS[@]} -gt 0 ]]; then + local temp_file + temp_file=$(mktemp) + printf "%s\n" "${DETAILED_RESULTS[@]}" > "$temp_file" + tests_json=$(jq -s '.' < "$temp_file" 2>/dev/null) || tests_json="[]" + rm -f "$temp_file" + fi + + jq -n \ + --arg timestamp "$timestamp" \ + --argjson total "$TOTAL_TESTS" \ + --argjson passed "$PASSED_TESTS" \ + --argjson failed "$FAILED_TESTS" \ + --argjson skipped "$SKIPPED_TESTS" \ + --argjson duration "$duration" \ + --arg rust_version "$rust_version" \ + --arg gnu_testsuite_dir "$GNU_TESTS_DIR" \ + --argjson tests "$tests_json" \ + '{ + timestamp: $timestamp, + summary: { + total: $total, + passed: $passed, + failed: $failed, + skipped: $skipped, + duration_seconds: $duration + }, + environment: { + rust_grep_version: $rust_version, + gnu_testsuite_dir: $gnu_testsuite_dir + }, + tests: $tests + }' > "$JSON_OUTPUT_FILE" + + log_info "JSON results written to: $JSON_OUTPUT_FILE" +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) usage; exit 0 ;; + -v|--verbose) VERBOSE=true; shift ;; + -q|--quiet) QUIET=true; shift ;; + --json-output) JSON_OUTPUT_FILE="$2"; shift 2 ;; + *) echo "Unknown argument: $1"; usage; exit 1 ;; + esac +done + +# Validate environment +if [[ -d "$GNU_GREP_DIR" ]]; then + GNU_GREP_DIR="$(cd "$GNU_GREP_DIR" && pwd)" + GNU_TESTS_DIR="$GNU_GREP_DIR/tests" +fi + +if [[ ! -f "$GNU_TESTS_DIR/init.sh" ]]; then + log_error "GNU grep testsuite not found at: $GNU_GREP_DIR" + log_error "Fetch it with:" + log_error " mkdir -p ${RUST_GREP_DIR}/../gnu.grep && (cd ${RUST_GREP_DIR}/../gnu.grep && bash ${RUST_GREP_DIR}/util/fetch-gnu.sh)" + exit 1 +fi + +if [[ ! -f "$RUST_GREP_DIR/Cargo.toml" ]]; then + log_error "Not in a Rust project directory: $RUST_GREP_DIR" + exit 1 +fi + +# Build the Rust grep implementation +log_info "Building Rust grep implementation..." +cd "$RUST_GREP_DIR" || exit 1 +if ! cargo build --release --quiet; then + log_error "Failed to build Rust grep implementation" + exit 1 +fi + +RUST_GREP_BIN="$RUST_GREP_DIR/target/release/grep" +if [[ ! -x "$RUST_GREP_BIN" ]]; then + log_error "Built grep binary not found at: $RUST_GREP_BIN" + exit 1 +fi +log_info "Using Rust grep binary: $RUST_GREP_BIN" + +# Create a temporary work tree that mimics a GNU grep build directory. +TEST_WORK_DIR=$(mktemp -d) +trap 'rm -rf "$TEST_WORK_DIR"' EXIT +log_info "Test working directory: $TEST_WORK_DIR" + +# A fake $abs_top_builddir whose src/ holds the binaries the tests expect. +BUILD_DIR="$TEST_WORK_DIR/build" +BIN_DIR="$BUILD_DIR/src" +mkdir -p "$BIN_DIR" + +# grep, plus the egrep/fgrep wrappers a handful of tests rely on. +cat > "$BIN_DIR/grep" < "$BIN_DIR/egrep" < "$BIN_DIR/fgrep" < "$BUILD_DIR/config.h" + +# get-mb-cur-max is a tiny standalone helper used by the locale require_ checks. +if [[ -f "$GNU_TESTS_DIR/get-mb-cur-max.c" ]]; then + if cc -I"$BUILD_DIR" -o "$BIN_DIR/get-mb-cur-max" "$GNU_TESTS_DIR/get-mb-cur-max.c" 2>/dev/null; then + log_info "Built get-mb-cur-max helper" + else + log_warning "Could not build get-mb-cur-max; multibyte/locale tests may skip" + fi +fi + +# Replicate the PCRE_WORKS probe from tests/Makefile.am's TESTS_ENVIRONMENT. +PCRE_WORKS=0 +if err=$(echo . | "$BIN_DIR/grep" -Pq . 2>&1); then + [[ -z "$err" ]] && PCRE_WORKS=1 +fi +log_info "PCRE_WORKS=$PCRE_WORKS" + +GREP_VERSION=$(basename "$GNU_GREP_DIR" | sed 's/^grep-//') +[[ "$GREP_VERSION" == "$(basename "$GNU_GREP_DIR")" ]] && GREP_VERSION="unknown" +HOST_TRIPLET="$(uname -m)-pc-linux-gnu" + +# Record a test result (for JSON output) +record_result() { + if [[ -n "$JSON_OUTPUT_FILE" ]]; then + DETAILED_RESULTS+=("$(jq -n \ + --arg name "$1" --arg status "$2" --arg error "$3" \ + '{name: $name, status: $status, error: $error}')") + fi +} + +# Run a single GNU testsuite script with the Rust grep on PATH. +run_gnu_test() { + local test_script="$1" + local test_name + test_name=$(basename "$test_script") + + TOTAL_TESTS=$((TOTAL_TESTS + 1)) + + local test_output_file="$TEST_WORK_DIR/test_output_$$" + local test_exit_code=0 + + # When not the process-group leader (e.g. in CI), GNU timeout falls back to + # "foreground" mode and SIGTERMs the whole group on timeout. Shield the + # parent script so a single hung test doesn't take the run down. + trap '' TERM + + ( + cd "$TEST_WORK_DIR" || exit 99 + # init.cfg refuses to run if these are set. + unset GREP_COLOR GREP_COLORS TERM CDPATH + export PATH="$BIN_DIR:$PATH" + export srcdir="$GNU_TESTS_DIR" abs_srcdir="$GNU_TESTS_DIR" + export abs_top_srcdir="$GNU_GREP_DIR" top_srcdir="$GNU_GREP_DIR" + export abs_top_builddir="$BUILD_DIR" + export CONFIG_HEADER="$BUILD_DIR/config.h" + export built_programs="grep egrep fgrep" + export AWK=awk PERL=perl SHELL=/bin/sh MAKE=make CC=cc + export LC_ALL=C MALLOC_PERTURB_=87 + export VERSION="$GREP_VERSION" PACKAGE_VERSION="$GREP_VERSION" + export host_triplet="$HOST_TRIPLET" + export PCRE_WORKS="$PCRE_WORKS" + export GREP_TEST_NAME="$test_name" + export RUN_EXPENSIVE_TESTS="${RUN_EXPENSIVE_TESTS:-no}" + + # fd 9 is the framework's stderr (init.cfg's stderr_fileno_=9). + if [[ "$test_name" == *.pl ]]; then + exec timeout --kill-after=5 "$PER_TEST_TIMEOUT" \ + perl -w -I"$GNU_TESTS_DIR" -MCoreutils -MCuSkip "$test_script" 9>&2 + else + exec timeout --kill-after=5 "$PER_TEST_TIMEOUT" \ + /bin/sh "$test_script" 9>&2 + fi + ) "$test_output_file" 2>&1 + test_exit_code=$? + + trap - TERM + + # Strip NUL bytes: some tests (e.g. z-anchor-newline) emit binary output, + # which would otherwise trigger a "ignored null byte" warning from $(...). + local test_output="" + [[ -f "$test_output_file" ]] && test_output=$(tr -d '\0' < "$test_output_file") + rm -f "$test_output_file" + + # 124 = GNU timeout, 125 = uutils timeout, >=128 = killed by signal. + if [[ $test_exit_code -eq 124 || $test_exit_code -eq 125 || $test_exit_code -ge 128 ]]; then + log_error "$test_name (timeout)" + FAILED_TESTS=$((FAILED_TESTS + 1)) + record_result "$test_name" "FAIL" "Test timed out after ${PER_TEST_TIMEOUT}s" + return + fi + + case $test_exit_code in + 0) + log_success "$test_name" + PASSED_TESTS=$((PASSED_TESTS + 1)) + record_result "$test_name" "PASS" "" + ;; + 77) + log_skip "$test_name" + SKIPPED_TESTS=$((SKIPPED_TESTS + 1)) + [[ "$VERBOSE" == "true" ]] && echo "$test_output" | head -3 | sed 's/^/ | /' + record_result "$test_name" "SKIP" "$test_output" + ;; + *) + log_error "$test_name (exit $test_exit_code)" + FAILED_TESTS=$((FAILED_TESTS + 1)) + [[ "$VERBOSE" == "true" ]] && echo "$test_output" | head -10 | sed 's/^/ | /' + record_result "$test_name" "FAIL" "Exit code: $test_exit_code" + ;; + esac +} + +# Discover the canonical test list from tests/Makefile.am's TESTS variable. +collect_tests() { + awk ' + /^TESTS *\+?=/ { collect=1; sub(/^TESTS *\+?=/, "") } + collect { + line=$0 + cont=sub(/\\[ \t]*$/, "", line) + n=split(line, a, /[ \t]+/) + for (i=1; i<=n; i++) if (a[i] != "") print a[i] + if (!cont) collect=0 + } + ' "$GNU_TESTS_DIR/Makefile.am" +} + +log_info "Discovering tests from $GNU_TESTS_DIR/Makefile.am" +mapfile -t TEST_LIST < <(collect_tests | sort -u) +log_info "Found ${#TEST_LIST[@]} tests" + +log_info "Starting test execution..." +start_time=$(date +%s) + +for t in "${TEST_LIST[@]}"; do + [[ -z "$t" ]] && continue + test_path="$GNU_TESTS_DIR/$t" + [[ -f "$test_path" ]] || { log_warning "Listed test not found: $t"; continue; } + run_gnu_test "$test_path" +done + +end_time=$(date +%s) +duration=$((end_time - start_time)) + +# Print summary +echo +echo "=========================================" +echo "GNU grep testsuite results" +echo "=========================================" +echo "Total tests: $TOTAL_TESTS" +echo "Passed: $PASSED_TESTS" +echo "Failed: $FAILED_TESTS" +echo "Skipped: $SKIPPED_TESTS" +echo "Duration: ${duration}s" + +if [[ -n "$JSON_OUTPUT_FILE" ]]; then + generate_json_output +fi + +if [[ $((PASSED_TESTS + FAILED_TESTS)) -gt 0 ]]; then + pass_rate=$(( (PASSED_TESTS * 100) / (PASSED_TESTS + FAILED_TESTS) )) + echo "Pass rate: ${pass_rate}%" +fi + +# Mirror the script's exit convention to ../sed: nonzero if anything failed. +[[ $FAILED_TESTS -eq 0 ]] && exit 0 || exit 1