diff --git a/.github/workflows/slow-tests.yml b/.github/workflows/slow-tests.yml index 73cdc58d4df..7472e972568 100644 --- a/.github/workflows/slow-tests.yml +++ b/.github/workflows/slow-tests.yml @@ -8,7 +8,7 @@ on: - "trl/**.py" - "examples/**.py" env: - SLACK_API_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} + CI_SLACK_CHANNEL: ${{ secrets.CI_PUSH_MAIN_CHANNEL }} HF_TOKEN: ${{ secrets.HF_TOKEN }} TRL_EXPERIMENTAL_SILENCE: 1 @@ -56,9 +56,18 @@ jobs: if: always() run: | source .venv/bin/activate - uv pip install slack_sdk tabulate + uv pip install tabulate python scripts/log_reports.py >> $GITHUB_STEP_SUMMARY + - name: Post to Slack + if: always() + uses: huggingface/hf-workflows/.github/actions/post-slack@main + with: + slack_channel: ${{ env.CI_SLACK_CHANNEL }} + title: Results of the slow tests on a single GPU + status: ${{ job.status }} + slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} + run_all_tests_multi_gpu: runs-on: group: aws-g4dn-2xlarge @@ -102,5 +111,14 @@ jobs: if: always() run: | source .venv/bin/activate - uv pip install slack_sdk tabulate + uv pip install tabulate python scripts/log_reports.py >> $GITHUB_STEP_SUMMARY + + - name: Post to Slack + if: always() + uses: huggingface/hf-workflows/.github/actions/post-slack@main + with: + slack_channel: ${{ env.CI_SLACK_CHANNEL }} + title: Results of the slow tests on multiple GPUs + status: ${{ job.status }} + slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }} diff --git a/scripts/log_reports.py b/scripts/log_reports.py index fdffdd1efa1..fd8709860da 100644 --- a/scripts/log_reports.py +++ b/scripts/log_reports.py @@ -12,21 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse import json import logging import os -from datetime import date from pathlib import Path from tabulate import tabulate -MAX_LEN_MESSAGE = 2900 # Slack endpoint has a limit of 3001 characters - -parser = argparse.ArgumentParser() -parser.add_argument("--slack_channel_name", default="trl-push-ci") - # Set up logging logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") @@ -34,7 +27,6 @@ def process_log_file(log): failed_tests = [] passed_tests = [] - section_num_failed = 0 try: with open(log) as f: @@ -47,7 +39,6 @@ def process_log_file(log): if test_name: if outcome == "failed": - section_num_failed += 1 failed_tests.append([test_name, duration, log.stem.split("_")[0]]) else: passed_tests.append([test_name, duration, log.stem.split("_")[0]]) @@ -59,26 +50,29 @@ def process_log_file(log): except Exception as e: logging.error(f"Error processing log file {log}: {e}") - return failed_tests, passed_tests, section_num_failed + return failed_tests, passed_tests -def main(slack_channel_name): - group_info = [] - total_num_failed = 0 - total_empty_files = [] +def main(): + print(f"## 🤗 Results of the {os.environ['TEST_TYPE']} TRL tests.") log_files = list(Path().glob("*.log")) if not log_files: - logging.info("No log files found.") + print("⚠️ No log file found! The tests did not run, check the GitHub action job.") return for log in log_files: - failed, passed, section_num_failed = process_log_file(log) - empty_file = not failed and not passed - - total_num_failed += section_num_failed - total_empty_files.append(empty_file) - group_info.append([str(log), section_num_failed, failed]) + failed, passed = process_log_file(log) + + if failed: + print(f"### ❌ {len(failed)} failed test(s) in `{log}`") + failed_table = [test[0].split("::")[:2] + [test[0].split("::")[-1][:30] + ".."] for test in failed] + table = tabulate(failed_table, headers=["File", "Class", "Test Name"], tablefmt="grid") + print(f"\n```\n{table}\n```\n") + elif passed: + print(f"### ✅ No failures in `{log}`") + else: + print(f"⚠️ Empty log file `{log}`! Check the GitHub action job.") # Clean up log file try: @@ -86,84 +80,6 @@ def main(slack_channel_name): except OSError as e: logging.warning(f"Could not remove log file {log}: {e}") - # Prepare Slack message payload - payload = [ - { - "type": "header", - "text": {"type": "plain_text", "text": f"🤗 Results of the {os.environ.get('TEST_TYPE', '')} TRL tests."}, - }, - ] - - if total_num_failed > 0: - message = "" - for name, num_failed, failed_tests in group_info: - if num_failed > 0: - message += f"*{name}: {num_failed} failed test(s)*\n" - failed_table = [ - test[0].split("::")[:2] + [test[0].split("::")[-1][:30] + ".."] for test in failed_tests - ] - message += ( - "\n```\n" - + tabulate(failed_table, headers=["File", "Class", "Test Name"], tablefmt="grid") - + "\n```\n" - ) - - if any(total_empty_files): - message += f"\n*{name}: Warning! Empty file - check GitHub action job*\n" - - # Logging - logging.info(f"Total failed tests: {total_num_failed}") - print(f"### {message}") - - if len(message) > MAX_LEN_MESSAGE: - message = ( - f"❌ There are {total_num_failed} failed tests in total! Please check the action results directly." - ) - - payload.append({"type": "section", "text": {"type": "mrkdwn", "text": message}}) - payload.append( - { - "type": "section", - "text": {"type": "mrkdwn", "text": "*For more details:*"}, - "accessory": { - "type": "button", - "text": {"type": "plain_text", "text": "Check Action results"}, - "url": f"https://github.com/huggingface/trl/actions/runs/{os.environ['GITHUB_RUN_ID']}", - }, - } - ) - payload.append( - { - "type": "context", - "elements": [ - { - "type": "plain_text", - "text": f"On Push main {os.environ.get('TEST_TYPE')} results for {date.today()}", - } - ], - } - ) - - # Send to Slack - from slack_sdk import WebClient - - slack_client = WebClient(token=os.environ.get("SLACK_API_TOKEN")) - slack_client.chat_postMessage(channel=f"#{slack_channel_name}", text=message, blocks=payload) - - else: - payload.append( - { - "type": "section", - "text": { - "type": "plain_text", - "text": "✅ No failures! All tests passed successfully.", - "emoji": True, - }, - } - ) - logging.info("All tests passed. No errors detected.") - if __name__ == "__main__": - args = parser.parse_args() - main(args.slack_channel_name) + main()