Report slow tests results to Slack based on the job status

.github/workflows/slow-tests.yml

@@ -8,7 +8,7 @@ on:
       - "trl/**.py"
       - "examples/**.py"
 env:
-  SLACK_API_TOKEN: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
+  CI_SLACK_CHANNEL: ${{ secrets.CI_PUSH_MAIN_CHANNEL }}
   HF_TOKEN: ${{ secrets.HF_TOKEN }}
   TRL_EXPERIMENTAL_SILENCE: 1
 
@@ -56,9 +56,18 @@ jobs:
         if: always()
         run: |
           source .venv/bin/activate
-          uv pip install slack_sdk tabulate
+          uv pip install tabulate
           python scripts/log_reports.py >> $GITHUB_STEP_SUMMARY
 
+      - name: Post to Slack
+        if: always()
+        uses: huggingface/hf-workflows/.github/actions/post-slack@main
+        with:
+          slack_channel: ${{ env.CI_SLACK_CHANNEL }}
+          title: Results of the slow tests on a single GPU
+          status: ${{ job.status }}
+          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}
+
   run_all_tests_multi_gpu:
     runs-on:
       group: aws-g4dn-2xlarge
@@ -102,5 +111,14 @@ jobs:
         if: always()
         run: |
           source .venv/bin/activate
-          uv pip install slack_sdk tabulate
+          uv pip install tabulate
           python scripts/log_reports.py >> $GITHUB_STEP_SUMMARY
+
+      - name: Post to Slack
+        if: always()
+        uses: huggingface/hf-workflows/.github/actions/post-slack@main
+        with:
+          slack_channel: ${{ env.CI_SLACK_CHANNEL }}
+          title: Results of the slow tests on multiple GPUs
+          status: ${{ job.status }}
+          slack_token: ${{ secrets.SLACK_CIFEEDBACK_BOT_TOKEN }}

scripts/log_reports.py

@@ -12,29 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import argparse
 import json
 import logging
 import os
-from datetime import date
 from pathlib import Path
 
 from tabulate import tabulate
 
 
-MAX_LEN_MESSAGE = 2900  # Slack endpoint has a limit of 3001 characters
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--slack_channel_name", default="trl-push-ci")
-
 # Set up logging
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 
 
 def process_log_file(log):
     failed_tests = []
     passed_tests = []
-    section_num_failed = 0
 
     try:
         with open(log) as f:
@@ -47,7 +39,6 @@ def process_log_file(log):
 
                     if test_name:
                         if outcome == "failed":
-                            section_num_failed += 1
                             failed_tests.append([test_name, duration, log.stem.split("_")[0]])
                         else:
                             passed_tests.append([test_name, duration, log.stem.split("_")[0]])
@@ -59,111 +50,36 @@ def process_log_file(log):
     except Exception as e:
         logging.error(f"Error processing log file {log}: {e}")
 
-    return failed_tests, passed_tests, section_num_failed
+    return failed_tests, passed_tests
 
 
-def main(slack_channel_name):
-    group_info = []
-    total_num_failed = 0
-    total_empty_files = []
+def main():
+    print(f"## 🤗 Results of the {os.environ['TEST_TYPE']} TRL tests.")
 
     log_files = list(Path().glob("*.log"))
     if not log_files:
-        logging.info("No log files found.")
+        print("⚠️ No log file found! The tests did not run, check the GitHub action job.")
         return
 
     for log in log_files:
-        failed, passed, section_num_failed = process_log_file(log)
-        empty_file = not failed and not passed
-
-        total_num_failed += section_num_failed
-        total_empty_files.append(empty_file)
-        group_info.append([str(log), section_num_failed, failed])
+        failed, passed = process_log_file(log)
+
+        if failed:
+            print(f"### ❌ {len(failed)} failed test(s) in `{log}`")
+            failed_table = [test[0].split("::")[:2] + [test[0].split("::")[-1][:30] + ".."] for test in failed]
+            table = tabulate(failed_table, headers=["File", "Class", "Test Name"], tablefmt="grid")
+            print(f"\n```\n{table}\n```\n")
+        elif passed:
+            print(f"### ✅ No failures in `{log}`")
+        else:
+            print(f"⚠️ Empty log file `{log}`! Check the GitHub action job.")
 
         # Clean up log file
         try:
             os.remove(log)
         except OSError as e:
             logging.warning(f"Could not remove log file {log}: {e}")
 
-    # Prepare Slack message payload
-    payload = [
-        {
-            "type": "header",
-            "text": {"type": "plain_text", "text": f"🤗 Results of the {os.environ.get('TEST_TYPE', '')} TRL tests."},
-        },
-    ]
-
-    if total_num_failed > 0:
-        message = ""
-        for name, num_failed, failed_tests in group_info:
-            if num_failed > 0:
-                message += f"*{name}: {num_failed} failed test(s)*\n"
-                failed_table = [
-                    test[0].split("::")[:2] + [test[0].split("::")[-1][:30] + ".."] for test in failed_tests
-                ]
-                message += (
-                    "\n```\n"
-                    + tabulate(failed_table, headers=["File", "Class", "Test Name"], tablefmt="grid")
-                    + "\n```\n"
-                )
-
-            if any(total_empty_files):
-                message += f"\n*{name}: Warning! Empty file - check GitHub action job*\n"
-
-        # Logging
-        logging.info(f"Total failed tests: {total_num_failed}")
-        print(f"### {message}")
-
-        if len(message) > MAX_LEN_MESSAGE:
-            message = (
-                f"❌ There are {total_num_failed} failed tests in total! Please check the action results directly."
-            )
-
-        payload.append({"type": "section", "text": {"type": "mrkdwn", "text": message}})
-        payload.append(
-            {
-                "type": "section",
-                "text": {"type": "mrkdwn", "text": "*For more details:*"},
-                "accessory": {
-                    "type": "button",
-                    "text": {"type": "plain_text", "text": "Check Action results"},
-                    "url": f"https://github.com/huggingface/trl/actions/runs/{os.environ['GITHUB_RUN_ID']}",
-                },
-            }
-        )
-        payload.append(
-            {
-                "type": "context",
-                "elements": [
-                    {
-                        "type": "plain_text",
-                        "text": f"On Push main {os.environ.get('TEST_TYPE')} results for {date.today()}",
-                    }
-                ],
-            }
-        )
-
-        # Send to Slack
-        from slack_sdk import WebClient
-
-        slack_client = WebClient(token=os.environ.get("SLACK_API_TOKEN"))
-        slack_client.chat_postMessage(channel=f"#{slack_channel_name}", text=message, blocks=payload)
-
-    else:
-        payload.append(
-            {
-                "type": "section",
-                "text": {
-                    "type": "plain_text",
-                    "text": "✅ No failures! All tests passed successfully.",
-                    "emoji": True,
-                },
-            }
-        )
-        logging.info("All tests passed. No errors detected.")
-
 
 if __name__ == "__main__":
-    args = parser.parse_args()
-    main(args.slack_channel_name)
+    main()