From 2c60796b3fff2190377de7a74d944375e8f14a48 Mon Sep 17 00:00:00 2001 From: Andoni Morales Alastruey Date: Thu, 11 Dec 2025 11:09:00 +0100 Subject: [PATCH] test: use unittest for tests --- Makefile | 38 +------ tests/__init__.py | 18 +++ tests/test_functional.py | 234 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+), 36 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/test_functional.py diff --git a/Makefile b/Makefile index 1d47fb11..b9b0f855 100644 --- a/Makefile +++ b/Makefile @@ -15,42 +15,8 @@ help: install_deps: ## install Python dependencies python3 -m pip install -r requirements-dev.txt -check: ## check that very basic tests run - @echo "Running dummy test..." - $(FLUSTER) list - $(FLUSTER) list -c - $(FLUSTER) -ne list -c - $(FLUSTER) list -ts dummy -tv - $(FLUSTER) download dummy dummy_fail -k - $(FLUSTER) run -ts dummy -tv one - $(FLUSTER) reference Dummy dummy - $(FLUSTER) run -ts dummy -tv one -j1 - $(FLUSTER) run -ts dummy -s - $(FLUSTER) -ne run -ts dummy -s - $(FLUSTER) run -ts dummy -so summary.log && cat summary.log && rm -rf summary.log - $(FLUSTER) run -ts dummy -j1 -s - $(FLUSTER) run -ts dummy -th 1 - $(FLUSTER) run -ts dummy -tth 10 -ifneq ($(OS),Windows_NT) - $(FLUSTER) run -ts dummy non_existing_test_suite; test $$? -ne 0 - $(FLUSTER) run -ts dummy -th 2; test $$? -eq 2 - $(FLUSTER) run -ts dummy -tth 0.000000001; test $$? -eq 3 - $(FLUSTER) run -ts dummy_fail -th 1 - $(FLUSTER) run -ts dummy_fail -th 2; test $$? -eq 2 - $(FLUSTER) run -ts dummy_fail -j1 -ff -s; test $$? -ne 0 - $(FLUSTER) download dummy non_existing_test_suite -k; test $$? -ne 0 - $(FLUSTER) download dummy dummy_download_fail -k; test $$? -ne 0 - $(FLUSTER) download H264-min H265-min -k - $(FLUSTER) run -ts H264-min -d GStreamer-H.264-Libav-Gst1.0 FFmpeg-H.264 -s - $(FLUSTER) run -ts H265-min -d GStreamer-H.265-Libav-Gst1.0 FFmpeg-H.265 -s -ifeq ($(KERNEL_NAME), Linux) - $(FLUSTER) download AV1-min VP8-min VP9-min -k - $(FLUSTER) run -ts AV1-min -d libaom-AV1 -s - $(FLUSTER) run -ts VP8-min -d libvpx-VP8 -s - $(FLUSTER) run -ts VP9-min -d libvpx-VP9 -s -endif -endif - @echo "\nAll test finished succesfully!" +check: ## run Python unit tests + python3 -m unittest discover -v -s tests -p "test_*.py" create_dirs=mkdir -p $(CONTRIB_DIR) $(DECODERS_DIR) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..1d8542a4 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,18 @@ +# Fluster - testing framework for decoders conformance +# Copyright (C) 2025, Fluendo, S.A. +# Author: Andoni Morales Alastruey , Fluendo, S.A. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public License +# as published by the Free Software Foundation, either version 3 +# of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +"""Fluster test package.""" diff --git a/tests/test_functional.py b/tests/test_functional.py new file mode 100644 index 00000000..a826399a --- /dev/null +++ b/tests/test_functional.py @@ -0,0 +1,234 @@ +# Fluster - testing framework for decoders conformance +# Copyright (C) 2025, Fluendo, S.A. +# Author: Andoni Morales Alastruey , Fluendo, S.A. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public License +# as published by the Free Software Foundation, either version 3 +# of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. If not, see . + +from __future__ import annotations + +import os +import platform +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path +from typing import List + +# Get the project root directory +PROJECT_ROOT = Path(__file__).parent.parent +FLUSTER_SCRIPT = PROJECT_ROOT / "fluster.py" +CHECK_DIR = PROJECT_ROOT / "check" + +# Platform detection +IS_WINDOWS = platform.system() == "Windows" +IS_LINUX = platform.system() == "Linux" + + +def run_fluster( + args: List[str], + no_emoji: bool = False, +) -> subprocess.CompletedProcess[str]: + """ + Run the fluster CLI with the given arguments. + + Args: + args: List of arguments to pass to fluster.py + no_emoji: If True, add --no-emoji flag + + Returns: + CompletedProcess with stdout and stderr + """ + cmd = [sys.executable, str(FLUSTER_SCRIPT), "-tsd", str(CHECK_DIR)] + # Always use --no-emoji on Windows to avoid encoding issues + if no_emoji or IS_WINDOWS: + cmd.append("-ne") + cmd.extend(args) + result = subprocess.run( + cmd, + capture_output=True, + text=True, + cwd=PROJECT_ROOT, + check=False, + ) + return result + + +class TestList(unittest.TestCase): + def test_list_basic(self) -> None: + result = run_fluster(["list"]) + self.assertIn("dummy", result.stdout) + self.assertIn("Dummy", result.stdout) + + def test_list_with_check(self) -> None: + result = run_fluster(["list", "-c"]) + self.assertIn("dummy", result.stdout) + + def test_list_no_emoji_with_check(self) -> None: + result = run_fluster(["list", "-c"], no_emoji=True) + self.assertIn("dummy", result.stdout) + # Should not contain emoji characters + self.assertNotIn("✔", result.stdout) + self.assertNotIn("❌", result.stdout) + + def test_list_test_suite_with_vectors(self) -> None: + result = run_fluster(["list", "-ts", "dummy", "-tv"]) + self.assertIn("dummy", result.stdout) + self.assertIn("one", result.stdout) + + +class TestDownload(unittest.TestCase): + def test_download_dummy_suites(self) -> None: + result = run_fluster(["download", "dummy", "dummy_fail", "-k"]) + self.assertEqual(result.returncode, 0) + + def test_download_non_existing_suite_fails(self) -> None: + result = run_fluster(["download", "dummy", "non_existing_test_suite", "-k"]) + self.assertNotEqual(result.returncode, 0) + + def test_download_checksum_failure(self) -> None: + result = run_fluster(["download", "dummy", "dummy_download_fail", "-k"]) + self.assertNotEqual(result.returncode, 0) + + def test_download_h264_h265(self) -> None: + result = run_fluster(["download", "H264-min", "H265-min", "-k"]) + self.assertEqual(result.returncode, 0) + + def test_download_av1_vp8_vp9(self) -> None: + result = run_fluster(["download", "AV1-min", "VP8-min", "VP9-min", "-k"]) + self.assertEqual(result.returncode, 0) + + +class TestReference(unittest.TestCase): + def setUp(self) -> None: + run_fluster(["download", "dummy", "-k"]) + + def test_reference_dummy(self) -> None: + result = run_fluster(["reference", "Dummy", "dummy"]) + self.assertIn("Reference run", result.stdout) + self.assertEqual(result.returncode, 0) + + +class TestRun(unittest.TestCase): + def setUp(self) -> None: + run_fluster(["download", "dummy", "dummy_fail", "-k"]) + + def test_run_single_vector(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "-tv", "one"]) + self.assertIn("Success", result.stdout) + self.assertEqual(result.returncode, 0) + + def test_run_single_vector_single_job(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "-tv", "one", "-j1"]) + self.assertIn("Success", result.stdout) + + def test_run_with_summary(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "-s"]) + self.assertIn("Success", result.stdout) + self.assertIn("TOTAL", result.stdout) + + def test_run_no_emoji_with_summary(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "-s"], no_emoji=True) + self.assertIn("Success", result.stdout) + # Should not contain emoji + self.assertNotIn("✔", result.stdout) + + def test_run_with_summary_output_file(self) -> None: + with tempfile.NamedTemporaryFile(mode="w", suffix=".log", delete=False) as f: + summary_file = f.name + + try: + result = run_fluster(["run", "-ts", "dummy", "-so", summary_file]) + self.assertEqual(result.returncode, 0) + + # Verify summary file was created and has content + self.assertTrue(os.path.exists(summary_file)) + with open(summary_file, "r") as f: + content = f.read() + self.assertIn("TOTAL", content) + finally: + if os.path.exists(summary_file): + os.unlink(summary_file) + + def test_run_single_job_with_summary(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "-j1", "-s"]) + self.assertIn("Success", result.stdout) + + def test_run_threshold_pass(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "-th", "1"]) + self.assertEqual(result.returncode, 0) + + def test_run_time_threshold_pass(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "-tth", "10"]) + self.assertEqual(result.returncode, 0) + + def test_run_non_existing_suite_fails(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "non_existing_test_suite"]) + self.assertNotEqual(result.returncode, 0) + + def test_run_threshold_below_fails_exit_2(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "-th", "2"]) + self.assertEqual(result.returncode, 2) + + def test_run_time_threshold_exceeded_exit_3(self) -> None: + result = run_fluster(["run", "-ts", "dummy", "-tth", "0.000000001"]) + self.assertEqual(result.returncode, 3) + + def test_run_dummy_fail_threshold_1(self) -> None: + result = run_fluster(["run", "-ts", "dummy_fail", "-th", "1"]) + self.assertEqual(result.returncode, 0) + + def test_run_dummy_fail_threshold_2_exit_2(self) -> None: + result = run_fluster(["run", "-ts", "dummy_fail", "-th", "2"]) + self.assertEqual(result.returncode, 2) + + def test_run_failfast(self) -> None: + result = run_fluster(["run", "-ts", "dummy_fail", "-j1", "-ff", "-s"]) + self.assertNotEqual(result.returncode, 0) + + @unittest.skipIf(IS_WINDOWS, "Unix-specific test") + def test_run_h264_decoders(self) -> None: + run_fluster(["download", "H264-min", "-k"]) + result = run_fluster(["run", "-ts", "H264-min", "-d", "GStreamer-H.264-Libav-Gst1.0", "FFmpeg-H.264", "-s"]) + # May fail if decoders are not available, but command should run + self.assertTrue("H264-min" in result.stdout or "Skipping" in result.stdout) + + @unittest.skipIf(IS_WINDOWS, "Unix-specific test") + def test_run_h265_decoders(self) -> None: + run_fluster(["download", "H265-min", "-k"]) + result = run_fluster(["run", "-ts", "H265-min", "-d", "GStreamer-H.265-Libav-Gst1.0", "FFmpeg-H.265", "-s"]) + # May fail if decoders are not available, but command should run + self.assertTrue("H265-min" in result.stdout or "Skipping" in result.stdout) + + @unittest.skipUnless(IS_LINUX, "Linux-specific test") + def test_run_av1_libaom(self) -> None: + run_fluster(["download", "AV1-min", "-k"]) + result = run_fluster(["run", "-ts", "AV1-min", "-d", "libaom-AV1", "-s"]) + self.assertTrue("AV1-min" in result.stdout or "Skipping" in result.stdout or "cannot be run" in result.stdout) + + @unittest.skipUnless(IS_LINUX, "Linux-specific test") + def test_run_vp8_libvpx(self) -> None: + run_fluster(["download", "VP8-min", "-k"]) + result = run_fluster(["run", "-ts", "VP8-min", "-d", "libvpx-VP8", "-s"]) + self.assertTrue("VP8-min" in result.stdout or "Skipping" in result.stdout or "cannot be run" in result.stdout) + + @unittest.skipUnless(IS_LINUX, "Linux-specific test") + def test_run_vp9_libvpx(self) -> None: + run_fluster(["download", "VP9-min", "-k"]) + result = run_fluster(["run", "-ts", "VP9-min", "-d", "libvpx-VP9", "-s"]) + self.assertTrue("VP9-min" in result.stdout or "Skipping" in result.stdout or "cannot be run" in result.stdout) + + +if __name__ == "__main__": + unittest.main(verbosity=2)