Merge branch 'master' into api-pr1-scaffolding

Author: pulk17

.github/workflows/sp-deployment-pipeline.yml

@@ -81,7 +81,7 @@ jobs:
               sudo git clean -f -d
               sudo git pull origin $DEPLOY_BRANCH
 
-              sudo python -m pip install -r requirements.txt
+              sudo python3 -m pip install -r requirements.txt --break-system-packages
               sudo FLASK_APP=./run.py flask db upgrade
 
               sudo cp "install/ci-vm/ci-linux/ci/bootstrap" "${SAMPLE_REPOSITORY}/TestData/ci-linux/bootstrap" 2>/dev/null || true
@@ -156,7 +156,7 @@ jobs:
                 PREV_COMMIT=$(cat /tmp/previous_commit.txt)
                 echo "Rolling back to commit: $PREV_COMMIT"
                 sudo git checkout "$PREV_COMMIT"
-                sudo python -m pip install -r requirements.txt
+                sudo python3 -m pip install -r requirements.txt --break-system-packages
                 sudo systemctl reload platform
                 echo "Rollback complete"
               else

README.md

@@ -32,7 +32,7 @@ regression tests useless. In these cases, the tests can give false negative. For
 result files by the following command.
 
 ```shell
-python manage.py update /path/to/ccextractor/executable
+python3 manage.py update /path/to/ccextractor/executable
 ```
 
 NOTE: Please backup old results and be sure about updating them as there is no method to go back.

install/deploy/deploy.sh

@@ -45,7 +45,7 @@ echo "✓ Updated to commit: $NEW_COMMIT"
 # Step 4: Update dependencies
 echo ""
 echo "--- Step 3: Updating dependencies ---"
-python3 -m pip install -r requirements.txt --quiet --disable-pip-version-check
+python3 -m pip install -r requirements.txt --quiet --disable-pip-version-check --break-system-packages
 echo "✓ Dependencies updated"
 
 # Step 5: Run database migrations

install/deploy/rollback.sh

@@ -84,7 +84,7 @@ fi
 # Step 3: Reinstall dependencies for rolled-back version
 echo ""
 echo "--- Step 3: Reinstalling dependencies ---"
-python3 -m pip install -r requirements.txt --quiet --disable-pip-version-check
+python3 -m pip install -r requirements.txt --quiet --disable-pip-version-check --break-system-packages
 echo "✓ Dependencies reinstalled"
 
 # Step 4: Reload application

install/init_db.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import sys
 from os import path

install/install.sh

@@ -26,11 +26,11 @@ echo "-------------------------------"
 echo ""
 echo "* Updating package list"
 apt-get update >> "$install_log" 2>&1
-echo "* Installing nginx, python, pip, mediainfo and gunicorn"
+echo "* Installing nginx, python3, pip, mediainfo and gunicorn"
 add-apt-repository ppa:deadsnakes/ppa -y >> "$install_log" 2>&1
 apt-get -q -y install python3.9 nginx python3.9-distutils python3-pip mediainfo gunicorn3 >> "$install_log" 2>&1
-update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 >> "$install_log" 2>&1
-update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 >> "$install_log" 2>&1
+update-alternatives --install /usr/bin/python python /usr/bin/python3 1 >> "$install_log" 2>&1
+update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3 1 >> "$install_log" 2>&1
 rm -f /etc/nginx/sites-available/default
 rm -f /etc/nginx/sites-enabled/default
 if [ ! -f /etc/init.d/mysql* ]; then
@@ -64,9 +64,9 @@ userInput () {
 }
 
 echo "* Update pip, setuptools and wheel"
-python -m pip install --upgrade pip setuptools wheel >> "$install_log" 2>&1
+python3 -m pip install --upgrade pip setuptools wheel --break-system-packages >> "$install_log" 2>&1
 echo "* Installing pip dependencies"
-python -m pip install -r "${root_dir}/requirements.txt" >> "$install_log" 2>&1
+python3 -m pip install -r "${root_dir}/requirements.txt" --break-system-packages >> "$install_log" 2>&1
 echo ""
 echo "-------------------------------"
 echo "|        Configuration        |"
@@ -204,13 +204,13 @@ while [[ $admin_password == "" ]];do
    fi
 done
 echo "Creating admin account: "
-python "${root_dir}/install/init_db.py" "${config_db_uri}" "${admin_name}" "${admin_email}" "${admin_password}"
+python3 "${root_dir}/install/init_db.py" "${config_db_uri}" "${admin_name}" "${admin_email}" "${admin_password}"
 # Create sample database if user wanted to
 if [ "${sample_response}" == 'y' ]; then
     echo "Creating sample database.."
     sample_dir="${dir}/sample_files/*"
     cp -r $sample_dir "${sample_repository}/TestFiles"
-    python "${dir}/sample_db.py" "${config_db_uri}"
+    python3 "${dir}/sample_db.py" "${config_db_uri}"
 fi
 echo ""
 echo "-------------------------------"

install/installation.md

@@ -162,7 +162,7 @@ determine what goes wrong. The snippet below shows how this can be done:
 
 ```
 cd /var/www/sample-platform/
-sudo python bootstrap_gunicorn.py
+sudo python3 bootstrap_gunicorn.py
 ```
 
 * If `gunicorn` boots up successfully, most relevant logs will be stored in
@@ -211,7 +211,7 @@ The file `mod_ci/cron.py` is to be run in periodic intervals. To setup a cron jo
 1. Open your terminal and enter the command `sudo crontab -e`.
 2. To setup a cron job that runs this file every 10 minutes, append this at the bottom of the file
     ```
-    */10 * * * * python /var/www/sample-platform/mod_ci/cron.py > /var/www/sample-platform/logs/cron.log 2>&1
+    */10 * * * * python3 /var/www/sample-platform/mod_ci/cron.py > /var/www/sample-platform/logs/cron.log 2>&1
     ```
     Change the `/var/www/sample-plaform` directory, if you have installed the platform in a different directory.
 

install/sample_db.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import sys
 from os import path

mod_ci/controllers.py

@@ -28,7 +28,6 @@
 from pymysql.err import IntegrityError
 from sqlalchemy import and_, func, select
 from sqlalchemy.sql import label
-from sqlalchemy.sql.functions import count
 from werkzeug.utils import secure_filename
 
 from database import DeclEnum, create_session
@@ -249,8 +248,8 @@ class Workflow_builds(DeclEnum):
 class Artifact_names(DeclEnum):
     """Define CCExtractor GitHub Artifacts names."""
 
-    linux = "CCExtractor Linux build"
-    windows = "CCExtractor Windows x64 Release build"
+    linux = "CCExtractor Linux build (with migrations)"
+    windows = "CCExtractor Windows x64 Release build (with migrations)"
 
 
 def is_valid_commit_hash(commit: Optional[str]) -> bool:
@@ -2276,14 +2275,16 @@ def start_ci():
         return json.dumps({'msg': 'EOL'})
 
 
-def update_build_badge(status, test) -> None:
+def update_build_badge(status, test, test_results=None) -> None:
     """
     Build status badge for current test to be displayed on sample-platform.
 
     :param status: current testing status
     :type status: str
     :param test: current commit that is tested
     :type test: Test
+    :param test_results: pre-computed results from get_test_results; if None, fetched internally
+    :type test_results: list | None
     :return: null
     :rtype: null
     """
@@ -2294,7 +2295,8 @@ def update_build_badge(status, test) -> None:
         shutil.copyfile(original_location, build_status_location)
         g.log.info('Build badge updated successfully!')
 
-        test_results = get_test_results(test)
+        if test_results is None:
+            test_results = get_test_results(test)
         test_ids_to_update = []
         for category_results in test_results:
             test_ids_to_update.extend([test['test'].id for test in category_results['tests'] if not test['error']])
@@ -2429,33 +2431,15 @@ def progress_type_request(log, test, test_id, request) -> bool:
         message = 'Tests aborted due to an error; please check'
 
     elif status == TestStatus.completed:
-        # Determine if success or failure
-        # It fails if any of these happen:
-        # - A crash (unexpected exit code)
-        # - A not None value on the "got" of a TestResultFile (
-        #       meaning the hashes do not match)
-        crashes = g.db.query(count(TestResult.exit_code)).filter(
-            and_(
-                TestResult.test_id == test.id,
-                TestResult.exit_code != TestResult.expected_rc
-            )).scalar()
-        results_zero_rc = select(RegressionTest.id).filter(
-            RegressionTest.expected_rc == 0
+        test_results = get_test_results(test)
+        has_failures = any(
+            t['error']
+            for category in test_results
+            for t in category['tests']
         )
-        results = g.db.query(count(TestResultFile.got)).filter(
-            and_(
-                TestResultFile.test_id == test.id,
-                TestResultFile.regression_test_id.in_(
-                    results_zero_rc.select()
-                ),
-                TestResultFile.got.isnot(None)
-            )
-        ).scalar()
-        log.debug(f'[Test: {test.id}] Test completed: {crashes} crashes, {results} results')
-        if crashes > 0 or results > 0:
+        if has_failures:
             state = Status.FAILURE
             message = 'Not all tests completed successfully, please check'
-
         else:
             state = Status.SUCCESS
             message = 'Tests completed'
@@ -2468,7 +2452,7 @@ def progress_type_request(log, test, test_id, request) -> bool:
                 message = 'All tests passed'
             else:
                 message = 'Not all tests completed successfully, please check'
-        update_build_badge(state, test)
+        update_build_badge(state, test, test_results)
 
     else:
         message = progress.message
@@ -2526,29 +2510,17 @@ def update_final_status():
                     TestProgress.test_id.in_(platform_tests.select())
                 )
             )
-            in_progress_statuses = [TestStatus.preparation, TestStatus.completed, TestStatus.canceled]
-            finished_tests_progress = g.db.query(TestProgress).filter(
-                and_(
-                    TestProgress.test_id.in_(finished_tests.select()),
-                    TestProgress.status.in_(in_progress_statuses)
-                )
-            ).subquery()
             times = g.db.query(
-                finished_tests_progress.c.test_id,
-                label('time', func.group_concat(finished_tests_progress.c.timestamp))
-            ).group_by(finished_tests_progress.c.test_id).all()
+                TestProgress.test_id,
+                label('start_time', func.min(TestProgress.timestamp)),
+                label('end_time', func.max(TestProgress.timestamp))
+            ).filter(
+                TestProgress.test_id.in_(finished_tests)
+            ).group_by(TestProgress.test_id).all()
 
             for p in times:
-                parts = p.time.split(',')
-                try:
-                    # Try parsing with microsecond precision first
-                    start = datetime.datetime.strptime(parts[0], '%Y-%m-%d %H:%M:%S.%f')
-                    end = datetime.datetime.strptime(parts[-1], '%Y-%m-%d %H:%M:%S.%f')
-                except ValueError:
-                    # Fall back to format without microseconds
-                    start = datetime.datetime.strptime(parts[0], '%Y-%m-%d %H:%M:%S')
-                    end = datetime.datetime.strptime(parts[-1], '%Y-%m-%d %H:%M:%S')
-                total_time += int((end - start).total_seconds())
+                duration = max(0, int((p.end_time - p.start_time).total_seconds()))
+                total_time += duration
 
             if len(times) != 0:
                 average_time = total_time // len(times)
@@ -2559,10 +2531,15 @@ def update_final_status():
             safe_db_commit(g.db, "setting new average time")
 
         else:
-            all_results = TestResult.query.count()
-            regression_test_count = RegressionTest.query.count()
-            number_test = all_results / regression_test_count
-            updated_average = float(current_average.value) * (number_test - 1)
+            # Count actual finished tests for this platform
+            platform_tests = g.db.query(Test.id).filter(Test.platform == test.platform).subquery()
+            number_test = g.db.query(func.count(func.distinct(TestProgress.test_id))).filter(
+                and_(
+                    TestProgress.status.in_([TestStatus.completed, TestStatus.canceled]),
+                    TestProgress.test_id.in_(platform_tests)
+                )
+            ).scalar() or 0
+
             pr = test.progress_data()
             end_time = pr['end']
             start_time = pr['start']
@@ -2573,9 +2550,14 @@ def update_final_status():
             if start_time.tzinfo is not None:
                 start_time = start_time.replace(tzinfo=None)
 
-            last_running_test = end_time - start_time
-            updated_average = updated_average + last_running_test.total_seconds()
-            current_average.value = 0 if number_test == 0 else updated_average // number_test
+            last_duration = max(0, int((end_time - start_time).total_seconds()))
+
+            if number_test > 1:
+                updated_average = (float(current_average.value) * (number_test - 1) + last_duration) / number_test
+            else:
+                updated_average = last_duration
+
+            current_average.value = max(0, int(updated_average))
             safe_db_commit(g.db, "updating average time")
             log.info(f'average time updated to {str(current_average.value)}')
 

mod_ci/cron.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 """handle cron logic for CI platform."""
 
 import sys