pull_request.yaml

name: Pull Request
on:
  pull_request:
  workflow_dispatch:

defaults:
  run:
    shell: bash

concurrency:
  group: pr-${{ github.ref }}
  cancel-in-progress: true

jobs:
  # This job is a hack because GitHub doesn't support referencing env vars for setting runners or container image
  # See https://github.com/orgs/community/discussions/26324
  vars:
    name: Set common variables
    runs-on: ubuntu-latest
    outputs:
      runner: self-hosted
      container-image: docker-registry-internal.aom-infra.org/aomediacodec/aom-testing/ubuntu2404:20260116205104
      container-image-multilib: docker-registry-internal.aom-infra.org/aomediacodec/aom-testing/ubuntu2404-multilib:20260116205104
      timeout-minutes: 660
    steps:
      - name: Do nothing
        run: |
          echo noop

  style-checks:
    name: Style Checks
    if: (!cancelled())
    needs:
      - vars
    timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
    runs-on: ${{ needs.vars.outputs.runner }}
    container:
      image: ${{ needs.vars.outputs.container-image }}
    env:
      DIFF_REF: ${{ github.event.pull_request.base.sha }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 50
          lfs: true

      - uses: ./.github/actions/common-setup

      - name: Run clang-format
        run: |
          echo "lang-format version: $(clang-format --version)"

          # Run clang-format check.
          for f in $(git diff --diff-filter=ACMR --name-only ${DIFF_REF} '*.[hc]pp' '*.cc' '*.[ch]' | grep -v third_party); do
            exit_code=0
            clang-format -i --style=file $f -n -Werror || exit_code=$?
            if [ ${exit_code} -ne 0 ]; then
              echo -e "\e[31mPlease format your code by following instructions here:\e[0m"
              echo -e "\e[31mhttps://github.com/AOMediaCodec/avm/wiki/Reproducing-CI-Test-Failures-Locally#style-check\e[0m"
              exit 1
            fi
          done

      - name: Run cmake-format
        run: |
          echo "cmake-format version: $(cmake-format --version)"

          # Run cmake-format check.
          for f in $(git diff --diff-filter=ACMR --name-only ${DIFF_REF} '*.cmake' 'CMakeLists.txt' | grep -v third_party); do
            exit_code=0
            cmake-format --check $f || exit_code=$?
            if [ ${exit_code} -ne 0 ]; then
              echo -e "\e[31mPlease format your code by following instructions here:\e[0m"
              echo -e "\e[31mhttps://github.com/AOMediaCodec/avm/wiki/Reproducing-CI-Test-Failures-Locally#style-check\e[0m"
              exit 1
            fi
          done

  pr-checks:
    name: PR Checks
    if: (!cancelled())
    needs:
      - vars
    timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
    runs-on: ${{ needs.vars.outputs.runner }}
    container:
      image: ${{ needs.vars.outputs.container-image }}
    env:
      DIFF_REF: ${{ github.event.pull_request.base.sha }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 50
          lfs: true

      - uses: ./.github/actions/common-setup

      - name: Validate files with the executable bit set
        run: |
          echo lol=${{ github.ref }}
          git diff "${DIFF_REF}" --check

          # Validate files with the executable bit set
          files=$(git diff --name-only "${DIFF_REF}" | tr '\n' ' ')
          git ls-tree -r HEAD $files | while read mode type sha1 file; do
            if [ "$mode" = "100755" ]; then
              case "$file" in
                configure|*.php|*.pl|*.py|*.sh)
                  ;;
                *)
                  echo "File $file should not be executable."
                  echo "Only configure|*.php|*.pl|*.py|*.sh are accepted."
                  exit 1
                  ;;
              esac
            fi
          done

  test-data-checks:
    name: Test Data Checks
    if: (!cancelled())
    needs:
      - vars
    timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
    runs-on: ${{ needs.vars.outputs.runner }}
    container:
      image: ${{ needs.vars.outputs.container-image }}
    env:
      DIFF_REF: ${{ github.event.pull_request.base.sha }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 50
          lfs: true

      - uses: ./.github/actions/common-setup

      - name: Ensure added test files add declared in test data
        run: |
          # TODO: replace this test with a proper job/rule filter
          if [ -z "$(git diff ${DIFF_REF} "test/")" ]; then
            echo "No test file added."
            exit 0
          fi
          ADDED_TEST_FILES="$(git diff ${DIFF_REF} "test/" \
            | perl -ne  '/^\+.*?"((?!\$).*?\.(ivf|webm|res|mkv|y4m|yuv))"/g and print "$1\n"')" \
          printf "Checking for files:\n%s\n\n" "$ADDED_TEST_FILES"
          echo "$ADDED_TEST_FILES" | while read -r f; do
            if [ -n "$(grep -L "${f}"  test/test-data.sha1 test/test_data_util.cmake)" ]; then
              echo "file: ${f} was not found in test-data.sha1 or test_data_util.cmake"
              exit 1
            fi
          done

  previous-build-x86_64-linux-gcc:
    name: Previous Build (x86_64-linux-gcc)
    uses: ./.github/workflows/build-job-reusable.yaml
    needs:
      - style-checks
      - pr-checks
      - test-data-checks
      - vars
    if: (!cancelled())
    with:
      avm-build-config: ${{ matrix.avm-build-config }}
      parent-job-name: previous-build-x86_64-linux-gcc
      runner: '["${{ needs.vars.outputs.runner }}"]'
      container-image: ${{ needs.vars.outputs.container-image }}
      show-github-context: true
      extra-env-vars: |
        DIFF_REF=${{ github.event.pull_request.base.sha }}
        INSTALLROOT_FOLDER=installroot_old
        # Allow build warnings.
        EXTRA_CMAKE_FLAGS=-DENABLE_WERROR=0
      before-script: |
        #TODO: git fetch --unshallow
        git checkout "${DIFF_REF}"
      artifact-paths: /installroot_old
    strategy:
      matrix:
        avm-build-config:
          - encode-only

  test-data:
    name: Test Data
    uses: ./.github/workflows/common-test-data-reusable.yaml
    needs:
      - vars
    if: (!cancelled())
    with:
      runner: '["${{ needs.vars.outputs.runner }}"]'
      container-image: ${{ needs.vars.outputs.container-image }}

  common-builds:
    name: Common Builds
    uses: ./.github/workflows/common-builds-reusable.yaml
    needs:
      - style-checks
      - pr-checks
      - test-data-checks
      - vars
    if: (!cancelled())
    with:
      runner: '["${{ needs.vars.outputs.runner }}"]'
      container-image: ${{ needs.vars.outputs.container-image }}
      container-image-multilib: ${{ needs.vars.outputs.container-image-multilib }}

  example-test-x86_64-gcc:
    name: Example Test (x86_64-gcc)
    needs:
      - common-builds
      - test-data
      - vars
    if: (!cancelled())
    timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
    runs-on: ${{ needs.vars.outputs.runner }}
    container:
      image: ${{ needs.vars.outputs.container-image }}
    env:
      LIBAVM_TEST_DATA_PATH: ${{ github.workspace }}/libavm-test-data
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 0
          fetch-tags: true
          lfs: true

      - uses: ./.github/actions/common-setup

      - name: Get test data
        uses: actions/download-artifact@v8
        with:
          name: test-data
          path: libavm-test-data

      - name: Get example build
        uses: actions/download-artifact@v8
        with:
          name: example-build
          path: avm_example_build

      - name: Fix executable bit permissions
        run: |
          # See https://github.com/actions/upload-artifact/issues/38 for details about why we need this

          # Look for files starting with ELF header and set executable but
          for file in $(find avm_example_build/ -type f); do
            if grep --quiet --binary --text --perl-regexp "^\x7f\x45\x4c\x46" "${file}"; then
              echo "Found binary ${file}, fixing permissions."
              chmod a+x "${file}"
            fi
          done

      - name: Run example test
        run: |
          cpu_cores=$(nproc)
          commands=/tmp/parallel.commands.$$
          ret=0

          printf "" >${commands}

          cd avm_example_build
          for test in $(../test/examples.sh --list-tests); do
            echo "${GITHUB_WORKSPACE}/.github/capture-logs.sh ${GITHUB_WORKSPACE}/example.test.${test}.log sh ../test/examples.sh --bin-path examples --verbose --show-program-output --filter \"\b${test}\b\"" >>${commands}
          done

          cat ${commands}
          time parallel --line-buffer --jobs ${cpu_cores} < ${commands} || ret=$?

          if [ ${ret} -ne 0 ]; then
            echo "Failure running example tests, see logs for details." >&2
            exit 1
          fi

      - name: Upload logs
        uses: actions/upload-artifact@v7
        if: ${{ always() }}
        with:
          name: example-test-logs
          path: example.test.*.log

  linux-sanitizer-test:
    name: Linux Sanitizer Test
    uses: ./.github/workflows/sanitizer-job-reusable.yaml
    needs:
      - common-builds
      - vars
    if: (!cancelled())
    with:
      avm-sanitizer-type: ${{ matrix.avm-sanitizer-type }}
      max-total-shards: ${{ matrix.max-total-shards }}
      runner: '["${{ needs.vars.outputs.runner }}"]'
      container-image: ${{ needs.vars.outputs.container-image }}
      show-github-context: true
    strategy:
      fail-fast: false
      matrix:
        avm-sanitizer-type:
          - address
          - integer
          - thread
          - undefined
          # CFI Sanitizer commented for now, as lto build does not work
          # - cfi
        include:
          # default to unlimited
          - max-total-shards: 0

  encdec-run:
    name: Enc/Dec
    needs:
      - common-builds
      - previous-build-x86_64-linux-gcc
      - vars
    if: (!cancelled())
    timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
    runs-on: ${{ needs.vars.outputs.runner }}
    container:
      image: ${{ needs.vars.outputs.container-image }}
    env:
      AVMENC_OUTPUT: ${{ matrix.avm-enc-output }}
      AVMENC_LIMIT: ${{ matrix.avm-enc-limit }}
      AVMENC_QP: 210
      AVMENC_INPUT: Vertical_Bayshore_270x480_2997.y4m
      AVMENC_COMMON_ARGS: "\
        --debug \
        --cpu-used=0 \
        --passes=1 \
        --deltaq-mode=0 \
        --enable-keyframe-filtering=0 \
        --enable-tpl-model=0 \
        --use-fixed-qp-offsets=1 \
        --end-usage=q \
        --obu \
        --psnr \
      "

    steps:
      - name: Configure AVM encoder
        run: |
          if [ -z "${AVMENC_OUTPUT}" ]; then
            echo "AVMENC_OUTPUT is not set." >&2
            exit 1
          fi

          # common encoder arguments to all outputs
          AVMENC_ARGS="${AVMENC_COMMON_ARGS} \
            --qp=${AVMENC_QP} \
            --limit=${AVMENC_LIMIT} \
          "

          # output-specific arguments
          case ${AVMENC_OUTPUT} in
            all-intra)
              AVMENC_ARGS="${AVMENC_ARGS} \
              --kf-max-dist=0 \
              --kf-min-dist=0 \
              "
              ;;

            random-access)
              AVMENC_ARGS="${AVMENC_ARGS} \
                --auto-alt-ref=1 \
                --gf-max-pyr-height=4 \
                --gf-min-pyr-height=4 \
                --kf-max-dist=65 \
                --kf-min-dist=65 \
                --lag-in-frames=19 \
                --max-gf-interval=16 \
                --min-gf-interval=16 \
              "
              ;;

            low-delay)
              AVMENC_ARGS="${AVMENC_ARGS} \
                --gf-max-pyr-height=4 \
                --gf-min-pyr-height=4 \
                --kf-max-dist=9999 \
                --kf-min-dist=9999 \
                --lag-in-frames=0 \
                --subgop-config-str=ld \
              "
              ;;

            *)
              echo "Unknown encoder output: '${AVMENC_OUTPUT}'" >&2
              exit 1
              ;;
          esac

          # Append to workflow environment
          echo "AVMENC_ARGS=${AVMENC_ARGS}" >>${GITHUB_ENV}

      - name: Pull input file
        run: |
          curl -s -S -f -O https://storage.googleapis.com/aom-test-data/${AVMENC_INPUT}.xz
          unxz ${AVMENC_INPUT}.xz

      - name: Pull previous encoder build
        uses: actions/download-artifact@v8
        with:
          name: previous-build-x86_64-linux-gcc-encode-only
          path: enc-previous

      - name: Pull current encoder build
        uses: actions/download-artifact@v8
        with:
          name: example-build
          path: enc-current

      - name: Fix executable bit permissions
        run: |
          # See https://github.com/actions/upload-artifact/issues/38 for details about why we need this

          # Look for files starting with ELF header and set executable but
          for file in $(find . -type f); do
            if grep --quiet --binary --text --perl-regexp "^\x7f\x45\x4c\x46" "${file}"; then
              echo "Found binary ${file}, fixing permissions."
              chmod a+x "${file}"
            fi
          done

      - name: Encode
        run: |
          printf -- "\n---------------------------------------------------\n"
          echo "Encoder input:  ${AVMENC_INPUT}"
          echo "Encoder output: ${AVMENC_OUTPUT}"
          echo "Encoder args:   ${AVMENC_ARGS}"
          printf -- "---------------------------------------------------\n\n"

          commands=/tmp/parallel.commands.$$
          cat >${commands} <<EOF
          enc-previous/usr/local/bin/avmenc ${AVMENC_ARGS} --output=${AVMENC_OUTPUT}.previous.obu ${AVMENC_INPUT} 2>&1 | tee ${AVMENC_OUTPUT}.previous.psnr.log
          enc-current/avmenc                ${AVMENC_ARGS} --output=${AVMENC_OUTPUT}.current.obu  ${AVMENC_INPUT} 2>&1 | tee ${AVMENC_OUTPUT}.current.psnr.log
          EOF

          time parallel --line-buffer < ${commands} || true

          for obu_file in ${AVMENC_OUTPUT}.previous.obu ${AVMENC_OUTPUT}.current.obu; do
            if [ ! -f ${obu_file} ]; then
              echo "Cannot find OBU file ${obu_file}" >&2
              exit 1
            fi

            md5sum --binary ${obu_file} | awk '{ print $1; }' > ${obu_file}.md5
          done

      - name: Decode
        run: |
          enc-current/avmdec \
            ${AVMENC_OUTPUT}.current.obu \
            --output=${AVMDEC_OUTPUT}.current.decoded.y4m \
            --summary \
            2>&1 | tee ${AVMDEC_OUTPUT}.current.summary.log

          if [ ! -f ${AVMDEC_OUTPUT}.current.decoded.y4m ]; then
            echo "Cannot find decoded Y4M file" >&2
            exit 1
          fi

          if [ ! -f ${AVMDEC_OUTPUT}.current.summary.log ]; then
            echo "Cannot find decoder summary log file" >&2
            exit 1
          fi

          for str in 'decoded frames' 'showed frames'; do
            frame_count=$(grep -E -o "[0-9]+ ${str}" "${AVMDEC_OUTPUT}.current.summary.log" | sed -E "s/([0-9]+) ${str}/\1/g")
            echo "${str} = ${frame_count}"
            if [ ${frame_count} -ne ${AVMENC_LIMIT} ]; then
              echo "ERROR: Unexpected number of ${str}. Got ${frame_count}, expected ${AVMENC_LIMIT}" >&2
              exit 1
            fi
          done

      - name: Upload artifacts
        uses: actions/upload-artifact@v7
        if: ${{ always() }}
        with:
          name: encdec-${{ matrix.avm-enc-output }}
          path: ${{ matrix.avm-enc-output }}.*

      - name: Compare hashes
        run: |
          ALLOW_FAIL=0
          FAILED=0

          if [ -n "$(git --no-pager log --grep STATS_CHANGED --format=format:"%H" "${{ github.event.pull_request.base.sha }}..${{ github.sha }}")" ]; then
            ALLOW_FAIL=1
          fi

          variant=${AVMDEC_OUTPUT}

          if diff -- "${AVMENC_OUTPUT}.previous.obu.md5" "${AVMENC_OUTPUT}.current.obu.md5" &>/dev/null; then
            echo "OK: No differences in ${AVMENC_OUTPUT} outputs found."
            echo ""
          else
            echo "WARNING: ${AVMENC_OUTPUT}.previous.obu.md5 and ${AVMENC_OUTPUT}.current.obu.md5 differ!"
            echo "${AVMENC_OUTPUT}.previous.obu.md5 : $(cat ${AVMENC_OUTPUT}.previous.obu.md5)"
            echo "${AVMENC_OUTPUT}.current.obu.md5  : $(cat ${AVMENC_OUTPUT}.current.obu.md5)"
            echo ""
            FAILED=1
          fi

          if [ ${FAILED} -eq 1 ]; then
            if [ ${ALLOW_FAIL} -eq 1 ]; then
              echo "SUCCESS: Even though a mismatch was detected, one or more"
              echo "         commits contain the STATS_CHANGED keyword, so mismatch"
              echo "         of encode outputs is allowed."
              echo "Commits containing STATS_CHANGED added in this MR:"
              git --no-pager log --grep STATS_CHANGED --format=format:"%h: %s; \"%aN\" <%aE>" "${{ github.event.pull_request.base.sha }}..${{ github.sha }}"
              echo ""
              exit 0
            fi

            echo "FAIL: Failing job due to mismatch and no STATS_CHANGED keyword"
            echo "      was found in any of the commits added in this MR!"
            exit 1
          fi

    strategy:
      fail-fast: false
      matrix:
        avm-enc-output:
          - all-intra
          - random-access
          - low-delay
        include:
          - avm-enc-limit: 30
          - avm-enc-limit: 15
            avm-enc-output: all-intra

  static-analyzer:
    name: Static Analyzer
    needs:
      - vars
    if: (!cancelled())
    timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
    runs-on: ${{ needs.vars.outputs.runner }}
    container:
      image: ${{ needs.vars.outputs.container-image }}
    env:
      ANALYZER_MODE: ${{ matrix.analyzer-mode }}
    steps:
      - uses: actions/checkout@v5
        with:
          fetch-depth: 50
          fetch-tags: true
          lfs: true

      - uses: ./.github/actions/common-setup

      - name: Analyze
        run: |
          mkdir analyzer-${ANALYZER_MODE}

          scan_build() {
            scan-build \
              --exclude third_party \
              --exclude _deps \
              --exclude abseil-cpp \
              --exclude benchmark \
              --exclude cpuinfo \
              --exclude eigen \
              --exclude farmhash \
              --exclude fft2d \
              --exclude flatbuffers \
              --exclude flatbuffers-flatc \
              --exclude fp16 \
              --exclude FP16 \
              --exclude FXdiv \
              --exclude psimd \
              --exclude ml_dtypes \
              --exclude neon2sse \
              --exclude protobuf \
              --exclude pthreadpool \
              --exclude pthreadpool-source \
              --exclude ruy \
              --exclude xnnpack \
              --exclude XNNPACK \
              --exclude googletest \
              -o analyzer-${ANALYZER_MODE} \
              -analyzer-config mode=${ANALYZER_MODE} \
              $*
          }

          scan_build cmake -B avm_build -GNinja -DCMAKE_BUILD_TYPE=Debug
          exit_code=0
          scan_build --status-bugs cmake --build avm_build || exit_code=$?
          if [ ${exit_code} -ne 0 ]; then
            echo -e "\e[31mMore info on static analysis can be found at the following this wiki page:\e[0m"
            echo -e "\e[31mhttps://github.com/AOMediaCodec/avm/wiki/Reproducing-CI-Test-Failures-Locally#static-analysis\e[0m"
            exit 1
          fi

      - name: Upload artifacts
        uses: actions/upload-artifact@v7
        if: ${{ failure() }}
        with:
          name: analyzer-${{ matrix.analyzer-mode }}
          path: analyzer-${{ matrix.analyzer-mode }}
          retention-days: 14

    strategy:
      fail-fast: false
      matrix:
        analyzer-mode:
          - deep
          - shallow

  so-checks:
    name: SO checks
    needs:
      - common-builds
      - vars
    if: (!cancelled())
    timeout-minutes: ${{ fromJson(needs.vars.outputs.timeout-minutes) }}
    runs-on: ${{ needs.vars.outputs.runner }}
    container:
      image: ${{ needs.vars.outputs.container-image }}
    steps:
      - name: Pull build
        uses: actions/download-artifact@v8
        with:
          name: build-x86_64-linux-gcc-shared

      - name: Check SO
        run: |
          for so in $(find -name '*.so*'); do
            echo "Checking ${so} for textrels..."
            if readelf -dW ${so} | grep TEXTREL; then
              textrels=$(eu-findtextrel ${so} | sort | uniq)
              if [ -n "${textrels}" ]; then
                echo "${textrels}"
                exit 1
              fi
            fi

            echo "Checking ${so} ELF header/section headers..."
            if readelf -lW ${so} | grep WE; then
              echo "Invalid ELF header/section headers"
              echo "https://android.googlesource.com/platform/bionic/+/master/android-changes-for-ndk-developers.md#Writable-and-Executable-Segments-Enforced-for-API-level-26"
              exit 1
            fi
          done