DRNavigator/.github/workflows/benchmark.yaml at main · Netcracker/DRNavigator · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
name: Benchmark check

on:
  workflow_dispatch:
    inputs:
      current_sm_version:
        description: 'SM version, that should be benchmarked'
        required: true
        type: 'string'
        default: '0.20.0'
      previous_sm_version:
        description: 'SM version, that should be used to benchmark with current one'
        required: true
        type: 'string'
        default: '0.19.0'

jobs:
  measure-time:
    runs-on: ubuntu-22.04
    strategy:
      fail-fast: false
      matrix:
        procedure: ["move", "stop"]
        sm-version: ["${{ inputs.current_sm_version }}", "${{ inputs.previous_sm_version }}"]
    env:
      benchmark_dir: ./benchmark
    steps:
      - name: Checkout Repo
        uses: actions/checkout@v4
      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Docker login
        run: echo ${{secrets.GITHUB_TOKEN}} | docker login https://ghcr.io -u ${GITHUB_ACTOR} --password-stdin
      - name: Pull and tag SM images
        run: |
          docker pull ghcr.io/netcracker/sm-dummy:${{ matrix.sm-version }}
          docker pull ghcr.io/netcracker/site-manager:${{ matrix.sm-version }}
          docker tag ghcr.io/netcracker/sm-dummy:${{ matrix.sm-version }} sm-dummy
          docker tag ghcr.io/netcracker/site-manager:${{ matrix.sm-version }} site-manager
      - name: Copy benchmark files to not git directory
        run: cp -r ./ci/benchmark ${{ env.benchmark_dir }}
      - name: Checkout Repo
        uses: actions/checkout@v4
        with:
          ref: ${{ matrix.sm-version }}
          clean: false
      - name: Run docker compose
        run: docker compose -f ${{ env.benchmark_dir }}/docker-compose.yaml up --detach
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements-sc.txt
      - name: Measure the time
        run: ${{ env.benchmark_dir }}/measure_time.sh ${{ matrix.procedure }} ${{ matrix.sm-version }}
      - name: Collect artifacts reports
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-${{ matrix.procedure }}-${{ matrix.sm-version }}
          path: report-${{ matrix.procedure }}-${{ matrix.sm-version }}.json
          retention-days: 1
      - name: Collect logs from docker-compose
        if: ${{ !cancelled() }}
        run: docker compose -f ${{ env.benchmark_dir }}/docker-compose.yaml logs > docker-compose.log
      - name: Down docker compose
        if: ${{ !cancelled() }}
        run: docker compose -f ${{ env.benchmark_dir }}/docker-compose.yaml down
      - name: Collect artifacts logs
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: docker-compose-logs-${{ matrix.procedure }}-${{ matrix.sm-version }}
          path: docker-compose.log
          retention-days: 7

  analyze-results:
    runs-on: ubuntu-22.04
    needs: [ measure-time ]
    strategy:
      fail-fast: false
      matrix:
        procedure: ["move", "stop"]
    steps:
    - name: Download benchmark reports
      uses: actions/download-artifact@v4
      with:
        path: benchmark
        pattern: benchmark-${{ matrix.procedure }}-*
        merge-multiple: true
    - name: Check downloaded reports
      run: ls -R benchmark
    - name: Compare performance
      run: |
        dir='./benchmark'
        previous_run_seconds=$(jq '.nanoseconds' $dir/report-${{ matrix.procedure }}-${{ inputs.previous_sm_version }}.json)
        if [ $? -ne 0 ]; then
          echo "Something went wrong during parsing report for procedure ${{ matrix.procedure }} version ${{ inputs.previous_sm_version }}"
          exit 1
        fi
        current_run_seconds=$(jq '.nanoseconds' $dir/report-${{ matrix.procedure }}-${{ inputs.current_sm_version }}.json)
        if [ $? -ne 0 ]; then
          echo "Something went wrong during parsing report for procedure ${{ matrix.procedure }} version ${{ inputs.current_sm_version }}"
          exit 1
        fi

        echo "Average time for procedure ${{ matrix.procedure }} for version ${{ inputs.previous_sm_version }}: $previous_run_seconds nanoseconds"
        echo "Average time for procedure ${{ matrix.procedure }} for version ${{ inputs.current_sm_version }}: $current_run_seconds nanoseconds"
        score=$(bc <<< "scale=2; $current_run_seconds * 100 /$previous_run_seconds")
        echo "Average score: $score%"
        delta=$(bc <<< "$score - 100")
        if (( $(echo "$delta > 5" |bc -l) )); then
          echo "Delta is too big: $delta%"
          exit 1
        fi