From 52109f4f8f3052280bd32213be4cc596c05df378 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Wed, 9 Jul 2025 21:24:30 +0000
Subject: [PATCH 1/3] Add dynamic model selection to CI workflow

- Add new step to fetch available models from API endpoint
- Parse JSON response to extract first model ID from data array
- Add error handling for empty data arrays
- Replace hardcoded MODEL_NAME with dynamically selected model
- Output chosen model name to console for visibility

Co-Authored-By: Liren <tuliren@gmail.com>
---
 .github/workflows/run-eval.yaml | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-eval.yaml b/.github/workflows/run-eval.yaml
index abce77a..1ff9302 100644
--- a/.github/workflows/run-eval.yaml
+++ b/.github/workflows/run-eval.yaml
@@ -33,9 +33,35 @@ jobs:
         run: |
           bin/api/prepare.sh
 
+      - name: Get Available Model
+        id: get_model
+        env:
+          MODEL_URL: ${{ vars.MODEL_URL }}
+          MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
+        run: |
+          echo "Fetching available models from $MODEL_URL/v1/models"
+          RESPONSE=$(curl -s "$MODEL_URL/v1/models" -H "Authorization: Bearer $MODEL_API_KEY")
+          echo "API Response: $RESPONSE"
+          
+          DATA_LENGTH=$(echo "$RESPONSE" | jq -r '.data | length')
+          if [ "$DATA_LENGTH" = "null" ] || [ "$DATA_LENGTH" = "0" ]; then
+            echo "Error: No models available in the data array"
+            exit 1
+          fi
+          
+          SELECTED_MODEL=$(echo "$RESPONSE" | jq -r '.data[0].id')
+          
+          if [ "$SELECTED_MODEL" = "null" ] || [ -z "$SELECTED_MODEL" ]; then
+            echo "Error: Could not extract model ID from API response"
+            exit 1
+          fi
+          
+          echo "Selected model: $SELECTED_MODEL"
+          echo "model_name=$SELECTED_MODEL" >> $GITHUB_OUTPUT
+
       - name: Run API Evaluation Script
         env:
-          MODEL_NAME: lfm-3b
+          MODEL_NAME: ${{ steps.get_model.outputs.model_name }}
           MODEL_URL: ${{ vars.MODEL_URL }}
           MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
         run: |
@@ -69,7 +95,7 @@ jobs:
 
       - name: Run OpenAI Judge
         env:
-          MODEL_NAME: lfm-3b
+          MODEL_NAME: ${{ steps.get_model.outputs.model_name }}
           MODEL_URL: ${{ vars.MODEL_URL }}
           MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
         run: |

From 53ff5f1b4eece79305f94e5ab2faae931ea00d30 Mon Sep 17 00:00:00 2001
From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Date: Wed, 9 Jul 2025 23:14:14 +0000
Subject: [PATCH 2/3] Fix API endpoint URL construction - remove double /v1

Co-Authored-By: Liren <tuliren@gmail.com>
---
 .github/workflows/run-eval.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-eval.yaml b/.github/workflows/run-eval.yaml
index 1ff9302..1a765cc 100644
--- a/.github/workflows/run-eval.yaml
+++ b/.github/workflows/run-eval.yaml
@@ -39,8 +39,8 @@ jobs:
           MODEL_URL: ${{ vars.MODEL_URL }}
           MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
         run: |
-          echo "Fetching available models from $MODEL_URL/v1/models"
-          RESPONSE=$(curl -s "$MODEL_URL/v1/models" -H "Authorization: Bearer $MODEL_API_KEY")
+          echo "Fetching available models from $MODEL_URL/models"
+          RESPONSE=$(curl -s "$MODEL_URL/models" -H "Authorization: Bearer $MODEL_API_KEY")
           echo "API Response: $RESPONSE"
           
           DATA_LENGTH=$(echo "$RESPONSE" | jq -r '.data | length')

From a600d55c268918bf286acd6ad5c6d66d9682b416 Mon Sep 17 00:00:00 2001
From: Liren Tu <tuliren@gmail.com>
Date: Wed, 9 Jul 2025 16:24:14 -0700
Subject: [PATCH 3/3] Add cron trigger

---
 .github/workflows/run-eval.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/run-eval.yaml b/.github/workflows/run-eval.yaml
index 1a765cc..187c69d 100644
--- a/.github/workflows/run-eval.yaml
+++ b/.github/workflows/run-eval.yaml
@@ -8,6 +8,8 @@ on:
     branches:
       - main
   workflow_dispatch:
+  schedule:
+    - cron: '29 2 * * 1'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref || github.run_id }}