Github Action to automatically update embeddings

.github/workflows/update_embeddings.yml

@@ -0,0 +1,50 @@
+name: Update Embeddings
+
+on:
+  pull_request:
+    paths:
+      - prompt-sentences-main/prompt_sentences.json
+  workflow_dispatch: # To allow manual trigger of this workflow
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  regenerate:
+    if: github.actor != 'ci-bot'
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.head_ref }}
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ runner.os }}-${{ hashFiles('requirements.txt') }}
+
+      - run: pip install -r requirements.txt
+
+      # Github actions can run for 6 hours maximum
+      # For small updates to the prompts file, this should be plenty
+      - name: Run generator
+        run: python customize/customize_embeddings.py
+
+      - name: Commit if changed
+        run: |
+          git status
+          if git diff --quiet; then
+            echo "No changes"
+          else
+            git config user.name "ci-bot"
+            git config user.email "ci-bot@users.noreply.github.com"
+            git add -u
+            git commit -m "chore: update embeddings" -s
+            git push
+          fi

customize/customize_embeddings.py

@@ -33,7 +33,7 @@
 
 # Sentence transformer model HF
 model_path = 'models/all-MiniLM-L6-v2'
-model_id = model_path.split("/")[1]
+model_id = model_path.split("/")[1].lower()
 
 # INPUT FILE
 # Default file with empty embeddings
@@ -43,13 +43,15 @@
 # OUTPUT FILE
 json_out_file_name = f'{json_in_file_name}-{model_id}.json'
 
+print(f'Loading existing data')
+existing_data = None
 # check if the output file already exists
 if os.path.exists(json_out_file_name):
     try:
         # Load existing data from the output file
         existing_data = customize_helper.load_json(json_out_file_name)
     except Exception as e:
-        existing_data = None
+        print(f"Error loading existing data: {e}")
 
 # hashmap 
 prompts_embeddings = {}
@@ -62,8 +64,10 @@
             prompts_embeddings[p["text"]] = p["embedding"]
 
 
-
+print("Populating embeddings and centroids")
 prompt_json = json.load(open(json_in_file))
 prompt_json_embeddings = customize_helper.populate_embeddings(prompt_json, model_path, prompts_embeddings)
 prompt_json_centroids = customize_helper.populate_centroids(prompt_json_embeddings)
+
+print("Saving the embeddings")
 customize_helper.save_json(prompt_json_centroids, json_out_file_name)