fix: add rate limiting and retry logic for Aurora and OSDG API calls

backend/app.py

@@ -1,4 +1,5 @@
 import os
+import time
 # import uuid
 # import json
 import requests
@@ -172,19 +173,36 @@ def osdg_external_api():
     if not projectDescription:
         return jsonify({'error': 'Project description is required'}), 400
 
-    # Call the external OSDG API
+    max_retries = 3
+    retry_delay = 1  # seconds between retries
+
     try:
-        osdg_response = requests.post(
-            "http://20.73.166.85/label_text",
-            json={
-                "text": projectDescription
-            },
-            headers={
-                "token": os.environ.get("OSDG_TOKEN")  # Ensure you have the OSDG token set in your environment variables
-            },
-            timeout=1000  # Set a timeout for the request
-        )
-        osdg_response.raise_for_status()  # Raise an error for bad status codes
+        for attempt in range(max_retries):
+            osdg_response = requests.post(
+                "http://20.73.166.85/label_text",
+                json={
+                    "text": projectDescription
+                },
+                headers={
+                    "token": os.environ.get("OSDG_TOKEN")
+                },
+                timeout=30
+            )
+
+            if osdg_response.status_code == 429:
+                wait = retry_delay * (2 ** attempt)
+                print(f"OSDG API rate limited (429). Retrying in {wait}s... (attempt {attempt + 1}/{max_retries})")
+                time.sleep(wait)
+                continue
+
+            osdg_response.raise_for_status()
+            break
+        else:
+            return jsonify({
+                "error": "OSDG API rate limit exceeded after 3 retries",
+                "message": "OSDG API classification failed"
+            }), 429
+
         osdg_result = osdg_response.json()
     except requests.exceptions.RequestException as e:
         print(f"OSDG API request failed: {str(e)}")

backend/aurora_api.py

@@ -1,4 +1,5 @@
 import requests
+import time
 import json
 from sdg_constants import SDG_LABELS_DICT as SDG_LABELS
 
@@ -19,9 +20,30 @@ def main(text: str, project_name: str = None, project_url: str = None):
         url = "https://aurora-sdg.labs.vu.nl/classifier/classify/elsevier-sdg-multi"
         payload = json.dumps({"text": text})
         headers = {'Content-Type': 'application/json'}
-        response = requests.request("POST", url, headers=headers, data=payload)
-        # response.raise_for_status()
-
+
+        max_retries = 3
+        retry_delay = 1  # seconds between retries
+
+        for attempt in range(max_retries):
+            response = requests.request("POST", url, headers=headers, data=payload, timeout=30)
+
+            if response.status_code == 429:
+                wait = retry_delay * (2 ** attempt)
+                print(f"Aurora API rate limited (429). Retrying in {wait}s... (attempt {attempt + 1}/{max_retries})")
+                time.sleep(wait)
+                continue
+
+            response.raise_for_status()
+            break
+        else:
+            return {
+                "project_name": project_name or "Unknown",
+                "project_url": project_url or "",
+                "sdg_predictions": {},
+                "error": "Aurora API rate limit exceeded after 3 retries",
+                "message": "Aurora API classification failed"
+            }
+
         raw_result = response.json()
 
         # Transform Aurora API response to match embedding model format